View Javadoc
1   /*
2    * This file is part of dependency-check-core.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   *
16   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
17   */
18  package org.owasp.dependencycheck.analyzer;
19  
20  import java.io.BufferedReader;
21  import java.io.IOException;
22  import java.io.InputStreamReader;
23  import java.nio.charset.StandardCharsets;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collections;
27  import java.util.Comparator;
28  import java.util.HashMap;
29  import java.util.HashSet;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.Objects;
33  import java.util.Set;
34  import java.util.concurrent.TimeUnit;
35  import java.util.stream.Collectors;
36  import javax.annotation.concurrent.ThreadSafe;
37  import org.apache.commons.lang3.StringUtils;
38  import org.apache.commons.lang3.builder.CompareToBuilder;
39  import org.apache.commons.lang3.builder.EqualsBuilder;
40  import org.apache.commons.lang3.builder.HashCodeBuilder;
41  import org.apache.commons.lang3.mutable.MutableInt;
42  import org.apache.lucene.analysis.CharArraySet;
43  import org.apache.lucene.document.Document;
44  import org.apache.lucene.index.CorruptIndexException;
45  import org.apache.lucene.queryparser.classic.ParseException;
46  import org.apache.lucene.search.Query;
47  import org.apache.lucene.search.ScoreDoc;
48  import org.apache.lucene.search.TopDocs;
49  import org.jetbrains.annotations.NotNull;
50  import org.jetbrains.annotations.Nullable;
51  import org.owasp.dependencycheck.Engine;
52  import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
53  import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
54  import org.owasp.dependencycheck.data.cpe.Fields;
55  import org.owasp.dependencycheck.data.cpe.IndexEntry;
56  import org.owasp.dependencycheck.data.cpe.IndexException;
57  import org.owasp.dependencycheck.data.cpe.MemoryIndex;
58  import org.owasp.dependencycheck.data.lucene.LuceneUtils;
59  import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
60  import org.owasp.dependencycheck.data.nvd.ecosystem.Ecosystem;
61  import org.owasp.dependencycheck.data.nvdcve.CveDB;
62  import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
63  import org.owasp.dependencycheck.data.update.cpe.CpePlus;
64  import org.owasp.dependencycheck.dependency.Confidence;
65  import org.owasp.dependencycheck.dependency.Dependency;
66  import org.owasp.dependencycheck.dependency.Evidence;
67  import org.owasp.dependencycheck.dependency.EvidenceType;
68  import org.owasp.dependencycheck.dependency.naming.CpeIdentifier;
69  import org.owasp.dependencycheck.dependency.naming.Identifier;
70  import org.owasp.dependencycheck.dependency.naming.PurlIdentifier;
71  import org.owasp.dependencycheck.exception.InitializationException;
72  import org.owasp.dependencycheck.utils.DependencyVersion;
73  import org.owasp.dependencycheck.utils.DependencyVersionUtil;
74  import org.owasp.dependencycheck.utils.Settings;
75  import org.slf4j.Logger;
76  import org.slf4j.LoggerFactory;
77  import us.springett.parsers.cpe.Cpe;
78  import us.springett.parsers.cpe.CpeBuilder;
79  import us.springett.parsers.cpe.exceptions.CpeValidationException;
80  import us.springett.parsers.cpe.values.Part;
81  
82  /**
83   * CPEAnalyzer is a utility class that takes a project dependency and attempts
84   * to discern if there is an associated CPE. It uses the evidence contained
85   * within the dependency to search the Lucene index.
86   *
87   * @author Jeremy Long
88   */
89  @ThreadSafe
90  public class CPEAnalyzer extends AbstractAnalyzer {
91  
92      /**
93       * The Logger.
94       */
95      private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
96      /**
97       * The weighting boost to give terms when constructing the Lucene query.
98       */
99      private static final int WEIGHTING_BOOST = 1;
100     /**
101      * A string representation of a regular expression defining characters
102      * utilized within the CPE Names. Note, the :/ are included so URLs are
103      * passed into the Lucene query so that the specialized tokenizer can parse
104      * them.
105      */
106     private static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._:/-]";
107     /**
108      * A string representation of a regular expression used to remove all but
109      * alpha characters.
110      */
111     private static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
112 
113     /**
114      * The CPE in memory index.
115      */
116     private MemoryIndex cpe;
117     /**
118      * The CVE Database.
119      */
120     private CveDB cve;
121     /**
122      * A reference to the ODC engine.
123      */
124     private Engine engine;
125     /**
126      * The list of ecosystems to skip during analysis. These are skipped because
127      * there is generally a more accurate vulnerability analyzer in the
128      * pipeline.
129      */
130     private List<String> skipEcosystems;
131     /**
132      * A reference to the ecosystem object; used to obtain the max query results
133      * for each ecosystem.
134      */
135     private Ecosystem ecosystemTools;
136     /**
137      * A reference to the suppression analyzer; for timing reasons we need to
138      * test for suppressions immediately after identifying the match because a
139      * higher confidence match on a FP can mask a lower confidence, yet valid
140      * match.
141      */
142     private CpeSuppressionAnalyzer suppression;
143 
144     /**
145      * Returns the name of this analyzer.
146      *
147      * @return the name of this analyzer.
148      */
149     @Override
150     public String getName() {
151         return "CPE Analyzer";
152     }
153 
154     /**
155      * Returns the analysis phase that this analyzer should run in.
156      *
157      * @return the analysis phase that this analyzer should run in.
158      */
159     @Override
160     public AnalysisPhase getAnalysisPhase() {
161         return AnalysisPhase.IDENTIFIER_ANALYSIS;
162     }
163 
164     /**
165      * Creates the CPE Lucene Index.
166      *
167      * @param engine a reference to the dependency-check engine
168      * @throws InitializationException is thrown if there is an issue opening
169      * the index.
170      */
171     @Override
172     public void prepareAnalyzer(Engine engine) throws InitializationException {
173         super.prepareAnalyzer(engine);
174         this.engine = engine;
175         try {
176             this.open(engine.getDatabase());
177         } catch (IOException ex) {
178             LOGGER.debug("Exception initializing the Lucene Index", ex);
179             throw new InitializationException("An exception occurred initializing the Lucene Index", ex);
180         } catch (DatabaseException ex) {
181             LOGGER.debug("Exception accessing the database", ex);
182             throw new InitializationException("An exception occurred accessing the database", ex);
183         }
184         final String[] tmp = engine.getSettings().getArray(Settings.KEYS.ECOSYSTEM_SKIP_CPEANALYZER);
185         if (tmp == null) {
186             skipEcosystems = new ArrayList<>();
187         } else {
188             LOGGER.debug("Skipping CPE Analysis for {}", StringUtils.join(tmp, ","));
189             skipEcosystems = Arrays.asList(tmp);
190         }
191         ecosystemTools = new Ecosystem(engine.getSettings());
192         suppression = new CpeSuppressionAnalyzer();
193         suppression.initialize(engine.getSettings());
194         suppression.prepareAnalyzer(engine);
195     }
196 
197     /**
198      * Opens the data source.
199      *
200      * @param cve a reference to the NVD CVE database
201      * @throws IOException when the Lucene directory to be queried does not
202      * exist or is corrupt.
203      * @throws DatabaseException when the database throws an exception. This
204      * usually occurs when the database is in use by another process.
205      */
206     public void open(CveDB cve) throws IOException, DatabaseException {
207         this.cve = cve;
208         this.cpe = CpeMemoryIndex.getInstance();
209         try {
210             final long creationStart = System.currentTimeMillis();
211             cpe.open(cve.getVendorProductList(), this.getSettings());
212             final long creationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - creationStart);
213             LOGGER.info("Created CPE Index ({} seconds)", creationSeconds);
214         } catch (IndexException ex) {
215             LOGGER.debug("IndexException", ex);
216             throw new DatabaseException(ex);
217         }
218     }
219 
220     /**
221      * Closes the data sources.
222      */
223     @Override
224     public void closeAnalyzer() {
225         if (cpe != null) {
226             cpe.close();
227             cpe = null;
228         }
229     }
230 
231     /**
232      * Searches the data store of CPE entries, trying to identify the CPE for
233      * the given dependency based on the evidence contained within. The
234      * dependency passed in is updated with any identified CPE values.
235      *
236      * @param dependency the dependency to search for CPE entries on
237      * @throws CorruptIndexException is thrown when the Lucene index is corrupt
238      * @throws IOException is thrown when an IOException occurs
239      * @throws ParseException is thrown when the Lucene query cannot be parsed
240      * @throws AnalysisException thrown if the suppression rules failed
241      */
242     protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException, AnalysisException {
243         boolean identifierAdded;
244 
245         final Set<String> majorVersions = dependency.getSoftwareIdentifiers()
246                 .stream()
247                 .filter(i -> i instanceof PurlIdentifier)
248                 .map(i -> {
249                     final PurlIdentifier p = (PurlIdentifier) i;
250                     final DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false);
251                     if (depVersion != null) {
252                         return depVersion.getVersionParts().get(0);
253                     }
254                     return null;
255                 }).collect(Collectors.toSet());
256 
257         final Map<String, MutableInt> vendors = new HashMap<>();
258         final Map<String, MutableInt> products = new HashMap<>();
259         final Set<Integer> previouslyFound = new HashSet<>();
260 
261         for (Confidence confidence : Confidence.values()) {
262             collectTerms(vendors, dependency.getIterator(EvidenceType.VENDOR, confidence));
263             LOGGER.trace("vendor search: {}", vendors);
264             collectTerms(products, dependency.getIterator(EvidenceType.PRODUCT, confidence));
265             addMajorVersionToTerms(majorVersions, products);
266             LOGGER.trace("product search: {}", products);
267             if (!vendors.isEmpty() && !products.isEmpty()) {
268                 final List<IndexEntry> entries = searchCPE(vendors, products,
269                         dependency.getVendorWeightings(), dependency.getProductWeightings(),
270                         dependency.getEcosystem());
271                 if (entries == null) {
272                     continue;
273                 }
274 
275                 identifierAdded = false;
276                 for (IndexEntry e : entries) {
277                     if (previouslyFound.contains(e.getDocumentId()) /*|| (filter > 0 && e.getSearchScore() < filter)*/) {
278                         continue;
279                     }
280                     previouslyFound.add(e.getDocumentId());
281                     if (verifyEntry(e, dependency, majorVersions)) {
282                         final String vendor = e.getVendor();
283                         final String product = e.getProduct();
284                         LOGGER.trace("identified vendor/product: {}/{}", vendor, product);
285                         identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
286                     }
287                 }
288                 if (identifierAdded) {
289                     break;
290                 }
291             }
292         }
293     }
294 
295     /**
296      * <p>
297      * Returns the text created by concatenating the text and the values from
298      * the EvidenceCollection (filtered for a specific confidence). This
299      * attempts to prevent duplicate terms from being added.</p>
300      * <p>
301      * Note, if the evidence is longer then 1000 characters it will be
302      * truncated.</p>
303      *
304      * @param terms the collection of terms
305      * @param evidence an iterable set of evidence to concatenate
306      */
307     @SuppressWarnings("null")
308 
309     protected void collectTerms(Map<String, MutableInt> terms, Iterable<Evidence> evidence) {
310         for (Evidence e : evidence) {
311             String value = cleanseText(e.getValue());
312             if (StringUtils.isBlank(value)) {
313                 continue;
314             }
315             if (value.length() > 1000) {
316                 boolean trimmed = false;
317                 int pos = value.lastIndexOf(" ", 1000);
318                 if (pos > 0) {
319                     value = value.substring(0, pos);
320                     trimmed = true;
321                 } else {
322                     pos = value.lastIndexOf(".", 1000);
323                 }
324                 if (!trimmed) {
325                     if (pos > 0) {
326                         value = value.substring(0, pos);
327                         trimmed = true;
328                     } else {
329                         pos = value.lastIndexOf("-", 1000);
330                     }
331                 }
332                 if (!trimmed) {
333                     if (pos > 0) {
334                         value = value.substring(0, pos);
335                         trimmed = true;
336                     } else {
337                         pos = value.lastIndexOf("_", 1000);
338                     }
339                 }
340                 if (!trimmed) {
341                     if (pos > 0) {
342                         value = value.substring(0, pos);
343                         trimmed = true;
344                     } else {
345                         pos = value.lastIndexOf("/", 1000);
346                     }
347                 }
348                 if (!trimmed && pos > 0) {
349                     value = value.substring(0, pos);
350                     trimmed = true;
351                 }
352                 if (!trimmed) {
353                     value = value.substring(0, 1000);
354                 }
355             }
356             addTerm(terms, value);
357         }
358     }
359 
360     private void addMajorVersionToTerms(Set<String> majorVersions, Map<String, MutableInt> products) {
361         final Map<String, MutableInt> temp = new HashMap<>();
362         products.entrySet().stream()
363                 .filter(term -> term.getKey() != null)
364                 .forEach(term -> majorVersions.stream()
365                 .filter(version -> version != null
366                 && (!term.getKey().endsWith(version)
367                 && !Character.isDigit(term.getKey().charAt(term.getKey().length() - 1))
368                 && !products.containsKey(term.getKey() + version)))
369                 .forEach(version -> {
370                     addTerm(temp, term.getKey() + version);
371                 }));
372         products.entrySet().stream()
373                 .filter(term -> term.getKey() != null)
374                 .forEach(term -> majorVersions.stream()
375                 .filter(Objects::nonNull)
376                 .map(version -> "v" + version)
377                 .filter(version -> (!term.getKey().endsWith(version)
378                 && !Character.isDigit(term.getKey().charAt(term.getKey().length() - 1))
379                 && !products.containsKey(term.getKey() + version)))
380                 .forEach(version -> {
381                     addTerm(temp, term.getKey() + version);
382                 }));
383         products.putAll(temp);
384     }
385 
386     /**
387      * Adds a term to the map of terms.
388      *
389      * @param terms the map of terms
390      * @param value the value of the term to add
391      */
392     private void addTerm(Map<String, MutableInt> terms, String value) {
393         final MutableInt count = terms.get(value);
394         if (count == null) {
395             terms.put(value, new MutableInt(1));
396         } else {
397             count.add(1);
398         }
399     }
400 
401     /**
402      * <p>
403      * Searches the Lucene CPE index to identify possible CPE entries associated
404      * with the supplied vendor, product, and version.</p>
405      *
406      * <p>
407      * If either the vendorWeightings or productWeightings lists have been
408      * populated this data is used to add weighting factors to the search.</p>
409      *
410      * @param vendor the text used to search the vendor field
411      * @param product the text used to search the product field
412      * @param vendorWeightings a list of strings to use to add weighting factors
413      * to the vendor field
414      * @param productWeightings Adds a list of strings that will be used to add
415      * weighting factors to the product search
416      * @param ecosystem the dependency's ecosystem
417      * @return a list of possible CPE values
418      */
419     protected List<IndexEntry> searchCPE(Map<String, MutableInt> vendor, Map<String, MutableInt> product,
420             Set<String> vendorWeightings, Set<String> productWeightings, String ecosystem) {
421 
422         final int maxQueryResults = ecosystemTools.getLuceneMaxQueryLimitFor(ecosystem);
423         final List<IndexEntry> ret = new ArrayList<>(maxQueryResults);
424 
425         final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
426         if (searchString == null) {
427             return ret;
428         }
429         try {
430             final Query query = cpe.parseQuery(searchString);
431             final TopDocs docs = cpe.search(query, maxQueryResults);
432 
433             for (ScoreDoc d : docs.scoreDocs) {
434                 //if (d.score >= minLuceneScore) {
435                 final Document doc = cpe.getDocument(d.doc);
436                 final IndexEntry entry = new IndexEntry();
437                 entry.setDocumentId(d.doc);
438                 entry.setVendor(doc.get(Fields.VENDOR));
439                 entry.setProduct(doc.get(Fields.PRODUCT));
440                 entry.setSearchScore(d.score);
441 
442 //                LOGGER.error("Explanation: ---------------------");
443 //                LOGGER.error("Explanation: " + entry.getVendor() + " " + entry.getProduct() + " " + entry.getSearchScore());
444 //                LOGGER.error("Explanation: " + searchString);
445 //                LOGGER.error("Explanation: " + cpe.explain(query, d.doc));
446                 if (!ret.contains(entry)) {
447                     ret.add(entry);
448                 }
449                 //}
450             }
451             return ret;
452         } catch (ParseException ex) {
453             LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
454             LOGGER.info("Unable to parse: {}", searchString, ex);
455         } catch (IndexException ex) {
456             LOGGER.warn("An error occurred resetting the CPE index searcher. See the log for more details.");
457             LOGGER.info("Unable to reset the search analyzer", ex);
458         } catch (IOException ex) {
459             LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
460             LOGGER.info("IO Error with search string: {}", searchString, ex);
461         }
462         return null;
463     }
464 
465     /**
466      * <p>
467      * Builds a Lucene search string by properly escaping data and constructing
468      * a valid search query.</p>
469      *
470      * <p>
471      * If either the possibleVendor or possibleProducts lists have been
472      * populated this data is used to add weighting factors to the search string
473      * generated.</p>
474      *
475      * @param vendor text to search the vendor field
476      * @param product text to search the product field
477      * @param vendorWeighting a list of strings to apply to the vendor to boost
478      * the terms weight
479      * @param productWeightings a list of strings to apply to the product to
480      * boost the terms weight
481      * @return the Lucene query
482      */
483     protected String buildSearch(Map<String, MutableInt> vendor, Map<String, MutableInt> product,
484             Set<String> vendorWeighting, Set<String> productWeightings) {
485 
486         final StringBuilder sb = new StringBuilder();
487 
488         if (!appendWeightedSearch(sb, Fields.PRODUCT, product, productWeightings)) {
489             return null;
490         }
491         sb.append(" AND ");
492         if (!appendWeightedSearch(sb, Fields.VENDOR, vendor, vendorWeighting)) {
493             return null;
494         }
495         return sb.toString();
496     }
497 
498     /**
499      * This method constructs a Lucene query for a given field. The searchText
500      * is split into separate words and if the word is within the list of
501      * weighted words then an additional weighting is applied to the term as it
502      * is appended into the query.
503      *
504      * @param sb a StringBuilder that the query text will be appended to.
505      * @param field the field within the Lucene index that the query is
506      * searching.
507      * @param terms text used to construct the query.
508      * @param weightedText a list of terms that will be considered higher
509      * importance when searching.
510      * @return if the append was successful.
511      */
512     @SuppressWarnings("StringSplitter")
513     private boolean appendWeightedSearch(StringBuilder sb, String field, Map<String, MutableInt> terms, Set<String> weightedText) {
514         if (terms.isEmpty()) {
515             return false;
516         }
517         sb.append(field).append(":(");
518         boolean addSpace = false;
519         boolean addedTerm = false;
520 
521         for (Map.Entry<String, MutableInt> entry : terms.entrySet()) {
522             final StringBuilder boostedTerms = new StringBuilder();
523             final int weighting = entry.getValue().intValue();
524             final String[] text = entry.getKey().split(" ");
525             for (String word : text) {
526                 if (word.isEmpty()) {
527                     continue;
528                 }
529                 if (addSpace) {
530                     sb.append(" ");
531                 } else {
532                     addSpace = true;
533                 }
534                 addedTerm = true;
535                 if (LuceneUtils.isKeyword(word)) {
536                     sb.append("\"");
537                     LuceneUtils.appendEscapedLuceneQuery(sb, word);
538                     sb.append("\"");
539                 } else {
540                     LuceneUtils.appendEscapedLuceneQuery(sb, word);
541                 }
542                 final String boostTerm = findBoostTerm(word, weightedText);
543 
544                 //The weighting is on a full phrase rather then at a term level for vendor or products
545                 //TODO - should the weighting be at a "word" level as opposed to phrase level? Or combined word and phrase?
546                 //remember the reason we are counting the frequency of "phrases" as opposed to terms is that
547                 //we need to keep the correct sequence of terms from the evidence so the term concatenating analyzer
548                 //works correctly and will causes searches to take spring framework and produce: spring springframework framework
549                 if (boostTerm != null) {
550                     sb.append("^").append(weighting + WEIGHTING_BOOST);
551                     if (!boostTerm.equals(word)) {
552                         boostedTerms.append(" ");
553                         LuceneUtils.appendEscapedLuceneQuery(boostedTerms, boostTerm);
554                         boostedTerms.append("^").append(weighting + WEIGHTING_BOOST);
555                     }
556                 } else if (weighting > 1) {
557                     sb.append("^").append(weighting);
558                 }
559             }
560             if (boostedTerms.length() > 0) {
561                 sb.append(boostedTerms);
562             }
563         }
564         sb.append(")");
565         return addedTerm;
566     }
567 
568     /**
569      * Removes characters from the input text that are not used within the CPE
570      * index.
571      *
572      * @param text is the text to remove the characters from.
573      * @return the text having removed some characters.
574      */
575     private String cleanseText(String text) {
576         return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
577     }
578 
579     /**
580      * Searches the collection of boost terms for the given term. The elements
581      * are case insensitive matched using only the alpha-numeric contents of the
582      * terms; all other characters are removed.
583      *
584      * @param term the term to search for
585      * @param boost the collection of boost terms
586      * @return the value identified
587      */
588     private String findBoostTerm(String term, Set<String> boost) {
589         for (String entry : boost) {
590             if (equalsIgnoreCaseAndNonAlpha(term, entry)) {
591                 return entry;
592             }
593         }
594         return null;
595     }
596 
597     /**
598      * Compares two strings after lower casing them and removing the non-alpha
599      * characters.
600      *
601      * @param l string one to compare.
602      * @param r string two to compare.
603      * @return whether or not the two strings are similar.
604      */
605     private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
606         if (l == null || r == null) {
607             return false;
608         }
609 
610         final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
611         final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
612         return left.equalsIgnoreCase(right);
613     }
614 
615     /**
616      * Ensures that the CPE Identified matches the dependency. This validates
617      * that the product, vendor, and version information for the CPE are
618      * contained within the dependencies evidence.
619      *
620      * @param entry a CPE entry
621      * @param dependency the dependency that the CPE entries could be for
622      * @param majorVersions the major versions detected for the dependency
623      * @return whether or not the entry is valid.
624      */
625     private boolean verifyEntry(final IndexEntry entry, final Dependency dependency,
626             final Set<String> majorVersions) {
627         boolean isValid = false;
628         //TODO - does this nullify some of the fuzzy matching that happens in the lucene search?
629         // for instance CPE some-component and in the evidence we have SomeComponent.
630 
631         //TODO - should this have a package manager only flag instead of just looking for NPM
632         if (Ecosystem.NODEJS.equals(dependency.getEcosystem())) {
633             for (Identifier i : dependency.getSoftwareIdentifiers()) {
634                 if (i instanceof PurlIdentifier) {
635                     final PurlIdentifier p = (PurlIdentifier) i;
636                     if (cleanPackageName(p.getName()).equals(cleanPackageName(entry.getProduct()))) {
637                         isValid = true;
638                     }
639                 }
640             }
641         } else if (collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) {
642             if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct())) {
643                 isValid = true;
644             } else {
645                 isValid = majorVersions.stream().filter(version
646                         -> version != null && entry.getProduct().endsWith("v" + version) && entry.getProduct().length() > version.length() + 1)
647                         .anyMatch(version
648                                 -> collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT),
649                                 entry.getProduct().substring(0, entry.getProduct().length() - version.length() - 1))
650                         );
651                 isValid |= majorVersions.stream().filter(version
652                         -> version != null && entry.getProduct().endsWith(version) && entry.getProduct().length() > version.length())
653                         .anyMatch(version
654                                 -> collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT),
655                                 entry.getProduct().substring(0, entry.getProduct().length() - version.length()))
656                         );
657             }
658         }
659         return isValid;
660     }
661 
662     /**
663      * Only returns alpha numeric characters contained in a given package name.
664      *
665      * @param name the package name to cleanse
666      * @return the cleansed packaage name
667      */
668     private String cleanPackageName(String name) {
669         if (name == null) {
670             return "";
671         }
672         return name.replaceAll("[^a-zA-Z0-9]+", "");
673     }
674 
675     /**
676      * Used to determine if the EvidenceCollection contains a specific string.
677      *
678      * @param evidence an of evidence object to check
679      * @param text the text to search for
680      * @return whether or not the EvidenceCollection contains the string
681      */
682     @SuppressWarnings("StringSplitter")
683     private boolean collectionContainsString(Set<Evidence> evidence, String text) {
684         //TODO - likely need to change the split... not sure if this will work for CPE with special chars
685         if (text == null) {
686             return false;
687         }
688         // Check if we have an exact match
689         final String textLC = text.toLowerCase();
690         for (Evidence e : evidence) {
691             if (e.getValue().toLowerCase().equals(textLC)) {
692                 return true;
693             }
694         }
695 
696         final String[] words = text.split("[\\s_-]+");
697         final List<String> list = new ArrayList<>();
698         String tempWord = null;
699         final CharArraySet stopWords = SearchFieldAnalyzer.getStopWords();
700         for (String word : words) {
701             /*
702              single letter words should be concatenated with the next word.
703              so { "m", "core", "sample" } -> { "mcore", "sample" }
704              */
705             if (tempWord != null) {
706                 list.add(tempWord + word);
707                 tempWord = null;
708             } else if (word.length() <= 2) {
709                 tempWord = word;
710             } else {
711                 if (stopWords.contains(word)) {
712                     continue;
713                 }
714                 list.add(word);
715             }
716         }
717         if (tempWord != null) {
718             if (!list.isEmpty()) {
719                 final String tmp = list.get(list.size() - 1) + tempWord;
720                 list.add(tmp);
721             } else {
722                 list.add(tempWord);
723             }
724         }
725         if (list.isEmpty()) {
726             return false;
727         }
728         boolean isValid = true;
729 
730         // Prepare the evidence values, e.g. remove the characters we used for splitting
731         final List<String> evidenceValues = new ArrayList<>(evidence.size());
732         evidence.forEach((e) -> evidenceValues.add(e.getValue().toLowerCase().replaceAll("[\\s_-]+", "")));
733 
734         for (String word : list) {
735             word = word.toLowerCase();
736             boolean found = false;
737             for (String e : evidenceValues) {
738                 if (e.contains(word)) {
739                     if ("http".equals(word) && e.contains("http:")) {
740                         continue;
741                     }
742                     found = true;
743                     break;
744                 }
745             }
746             isValid &= found;
747         }
748         return isValid;
749     }
750 
751     /**
752      * Analyzes a dependency and attempts to determine if there are any CPE
753      * identifiers for this dependency.
754      *
755      * @param dependency The Dependency to analyze.
756      * @param engine The analysis engine
757      * @throws AnalysisException is thrown if there is an issue analyzing the
758      * dependency.
759      */
760     @Override
761     protected void analyzeDependency(Dependency dependency, Engine engine) throws AnalysisException {
762         if (skipEcosystems.contains(dependency.getEcosystem())) {
763             return;
764         }
765         try {
766             determineCPE(dependency);
767         } catch (CorruptIndexException ex) {
768             throw new AnalysisException("CPE Index is corrupt.", ex);
769         } catch (IOException ex) {
770             throw new AnalysisException("Failure opening the CPE Index.", ex);
771         } catch (ParseException ex) {
772             throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
773         }
774     }
775 
776     /**
777      * Retrieves a list of CPE values from the CveDB based on the vendor and
778      * product passed in. The list is then validated to find only CPEs that are
779      * valid for the given dependency. It is possible that the CPE identified is
780      * a best effort "guess" based on the vendor, product, and version
781      * information.
782      *
783      * @param dependency the Dependency being analyzed
784      * @param vendor the vendor for the CPE being analyzed
785      * @param product the product for the CPE being analyzed
786      * @param currentConfidence the current confidence being used during
787      * analysis
788      * @return <code>true</code> if an identifier was added to the dependency;
789      * otherwise <code>false</code>
790      * @throws AnalysisException thrown if the suppression rules failed
791      */
792     @SuppressWarnings("StringSplitter")
793     protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
794             Confidence currentConfidence) throws AnalysisException {
795 
796         final CpeBuilder cpeBuilder = new CpeBuilder();
797 
798         final Set<CpePlus> cpePlusEntries = cve.getCPEs(vendor, product);
799         final Set<Cpe> cpes = filterEcosystem(dependency.getEcosystem(), cpePlusEntries);
800         if (cpes == null || cpes.isEmpty()) {
801             return false;
802         }
803 
804         DependencyVersion bestGuess;
805         if ("Golang".equals(dependency.getEcosystem()) && dependency.getVersion() == null) {
806             bestGuess = new DependencyVersion("*");
807         } else {
808             bestGuess = new DependencyVersion("-");
809         }
810         String bestGuessUpdate = null;
811         Confidence bestGuessConf = null;
812         String bestGuessURL = null;
813         final Set<IdentifierMatch> collected = new HashSet<>();
814 
815         considerDependencyVersion(dependency, vendor, product, currentConfidence, collected);
816 
817         //TODO the following algorithm incorrectly identifies things as a lower version
818         // if there lower confidence evidence when the current (highest) version number
819         // is newer then anything in the NVD.
820         for (Confidence conf : Confidence.values()) {
821             for (Evidence evidence : dependency.getIterator(EvidenceType.VERSION, conf)) {
822                 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue(), true);
823                 if (evVer == null) {
824                     continue;
825                 }
826                 DependencyVersion evBaseVer = null;
827                 String evBaseVerUpdate = null;
828                 final int idx = evVer.getVersionParts().size() - 1;
829                 if (evVer.getVersionParts().get(idx)
830                         .matches("^(v|release|final|snapshot|beta|alpha|u|rc|m|20\\d\\d).*$")) {
831                     //store the update version
832                     final String checkUpdate = evVer.getVersionParts().get(idx);
833                     if (checkUpdate.matches("^(v|release|final|snapshot|beta|alpha|u|rc|m|20\\d\\d).*$")) {
834                         evBaseVerUpdate = checkUpdate;
835                         evBaseVer = new DependencyVersion();
836                         evBaseVer.setVersionParts(evVer.getVersionParts().subList(0, idx));
837                     }
838                 }
839                 //TODO - review and update for new JSON data
840                 for (Cpe vs : cpes) {
841                     final DependencyVersion dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
842                     DependencyVersion dbVerUpdate = dbVer;
843                     if (vs.getUpdate() != null && !vs.getUpdate().isEmpty() && !vs.getUpdate().startsWith("*") && !vs.getUpdate().startsWith("-")) {
844                         dbVerUpdate = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate(), true);
845                     }
846                     if (dbVer == null) { //special case, no version specified - everything is vulnerable
847                         final String url = CpeIdentifier.nvdProductSearchUrlFor(vs);
848                         final IdentifierMatch match = new IdentifierMatch(vs, url, IdentifierConfidence.BROAD_MATCH, conf);
849                         collected.add(match);
850                     } else if (evVer.equals(dbVer)) {
851                         addExactMatch(vs, evBaseVerUpdate, conf, collected);
852                     } else if (evBaseVer != null && evBaseVer.equals(dbVer)
853                             && (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0)) {
854                         bestGuessConf = conf;
855                         bestGuess = dbVer;
856                         bestGuessUpdate = evBaseVerUpdate;
857                         bestGuessURL = CpeIdentifier.nvdSearchUrlFor(vs);
858                     } else if (dbVerUpdate != null && evVer.getVersionParts().size() <= dbVerUpdate.getVersionParts().size()
859                             && evVer.matchesAtLeastThreeLevels(dbVerUpdate)) {
860                         if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
861                             if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
862                                 bestGuess = dbVer;
863                                 bestGuessUpdate = evBaseVerUpdate;
864                                 bestGuessConf = conf;
865                             }
866                         }
867                     }
868                 }
869                 if ((bestGuessConf == null || bestGuessConf.compareTo(conf) > 0)
870                         && bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
871                     bestGuess = evVer;
872                     bestGuessUpdate = evBaseVerUpdate;
873                     bestGuessConf = conf;
874                 }
875             }
876         }
877 
878         cpeBuilder.part(Part.APPLICATION).vendor(vendor).product(product);
879         final int idx = bestGuess.getVersionParts().size() - 1;
880         if (bestGuess.getVersionParts().get(idx)
881                 .matches("^(v|release|final|snapshot|beta|alpha|u|rc|m|20\\d\\d).*$")) {
882             cpeBuilder.version(StringUtils.join(bestGuess.getVersionParts().subList(0, idx), "."));
883             //when written - no update versions in the NVD start with v### - they all strip the v off
884             if (bestGuess.getVersionParts().get(idx).matches("^v\\d.*$")) {
885                 cpeBuilder.update(bestGuess.getVersionParts().get(idx).substring(1));
886             } else {
887                 cpeBuilder.update(bestGuess.getVersionParts().get(idx));
888             }
889         } else {
890             cpeBuilder.version(bestGuess.toString());
891             if (bestGuessUpdate != null) {
892                 cpeBuilder.update(bestGuessUpdate);
893             }
894         }
895         final Cpe guessCpe;
896 
897         try {
898             guessCpe = cpeBuilder.build();
899         } catch (CpeValidationException ex) {
900             throw new AnalysisException(String.format("Unable to create a CPE for %s:%s:%s", vendor, product, bestGuess));
901         }
902         if (!"-".equals(guessCpe.getVersion())) {
903             String url = null;
904             if (bestGuessURL != null) {
905                 url = bestGuessURL;
906             }
907             if (bestGuessConf == null) {
908                 bestGuessConf = Confidence.LOW;
909             }
910             final IdentifierMatch match = new IdentifierMatch(guessCpe, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
911 
912             collected.add(match);
913         }
914         boolean identifierAdded = false;
915         if (!collected.isEmpty()) {
916             final List<IdentifierMatch> items = new ArrayList<>(collected);
917 
918             Collections.sort(items);
919             final IdentifierConfidence bestIdentifierQuality = items.get(0).getIdentifierConfidence();
920             final Confidence bestEvidenceQuality = items.get(0).getEvidenceConfidence();
921             boolean addedNonGuess = false;
922             final Confidence prevAddedConfidence = dependency.getVulnerableSoftwareIdentifiers().stream().map(Identifier::getConfidence)
923                     .min(Comparator.comparing(Confidence::ordinal))
924                     .orElse(Confidence.LOW);
925 
926             for (IdentifierMatch m : items) {
927                 if (bestIdentifierQuality.equals(m.getIdentifierConfidence())
928                         && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
929                     final CpeIdentifier i = m.getIdentifier();
930                     if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
931                         if (addedNonGuess) {
932                             continue;
933                         }
934                         i.setConfidence(Confidence.LOW);
935                     } else {
936                         i.setConfidence(bestEvidenceQuality);
937                     }
938                     if (prevAddedConfidence.compareTo(i.getConfidence()) < 0) {
939                         continue;
940                     }
941 
942                     //TODO - while this gets the job down it is slow; consider refactoring
943                     dependency.addVulnerableSoftwareIdentifier(i);
944                     suppression.analyze(dependency, engine);
945                     if (dependency.getVulnerableSoftwareIdentifiers().contains(i)) {
946                         identifierAdded = true;
947                         if (!addedNonGuess && bestIdentifierQuality != IdentifierConfidence.BEST_GUESS) {
948                             addedNonGuess = true;
949                         }
950                     }
951                 }
952             }
953         }
954         return identifierAdded;
955     }
956 
957     /**
958      * Adds a new CPE to the identifier match collection.
959      *
960      * @param vs a reference to the vulnerable software
961      * @param updateVersion the update version
962      * @param conf the current confidence
963      * @param collected a reference to the collected identifiers
964      */
965     private void addExactMatch(Cpe vs, String updateVersion, Confidence conf,
966             final Set<IdentifierMatch> collected) {
967 
968         final CpeBuilder cpeBuilder = new CpeBuilder();
969         final String url = CpeIdentifier.nvdSearchUrlFor(vs);
970         Cpe useCpe;
971         if (updateVersion != null && "*".equals(vs.getUpdate())) {
972             try {
973                 useCpe = cpeBuilder.part(vs.getPart()).wfVendor(vs.getWellFormedVendor())
974                         .wfProduct(vs.getWellFormedProduct()).wfVersion(vs.getWellFormedVersion())
975                         .wfEdition(vs.getWellFormedEdition()).wfLanguage(vs.getWellFormedLanguage())
976                         .wfOther(vs.getWellFormedOther()).wfSwEdition(vs.getWellFormedSwEdition())
977                         .update(updateVersion).build();
978             } catch (CpeValidationException ex) {
979                 LOGGER.debug("Error building cpe with update:" + updateVersion, ex);
980                 useCpe = vs;
981             }
982         } else {
983             useCpe = vs;
984         }
985         final IdentifierMatch match = new IdentifierMatch(useCpe, url, IdentifierConfidence.EXACT_MATCH, conf);
986         collected.add(match);
987     }
988 
989     /**
990      * Evaluates whether or not to use the `version` of the dependency instead
991      * of the version evidence. The dependency should not always be used as it
992      * can cause FP.
993      *
994      * @param dependency the dependency being analyzed
995      * @param product the product name
996      * @param vendor the vendor name
997      * @param confidence the current confidence level
998      * @param collected a reference to the identifiers matched
999      * @throws AnalysisException thrown if aliens attacked and valid input could
1000      * not be used to construct a CPE
1001      */
1002     private void considerDependencyVersion(Dependency dependency,
1003             String vendor, String product, Confidence confidence,
1004             final Set<IdentifierMatch> collected)
1005             throws AnalysisException {
1006 
1007         if (dependency.getVersion() != null && !dependency.getVersion().isEmpty()) {
1008             final CpeBuilder cpeBuilder = new CpeBuilder();
1009             boolean useDependencyVersion = true;
1010             final CharArraySet stopWords = SearchFieldAnalyzer.getStopWords();
1011             if (dependency.getName() != null && !dependency.getName().isEmpty()) {
1012                 final String name = dependency.getName();
1013                 for (String word : product.split("[^a-zA-Z0-9]")) {
1014                     useDependencyVersion &= name.contains(word) || stopWords.contains(word)
1015                             || wordMatchesEcosystem(dependency.getEcosystem(), word);
1016                 }
1017             }
1018 
1019             if (useDependencyVersion) {
1020                 //TODO - we need to filter this so that we only use this if something in the
1021                 //dependency.getName() matches the vendor/product in some way
1022                 final DependencyVersion depVersion = new DependencyVersion(dependency.getVersion());
1023                 if (depVersion.getVersionParts().size() > 0) {
1024                     cpeBuilder.part(Part.APPLICATION).vendor(vendor).product(product);
1025                     addVersionAndUpdate(depVersion, cpeBuilder);
1026                     try {
1027                         final Cpe depCpe = cpeBuilder.build();
1028                         final String url = CpeIdentifier.nvdSearchUrlFor(vendor, product, depCpe.getVersion());
1029                         final IdentifierMatch match = new IdentifierMatch(depCpe, url, IdentifierConfidence.EXACT_MATCH, confidence);
1030                         collected.add(match);
1031                     } catch (CpeValidationException ex) {
1032                         throw new AnalysisException(String.format("Unable to create a CPE for %s:%s:%s", vendor, product, depVersion));
1033                     }
1034                 }
1035             }
1036         }
1037     }
1038 
1039     /**
1040      * If a CPE product word represents the ecosystem of a dependency it is not required
1041      * to appear in the dependencyName to still consider the CPE product a match.
1042      *
1043      * @param ecosystem The ecosystem of the dependency
1044      * @param word       The word from the CPE product to check
1045      * @return {@code true} when the CPE product word is known to match the ecosystem of the dependency
1046      * @implNote This method is not intended to cover every possible case where the ecosystem is represented by the word. It is a
1047      * best-effort attempt to prevent {@link #considerDependencyVersion(Dependency, String, String, Confidence, Set)}
1048      * from not taking an exact-match versioned CPE into account because the ecosystem-related word does not appear in
1049      * the dependencyName. It helps prevent false-positive cases like https://github.com/dependency-check/DependencyCheck/issues/5545
1050      * @see #considerDependencyVersion(Dependency, String, String, Confidence, Set)
1051      */
1052     private boolean wordMatchesEcosystem(@Nullable String ecosystem, String word) {
1053         if (Ecosystem.JAVA.equalsIgnoreCase(word)) {
1054             return Ecosystem.JAVA.equals(ecosystem);
1055         }
1056         return false;
1057     }
1058 
1059     /**
1060      * <p>
1061      * Returns the setting key to determine if the analyzer is enabled.</p>
1062      *
1063      * @return the key for the analyzer's enabled property
1064      */
1065     @Override
1066     protected String getAnalyzerEnabledSettingKey() {
1067         return Settings.KEYS.ANALYZER_CPE_ENABLED;
1068     }
1069 
1070     /**
1071      * Filters the given list of CPE Entries (plus ecosystem) for the given
1072      * dependencies ecosystem.
1073      *
1074      * @param ecosystem the dependencies ecosystem
1075      * @param entries the CPE Entries (plus ecosystem)
1076      * @return the filtered list of CPE entries
1077      */
1078     private Set<Cpe> filterEcosystem(String ecosystem, Set<CpePlus> entries) {
1079         if (entries == null || entries.isEmpty()) {
1080             return null;
1081         }
1082         if (ecosystem != null) {
1083             return entries.stream().filter(c
1084                     -> c.getEcosystem() == null
1085                     || c.getEcosystem().equals(ecosystem)
1086                     //some ios CVE/CPEs are listed under native
1087                     || (Ecosystem.IOS.equals(ecosystem) && Ecosystem.NATIVE.equals(c.getEcosystem())))
1088                     .map(CpePlus::getCpe)
1089                     .collect(Collectors.toSet());
1090         }
1091         return entries.stream()
1092                 .map(CpePlus::getCpe)
1093                 .collect(Collectors.toSet());
1094     }
1095 
1096     /**
1097      * Add the given version to the CpeBuilder - this method attempts to parse
1098      * out the update from the version and correctly set the value in the CPE.
1099      *
1100      * @param depVersion the version to add
1101      * @param cpeBuilder a reference to the CPE Builder
1102      */
1103     private void addVersionAndUpdate(DependencyVersion depVersion, final CpeBuilder cpeBuilder) {
1104         final int idx = depVersion.getVersionParts().size() - 1;
1105         if (idx > 0 && depVersion.getVersionParts().get(idx)
1106                 .matches("^(v|final|release|snapshot|r|b|beta|a|alpha|u|rc|sp|dev|revision|service|build|pre|p|patch|update|m|20\\d\\d).*$")) {
1107             cpeBuilder.version(StringUtils.join(depVersion.getVersionParts().subList(0, idx), "."));
1108             //when written - no update versions in the NVD start with v### - they all strip the v off
1109             if (depVersion.getVersionParts().get(idx).matches("^v\\d.*$")) {
1110                 cpeBuilder.update(depVersion.getVersionParts().get(idx).substring(1));
1111             } else {
1112                 cpeBuilder.update(depVersion.getVersionParts().get(idx));
1113             }
1114         } else {
1115             cpeBuilder.version(depVersion.toString());
1116         }
1117     }
1118 
1119     /**
1120      * The confidence whether the identifier is an exact match, or a best guess.
1121      */
1122     private enum IdentifierConfidence {
1123 
1124         /**
1125          * An exact match for the CPE.
1126          */
1127         EXACT_MATCH,
1128         /**
1129          * A best guess for the CPE.
1130          */
1131         BEST_GUESS,
1132         /**
1133          * The entire vendor/product group must be added (without a guess at
1134          * version) because there is a CVE with a VS that only specifies
1135          * vendor/product.
1136          */
1137         BROAD_MATCH
1138     }
1139 
1140     /**
1141      * A simple object to hold an identifier and carry information about the
1142      * confidence in the identifier.
1143      */
1144     private static class IdentifierMatch implements Comparable<IdentifierMatch> {
1145 
1146         /**
1147          * The confidence whether this is an exact match, or a best guess.
1148          */
1149         private IdentifierConfidence identifierConfidence;
1150         /**
1151          * The CPE identifier.
1152          */
1153         private CpeIdentifier identifier;
1154 
1155         /**
1156          * Constructs an IdentifierMatch.
1157          *
1158          * @param cpe the CPE value for the match
1159          * @param url the URL of the identifier
1160          * @param identifierConfidence the confidence in the identifier: best
1161          * guess or exact match
1162          * @param evidenceConfidence the confidence of the evidence used to find
1163          * the identifier
1164          */
1165         IdentifierMatch(Cpe cpe, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
1166             this.identifier = new CpeIdentifier(cpe, url, evidenceConfidence);
1167             this.identifierConfidence = identifierConfidence;
1168         }
1169 
1170         //<editor-fold defaultstate="collapsed" desc="Property implementations: evidenceConfidence, confidence, identifier">
1171         /**
1172          * Get the value of evidenceConfidence
1173          *
1174          * @return the value of evidenceConfidence
1175          */
1176         public Confidence getEvidenceConfidence() {
1177             return this.identifier.getConfidence();
1178         }
1179 
1180         /**
1181          * Set the value of evidenceConfidence
1182          *
1183          * @param evidenceConfidence new value of evidenceConfidence
1184          */
1185         public void setEvidenceConfidence(Confidence evidenceConfidence) {
1186             this.identifier.setConfidence(evidenceConfidence);
1187         }
1188 
1189         /**
1190          * Get the value of confidence.
1191          *
1192          * @return the value of confidence
1193          */
1194         public IdentifierConfidence getIdentifierConfidence() {
1195             return identifierConfidence;
1196         }
1197 
1198         /**
1199          * Set the value of confidence.
1200          *
1201          * @param confidence new value of confidence
1202          */
1203         public void setIdentifierConfidence(IdentifierConfidence confidence) {
1204             this.identifierConfidence = confidence;
1205         }
1206 
1207         /**
1208          * Get the value of identifier.
1209          *
1210          * @return the value of identifier
1211          */
1212         public CpeIdentifier getIdentifier() {
1213             return identifier;
1214         }
1215 
1216         /**
1217          * Set the value of identifier.
1218          *
1219          * @param identifier new value of identifier
1220          */
1221         public void setIdentifier(CpeIdentifier identifier) {
1222             this.identifier = identifier;
1223         }
1224         //</editor-fold>
1225         //<editor-fold defaultstate="collapsed" desc="Standard implementations of toString, hashCode, and equals">
1226 
1227         /**
1228          * Standard toString() implementation.
1229          *
1230          * @return the string representation of the object
1231          */
1232         @Override
1233         public String toString() {
1234             return "IdentifierMatch{ IdentifierConfidence=" + identifierConfidence + ", identifier=" + identifier + '}';
1235         }
1236 
1237         /**
1238          * Standard hashCode() implementation.
1239          *
1240          * @return the hashCode
1241          */
1242         @Override
1243         public int hashCode() {
1244             return new HashCodeBuilder(115, 303)
1245                     .append(identifierConfidence)
1246                     .append(identifier)
1247                     .toHashCode();
1248         }
1249 
1250         /**
1251          * Standard equals implementation.
1252          *
1253          * @param obj the object to compare
1254          * @return true if the objects are equal, otherwise false
1255          */
1256         @Override
1257         public boolean equals(Object obj) {
1258             if (obj == null || !(obj instanceof IdentifierMatch)) {
1259                 return false;
1260             }
1261             if (this == obj) {
1262                 return true;
1263             }
1264             final IdentifierMatch other = (IdentifierMatch) obj;
1265             return new EqualsBuilder()
1266                     .append(identifierConfidence, other.identifierConfidence)
1267                     .append(identifier, other.identifier)
1268                     .build();
1269         }
1270         //</editor-fold>
1271 
1272         /**
1273          * Standard implementation of compareTo that compares identifier
1274          * confidence, evidence confidence, and then the identifier.
1275          *
1276          * @param o the IdentifierMatch to compare to
1277          * @return the natural ordering of IdentifierMatch
1278          */
1279         @Override
1280         public int compareTo(@NotNull IdentifierMatch o) {
1281             return new CompareToBuilder()
1282                     .append(identifierConfidence, o.identifierConfidence)
1283                     .append(identifier, o.identifier)
1284                     .toComparison();
1285         }
1286     }
1287 
1288     /**
1289      * Command line tool for querying the Lucene CPE Index.
1290      *
1291      * @param args not used
1292      */
1293     @SuppressWarnings("InfiniteLoopStatement")
1294     public static void main(String[] args) {
1295         final Settings props = new Settings();
1296         try (Engine en = new Engine(Engine.Mode.EVIDENCE_PROCESSING, props)) {
1297             en.openDatabase(false, false);
1298             final CPEAnalyzer analyzer = new CPEAnalyzer();
1299             analyzer.initialize(props);
1300             analyzer.prepareAnalyzer(en);
1301             LOGGER.error("test");
1302             System.out.println("Memory index query for ODC");
1303             try (BufferedReader br = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8))) {
1304                 while (true) {
1305 
1306                     final Map<String, MutableInt> vendor = new HashMap<>();
1307                     final Map<String, MutableInt> product = new HashMap<>();
1308                     System.out.print("Vendor: ");
1309                     String[] parts = br.readLine().split(" ");
1310                     for (String term : parts) {
1311                         final MutableInt count = vendor.get(term);
1312                         if (count == null) {
1313                             vendor.put(term, new MutableInt(0));
1314                         } else {
1315                             count.add(1);
1316                         }
1317                     }
1318                     System.out.print("Product: ");
1319                     parts = br.readLine().split(" ");
1320                     for (String term : parts) {
1321                         final MutableInt count = product.get(term);
1322                         if (count == null) {
1323                             product.put(term, new MutableInt(0));
1324                         } else {
1325                             count.add(1);
1326                         }
1327                     }
1328                     final List<IndexEntry> list = analyzer.searchCPE(vendor, product, new HashSet<>(), new HashSet<>(), "default");
1329                     if (list == null || list.isEmpty()) {
1330                         System.out.println("No results found");
1331                     } else {
1332                         list.forEach((e) -> System.out.printf("%s:%s (%f)%n", e.getVendor(), e.getProduct(),
1333                                 e.getSearchScore()));
1334                     }
1335                     System.out.println();
1336                     System.out.println();
1337                 }
1338             }
1339         } catch (InitializationException | IOException ex) {
1340             System.err.println("Lucene ODC search tool failed:");
1341             System.err.println(ex.getMessage());
1342         }
1343     }
1344 
1345     /**
1346      * Sets the reference to the CveDB.
1347      *
1348      * @param cveDb the CveDB
1349      */
1350     protected void setCveDB(CveDB cveDb) {
1351         this.cve = cveDb;
1352     }
1353 
1354     /**
1355      * returns a reference to the CveDB.
1356      *
1357      * @return a reference to the CveDB
1358      */
1359     protected CveDB getCveDB() {
1360         return this.cve;
1361     }
1362 
1363     /**
1364      * Sets the MemoryIndex.
1365      *
1366      * @param idx the memory index
1367      */
1368     protected void setMemoryIndex(MemoryIndex idx) {
1369         cpe = idx;
1370     }
1371 
1372     /**
1373      * Returns the memory index.
1374      *
1375      * @return the memory index
1376      */
1377     protected MemoryIndex getMemoryIndex() {
1378         return cpe;
1379     }
1380 
1381     /**
1382      * Sets the CPE Suppression Analyzer.
1383      *
1384      * @param suppression the CPE Suppression Analyzer
1385      */
1386     protected void setCpeSuppressionAnalyzer(CpeSuppressionAnalyzer suppression) {
1387         this.suppression = suppression;
1388     }
1389 }