1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.BufferedReader;
21 import java.io.IOException;
22 import java.io.InputStreamReader;
23 import java.nio.charset.StandardCharsets;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Collections;
27 import java.util.Comparator;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Objects;
33 import java.util.Set;
34 import java.util.concurrent.TimeUnit;
35 import java.util.stream.Collectors;
36 import javax.annotation.concurrent.ThreadSafe;
37 import org.apache.commons.lang3.StringUtils;
38 import org.apache.commons.lang3.builder.CompareToBuilder;
39 import org.apache.commons.lang3.builder.EqualsBuilder;
40 import org.apache.commons.lang3.builder.HashCodeBuilder;
41 import org.apache.commons.lang3.mutable.MutableInt;
42 import org.apache.lucene.analysis.CharArraySet;
43 import org.apache.lucene.document.Document;
44 import org.apache.lucene.index.CorruptIndexException;
45 import org.apache.lucene.queryparser.classic.ParseException;
46 import org.apache.lucene.search.Query;
47 import org.apache.lucene.search.ScoreDoc;
48 import org.apache.lucene.search.TopDocs;
49 import org.jetbrains.annotations.NotNull;
50 import org.jetbrains.annotations.Nullable;
51 import org.owasp.dependencycheck.Engine;
52 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
53 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
54 import org.owasp.dependencycheck.data.cpe.Fields;
55 import org.owasp.dependencycheck.data.cpe.IndexEntry;
56 import org.owasp.dependencycheck.data.cpe.IndexException;
57 import org.owasp.dependencycheck.data.cpe.MemoryIndex;
58 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
59 import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
60 import org.owasp.dependencycheck.data.nvd.ecosystem.Ecosystem;
61 import org.owasp.dependencycheck.data.nvdcve.CveDB;
62 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
63 import org.owasp.dependencycheck.data.update.cpe.CpePlus;
64 import org.owasp.dependencycheck.dependency.Confidence;
65 import org.owasp.dependencycheck.dependency.Dependency;
66 import org.owasp.dependencycheck.dependency.Evidence;
67 import org.owasp.dependencycheck.dependency.EvidenceType;
68 import org.owasp.dependencycheck.dependency.naming.CpeIdentifier;
69 import org.owasp.dependencycheck.dependency.naming.Identifier;
70 import org.owasp.dependencycheck.dependency.naming.PurlIdentifier;
71 import org.owasp.dependencycheck.exception.InitializationException;
72 import org.owasp.dependencycheck.utils.DependencyVersion;
73 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
74 import org.owasp.dependencycheck.utils.Settings;
75 import org.slf4j.Logger;
76 import org.slf4j.LoggerFactory;
77 import us.springett.parsers.cpe.Cpe;
78 import us.springett.parsers.cpe.CpeBuilder;
79 import us.springett.parsers.cpe.exceptions.CpeValidationException;
80 import us.springett.parsers.cpe.values.Part;
81
82
83
84
85
86
87
88
89 @ThreadSafe
90 public class CPEAnalyzer extends AbstractAnalyzer {
91
92
93
94
95 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
96
97
98
99 private static final int WEIGHTING_BOOST = 1;
100
101
102
103
104
105
106 private static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._:/-]";
107
108
109
110
111 private static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
112
113
114
115
116 private MemoryIndex cpe;
117
118
119
120 private CveDB cve;
121
122
123
124 private Engine engine;
125
126
127
128
129
130 private List<String> skipEcosystems;
131
132
133
134
135 private Ecosystem ecosystemTools;
136
137
138
139
140
141
142 private CpeSuppressionAnalyzer suppression;
143
144
145
146
147
148
149 @Override
150 public String getName() {
151 return "CPE Analyzer";
152 }
153
154
155
156
157
158
159 @Override
160 public AnalysisPhase getAnalysisPhase() {
161 return AnalysisPhase.IDENTIFIER_ANALYSIS;
162 }
163
164
165
166
167
168
169
170
171 @Override
172 public void prepareAnalyzer(Engine engine) throws InitializationException {
173 super.prepareAnalyzer(engine);
174 this.engine = engine;
175 try {
176 this.open(engine.getDatabase());
177 } catch (IOException ex) {
178 LOGGER.debug("Exception initializing the Lucene Index", ex);
179 throw new InitializationException("An exception occurred initializing the Lucene Index", ex);
180 } catch (DatabaseException ex) {
181 LOGGER.debug("Exception accessing the database", ex);
182 throw new InitializationException("An exception occurred accessing the database", ex);
183 }
184 final String[] tmp = engine.getSettings().getArray(Settings.KEYS.ECOSYSTEM_SKIP_CPEANALYZER);
185 if (tmp == null) {
186 skipEcosystems = new ArrayList<>();
187 } else {
188 LOGGER.debug("Skipping CPE Analysis for {}", StringUtils.join(tmp, ","));
189 skipEcosystems = Arrays.asList(tmp);
190 }
191 ecosystemTools = new Ecosystem(engine.getSettings());
192 suppression = new CpeSuppressionAnalyzer();
193 suppression.initialize(engine.getSettings());
194 suppression.prepareAnalyzer(engine);
195 }
196
197
198
199
200
201
202
203
204
205
206 public void open(CveDB cve) throws IOException, DatabaseException {
207 this.cve = cve;
208 this.cpe = CpeMemoryIndex.getInstance();
209 try {
210 final long creationStart = System.currentTimeMillis();
211 cpe.open(cve.getVendorProductList(), this.getSettings());
212 final long creationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - creationStart);
213 LOGGER.info("Created CPE Index ({} seconds)", creationSeconds);
214 } catch (IndexException ex) {
215 LOGGER.debug("IndexException", ex);
216 throw new DatabaseException(ex);
217 }
218 }
219
220
221
222
223 @Override
224 public void closeAnalyzer() {
225 if (cpe != null) {
226 cpe.close();
227 cpe = null;
228 }
229 }
230
231
232
233
234
235
236
237
238
239
240
241
242 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException, AnalysisException {
243 boolean identifierAdded;
244
245 final Set<String> majorVersions = dependency.getSoftwareIdentifiers()
246 .stream()
247 .filter(i -> i instanceof PurlIdentifier)
248 .map(i -> {
249 final PurlIdentifier p = (PurlIdentifier) i;
250 final DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false);
251 if (depVersion != null) {
252 return depVersion.getVersionParts().get(0);
253 }
254 return null;
255 }).collect(Collectors.toSet());
256
257 final Map<String, MutableInt> vendors = new HashMap<>();
258 final Map<String, MutableInt> products = new HashMap<>();
259 final Set<Integer> previouslyFound = new HashSet<>();
260
261 for (Confidence confidence : Confidence.values()) {
262 collectTerms(vendors, dependency.getIterator(EvidenceType.VENDOR, confidence));
263 LOGGER.trace("vendor search: {}", vendors);
264 collectTerms(products, dependency.getIterator(EvidenceType.PRODUCT, confidence));
265 addMajorVersionToTerms(majorVersions, products);
266 LOGGER.trace("product search: {}", products);
267 if (!vendors.isEmpty() && !products.isEmpty()) {
268 final List<IndexEntry> entries = searchCPE(vendors, products,
269 dependency.getVendorWeightings(), dependency.getProductWeightings(),
270 dependency.getEcosystem());
271 if (entries == null) {
272 continue;
273 }
274
275 identifierAdded = false;
276 for (IndexEntry e : entries) {
277 if (previouslyFound.contains(e.getDocumentId()) ) {
278 continue;
279 }
280 previouslyFound.add(e.getDocumentId());
281 if (verifyEntry(e, dependency, majorVersions)) {
282 final String vendor = e.getVendor();
283 final String product = e.getProduct();
284 LOGGER.trace("identified vendor/product: {}/{}", vendor, product);
285 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
286 }
287 }
288 if (identifierAdded) {
289 break;
290 }
291 }
292 }
293 }
294
295
296
297
298
299
300
301
302
303
304
305
306
307 @SuppressWarnings("null")
308
309 protected void collectTerms(Map<String, MutableInt> terms, Iterable<Evidence> evidence) {
310 for (Evidence e : evidence) {
311 String value = cleanseText(e.getValue());
312 if (StringUtils.isBlank(value)) {
313 continue;
314 }
315 if (value.length() > 1000) {
316 boolean trimmed = false;
317 int pos = value.lastIndexOf(" ", 1000);
318 if (pos > 0) {
319 value = value.substring(0, pos);
320 trimmed = true;
321 } else {
322 pos = value.lastIndexOf(".", 1000);
323 }
324 if (!trimmed) {
325 if (pos > 0) {
326 value = value.substring(0, pos);
327 trimmed = true;
328 } else {
329 pos = value.lastIndexOf("-", 1000);
330 }
331 }
332 if (!trimmed) {
333 if (pos > 0) {
334 value = value.substring(0, pos);
335 trimmed = true;
336 } else {
337 pos = value.lastIndexOf("_", 1000);
338 }
339 }
340 if (!trimmed) {
341 if (pos > 0) {
342 value = value.substring(0, pos);
343 trimmed = true;
344 } else {
345 pos = value.lastIndexOf("/", 1000);
346 }
347 }
348 if (!trimmed && pos > 0) {
349 value = value.substring(0, pos);
350 trimmed = true;
351 }
352 if (!trimmed) {
353 value = value.substring(0, 1000);
354 }
355 }
356 addTerm(terms, value);
357 }
358 }
359
360 private void addMajorVersionToTerms(Set<String> majorVersions, Map<String, MutableInt> products) {
361 final Map<String, MutableInt> temp = new HashMap<>();
362 products.entrySet().stream()
363 .filter(term -> term.getKey() != null)
364 .forEach(term -> majorVersions.stream()
365 .filter(version -> version != null
366 && (!term.getKey().endsWith(version)
367 && !Character.isDigit(term.getKey().charAt(term.getKey().length() - 1))
368 && !products.containsKey(term.getKey() + version)))
369 .forEach(version -> {
370 addTerm(temp, term.getKey() + version);
371 }));
372 products.entrySet().stream()
373 .filter(term -> term.getKey() != null)
374 .forEach(term -> majorVersions.stream()
375 .filter(Objects::nonNull)
376 .map(version -> "v" + version)
377 .filter(version -> (!term.getKey().endsWith(version)
378 && !Character.isDigit(term.getKey().charAt(term.getKey().length() - 1))
379 && !products.containsKey(term.getKey() + version)))
380 .forEach(version -> {
381 addTerm(temp, term.getKey() + version);
382 }));
383 products.putAll(temp);
384 }
385
386
387
388
389
390
391
392 private void addTerm(Map<String, MutableInt> terms, String value) {
393 final MutableInt count = terms.get(value);
394 if (count == null) {
395 terms.put(value, new MutableInt(1));
396 } else {
397 count.add(1);
398 }
399 }
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419 protected List<IndexEntry> searchCPE(Map<String, MutableInt> vendor, Map<String, MutableInt> product,
420 Set<String> vendorWeightings, Set<String> productWeightings, String ecosystem) {
421
422 final int maxQueryResults = ecosystemTools.getLuceneMaxQueryLimitFor(ecosystem);
423 final List<IndexEntry> ret = new ArrayList<>(maxQueryResults);
424
425 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
426 if (searchString == null) {
427 return ret;
428 }
429 try {
430 final Query query = cpe.parseQuery(searchString);
431 final TopDocs docs = cpe.search(query, maxQueryResults);
432
433 for (ScoreDoc d : docs.scoreDocs) {
434
435 final Document doc = cpe.getDocument(d.doc);
436 final IndexEntry entry = new IndexEntry();
437 entry.setDocumentId(d.doc);
438 entry.setVendor(doc.get(Fields.VENDOR));
439 entry.setProduct(doc.get(Fields.PRODUCT));
440 entry.setSearchScore(d.score);
441
442
443
444
445
446 if (!ret.contains(entry)) {
447 ret.add(entry);
448 }
449
450 }
451 return ret;
452 } catch (ParseException ex) {
453 LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
454 LOGGER.info("Unable to parse: {}", searchString, ex);
455 } catch (IndexException ex) {
456 LOGGER.warn("An error occurred resetting the CPE index searcher. See the log for more details.");
457 LOGGER.info("Unable to reset the search analyzer", ex);
458 } catch (IOException ex) {
459 LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
460 LOGGER.info("IO Error with search string: {}", searchString, ex);
461 }
462 return null;
463 }
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483 protected String buildSearch(Map<String, MutableInt> vendor, Map<String, MutableInt> product,
484 Set<String> vendorWeighting, Set<String> productWeightings) {
485
486 final StringBuilder sb = new StringBuilder();
487
488 if (!appendWeightedSearch(sb, Fields.PRODUCT, product, productWeightings)) {
489 return null;
490 }
491 sb.append(" AND ");
492 if (!appendWeightedSearch(sb, Fields.VENDOR, vendor, vendorWeighting)) {
493 return null;
494 }
495 return sb.toString();
496 }
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512 @SuppressWarnings("StringSplitter")
513 private boolean appendWeightedSearch(StringBuilder sb, String field, Map<String, MutableInt> terms, Set<String> weightedText) {
514 if (terms.isEmpty()) {
515 return false;
516 }
517 sb.append(field).append(":(");
518 boolean addSpace = false;
519 boolean addedTerm = false;
520
521 for (Map.Entry<String, MutableInt> entry : terms.entrySet()) {
522 final StringBuilder boostedTerms = new StringBuilder();
523 final int weighting = entry.getValue().intValue();
524 final String[] text = entry.getKey().split(" ");
525 for (String word : text) {
526 if (word.isEmpty()) {
527 continue;
528 }
529 if (addSpace) {
530 sb.append(" ");
531 } else {
532 addSpace = true;
533 }
534 addedTerm = true;
535 if (LuceneUtils.isKeyword(word)) {
536 sb.append("\"");
537 LuceneUtils.appendEscapedLuceneQuery(sb, word);
538 sb.append("\"");
539 } else {
540 LuceneUtils.appendEscapedLuceneQuery(sb, word);
541 }
542 final String boostTerm = findBoostTerm(word, weightedText);
543
544
545
546
547
548
549 if (boostTerm != null) {
550 sb.append("^").append(weighting + WEIGHTING_BOOST);
551 if (!boostTerm.equals(word)) {
552 boostedTerms.append(" ");
553 LuceneUtils.appendEscapedLuceneQuery(boostedTerms, boostTerm);
554 boostedTerms.append("^").append(weighting + WEIGHTING_BOOST);
555 }
556 } else if (weighting > 1) {
557 sb.append("^").append(weighting);
558 }
559 }
560 if (boostedTerms.length() > 0) {
561 sb.append(boostedTerms);
562 }
563 }
564 sb.append(")");
565 return addedTerm;
566 }
567
568
569
570
571
572
573
574
575 private String cleanseText(String text) {
576 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
577 }
578
579
580
581
582
583
584
585
586
587
588 private String findBoostTerm(String term, Set<String> boost) {
589 for (String entry : boost) {
590 if (equalsIgnoreCaseAndNonAlpha(term, entry)) {
591 return entry;
592 }
593 }
594 return null;
595 }
596
597
598
599
600
601
602
603
604
605 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
606 if (l == null || r == null) {
607 return false;
608 }
609
610 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
611 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
612 return left.equalsIgnoreCase(right);
613 }
614
615
616
617
618
619
620
621
622
623
624
625 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency,
626 final Set<String> majorVersions) {
627 boolean isValid = false;
628
629
630
631
632 if (Ecosystem.NODEJS.equals(dependency.getEcosystem())) {
633 for (Identifier i : dependency.getSoftwareIdentifiers()) {
634 if (i instanceof PurlIdentifier) {
635 final PurlIdentifier p = (PurlIdentifier) i;
636 if (cleanPackageName(p.getName()).equals(cleanPackageName(entry.getProduct()))) {
637 isValid = true;
638 }
639 }
640 }
641 } else if (collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) {
642 if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct())) {
643 isValid = true;
644 } else {
645 isValid = majorVersions.stream().filter(version
646 -> version != null && entry.getProduct().endsWith("v" + version) && entry.getProduct().length() > version.length() + 1)
647 .anyMatch(version
648 -> collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT),
649 entry.getProduct().substring(0, entry.getProduct().length() - version.length() - 1))
650 );
651 isValid |= majorVersions.stream().filter(version
652 -> version != null && entry.getProduct().endsWith(version) && entry.getProduct().length() > version.length())
653 .anyMatch(version
654 -> collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT),
655 entry.getProduct().substring(0, entry.getProduct().length() - version.length()))
656 );
657 }
658 }
659 return isValid;
660 }
661
662
663
664
665
666
667
668 private String cleanPackageName(String name) {
669 if (name == null) {
670 return "";
671 }
672 return name.replaceAll("[^a-zA-Z0-9]+", "");
673 }
674
675
676
677
678
679
680
681
682 @SuppressWarnings("StringSplitter")
683 private boolean collectionContainsString(Set<Evidence> evidence, String text) {
684
685 if (text == null) {
686 return false;
687 }
688
689 final String textLC = text.toLowerCase();
690 for (Evidence e : evidence) {
691 if (e.getValue().toLowerCase().equals(textLC)) {
692 return true;
693 }
694 }
695
696 final String[] words = text.split("[\\s_-]+");
697 final List<String> list = new ArrayList<>();
698 String tempWord = null;
699 final CharArraySet stopWords = SearchFieldAnalyzer.getStopWords();
700 for (String word : words) {
701
702
703
704
705 if (tempWord != null) {
706 list.add(tempWord + word);
707 tempWord = null;
708 } else if (word.length() <= 2) {
709 tempWord = word;
710 } else {
711 if (stopWords.contains(word)) {
712 continue;
713 }
714 list.add(word);
715 }
716 }
717 if (tempWord != null) {
718 if (!list.isEmpty()) {
719 final String tmp = list.get(list.size() - 1) + tempWord;
720 list.add(tmp);
721 } else {
722 list.add(tempWord);
723 }
724 }
725 if (list.isEmpty()) {
726 return false;
727 }
728 boolean isValid = true;
729
730
731 final List<String> evidenceValues = new ArrayList<>(evidence.size());
732 evidence.forEach((e) -> evidenceValues.add(e.getValue().toLowerCase().replaceAll("[\\s_-]+", "")));
733
734 for (String word : list) {
735 word = word.toLowerCase();
736 boolean found = false;
737 for (String e : evidenceValues) {
738 if (e.contains(word)) {
739 if ("http".equals(word) && e.contains("http:")) {
740 continue;
741 }
742 found = true;
743 break;
744 }
745 }
746 isValid &= found;
747 }
748 return isValid;
749 }
750
751
752
753
754
755
756
757
758
759
760 @Override
761 protected void analyzeDependency(Dependency dependency, Engine engine) throws AnalysisException {
762 if (skipEcosystems.contains(dependency.getEcosystem())) {
763 return;
764 }
765 try {
766 determineCPE(dependency);
767 } catch (CorruptIndexException ex) {
768 throw new AnalysisException("CPE Index is corrupt.", ex);
769 } catch (IOException ex) {
770 throw new AnalysisException("Failure opening the CPE Index.", ex);
771 } catch (ParseException ex) {
772 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
773 }
774 }
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792 @SuppressWarnings("StringSplitter")
793 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
794 Confidence currentConfidence) throws AnalysisException {
795
796 final CpeBuilder cpeBuilder = new CpeBuilder();
797
798 final Set<CpePlus> cpePlusEntries = cve.getCPEs(vendor, product);
799 final Set<Cpe> cpes = filterEcosystem(dependency.getEcosystem(), cpePlusEntries);
800 if (cpes == null || cpes.isEmpty()) {
801 return false;
802 }
803
804 DependencyVersion bestGuess;
805 if ("Golang".equals(dependency.getEcosystem()) && dependency.getVersion() == null) {
806 bestGuess = new DependencyVersion("*");
807 } else {
808 bestGuess = new DependencyVersion("-");
809 }
810 String bestGuessUpdate = null;
811 Confidence bestGuessConf = null;
812 String bestGuessURL = null;
813 final Set<IdentifierMatch> collected = new HashSet<>();
814
815 considerDependencyVersion(dependency, vendor, product, currentConfidence, collected);
816
817
818
819
820 for (Confidence conf : Confidence.values()) {
821 for (Evidence evidence : dependency.getIterator(EvidenceType.VERSION, conf)) {
822 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue(), true);
823 if (evVer == null) {
824 continue;
825 }
826 DependencyVersion evBaseVer = null;
827 String evBaseVerUpdate = null;
828 final int idx = evVer.getVersionParts().size() - 1;
829 if (evVer.getVersionParts().get(idx)
830 .matches("^(v|release|final|snapshot|beta|alpha|u|rc|m|20\\d\\d).*$")) {
831
832 final String checkUpdate = evVer.getVersionParts().get(idx);
833 if (checkUpdate.matches("^(v|release|final|snapshot|beta|alpha|u|rc|m|20\\d\\d).*$")) {
834 evBaseVerUpdate = checkUpdate;
835 evBaseVer = new DependencyVersion();
836 evBaseVer.setVersionParts(evVer.getVersionParts().subList(0, idx));
837 }
838 }
839
840 for (Cpe vs : cpes) {
841 final DependencyVersion dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
842 DependencyVersion dbVerUpdate = dbVer;
843 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty() && !vs.getUpdate().startsWith("*") && !vs.getUpdate().startsWith("-")) {
844 dbVerUpdate = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate(), true);
845 }
846 if (dbVer == null) {
847 final String url = CpeIdentifier.nvdProductSearchUrlFor(vs);
848 final IdentifierMatch match = new IdentifierMatch(vs, url, IdentifierConfidence.BROAD_MATCH, conf);
849 collected.add(match);
850 } else if (evVer.equals(dbVer)) {
851 addExactMatch(vs, evBaseVerUpdate, conf, collected);
852 } else if (evBaseVer != null && evBaseVer.equals(dbVer)
853 && (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0)) {
854 bestGuessConf = conf;
855 bestGuess = dbVer;
856 bestGuessUpdate = evBaseVerUpdate;
857 bestGuessURL = CpeIdentifier.nvdSearchUrlFor(vs);
858 } else if (dbVerUpdate != null && evVer.getVersionParts().size() <= dbVerUpdate.getVersionParts().size()
859 && evVer.matchesAtLeastThreeLevels(dbVerUpdate)) {
860 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
861 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
862 bestGuess = dbVer;
863 bestGuessUpdate = evBaseVerUpdate;
864 bestGuessConf = conf;
865 }
866 }
867 }
868 }
869 if ((bestGuessConf == null || bestGuessConf.compareTo(conf) > 0)
870 && bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
871 bestGuess = evVer;
872 bestGuessUpdate = evBaseVerUpdate;
873 bestGuessConf = conf;
874 }
875 }
876 }
877
878 cpeBuilder.part(Part.APPLICATION).vendor(vendor).product(product);
879 final int idx = bestGuess.getVersionParts().size() - 1;
880 if (bestGuess.getVersionParts().get(idx)
881 .matches("^(v|release|final|snapshot|beta|alpha|u|rc|m|20\\d\\d).*$")) {
882 cpeBuilder.version(StringUtils.join(bestGuess.getVersionParts().subList(0, idx), "."));
883
884 if (bestGuess.getVersionParts().get(idx).matches("^v\\d.*$")) {
885 cpeBuilder.update(bestGuess.getVersionParts().get(idx).substring(1));
886 } else {
887 cpeBuilder.update(bestGuess.getVersionParts().get(idx));
888 }
889 } else {
890 cpeBuilder.version(bestGuess.toString());
891 if (bestGuessUpdate != null) {
892 cpeBuilder.update(bestGuessUpdate);
893 }
894 }
895 final Cpe guessCpe;
896
897 try {
898 guessCpe = cpeBuilder.build();
899 } catch (CpeValidationException ex) {
900 throw new AnalysisException(String.format("Unable to create a CPE for %s:%s:%s", vendor, product, bestGuess));
901 }
902 if (!"-".equals(guessCpe.getVersion())) {
903 String url = null;
904 if (bestGuessURL != null) {
905 url = bestGuessURL;
906 }
907 if (bestGuessConf == null) {
908 bestGuessConf = Confidence.LOW;
909 }
910 final IdentifierMatch match = new IdentifierMatch(guessCpe, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
911
912 collected.add(match);
913 }
914 boolean identifierAdded = false;
915 if (!collected.isEmpty()) {
916 final List<IdentifierMatch> items = new ArrayList<>(collected);
917
918 Collections.sort(items);
919 final IdentifierConfidence bestIdentifierQuality = items.get(0).getIdentifierConfidence();
920 final Confidence bestEvidenceQuality = items.get(0).getEvidenceConfidence();
921 boolean addedNonGuess = false;
922 final Confidence prevAddedConfidence = dependency.getVulnerableSoftwareIdentifiers().stream().map(Identifier::getConfidence)
923 .min(Comparator.comparing(Confidence::ordinal))
924 .orElse(Confidence.LOW);
925
926 for (IdentifierMatch m : items) {
927 if (bestIdentifierQuality.equals(m.getIdentifierConfidence())
928 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
929 final CpeIdentifier i = m.getIdentifier();
930 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
931 if (addedNonGuess) {
932 continue;
933 }
934 i.setConfidence(Confidence.LOW);
935 } else {
936 i.setConfidence(bestEvidenceQuality);
937 }
938 if (prevAddedConfidence.compareTo(i.getConfidence()) < 0) {
939 continue;
940 }
941
942
943 dependency.addVulnerableSoftwareIdentifier(i);
944 suppression.analyze(dependency, engine);
945 if (dependency.getVulnerableSoftwareIdentifiers().contains(i)) {
946 identifierAdded = true;
947 if (!addedNonGuess && bestIdentifierQuality != IdentifierConfidence.BEST_GUESS) {
948 addedNonGuess = true;
949 }
950 }
951 }
952 }
953 }
954 return identifierAdded;
955 }
956
957
958
959
960
961
962
963
964
965 private void addExactMatch(Cpe vs, String updateVersion, Confidence conf,
966 final Set<IdentifierMatch> collected) {
967
968 final CpeBuilder cpeBuilder = new CpeBuilder();
969 final String url = CpeIdentifier.nvdSearchUrlFor(vs);
970 Cpe useCpe;
971 if (updateVersion != null && "*".equals(vs.getUpdate())) {
972 try {
973 useCpe = cpeBuilder.part(vs.getPart()).wfVendor(vs.getWellFormedVendor())
974 .wfProduct(vs.getWellFormedProduct()).wfVersion(vs.getWellFormedVersion())
975 .wfEdition(vs.getWellFormedEdition()).wfLanguage(vs.getWellFormedLanguage())
976 .wfOther(vs.getWellFormedOther()).wfSwEdition(vs.getWellFormedSwEdition())
977 .update(updateVersion).build();
978 } catch (CpeValidationException ex) {
979 LOGGER.debug("Error building cpe with update:" + updateVersion, ex);
980 useCpe = vs;
981 }
982 } else {
983 useCpe = vs;
984 }
985 final IdentifierMatch match = new IdentifierMatch(useCpe, url, IdentifierConfidence.EXACT_MATCH, conf);
986 collected.add(match);
987 }
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002 private void considerDependencyVersion(Dependency dependency,
1003 String vendor, String product, Confidence confidence,
1004 final Set<IdentifierMatch> collected)
1005 throws AnalysisException {
1006
1007 if (dependency.getVersion() != null && !dependency.getVersion().isEmpty()) {
1008 final CpeBuilder cpeBuilder = new CpeBuilder();
1009 boolean useDependencyVersion = true;
1010 final CharArraySet stopWords = SearchFieldAnalyzer.getStopWords();
1011 if (dependency.getName() != null && !dependency.getName().isEmpty()) {
1012 final String name = dependency.getName();
1013 for (String word : product.split("[^a-zA-Z0-9]")) {
1014 useDependencyVersion &= name.contains(word) || stopWords.contains(word)
1015 || wordMatchesEcosystem(dependency.getEcosystem(), word);
1016 }
1017 }
1018
1019 if (useDependencyVersion) {
1020
1021
1022 final DependencyVersion depVersion = new DependencyVersion(dependency.getVersion());
1023 if (depVersion.getVersionParts().size() > 0) {
1024 cpeBuilder.part(Part.APPLICATION).vendor(vendor).product(product);
1025 addVersionAndUpdate(depVersion, cpeBuilder);
1026 try {
1027 final Cpe depCpe = cpeBuilder.build();
1028 final String url = CpeIdentifier.nvdSearchUrlFor(vendor, product, depCpe.getVersion());
1029 final IdentifierMatch match = new IdentifierMatch(depCpe, url, IdentifierConfidence.EXACT_MATCH, confidence);
1030 collected.add(match);
1031 } catch (CpeValidationException ex) {
1032 throw new AnalysisException(String.format("Unable to create a CPE for %s:%s:%s", vendor, product, depVersion));
1033 }
1034 }
1035 }
1036 }
1037 }
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052 private boolean wordMatchesEcosystem(@Nullable String ecosystem, String word) {
1053 if (Ecosystem.JAVA.equalsIgnoreCase(word)) {
1054 return Ecosystem.JAVA.equals(ecosystem);
1055 }
1056 return false;
1057 }
1058
1059
1060
1061
1062
1063
1064
1065 @Override
1066 protected String getAnalyzerEnabledSettingKey() {
1067 return Settings.KEYS.ANALYZER_CPE_ENABLED;
1068 }
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078 private Set<Cpe> filterEcosystem(String ecosystem, Set<CpePlus> entries) {
1079 if (entries == null || entries.isEmpty()) {
1080 return null;
1081 }
1082 if (ecosystem != null) {
1083 return entries.stream().filter(c
1084 -> c.getEcosystem() == null
1085 || c.getEcosystem().equals(ecosystem)
1086
1087 || (Ecosystem.IOS.equals(ecosystem) && Ecosystem.NATIVE.equals(c.getEcosystem())))
1088 .map(CpePlus::getCpe)
1089 .collect(Collectors.toSet());
1090 }
1091 return entries.stream()
1092 .map(CpePlus::getCpe)
1093 .collect(Collectors.toSet());
1094 }
1095
1096
1097
1098
1099
1100
1101
1102
1103 private void addVersionAndUpdate(DependencyVersion depVersion, final CpeBuilder cpeBuilder) {
1104 final int idx = depVersion.getVersionParts().size() - 1;
1105 if (idx > 0 && depVersion.getVersionParts().get(idx)
1106 .matches("^(v|final|release|snapshot|r|b|beta|a|alpha|u|rc|sp|dev|revision|service|build|pre|p|patch|update|m|20\\d\\d).*$")) {
1107 cpeBuilder.version(StringUtils.join(depVersion.getVersionParts().subList(0, idx), "."));
1108
1109 if (depVersion.getVersionParts().get(idx).matches("^v\\d.*$")) {
1110 cpeBuilder.update(depVersion.getVersionParts().get(idx).substring(1));
1111 } else {
1112 cpeBuilder.update(depVersion.getVersionParts().get(idx));
1113 }
1114 } else {
1115 cpeBuilder.version(depVersion.toString());
1116 }
1117 }
1118
1119
1120
1121
1122 private enum IdentifierConfidence {
1123
1124
1125
1126
1127 EXACT_MATCH,
1128
1129
1130
1131 BEST_GUESS,
1132
1133
1134
1135
1136
1137 BROAD_MATCH
1138 }
1139
1140
1141
1142
1143
1144 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
1145
1146
1147
1148
1149 private IdentifierConfidence identifierConfidence;
1150
1151
1152
1153 private CpeIdentifier identifier;
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165 IdentifierMatch(Cpe cpe, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
1166 this.identifier = new CpeIdentifier(cpe, url, evidenceConfidence);
1167 this.identifierConfidence = identifierConfidence;
1168 }
1169
1170
1171
1172
1173
1174
1175
1176 public Confidence getEvidenceConfidence() {
1177 return this.identifier.getConfidence();
1178 }
1179
1180
1181
1182
1183
1184
1185 public void setEvidenceConfidence(Confidence evidenceConfidence) {
1186 this.identifier.setConfidence(evidenceConfidence);
1187 }
1188
1189
1190
1191
1192
1193
1194 public IdentifierConfidence getIdentifierConfidence() {
1195 return identifierConfidence;
1196 }
1197
1198
1199
1200
1201
1202
1203 public void setIdentifierConfidence(IdentifierConfidence confidence) {
1204 this.identifierConfidence = confidence;
1205 }
1206
1207
1208
1209
1210
1211
1212 public CpeIdentifier getIdentifier() {
1213 return identifier;
1214 }
1215
1216
1217
1218
1219
1220
1221 public void setIdentifier(CpeIdentifier identifier) {
1222 this.identifier = identifier;
1223 }
1224
1225
1226
1227
1228
1229
1230
1231
1232 @Override
1233 public String toString() {
1234 return "IdentifierMatch{ IdentifierConfidence=" + identifierConfidence + ", identifier=" + identifier + '}';
1235 }
1236
1237
1238
1239
1240
1241
1242 @Override
1243 public int hashCode() {
1244 return new HashCodeBuilder(115, 303)
1245 .append(identifierConfidence)
1246 .append(identifier)
1247 .toHashCode();
1248 }
1249
1250
1251
1252
1253
1254
1255
1256 @Override
1257 public boolean equals(Object obj) {
1258 if (obj == null || !(obj instanceof IdentifierMatch)) {
1259 return false;
1260 }
1261 if (this == obj) {
1262 return true;
1263 }
1264 final IdentifierMatch other = (IdentifierMatch) obj;
1265 return new EqualsBuilder()
1266 .append(identifierConfidence, other.identifierConfidence)
1267 .append(identifier, other.identifier)
1268 .build();
1269 }
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279 @Override
1280 public int compareTo(@NotNull IdentifierMatch o) {
1281 return new CompareToBuilder()
1282 .append(identifierConfidence, o.identifierConfidence)
1283 .append(identifier, o.identifier)
1284 .toComparison();
1285 }
1286 }
1287
1288
1289
1290
1291
1292
1293 @SuppressWarnings("InfiniteLoopStatement")
1294 public static void main(String[] args) {
1295 final Settings props = new Settings();
1296 try (Engine en = new Engine(Engine.Mode.EVIDENCE_PROCESSING, props)) {
1297 en.openDatabase(false, false);
1298 final CPEAnalyzer analyzer = new CPEAnalyzer();
1299 analyzer.initialize(props);
1300 analyzer.prepareAnalyzer(en);
1301 LOGGER.error("test");
1302 System.out.println("Memory index query for ODC");
1303 try (BufferedReader br = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8))) {
1304 while (true) {
1305
1306 final Map<String, MutableInt> vendor = new HashMap<>();
1307 final Map<String, MutableInt> product = new HashMap<>();
1308 System.out.print("Vendor: ");
1309 String[] parts = br.readLine().split(" ");
1310 for (String term : parts) {
1311 final MutableInt count = vendor.get(term);
1312 if (count == null) {
1313 vendor.put(term, new MutableInt(0));
1314 } else {
1315 count.add(1);
1316 }
1317 }
1318 System.out.print("Product: ");
1319 parts = br.readLine().split(" ");
1320 for (String term : parts) {
1321 final MutableInt count = product.get(term);
1322 if (count == null) {
1323 product.put(term, new MutableInt(0));
1324 } else {
1325 count.add(1);
1326 }
1327 }
1328 final List<IndexEntry> list = analyzer.searchCPE(vendor, product, new HashSet<>(), new HashSet<>(), "default");
1329 if (list == null || list.isEmpty()) {
1330 System.out.println("No results found");
1331 } else {
1332 list.forEach((e) -> System.out.printf("%s:%s (%f)%n", e.getVendor(), e.getProduct(),
1333 e.getSearchScore()));
1334 }
1335 System.out.println();
1336 System.out.println();
1337 }
1338 }
1339 } catch (InitializationException | IOException ex) {
1340 System.err.println("Lucene ODC search tool failed:");
1341 System.err.println(ex.getMessage());
1342 }
1343 }
1344
1345
1346
1347
1348
1349
1350 protected void setCveDB(CveDB cveDb) {
1351 this.cve = cveDb;
1352 }
1353
1354
1355
1356
1357
1358
1359 protected CveDB getCveDB() {
1360 return this.cve;
1361 }
1362
1363
1364
1365
1366
1367
1368 protected void setMemoryIndex(MemoryIndex idx) {
1369 cpe = idx;
1370 }
1371
1372
1373
1374
1375
1376
1377 protected MemoryIndex getMemoryIndex() {
1378 return cpe;
1379 }
1380
1381
1382
1383
1384
1385
1386 protected void setCpeSuppressionAnalyzer(CpeSuppressionAnalyzer suppression) {
1387 this.suppression = suppression;
1388 }
1389 }