View Javadoc
1   /*
2    * This file is part of dependency-check-core.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   *
16   * Copyright (c) 2019 Jason Dillon. All Rights Reserved.
17   */
18  package org.owasp.dependencycheck.analyzer;
19  
20  import io.github.jeremylong.openvulnerability.client.nvd.CvssV2;
21  import io.github.jeremylong.openvulnerability.client.nvd.CvssV2Data;
22  import io.github.jeremylong.openvulnerability.client.nvd.CvssV4;
23  import org.apache.commons.lang3.StringUtils;
24  import org.jspecify.annotations.NonNull;
25  import org.owasp.dependencycheck.Engine;
26  import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
27  import org.owasp.dependencycheck.data.ossindex.OssIndexClientProvider;
28  import org.owasp.dependencycheck.dependency.Dependency;
29  import org.owasp.dependencycheck.dependency.Vulnerability;
30  import org.owasp.dependencycheck.dependency.VulnerableSoftware;
31  import org.owasp.dependencycheck.dependency.VulnerableSoftwareBuilder;
32  import org.owasp.dependencycheck.dependency.naming.Identifier;
33  import org.owasp.dependencycheck.dependency.naming.PurlIdentifier;
34  import org.owasp.dependencycheck.exception.InitializationException;
35  import org.owasp.dependencycheck.utils.CvssUtil;
36  import org.owasp.dependencycheck.utils.Settings;
37  import org.owasp.dependencycheck.utils.Settings.KEYS;
38  import org.slf4j.Logger;
39  import org.slf4j.LoggerFactory;
40  import org.sonatype.goodies.packageurl.InvalidException;
41  import org.sonatype.goodies.packageurl.PackageUrl;
42  import org.sonatype.ossindex.service.api.componentreport.ComponentReport;
43  import org.sonatype.ossindex.service.api.componentreport.ComponentReportVulnerability;
44  import org.sonatype.ossindex.service.api.cvss.Cvss2Severity;
45  import org.sonatype.ossindex.service.api.cvss.Cvss2Vector;
46  import org.sonatype.ossindex.service.api.cvss.CvssVector;
47  import org.sonatype.ossindex.service.api.cvss.CvssVectorFactory;
48  import org.sonatype.ossindex.service.client.OssindexClient;
49  import us.springett.parsers.cpe.exceptions.CpeValidationException;
50  import us.springett.parsers.cpe.values.Part;
51  
52  import javax.annotation.Nullable;
53  import java.net.SocketTimeoutException;
54  import java.util.Arrays;
55  import java.util.Collections;
56  import java.util.List;
57  import java.util.Map;
58  import java.util.Optional;
59  import java.util.concurrent.TimeUnit;
60  import java.util.function.Function;
61  import java.util.regex.Matcher;
62  import java.util.regex.Pattern;
63  
64  import static java.util.Optional.ofNullable;
65  import static java.util.stream.Collectors.toList;
66  import static org.apache.hc.core5.http.HttpStatus.SC_FORBIDDEN;
67  import static org.apache.hc.core5.http.HttpStatus.SC_PAYMENT_REQUIRED;
68  import static org.apache.hc.core5.http.HttpStatus.SC_TOO_MANY_REQUESTS;
69  import static org.apache.hc.core5.http.HttpStatus.SC_UNAUTHORIZED;
70  
71  /**
72   * Enrich dependency information from Sonatype OSS index.
73   *
74   * @author Jason Dillon
75   * @since 5.0.0
76   */
77  public class OssIndexAnalyzer extends AbstractAnalyzer {
78  
79      /**
80       * A reference to the logger.
81       */
82      private static final Logger LOG = LoggerFactory.getLogger(OssIndexAnalyzer.class);
83  
84      /**
85       * A pattern to match CVE identifiers.
86       */
87      private static final Pattern CVE_PATTERN = Pattern.compile("\\bCVE-\\d{4}-\\d{4,10}\\b");
88  
89      /**
90       * The reference type.
91       */
92      public static final String REFERENCE_TYPE = "OSSINDEX";
93  
94      /**
95       * Fetched reports.
96       */
97      private static Map<PackageUrl, ComponentReport> reports;
98  
99      /**
100      * Lock to protect fetching state.
101      */
102     private static final Object FETCH_MUTEX = new Object();
103 
104     @Override
105     public String getName() {
106         return "Sonatype OSS Index Analyzer";
107     }
108 
109     @Override
110     public AnalysisPhase getAnalysisPhase() {
111         return AnalysisPhase.FINDING_ANALYSIS_PHASE2;
112     }
113 
114     @Override
115     protected String getAnalyzerEnabledSettingKey() {
116         return Settings.KEYS.ANALYZER_OSSINDEX_ENABLED;
117     }
118 
119     /**
120      * Run without parallel support.
121      *
122      * @return false
123      */
124     @Override
125     public boolean supportsParallelProcessing() {
126         return true;
127     }
128 
129     @Override
130     protected void closeAnalyzer() throws Exception {
131         synchronized (FETCH_MUTEX) {
132             reports = null;
133         }
134     }
135 
136     @Override
137     protected void prepareAnalyzer(Engine engine) throws InitializationException {
138         synchronized (FETCH_MUTEX) {
139             if (getSettings().getString(KEYS.ANALYZER_OSSINDEX_URL, "").contains("ossindex.sonatype.org")) {
140                 LOG.warn("{} capabilities are being migrated to Sonatype Guide. All integrations must migrate to using " +
141                         "a Sonatype Guide base URL or proxy. See " +
142                         "https://dependency-check.github.io/DependencyCheck/analyzers/oss-index-analyzer.html " +
143                         "for more information on migration to Sonatype Guide.", getName());
144             }
145             if (password().isEmpty() || (user().isEmpty() && passwordNotSonatypeGuideToken())) {
146                 LOG.warn("{} disabled due to missing credentials. Authentication with token is now required, and OSS Index " +
147                         "is migrating to Sonatype Guide. See https://dependency-check.github.io/DependencyCheck/analyzers/oss-index-analyzer.html " +
148                         "for more information on authentication with Sonatype Guide OSS Index.", getName());
149                 setEnabled(false);
150             } else if (passwordNotSonatypeGuideToken()) {
151                 LOG.warn("Sonatype OSS Index is migrating to Sonatype Guide, but it looks like you're not yet using a Sonatype Guide personal " +
152                         "access token. Legacy OSS Index API tokens should be replaced with Sonatype Guide Personal Access Tokens " +
153                         "before December 31, 2026. See https://dependency-check.github.io/DependencyCheck/analyzers/oss-index-analyzer.html " +
154                         "for more information on authentication with Sonatype Guide OSS Index.");
155             }
156         }
157     }
158 
159     private boolean passwordNotSonatypeGuideToken() {
160         return !password().startsWith("sonatype_pat_");
161     }
162 
163     private @NonNull String user() {
164         return getSettings().getString(KEYS.ANALYZER_OSSINDEX_USER, "");
165     }
166 
167     private @NonNull String password() {
168         return getSettings().getString(KEYS.ANALYZER_OSSINDEX_PASSWORD, "").trim();
169     }
170 
171     @Override
172     protected void analyzeDependency(final Dependency dependency, final Engine engine) throws AnalysisException {
173         // batch request component-reports for all dependencies
174         synchronized (FETCH_MUTEX) {
175             if (reports == null && isEnabled()) {
176                 try {
177                     requestDelay();
178                     reports = requestReports(engine.getDependencies());
179                 } catch (SocketTimeoutException e) {
180                     if (getSettings().getBoolean(KEYS.ANALYZER_OSSINDEX_WARN_ONLY_ON_REMOTE_ERRORS, false)) {
181                         LOG.warn("Sonatype OSS Index / Guide socket timeout", e);
182                     } else {
183                         throw new AnalysisException("Failed to establish socket to Sonatype OSS Index / Guide", e);
184                     }
185                 } catch (Exception ex) {
186                     OssIndexKnownError error = Arrays.stream(OssIndexKnownError.values())
187                             .filter(e -> e.matches(ex))
188                             .findFirst()
189                             .orElse(OssIndexKnownError.Unknown);
190 
191                     this.setEnabled(!error.fatal);
192                     String logMessage = error.errorMessage(ex);
193 
194                     if (getSettings().getBoolean(KEYS.ANALYZER_OSSINDEX_WARN_ONLY_ON_REMOTE_ERRORS, false)) {
195                         LOG.warn(logMessage);
196                     } else {
197                         throw new AnalysisException(logMessage, ex);
198                     }
199                 }
200             }
201 
202             // skip enrichment if we failed to fetch reports
203             if (reports != null) {
204                 enrich(dependency);
205             }
206         }
207     }
208 
209     /**
210      * Delays each request (thread) by the configured amount of seconds, if the
211      * configuration is present.
212      */
213     private void requestDelay() throws InterruptedException {
214         final int delay = getSettings().getInt(Settings.KEYS.ANALYZER_OSSINDEX_REQUEST_DELAY, 0);
215         if (delay > 0) {
216             LOG.debug("Request delay: {}", delay);
217             sleepSeconds(delay);
218         }
219     }
220 
221     void sleepSeconds(int delay) throws InterruptedException {
222         TimeUnit.SECONDS.sleep(delay);
223     }
224 
225     /**
226      * Helper to complain if unable to parse Package-URL.
227      *
228      * @param value the url to parse
229      * @return the package url
230      */
231     @Nullable
232     private PackageUrl parsePackageUrl(final String value) {
233         try {
234             return PackageUrl.parse(value);
235         } catch (InvalidException e) {
236             LOG.debug("Invalid Package-URL: {}", value, e);
237             return null;
238         }
239     }
240 
241     /**
242      * Batch request component-reports for all dependencies.
243      *
244      * @param dependencies the collection of dependencies
245      * @return the map of dependency to OSS Index's component-report
246      * @throws Exception thrown if there is an exception requesting the report
247      */
248     private Map<PackageUrl, ComponentReport> requestReports(final Dependency[] dependencies) throws Exception {
249         // create requests for each dependency which has a PURL identifier
250         final List<PackageUrl> packages = Arrays.stream(dependencies)
251                 .flatMap(dependency -> dependency.getSoftwareIdentifiers().stream())
252                 .filter(id -> id instanceof PurlIdentifier)
253                 .map(id -> parsePackageUrl(id.getValue()))
254                 .filter(id -> id != null && StringUtils.isNotBlank(id.getVersion()))
255                 .distinct()
256                 .collect(toList());
257 
258         LOG.debug("Requesting component-reports for {} dependencies with {} unique Package-URL identifiers", dependencies.length, packages.size());
259         // only attempt if we have been able to collect some packages
260         if (!packages.isEmpty()) {
261             try (OssindexClient client = OssIndexClientProvider.create(getSettings())) {
262                 LOG.debug("OSS Index Analyzer submitting: {}", packages);
263                 return client.requestComponentReports(packages);
264             }
265         }
266         LOG.warn("Unable to determine Package-URL identifiers for {} dependencies", dependencies.length);
267         return Collections.emptyMap();
268     }
269 
270     /**
271      * Known mappings of HTTP error codes to user messages
272      */
273     enum OssIndexKnownError {
274         Unauthorized(SC_UNAUTHORIZED, "has invalid credentials", true),
275         Forbidden(SC_FORBIDDEN, "access forbidden", true),
276         TooManyRequests(SC_TOO_MANY_REQUESTS, "rate limit exceeded", false),
277         InsufficientCredits(SC_PAYMENT_REQUIRED, "credits insufficient / payment required", true),
278         Unknown(999, "had unknown error", false, Exception::getMessage);
279 
280         final int statusCode;
281         final String userMessage;
282         final boolean fatal;
283         final Function<Exception, String> messageSuffix;
284 
285         OssIndexKnownError(int statusCode, String userMessage, boolean fatal) {
286             this(statusCode, userMessage, fatal, ex -> "");
287         }
288 
289         OssIndexKnownError(int statusCode, String userMessage, boolean fatal, Function<Exception, String> messageSuffix) {
290             this.statusCode = statusCode;
291             this.userMessage = userMessage;
292             this.fatal = fatal;
293             this.messageSuffix = messageSuffix;
294         }
295 
296         private String errorMessage(Exception ex) {
297             return String.format("Sonatype OSS Index / Guide %s%s. %s",
298                     userMessage,
299                     fatal ? ", disabling the analyzer" : "",
300                     messageSuffix.apply(ex)
301             ).trim();
302         }
303 
304         private boolean matches(Exception ex) {
305             return ex.toString().contains(Integer.toString(statusCode));
306         }
307     }
308 
309     /**
310      * Attempt to enrich given dependency with vulnerability details from OSS
311      * Index component-report.
312      *
313      * @param dependency the dependency to enrich
314      */
315     void enrich(final Dependency dependency) {
316         LOG.debug("Enrich dependency: {}", dependency);
317 
318         for (Identifier id : dependency.getSoftwareIdentifiers()) {
319             if (id instanceof PurlIdentifier) {
320                 LOG.debug("  Package: {} -> {}", id, id.getConfidence());
321 
322                 final PackageUrl purl = parsePackageUrl(id.getValue());
323                 if (purl != null && StringUtils.isNotBlank(purl.getVersion())) {
324                     try {
325                         final ComponentReport report = reports.get(purl);
326                         if (report == null) {
327                             LOG.debug("Missing component-report for: {}", purl);
328                             continue;
329                         }
330 
331                         // expose the URL to the package details for report generation
332                         id.setUrl(report.getReference().toString());
333 
334                         report.getVulnerabilities().stream()
335                                 .map((vuln) -> transform(report, vuln))
336                                 .forEachOrdered((v) -> {
337                                     final Vulnerability existing = dependency.getVulnerabilities().stream()
338                                             .filter(e -> e.getName().equals(v.getName())).findFirst()
339                                             .orElse(null);
340                                     if (existing != null) {
341                                         //TODO - can we enhance anything other than the references?
342                                         existing.addReferences(v.getReferences());
343                                     } else {
344                                         dependency.addVulnerability(v);
345                                     }
346                                 });
347                     } catch (Exception e) {
348                         LOG.warn("Failed to fetch component-report for: {}", purl, e);
349                     }
350                 }
351             }
352         }
353     }
354 
355     /**
356      * Transform OSS Index component-report to ODC vulnerability.
357      *
358      * @param report the component report
359      * @param source the vulnerability from the report to transform
360      * @return the transformed vulnerability
361      */
362     private Vulnerability transform(final ComponentReport report, final ComponentReportVulnerability source) {
363         final Vulnerability result = new Vulnerability();
364         result.setSource(Vulnerability.Source.OSSINDEX);
365         result.setName(nameFrom(source));
366         result.setDescription(source.getDescription());
367         result.addCwe(source.getCwe());
368 
369         final double cvssScore = source.getCvssScore() != null ? source.getCvssScore().doubleValue() : -1;
370 
371         if (source.getCvssVector() != null) {
372             if (source.getCvssVector().startsWith("CVSS:4")) {
373                 result.setCvssV4(CvssUtil.vectorToCvssV4("ossindex", CvssV4.Type.PRIMARY, cvssScore, source.getCvssVector()));
374             } else if (source.getCvssVector().startsWith("CVSS:3")) {
375                 result.setCvssV3(CvssUtil.vectorToCvssV3(source.getCvssVector(), cvssScore));
376             } else {
377                 // convert cvss details
378                 final CvssVector cvssVector = CvssVectorFactory.create(source.getCvssVector());
379                 final Map<String, String> metrics = cvssVector.getMetrics();
380                 if (cvssVector instanceof Cvss2Vector) {
381                     String tmp = metrics.get(Cvss2Vector.ACCESS_VECTOR);
382                     CvssV2Data.AccessVectorType accessVector = null;
383                     if (tmp != null) {
384                         accessVector = CvssV2Data.AccessVectorType.fromValue(tmp);
385                     }
386                     tmp = metrics.get(Cvss2Vector.ACCESS_COMPLEXITY);
387                     CvssV2Data.AccessComplexityType accessComplexity = null;
388                     if (tmp != null) {
389                         accessComplexity = CvssV2Data.AccessComplexityType.fromValue(tmp);
390                     }
391                     tmp = metrics.get(Cvss2Vector.AUTHENTICATION);
392                     CvssV2Data.AuthenticationType authentication = null;
393                     if (tmp != null) {
394                         authentication = CvssV2Data.AuthenticationType.fromValue(tmp);
395                     }
396                     tmp = metrics.get(Cvss2Vector.CONFIDENTIALITY_IMPACT);
397                     CvssV2Data.CiaType confidentialityImpact = null;
398                     if (tmp != null) {
399                         confidentialityImpact = CvssV2Data.CiaType.fromValue(tmp);
400                     }
401                     tmp = metrics.get(Cvss2Vector.INTEGRITY_IMPACT);
402                     CvssV2Data.CiaType integrityImpact = null;
403                     if (tmp != null) {
404                         integrityImpact = CvssV2Data.CiaType.fromValue(tmp);
405                     }
406                     tmp = metrics.get(Cvss2Vector.AVAILABILITY_IMPACT);
407                     CvssV2Data.CiaType availabilityImpact = null;
408                     if (tmp != null) {
409                         availabilityImpact = CvssV2Data.CiaType.fromValue(tmp);
410                     }
411                     final String severity = Cvss2Severity.of((float) cvssScore).name().toUpperCase();
412                     final CvssV2Data cvssData = new CvssV2Data(CvssV2Data.Version._2_0, source.getCvssVector(), accessVector,
413                             accessComplexity, authentication, confidentialityImpact,
414                             integrityImpact, availabilityImpact, cvssScore,
415                             severity, null, null, null, null, null, null, null, null, null, null);
416                     final CvssV2 cvssV2 = new CvssV2(null, null, cvssData, severity, null, null, null, null, null, null, null);
417                     result.setCvssV2(cvssV2);
418                 } else {
419                     LOG.warn("Unsupported CVSS vector: {}", cvssVector);
420                     result.setUnscoredSeverity(Double.toString(cvssScore));
421                 }
422             }
423         } else {
424             LOG.debug("OSS has no vector for {}", result.getName());
425             result.setUnscoredSeverity(Double.toString(cvssScore));
426         }
427         // generate a reference to the vulnerability details on OSS Index
428         result.addReference(REFERENCE_TYPE, source.getTitle(), source.getReference().toString());
429 
430         // generate references to other references reported by OSS Index
431         source.getExternalReferences().forEach(externalReference
432                 -> result.addReference("OSSIndex", externalReference.toString(), externalReference.toString()));
433 
434         // attach vulnerable software details as best we can
435         final PackageUrl purl = report.getCoordinates();
436         try {
437             final VulnerableSoftwareBuilder builder = new VulnerableSoftwareBuilder()
438                     .part(Part.APPLICATION)
439                     .vendor(purl.getNamespaceAsString())
440                     .product(purl.getName())
441                     .version(purl.getVersion());
442 
443             // TODO: consider if we want/need to extract version-ranges to apply to vulnerable-software?
444             final VulnerableSoftware software = builder.build();
445             result.addVulnerableSoftware(software);
446             result.setMatchedVulnerableSoftware(software);
447         } catch (CpeValidationException e) {
448             LOG.warn("Unable to construct vulnerable-software for: {}", purl, e);
449         }
450 
451         return result;
452     }
453 
454     private static String nameFrom(ComponentReportVulnerability vuln) {
455         return ofNullable(vuln.getCve())
456                 .or(() -> ofNullable(vuln.getTitle()).map(title -> matchesCveRegex(title)
457                                 .or(() -> ofNullable(vuln.getReference()).flatMap(reference -> matchesCveRegex(reference.toString())))
458                                 .orElse(title)))
459                 .orElse(vuln.getId());
460     }
461 
462     private static Optional<String> matchesCveRegex(String value) {
463         final Matcher matcher = CVE_PATTERN.matcher(value);
464         if (matcher.find()) {
465             return Optional.of(matcher.group());
466         }
467         return Optional.empty();
468     }
469 }