1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.update;
19
20 import com.fasterxml.jackson.databind.ObjectMapper;
21 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
22 import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
23 import io.github.jeremylong.openvulnerability.client.nvd.NvdApiException;
24 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient;
25 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder;
26 import java.io.File;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.StringReader;
30 import java.net.MalformedURLException;
31 import java.net.URI;
32 import java.net.URISyntaxException;
33 import java.net.URL;
34 import java.text.MessageFormat;
35 import java.time.Duration;
36 import java.time.LocalDate;
37 import java.time.ZoneId;
38 import java.time.ZoneOffset;
39 import java.time.ZonedDateTime;
40 import java.util.ArrayList;
41 import java.util.Collection;
42 import java.util.HashMap;
43 import java.util.HashSet;
44 import java.util.LinkedHashMap;
45 import java.util.List;
46 import java.util.Map;
47 import java.util.Objects;
48 import java.util.Optional;
49 import java.util.Properties;
50 import java.util.Set;
51 import java.util.concurrent.ExecutionException;
52 import java.util.concurrent.ExecutorService;
53 import java.util.concurrent.Executors;
54 import java.util.concurrent.Future;
55 import java.util.function.Function;
56 import java.util.zip.GZIPOutputStream;
57
58 import org.jetbrains.annotations.NotNull;
59 import org.owasp.dependencycheck.Engine;
60 import org.owasp.dependencycheck.data.nvdcve.CveDB;
61 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
62 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
63 import org.owasp.dependencycheck.data.update.exception.UpdateException;
64 import org.owasp.dependencycheck.data.update.nvd.api.DownloadTask;
65 import org.owasp.dependencycheck.data.update.nvd.api.NvdApiProcessor;
66 import org.owasp.dependencycheck.utils.DateUtil;
67 import org.owasp.dependencycheck.utils.DownloadFailedException;
68 import org.owasp.dependencycheck.utils.Downloader;
69 import org.owasp.dependencycheck.utils.InvalidSettingException;
70 import org.owasp.dependencycheck.utils.Pair;
71 import org.owasp.dependencycheck.utils.ResourceNotFoundException;
72 import org.owasp.dependencycheck.utils.Settings;
73 import org.owasp.dependencycheck.utils.TooManyRequestsException;
74 import org.slf4j.Logger;
75 import org.slf4j.LoggerFactory;
76
77 import static java.nio.charset.StandardCharsets.UTF_8;
78
79
80
81
82
83 public class NvdApiDataSource implements CachedWebDataSource {
84
85
86
87
88 private static final Logger LOGGER = LoggerFactory.getLogger(NvdApiDataSource.class);
89
90
91
92 private static final int PROCESSING_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors();
93
94
95
96 private Settings settings;
97
98
99
100 private CveDB cveDb = null;
101
102
103
104 private DatabaseProperties dbProperties = null;
105
106
107
108 private static final String NVD_API_CACHE_MODIFIED_DATE = "lastModifiedDate";
109
110
111
112
113 private static final int RESULTS_PER_PAGE = 2000;
114
115 @Override
116 public boolean update(Engine engine) throws UpdateException {
117 this.settings = engine.getSettings();
118 this.cveDb = engine.getDatabase();
119 if (isUpdateConfiguredFalse()) {
120 return false;
121 }
122 dbProperties = cveDb.getDatabaseProperties();
123
124 final String nvdDataFeedUrl = settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL);
125 if (nvdDataFeedUrl != null) {
126 return processDatafeed(nvdDataFeedUrl);
127 }
128 return processApi();
129 }
130
131 private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException {
132 boolean updatesMade = false;
133 try {
134 dbProperties = cveDb.getDatabaseProperties();
135 if (checkUpdate()) {
136 FeedUrl urlData = FeedUrl.extractFromUrlOptionalPattern(nvdDataFeedUrl);
137 final Properties cacheProperties = getRemoteDataFeedCacheProperties(urlData);
138 urlData = urlData.withPattern(p -> p.orElse(cacheProperties.getProperty("prefix", FeedUrl.DEFAULT_FILE_PATTERN_PREFIX) + FeedUrl.DEFAULT_FILE_PATTERN_SUFFIX));
139
140 final ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
141 final Map<String, String> updateable = getUpdatesNeeded(urlData, cacheProperties, now);
142 if (!updateable.isEmpty()) {
143 final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1);
144 final int downloadPoolSize = Math.min(Runtime.getRuntime().availableProcessors(), max);
145
146 final int execPoolSize = Math.min(PROCESSING_THREAD_POOL_SIZE, 2);
147
148 ExecutorService processingExecutorService = null;
149 ExecutorService downloadExecutorService = null;
150 try {
151 downloadExecutorService = Executors.newFixedThreadPool(downloadPoolSize);
152 processingExecutorService = Executors.newFixedThreadPool(execPoolSize);
153
154 DownloadTask runLast = null;
155 final Set<Future<Future<NvdApiProcessor>>> downloadFutures = new HashSet<>(updateable.size());
156 runLast = startDownloads(updateable, processingExecutorService, runLast, downloadFutures, downloadExecutorService);
157
158
159 final Set<Future<NvdApiProcessor>> processFutures = new HashSet<>(updateable.size());
160 for (Future<Future<NvdApiProcessor>> future : downloadFutures) {
161 processDownload(future, processFutures);
162 }
163
164 processFuture(processFutures);
165 processFutures.clear();
166
167
168 if (runLast != null) {
169 final Future<Future<NvdApiProcessor>> modified = downloadExecutorService.submit(runLast);
170 processDownload(modified, processFutures);
171 processFuture(processFutures);
172 }
173
174 } finally {
175 if (processingExecutorService != null) {
176 processingExecutorService.shutdownNow();
177 }
178 if (downloadExecutorService != null) {
179 downloadExecutorService.shutdownNow();
180 }
181 }
182 updatesMade = true;
183 }
184 storeLastModifiedDates(now, cacheProperties, updateable);
185 if (updatesMade) {
186 cveDb.persistEcosystemCache();
187 }
188 final int updateCount = cveDb.updateEcosystemCache();
189 LOGGER.debug("Corrected the ecosystem for {} ecoSystemCache entries", updateCount);
190 if (updatesMade || updateCount > 0) {
191 cveDb.cleanupDatabase();
192 }
193 }
194 } catch (UpdateException ex) {
195 if (ex.getCause() != null && ex.getCause() instanceof DownloadFailedException) {
196 final String jre = System.getProperty("java.version");
197 if (jre == null || jre.startsWith("1.4") || jre.startsWith("1.5") || jre.startsWith("1.6") || jre.startsWith("1.7")) {
198 LOGGER.error("An old JRE is being used ({} {}), and likely does not have the correct root certificates or algorithms "
199 + "to connect to the NVD - consider upgrading your JRE.", System.getProperty("java.vendor"), jre);
200 }
201 }
202 throw ex;
203 } catch (DatabaseException ex) {
204 throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex);
205 }
206 return updatesMade;
207 }
208
209 private void storeLastModifiedDates(final ZonedDateTime now, final Properties cacheProperties,
210 final Map<String, String> updateable) throws UpdateException {
211
212 final ZonedDateTime lastModifiedRequest = DatabaseProperties.getTimestamp(cacheProperties,
213 NVD_API_CACHE_MODIFIED_DATE + ".modified");
214 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_CHECKED, now);
215 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED, lastModifiedRequest);
216
217 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, now);
218 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
219
220 for (String entry : updateable.keySet()) {
221 final ZonedDateTime date = DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE + "." + entry);
222 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + entry, date);
223 }
224 }
225
226 private DownloadTask startDownloads(final Map<String, String> updateable, ExecutorService processingExecutorService, DownloadTask runLast,
227 final Set<Future<Future<NvdApiProcessor>>> downloadFutures, ExecutorService downloadExecutorService) throws UpdateException {
228 DownloadTask lastCall = runLast;
229 for (Map.Entry<String, String> cve : updateable.entrySet()) {
230 final DownloadTask call = new DownloadTask(cve.getValue(), processingExecutorService, cveDb, settings);
231 if (call.isModified()) {
232 lastCall = call;
233 } else {
234 final boolean added = downloadFutures.add(downloadExecutorService.submit(call));
235 if (!added) {
236 throw new UpdateException("Unable to add the download task for " + cve);
237 }
238 }
239 }
240 return lastCall;
241 }
242
243 private void processFuture(final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
244
245 for (Future<NvdApiProcessor> future : processFutures) {
246 try {
247 final NvdApiProcessor task = future.get();
248 } catch (InterruptedException ex) {
249 LOGGER.debug("Thread was interrupted during processing", ex);
250 Thread.currentThread().interrupt();
251 throw new UpdateException(ex);
252 } catch (ExecutionException ex) {
253 LOGGER.debug("Execution Exception during process", ex);
254 throw new UpdateException(ex);
255 }
256 }
257 }
258
259 private void processDownload(Future<Future<NvdApiProcessor>> future, final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
260 final Future<NvdApiProcessor> task;
261 try {
262 task = future.get();
263 if (task != null) {
264 processFutures.add(task);
265 }
266 } catch (InterruptedException ex) {
267 LOGGER.debug("Thread was interrupted during download", ex);
268 Thread.currentThread().interrupt();
269 throw new UpdateException("The download was interrupted", ex);
270 } catch (ExecutionException ex) {
271 LOGGER.debug("Thread was interrupted during download execution", ex);
272 throw new UpdateException("The execution of the download was interrupted", ex);
273 }
274 }
275
276 private boolean processApi() throws UpdateException {
277 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_CHECKED);
278 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
279 if (cveDb.dataExists() && lastChecked != null && validForHours > 0) {
280
281 final long validForSeconds = validForHours * 60L * 60L;
282 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
283 final Duration duration = Duration.between(lastChecked, now);
284 final long difference = duration.getSeconds();
285 if (difference < validForSeconds) {
286 LOGGER.info("Skipping the NVD API Update as it was completed within the last {} minutes", validForSeconds / 60);
287 return false;
288 }
289 }
290
291 ZonedDateTime lastModifiedRequest = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_MODIFIED);
292 final NvdCveClientBuilder builder = NvdCveClientBuilder.aNvdCveApi();
293 final String endpoint = settings.getString(Settings.KEYS.NVD_API_ENDPOINT);
294 if (endpoint != null) {
295 builder.withEndpoint(endpoint);
296 }
297 if (lastModifiedRequest != null) {
298
299 lastModifiedRequest = lastModifiedRequest.withZoneSameInstant(ZoneId.of("UTC"));
300 final ZonedDateTime end = lastModifiedRequest.plusDays(120);
301 builder.withLastModifiedFilter(lastModifiedRequest, end);
302 }
303 final String key = settings.getString(Settings.KEYS.NVD_API_KEY);
304 if (key != null) {
305
306 builder.withApiKey(key)
307 .withrequestsPerThirtySeconds(settings.getInt(Settings.KEYS.NVD_API_REQUESTS_PER_30_SECONDS_WITH_API_KEY, 50));
308 } else {
309 LOGGER.warn("An NVD API Key was not provided - it is highly recommended to use "
310 + "an NVD API key as the update can take a VERY long time without an API Key");
311 builder.withrequestsPerThirtySeconds(settings.getInt(Settings.KEYS.NVD_API_REQUESTS_PER_30_SECONDS_WITHOUT_API_KEY, 5));
312 }
313
314 final int resultsPerPage = Math.min(settings.getInt(Settings.KEYS.NVD_API_RESULTS_PER_PAGE, RESULTS_PER_PAGE), RESULTS_PER_PAGE);
315
316 builder.withResultsPerPage(resultsPerPage);
317
318
319
320
321
322
323 final int retryCount = settings.getInt(Settings.KEYS.NVD_API_MAX_RETRY_COUNT, 10);
324 builder.withMaxRetryCount(retryCount);
325 long delay = 0;
326 try {
327 delay = settings.getLong(Settings.KEYS.NVD_API_DELAY);
328 } catch (InvalidSettingException ex) {
329 LOGGER.warn("Invalid setting `NVD_API_DELAY`? ({}), using default delay", settings.getString(Settings.KEYS.NVD_API_DELAY));
330 }
331 if (delay > 0) {
332 builder.withDelay(delay);
333 }
334
335 ExecutorService processingExecutorService = null;
336 try {
337 processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE);
338 final List<Future<NvdApiProcessor>> submitted = new ArrayList<>();
339 int max = -1;
340 int ctr = 0;
341 try (NvdCveClient api = builder.build()) {
342 while (api.hasNext()) {
343 final Collection<DefCveItem> items = api.next();
344 max = api.getTotalAvailable();
345 if (ctr == 0) {
346 LOGGER.info(String.format("NVD API has %,d records in this update", max));
347 }
348 if (items != null && !items.isEmpty()) {
349 final ObjectMapper objectMapper = new ObjectMapper();
350 objectMapper.registerModule(new JavaTimeModule());
351 final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
352 try (FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream out = new GZIPOutputStream(fos);) {
353 objectMapper.writeValue(out, items);
354 final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
355 submitted.add(f);
356 }
357 ctr += 1;
358 if ((ctr % 5) == 0) {
359
360 final double percent = (double) (ctr * resultsPerPage) / max * 100;
361 if (percent < 100) {
362 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", ctr * resultsPerPage, max, percent));
363 }
364 }
365 }
366 final ZonedDateTime last = api.getLastUpdated();
367 if (last != null && (lastModifiedRequest == null || lastModifiedRequest.compareTo(last) < 0)) {
368 lastModifiedRequest = last;
369 }
370 }
371
372 } catch (Exception e) {
373 if (e instanceof NvdApiException && (e.getMessage().equals("NVD Returned Status Code: 404")
374 || e.getMessage().equals("NVD Returned Status Code: 403"))) {
375 final String msg;
376 if (key != null) {
377 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nPlease ensure your API Key is valid; "
378 + "see https://github.com/jeremylong/open-vulnerability-cli/blob/main/README.md#api-key-is-used-and-a-403-or-404-error-occurs\n\n"
379 + "If your NVD API Key is valid try increasing the NVD API Delay.\n\n"
380 + "If this is occurring in a CI environment";
381 } else {
382 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nConsider using an NVD API Key; "
383 + "see https://github.com/dependency-check/DependencyCheck?tab=readme-ov-file#nvd-api-key-highly-recommended";
384 }
385 throw new UpdateException(msg);
386 } else {
387 throw new UpdateException("Error updating the NVD Data", e);
388 }
389 }
390 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", max, max, 100f));
391 max = submitted.size();
392 final boolean updated = max > 0;
393 ctr = 0;
394 for (Future<NvdApiProcessor> f : submitted) {
395 try {
396 final NvdApiProcessor proc = f.get();
397 ctr += 1;
398 final double percent = (double) ctr / max * 100;
399 LOGGER.info(String.format("Completed processing batch %d/%d (%.0f%%) in %,dms", ctr, max, percent, proc.getDurationMillis()));
400 } catch (InterruptedException ex) {
401 Thread.currentThread().interrupt();
402 throw new RuntimeException(ex);
403 } catch (ExecutionException ex) {
404 LOGGER.error("Exception processing NVD API Results", ex);
405 throw new RuntimeException(ex);
406 }
407 }
408 if (lastModifiedRequest != null) {
409 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, ZonedDateTime.now());
410 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
411 }
412 return updated;
413 } finally {
414 if (processingExecutorService != null) {
415 processingExecutorService.shutdownNow();
416 }
417 }
418 }
419
420
421
422
423
424
425
426 private boolean isUpdateConfiguredFalse() {
427 if (!settings.getBoolean(Settings.KEYS.UPDATE_NVDCVE_ENABLED, true)) {
428 return true;
429 }
430 boolean autoUpdate = true;
431 try {
432 autoUpdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
433 } catch (InvalidSettingException ex) {
434 LOGGER.debug("Invalid setting for auto-update; using true.");
435 }
436 return !autoUpdate;
437 }
438
439 @Override
440 public boolean purge(Engine engine) {
441 boolean result = true;
442 try {
443 final File dataDir = engine.getSettings().getDataDirectory();
444 final File db = new File(dataDir, engine.getSettings().getString(Settings.KEYS.DB_FILE_NAME, "odc.mv.db"));
445 if (db.exists()) {
446 if (db.delete()) {
447 LOGGER.info("Database file purged; local copy of the NVD has been removed");
448 } else {
449 LOGGER.error("Unable to delete '{}'; please delete the file manually", db.getAbsolutePath());
450 result = false;
451 }
452 } else {
453 LOGGER.info("Unable to purge database; the database file does not exist: {}", db.getAbsolutePath());
454 result = false;
455 }
456 final File traceFile = new File(dataDir, "odc.trace.db");
457 if (traceFile.exists() && !traceFile.delete()) {
458 LOGGER.error("Unable to delete '{}'; please delete the file manually", traceFile.getAbsolutePath());
459 result = false;
460 }
461 final File lockFile = new File(dataDir, "odc.update.lock");
462 if (lockFile.exists() && !lockFile.delete()) {
463 LOGGER.error("Unable to delete '{}'; please delete the file manually", lockFile.getAbsolutePath());
464 result = false;
465 }
466 } catch (IOException ex) {
467 final String msg = "Unable to delete the database";
468 LOGGER.error(msg, ex);
469 result = false;
470 }
471 return result;
472 }
473
474
475
476
477
478
479
480
481
482 private boolean checkUpdate() throws UpdateException {
483 boolean proceed = true;
484
485 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
486 if (dataExists() && 0 < validForHours) {
487
488 final long validForSeconds = validForHours * 60L * 60L;
489 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_CHECKED);
490 if (lastChecked != null) {
491 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
492 final Duration duration = Duration.between(lastChecked, now);
493 final long difference = duration.getSeconds();
494 proceed = difference > validForSeconds;
495 if (!proceed) {
496 LOGGER.info("Skipping NVD API Cache check since last check was within {} hours.", validForHours);
497 LOGGER.debug("Last NVD API was at {}, and now {} is within {} s.", lastChecked, now, validForSeconds);
498 }
499 } else {
500 LOGGER.warn("NVD cache last checked not present; updating the entire database. This could occur if you are "
501 + "switching back and forth from using the API vs a datafeed or if you are using a database created prior to ODC 9.x");
502 }
503 }
504 return proceed;
505 }
506
507
508
509
510
511
512 private boolean dataExists() {
513 return cveDb.dataExists();
514 }
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529 protected final Map<String, String> getUpdatesNeeded(FeedUrl feedUrl, Properties cacheProperties, ZonedDateTime now) throws UpdateException {
530 LOGGER.debug("starting getUpdatesNeeded() ...");
531 final Map<String, String> updates = new HashMap<>();
532 if (dbProperties != null && !dbProperties.isEmpty()) {
533 Pair<Integer, Integer> yearRange = FeedUrl.toYearRange(settings, now);
534 int startYear = yearRange.getLeft();
535 int endYear = yearRange.getRight();
536 boolean needsFullUpdate = false;
537 for (int y = startYear; y <= endYear; y++) {
538 final ZonedDateTime val = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + y);
539 if (val == null && FeedUrl.isMandatoryFeedYear(now, y)) {
540 needsFullUpdate = true;
541 break;
542 }
543 }
544 final ZonedDateTime lastUpdated = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED);
545 final int days = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_VALID_FOR_DAYS, 7);
546
547 if (!needsFullUpdate && lastUpdated.equals(DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE))) {
548 return updates;
549 } else {
550 updates.put("modified", feedUrl.toFormattedUrlString("modified"));
551 if (needsFullUpdate) {
552 for (int i = startYear; i <= endYear; i++) {
553 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
554 updates.put(String.valueOf(i), feedUrl.toFormattedUrlString(i));
555 }
556 }
557 } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) {
558 for (int i = startYear; i <= endYear; i++) {
559 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
560 final ZonedDateTime lastModifiedCache = DatabaseProperties.getTimestamp(cacheProperties,
561 NVD_API_CACHE_MODIFIED_DATE + "." + i);
562 final ZonedDateTime lastModifiedDB = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + i);
563 if (lastModifiedDB == null || (lastModifiedCache != null && lastModifiedCache.compareTo(lastModifiedDB) > 0)) {
564 updates.put(String.valueOf(i), feedUrl.toFormattedUrlString(i));
565 }
566 }
567 }
568 }
569 }
570 }
571 if (updates.size() > 3) {
572 LOGGER.info("NVD API Cache / Data Feed requires several updates; this could take a couple of minutes.");
573 }
574 return updates;
575 }
576
577
578
579
580
581
582
583
584 protected final Properties getRemoteDataFeedCacheProperties(FeedUrl dataFeedUrl) throws UpdateException {
585 try {
586 final Properties properties = new Properties();
587 final String content = Downloader.getInstance().fetchContent(dataFeedUrl.toSuffixedUrl("cache.properties"), UTF_8);
588 properties.load(new StringReader(content));
589 return properties;
590
591 } catch (DownloadFailedException | ResourceNotFoundException ex) {
592 LOGGER.debug("Unable to download the NVD API cache.properties due to [{}]; attempting to build from data feed metadata files instead...", ex.toString());
593 return generateRemoteDataFeedCachePropertiesFromMetadata(dataFeedUrl);
594 } catch (URISyntaxException | MalformedURLException ex) {
595 throw new UpdateException("Invalid NVD Cache / Data Feed URL", ex);
596 } catch (TooManyRequestsException ex) {
597 throw new UpdateException("Unable to download the NVD API cache.properties", ex);
598 } catch (IOException ex) {
599 throw new UpdateException("Invalid NVD Cache properties file contents", ex);
600 }
601 }
602
603
604
605
606
607
608
609
610 private Properties generateRemoteDataFeedCachePropertiesFromMetadata(FeedUrl dataFeedUrl) throws UpdateException {
611 FeedUrl metaFeedUrl = dataFeedUrl.withPattern(p -> p
612 .orElse(FeedUrl.DEFAULT_FILE_PATTERN)
613 .replace(".json.gz", ".meta")
614 );
615
616 final Properties properties = new Properties();
617 ZonedDateTime lmd = metaFeedUrl.getLastModifiedFor("modified");
618 DatabaseProperties.setTimestamp(properties, NVD_API_CACHE_MODIFIED_DATE + ".modified", lmd);
619 DatabaseProperties.setTimestamp(properties, NVD_API_CACHE_MODIFIED_DATE, lmd);
620
621 metaFeedUrl.getLastModifiedDatePropertiesByYear(this.settings, ZonedDateTime.now(ZoneOffset.UTC))
622 .forEach((k, v) -> DatabaseProperties.setTimestamp(properties, k, v));
623 return properties;
624 }
625
626 protected static class FeedUrl {
627
628
629
630
631 static final String DEFAULT_FILE_PATTERN_PREFIX = "nvdcve-";
632
633
634
635 static final String DEFAULT_FILE_PATTERN_SUFFIX = "{0}.json.gz";
636
637
638
639 static final String DEFAULT_FILE_PATTERN = DEFAULT_FILE_PATTERN_PREFIX + DEFAULT_FILE_PATTERN_SUFFIX;
640
641
642
643
644 static final ZoneId ZONE_GLOBAL_EARLIEST = ZoneId.of("UTC+14:00");
645
646
647
648 static final ZoneId ZONE_GLOBAL_LATEST = ZoneId.of("UTC-12:00");
649
650
651
652
653 private final String url;
654
655
656
657
658 private final String pattern;
659
660 public FeedUrl(String url, String pattern) {
661 this.url = url;
662 this.pattern = pattern;
663 }
664
665 public FeedUrl withPattern(Function<Optional<String>, String> patternTransformer) {
666 return new FeedUrl(url, patternTransformer.apply(Optional.ofNullable(pattern)));
667 }
668
669 @NotNull String toFormattedUrlString(String formatArg) {
670 return url + MessageFormat.format(Optional.ofNullable(pattern).orElseThrow(), formatArg);
671 }
672
673 @NotNull String toFormattedUrlString(int formatArg) {
674 return toFormattedUrlString(String.valueOf(formatArg));
675 }
676
677 @NotNull URL toFormattedUrl(@NotNull String formatArg) throws MalformedURLException, URISyntaxException {
678 return new URI(toFormattedUrlString(formatArg)).toURL();
679 }
680
681 @SuppressWarnings("SameParameterValue")
682 @NotNull URL toSuffixedUrl(String suffix) throws MalformedURLException, URISyntaxException {
683 return new URI(url + suffix).toURL();
684 }
685
686
687
688
689
690
691 @SuppressWarnings("JavadocLinkAsPlainText")
692 protected static FeedUrl extractFromUrlOptionalPattern(String url) {
693 String baseUrl;
694 String pattern = null;
695 if (url.endsWith(".json.gz")) {
696 final int lio = url.lastIndexOf("/");
697 pattern = url.substring(lio + 1);
698 baseUrl = url.substring(0, lio);
699 } else {
700 baseUrl = url;
701 }
702 if (!baseUrl.endsWith("/")) {
703 baseUrl += "/";
704 }
705 return new FeedUrl(baseUrl, pattern);
706 }
707
708 private static @NotNull Pair<Integer, Integer> toYearRange(Settings settings, ZonedDateTime now) {
709
710
711
712 final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
713 final int endYear = now.withZoneSameInstant(ZONE_GLOBAL_EARLIEST).getYear();
714 return new Pair<>(startYear, endYear);
715 }
716
717 private @NotNull ZonedDateTime getLastModifiedFor(int year) throws UpdateException {
718 return getLastModifiedFor(String.valueOf(year));
719 }
720
721 private @NotNull ZonedDateTime getLastModifiedFor(String fileVersion) throws UpdateException {
722 try {
723 String content = Downloader.getInstance().fetchContent(toFormattedUrl(fileVersion), UTF_8);
724 Properties props = new Properties();
725 props.load(new StringReader(content));
726 return Objects.requireNonNull(DatabaseProperties.getIsoTimestamp(props, NVD_API_CACHE_MODIFIED_DATE));
727 } catch (Exception ex) {
728 throw new UpdateException("Unable to download & parse the data feed .meta file for " + fileVersion, ex);
729 }
730 }
731
732 Map<String, ZonedDateTime> getLastModifiedDatePropertiesByYear(Settings settings, ZonedDateTime now) throws UpdateException {
733 Pair<Integer, Integer> yearRange = toYearRange(settings, now);
734 Map<String, ZonedDateTime> lastModifiedDateProperties = new LinkedHashMap<>();
735 for (int y = yearRange.getLeft(); y <= yearRange.getRight(); y++) {
736 try {
737 lastModifiedDateProperties.put(NVD_API_CACHE_MODIFIED_DATE + "." + y, getLastModifiedFor(y));
738 } catch (UpdateException e) {
739 if (isMandatoryFeedYear(now, y)) {
740 throw e;
741 }
742 LOGGER.debug("Ignoring data feed metadata retrieval failure for {}, it is still January 1st in some TZ; so feed files may not yet be generated. Error was {}", y, e.toString());
743 }
744 }
745 return lastModifiedDateProperties;
746 }
747
748
749
750
751
752
753 static boolean isMandatoryFeedYear(ZonedDateTime now, int targetYear) {
754 return isNotTargetYearInAnyTZ(now, targetYear) || isAfterJanuary1InEveryTZ(now, targetYear);
755 }
756
757 private static boolean isNotTargetYearInAnyTZ(ZonedDateTime now, int targetYear) {
758 return targetYear != now.withZoneSameInstant(ZONE_GLOBAL_EARLIEST).getYear();
759 }
760
761 private static boolean isAfterJanuary1InEveryTZ(ZonedDateTime now, int targetYear) {
762 return now.isAfter(LocalDate.of(targetYear, 1, 2).atStartOfDay().atZone(ZONE_GLOBAL_LATEST));
763 }
764 }
765 }