1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.central;
19
20 import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler;
21 import org.apache.hc.core5.http.message.BasicHeader;
22 import org.owasp.dependencycheck.utils.DownloadFailedException;
23 import org.owasp.dependencycheck.utils.Downloader;
24 import org.owasp.dependencycheck.utils.ResourceNotFoundException;
25 import org.owasp.dependencycheck.utils.TooManyRequestsException;
26 import java.io.FileNotFoundException;
27 import java.io.IOException;
28 import java.net.MalformedURLException;
29 import java.net.URISyntaxException;
30 import java.net.URL;
31 import java.util.ArrayList;
32 import java.util.List;
33 import javax.annotation.concurrent.ThreadSafe;
34 import javax.xml.xpath.XPath;
35 import javax.xml.xpath.XPathConstants;
36 import javax.xml.xpath.XPathExpressionException;
37 import javax.xml.xpath.XPathFactory;
38 import org.apache.commons.jcs3.access.exception.CacheException;
39 import org.owasp.dependencycheck.data.cache.DataCache;
40 import org.owasp.dependencycheck.data.cache.DataCacheFactory;
41 import org.owasp.dependencycheck.data.nexus.MavenArtifact;
42 import org.owasp.dependencycheck.utils.Settings;
43 import org.owasp.dependencycheck.utils.ToXMLDocumentResponseHandler;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46 import org.w3c.dom.Document;
47 import org.w3c.dom.NodeList;
48
49
50
51
52
53
54 @ThreadSafe
55 public class CentralSearch {
56
57
58
59
60 private final String rootURL;
61
62
63
64
65 private final String query;
66
67
68
69
70 private final boolean useProxy;
71
72
73
74
75 private static final Logger LOGGER = LoggerFactory.getLogger(CentralSearch.class);
76
77
78
79 private final Settings settings;
80
81
82
83 private DataCache<List<MavenArtifact>> cache;
84
85
86
87
88
89
90
91 public CentralSearch(Settings settings) throws MalformedURLException {
92 this.settings = settings;
93
94 final String searchUrl = settings.getString(Settings.KEYS.ANALYZER_CENTRAL_URL);
95 LOGGER.debug("Central Search URL: {}", searchUrl);
96 if (isInvalidURL(searchUrl)) {
97 throw new MalformedURLException(String.format("The configured central analyzer URL is invalid: %s", searchUrl));
98 }
99 this.rootURL = searchUrl;
100 final String queryStr = settings.getString(Settings.KEYS.ANALYZER_CENTRAL_QUERY);
101 LOGGER.debug("Central Search Query: {}", queryStr);
102 if (!queryStr.matches("^%s.*%s.*$")) {
103 final String msg = String.format("The configured central analyzer query parameter is invalid (it must have two %%s): %s", queryStr);
104 throw new MalformedURLException(msg);
105 }
106 this.query = queryStr;
107 LOGGER.debug("Central Search Full URL: {}", String.format(query, rootURL, "[SHA1]"));
108 if (null != settings.getString(Settings.KEYS.PROXY_SERVER) || null != System.getProperty("https.proxyHost")) {
109 useProxy = true;
110 LOGGER.debug("Using proxy");
111 } else {
112 useProxy = false;
113 LOGGER.debug("Not using proxy");
114 }
115 if (settings.getBoolean(Settings.KEYS.ANALYZER_CENTRAL_USE_CACHE, true)) {
116 try {
117 final DataCacheFactory factory = new DataCacheFactory(settings);
118 cache = factory.getCentralCache();
119 } catch (CacheException ex) {
120 settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_USE_CACHE, false);
121 LOGGER.debug("Error creating cache, disabling caching", ex);
122 }
123 }
124 }
125
126
127
128
129
130
131
132
133
134
135
136
137
138 public List<MavenArtifact> searchSha1(String sha1) throws IOException, TooManyRequestsException {
139 if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) {
140 throw new IllegalArgumentException("Invalid SHA1 format");
141 }
142 if (cache != null) {
143 final List<MavenArtifact> cached = cache.get(sha1);
144 if (cached != null) {
145 LOGGER.debug("cache hit for Central: " + sha1);
146 if (cached.isEmpty()) {
147 throw new FileNotFoundException("Artifact not found in Central");
148 }
149 return cached;
150 }
151 }
152 final List<MavenArtifact> result = new ArrayList<>();
153 final URL url = new URL(String.format(query, rootURL, sha1));
154
155 LOGGER.trace("Searching Central url {}", url);
156
157
158
159 final BasicHeader acceptHeader = new BasicHeader("Accept", "application/xml");
160 final AbstractHttpClientResponseHandler<Document> handler = new ToXMLDocumentResponseHandler();
161 try {
162 final Document doc = Downloader.getInstance().fetchAndHandle(url, handler, List.of(acceptHeader), useProxy);
163 final boolean missing = addMavenArtifacts(doc, result);
164
165 if (missing) {
166 if (cache != null) {
167 cache.put(sha1, result);
168 }
169 throw new FileNotFoundException("Artifact not found in Central");
170 }
171 } catch (XPathExpressionException e) {
172 final String errorMessage = "Failed to parse MavenCentral XML Response: " + e.getMessage();
173 throw new IOException(errorMessage, e);
174 } catch (TooManyRequestsException e) {
175 final String errorMessage = "Too many requests sent to MavenCentral; additional requests are being rejected.";
176 throw new TooManyRequestsException(errorMessage, e);
177 } catch (ResourceNotFoundException | DownloadFailedException e) {
178 final String errorMessage = "Could not connect to MavenCentral " + e.getMessage();
179 throw new IOException(errorMessage, e);
180 } catch (URISyntaxException e) {
181 final String errorMessage = "Could not convert central search URL to a URI " + e.getMessage();
182 throw new IOException(errorMessage, e);
183 }
184 if (cache != null) {
185 cache.put(sha1, result);
186 }
187 return result;
188 }
189
190
191
192
193
194
195
196 private boolean addMavenArtifacts(Document doc, List<MavenArtifact> result) throws XPathExpressionException {
197 boolean missing = false;
198 final XPath xpath = XPathFactory.newInstance().newXPath();
199 final String numFound = xpath.evaluate("/response/result/@numFound", doc);
200 if ("0".equals(numFound)) {
201 missing = true;
202 } else {
203 final NodeList docs = (NodeList) xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET);
204 for (int i = 0; i < docs.getLength(); i++) {
205 final String g = xpath.evaluate("./str[@name='g']", docs.item(i));
206 LOGGER.trace("GroupId: {}", g);
207 final String a = xpath.evaluate("./str[@name='a']", docs.item(i));
208 LOGGER.trace("ArtifactId: {}", a);
209 final String v = xpath.evaluate("./str[@name='v']", docs.item(i));
210 final NodeList attributes = (NodeList) xpath.evaluate("./arr[@name='ec']/str", docs.item(i), XPathConstants.NODESET);
211 boolean pomAvailable = false;
212 boolean jarAvailable = false;
213 for (int x = 0; x < attributes.getLength(); x++) {
214 final String tmp = xpath.evaluate(".", attributes.item(x));
215 if (".pom".equals(tmp)) {
216 pomAvailable = true;
217 } else if (".jar".equals(tmp)) {
218 jarAvailable = true;
219 }
220 }
221 final String centralContentUrl = settings.getString(Settings.KEYS.CENTRAL_CONTENT_URL);
222 String artifactUrl = null;
223 String pomUrl = null;
224 if (jarAvailable) {
225
226 artifactUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/'
227 + v + '/' + a + '-' + v + ".jar";
228 }
229 if (pomAvailable) {
230
231 pomUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/'
232 + v + '/' + a + '-' + v + ".pom";
233 }
234 result.add(new MavenArtifact(g, a, v, artifactUrl, pomUrl));
235 }
236 }
237 return missing;
238 }
239
240
241
242
243
244
245
246 private boolean isInvalidURL(String url) {
247 try {
248 final URL u = new URL(url);
249 u.toURI();
250 } catch (MalformedURLException | URISyntaxException e) {
251 LOGGER.trace("URL is invalid: {}", url);
252 return true;
253 }
254 return false;
255 }
256
257 }