1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.central;
19
20 import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler;
21 import org.apache.hc.core5.http.message.BasicHeader;
22 import org.owasp.dependencycheck.utils.DownloadFailedException;
23 import org.owasp.dependencycheck.utils.Downloader;
24 import org.owasp.dependencycheck.utils.ForbiddenException;
25 import org.owasp.dependencycheck.utils.ResourceNotFoundException;
26 import org.owasp.dependencycheck.utils.TooManyRequestsException;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.net.MalformedURLException;
30 import java.net.URISyntaxException;
31 import java.net.URL;
32 import java.util.ArrayList;
33 import java.util.List;
34 import javax.annotation.concurrent.ThreadSafe;
35 import javax.xml.xpath.XPath;
36 import javax.xml.xpath.XPathConstants;
37 import javax.xml.xpath.XPathExpressionException;
38 import javax.xml.xpath.XPathFactory;
39 import org.apache.commons.jcs3.access.exception.CacheException;
40 import org.owasp.dependencycheck.data.cache.DataCache;
41 import org.owasp.dependencycheck.data.cache.DataCacheFactory;
42 import org.owasp.dependencycheck.data.nexus.MavenArtifact;
43 import org.owasp.dependencycheck.utils.Settings;
44 import org.owasp.dependencycheck.utils.ToXMLDocumentResponseHandler;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47 import org.w3c.dom.Document;
48 import org.w3c.dom.NodeList;
49
50
51
52
53
54
55 @ThreadSafe
56 public class CentralSearch {
57
58
59
60
61 private final String rootURL;
62
63
64
65
66 private final String query;
67
68
69
70
71 private final boolean useProxy;
72
73
74
75
76 private static final Logger LOGGER = LoggerFactory.getLogger(CentralSearch.class);
77
78
79
80 private final Settings settings;
81
82
83
84 private DataCache<List<MavenArtifact>> cache;
85
86
87
88
89
90
91
92 public CentralSearch(Settings settings) throws MalformedURLException {
93 this.settings = settings;
94
95 final String searchUrl = settings.getString(Settings.KEYS.ANALYZER_CENTRAL_URL);
96 LOGGER.debug("Central Search URL: {}", searchUrl);
97 if (isInvalidURL(searchUrl)) {
98 throw new MalformedURLException(String.format("The configured central analyzer URL is invalid: %s", searchUrl));
99 }
100 this.rootURL = searchUrl;
101 final String queryStr = settings.getString(Settings.KEYS.ANALYZER_CENTRAL_QUERY);
102 LOGGER.debug("Central Search Query: {}", queryStr);
103 if (!queryStr.matches("^%s.*%s.*$")) {
104 final String msg = String.format("The configured central analyzer query parameter is invalid (it must have two %%s): %s", queryStr);
105 throw new MalformedURLException(msg);
106 }
107 this.query = queryStr;
108 LOGGER.debug("Central Search Full URL: {}", String.format(query, rootURL, "[SHA1]"));
109 if (null != settings.getString(Settings.KEYS.PROXY_SERVER) || null != System.getProperty("https.proxyHost")) {
110 useProxy = true;
111 LOGGER.debug("Using proxy");
112 } else {
113 useProxy = false;
114 LOGGER.debug("Not using proxy");
115 }
116 if (settings.getBoolean(Settings.KEYS.ANALYZER_CENTRAL_USE_CACHE, true)) {
117 try {
118 final DataCacheFactory factory = new DataCacheFactory(settings);
119 cache = factory.getCentralCache();
120 } catch (CacheException ex) {
121 settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_USE_CACHE, false);
122 LOGGER.debug("Error creating cache, disabling caching", ex);
123 }
124 }
125 }
126
127
128
129
130
131
132
133
134
135
136
137
138
139 public List<MavenArtifact> searchSha1(String sha1) throws IOException, TooManyRequestsException, ForbiddenException {
140 if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) {
141 throw new IllegalArgumentException("Invalid SHA1 format");
142 }
143 if (cache != null) {
144 final List<MavenArtifact> cached = cache.get(sha1);
145 if (cached != null) {
146 LOGGER.debug("cache hit for Central: {}", sha1);
147 if (cached.isEmpty()) {
148 throw new FileNotFoundException("Artifact not found in Central");
149 }
150 return cached;
151 }
152 }
153 final List<MavenArtifact> result = new ArrayList<>();
154 final URL url = new URL(String.format(query, rootURL, sha1));
155
156 LOGGER.trace("Searching Central url {}", url);
157
158
159
160 final BasicHeader acceptHeader = new BasicHeader("Accept", "application/xml");
161 final AbstractHttpClientResponseHandler<Document> handler = new ToXMLDocumentResponseHandler();
162 try {
163 final Document doc = Downloader.getInstance().fetchAndHandle(url, handler, List.of(acceptHeader), useProxy);
164 final boolean missing = addMavenArtifacts(doc, result);
165
166 if (missing) {
167 if (cache != null) {
168 cache.put(sha1, result);
169 }
170 throw new FileNotFoundException("Artifact not found in Central");
171 }
172 } catch (XPathExpressionException e) {
173 final String errorMessage = "Failed to parse MavenCentral XML Response: " + e.getMessage();
174 throw new IOException(errorMessage, e);
175 } catch (TooManyRequestsException e) {
176 final String errorMessage = "Too many requests sent to MavenCentral; additional requests are being rejected.";
177 throw new TooManyRequestsException(errorMessage, e);
178 } catch (ResourceNotFoundException | DownloadFailedException e) {
179 final String errorMessage = "Could not connect to MavenCentral " + e.getMessage();
180 throw new IOException(errorMessage, e);
181 } catch (URISyntaxException e) {
182 final String errorMessage = "Could not convert central search URL to a URI " + e.getMessage();
183 throw new IOException(errorMessage, e);
184 } catch (ForbiddenException e) {
185 final String errorMessage = "Forbidden access to MavenCentral " + e.getMessage();
186 throw new ForbiddenException(errorMessage, e);
187 }
188 if (cache != null) {
189 cache.put(sha1, result);
190 }
191 return result;
192 }
193
194
195
196
197
198
199
200 private boolean addMavenArtifacts(Document doc, List<MavenArtifact> result) throws XPathExpressionException {
201 boolean missing = false;
202 final XPath xpath = XPathFactory.newInstance().newXPath();
203 final String numFound = xpath.evaluate("/response/result/@numFound", doc);
204 if ("0".equals(numFound)) {
205 missing = true;
206 } else {
207 final NodeList docs = (NodeList) xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET);
208 for (int i = 0; i < docs.getLength(); i++) {
209 final String g = xpath.evaluate("./str[@name='g']", docs.item(i));
210 LOGGER.trace("GroupId: {}", g);
211 final String a = xpath.evaluate("./str[@name='a']", docs.item(i));
212 LOGGER.trace("ArtifactId: {}", a);
213 final String v = xpath.evaluate("./str[@name='v']", docs.item(i));
214 final NodeList attributes = (NodeList) xpath.evaluate("./arr[@name='ec']/str", docs.item(i), XPathConstants.NODESET);
215 boolean pomAvailable = false;
216 boolean jarAvailable = false;
217 for (int x = 0; x < attributes.getLength(); x++) {
218 final String tmp = xpath.evaluate(".", attributes.item(x));
219 if (".pom".equals(tmp)) {
220 pomAvailable = true;
221 } else if (".jar".equals(tmp)) {
222 jarAvailable = true;
223 }
224 }
225 final String centralContentUrl = settings.getString(Settings.KEYS.CENTRAL_CONTENT_URL);
226 String artifactUrl = null;
227 String pomUrl = null;
228 if (jarAvailable) {
229
230 artifactUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/'
231 + v + '/' + a + '-' + v + ".jar";
232 }
233 if (pomAvailable) {
234
235 pomUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/'
236 + v + '/' + a + '-' + v + ".pom";
237 }
238 result.add(new MavenArtifact(g, a, v, artifactUrl, pomUrl));
239 }
240 }
241 return missing;
242 }
243
244
245
246
247
248
249
250 private boolean isInvalidURL(String url) {
251 try {
252 final URL u = new URL(url);
253 u.toURI();
254 } catch (MalformedURLException | URISyntaxException e) {
255 LOGGER.trace("URL is invalid: {}", url);
256 return true;
257 }
258 return false;
259 }
260
261 }