1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.creadur.tentacles;
18
19 import static org.apache.creadur.tentacles.LicenseType.loadLicensesFrom;
20 import static org.apache.creadur.tentacles.RepositoryType.HTTP;
21 import static org.apache.creadur.tentacles.RepositoryType.LOCAL_FILE_SYSTEM;
22
23 import java.io.File;
24 import java.io.IOException;
25 import java.net.URI;
26 import java.nio.file.Path;
27 import java.util.ArrayList;
28 import java.util.Collection;
29 import java.util.HashMap;
30 import java.util.HashSet;
31 import java.util.LinkedHashSet;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Set;
35 import java.util.zip.ZipEntry;
36 import java.util.zip.ZipInputStream;
37
38 import org.apache.logging.log4j.*;
39
40 public class Main {
41
42
43
44
45
46
47
48
49
50
51 private static final Logger log = LogManager.getLogger(Main.class);
52 private static final String CRAWL_PATTERN = ".*\\.(jar|zip|war|ear|rar|tar.gz)";
53
54 private final Reports reports;
55 private final Licenses licenses;
56
57 private final Layout layout;
58 private final Platform platform;
59 private final Configuration configuration;
60 private final FileSystem fileSystem;
61 private final IOSystem ioSystem;
62 private final TentaclesResources tentaclesResources;
63 private final Templates templates;
64
65 public Main(final String... args) throws Exception {
66 this(new Configuration(args), Platform.aPlatform());
67 }
68
69 public Main(final Configuration configuration, final Platform platform)
70 throws Exception {
71 this(configuration, platform, new Templates(platform), new Layout(
72 platform, configuration));
73 }
74
75 public Main(final Configuration configuration, final Platform platform,
76 final Templates templates, final Layout layout) throws Exception {
77 this.platform = platform;
78 this.configuration = configuration;
79 this.layout = layout;
80 this.fileSystem = platform.fileSystem();
81 this.ioSystem = platform.ioSystem();
82 this.tentaclesResources = platform.tentaclesResources();
83 this.templates = templates;
84
85 this.reports = new Reports();
86
87 log.info("Remote repository: {}", this.configuration.getStagingRepositoryURI());
88 log.info("Local root directory: {}", this.layout.getLocalRootDirectory());
89
90 this.tentaclesResources.copyTo("legal/style.css",
91 new File(this.layout.getOutputDirectory(), "style.css"));
92
93 this.licenses = loadLicensesFrom(platform);
94 }
95
96 public static void main(final String[] args) throws Exception {
97
98 log.info("Launching Apache Tentacles ...");
99
100 if(args == null || args.length < 1) {
101 log.error("Error: Input parameter missing - you did not specify any component to run Apache Tentacles on.");
102 log.error("Please launch Apache Tentacles with an URI to work on such as 'https://repository.apache.org/content/repositories/orgapachecreadur-1000/'.");
103 } else {
104 new Main(args).run();
105 }
106
107 }
108
109 private void run() throws Exception {
110 unpackContents(mirrorRepositoryFrom(this.configuration));
111 reportOn(archivesIn(this.layout.getRepositoryDirectory()));
112 }
113
114 private List<Archive> archivesIn(final File repository) throws IOException {
115 final List<File> jars = this.fileSystem.documentsFrom(repository);
116
117 final List<Archive> archives = new ArrayList<>();
118 for (final File file : jars) {
119 final Archive archive =
120 new Archive(file, this.fileSystem, this.layout);
121 archives.add(archive);
122 }
123 return archives;
124 }
125
126 private void reportOn(final List<Archive> archives) throws IOException {
127 this.templates
128 .template("legal/archives.vm")
129 .add("archives", archives)
130 .add("reports", this.reports)
131 .write(new File(this.layout.getOutputDirectory(),
132 "archives.html"));
133
134 reportLicenses(archives);
135 reportNotices(archives);
136 reportDeclaredLicenses(archives);
137 reportDeclaredNotices(archives);
138 }
139
140 private void reportLicenses(final List<Archive> archives)
141 throws IOException {
142 initLicenses(archives);
143
144 this.templates
145 .template("legal/licenses.vm")
146 .add("licenses", getLicenses(archives))
147 .add("reports", this.reports)
148 .write(new File(this.layout.getOutputDirectory(),
149 "licenses.html"));
150 }
151
152 private void initLicenses(final List<Archive> archives) throws IOException {
153 final Map<License, License> licenses = new HashMap<>();
154
155 for (final Archive archive : archives) {
156 final List<File> files =
157 this.fileSystem.licensesFrom(archive.contentsDirectory());
158 for (final File file : files) {
159 final License license = this.licenses.from(file);
160
161 License existing = licenses.get(license);
162 if (existing == null) {
163 licenses.put(license, license);
164 existing = license;
165 }
166
167 existing.getLocations().add(file);
168 existing.getArchives().add(archive);
169 archive.getLicenses().add(existing);
170 }
171 }
172 }
173
174 private Collection<License> getLicenses(final List<Archive> archives) {
175 final Set<License> licenses = new LinkedHashSet<>();
176 for (final Archive archive : archives) {
177 licenses.addAll(archive.getLicenses());
178 }
179 return licenses;
180 }
181
182 private void reportDeclaredLicenses(final List<Archive> archives)
183 throws IOException {
184
185 for (final Archive archive : archives) {
186
187 classifyLicenses(archive);
188 }
189 for (final Archive archive : archives) {
190
191 this.templates
192 .template("legal/archive-licenses.vm")
193 .add("archive", archive)
194 .add("reports", this.reports)
195 .write(new File(this.layout.getOutputDirectory(),
196 this.reports.licenses(archive)));
197 }
198
199 }
200
201 private void classifyLicenses(final Archive archive) throws IOException {
202 final Set<License> undeclared =
203 new HashSet<>(archive.getLicenses());
204
205 final File contents = archive.contentsDirectory();
206 final List<File> files = this.fileSystem.licensesDeclaredIn(contents);
207
208 for (final File file : files) {
209 undeclared.remove(this.licenses.from(file));
210 }
211
212 archive.getOtherLicenses().addAll(undeclared);
213
214 final Set<License> declared =
215 new HashSet<>(archive.getLicenses());
216 declared.removeAll(undeclared);
217 archive.getDeclaredLicenses().addAll(declared);
218
219 for (final License license : undeclared) {
220 for (final License declare : declared) {
221 if (license.implies(declare)) {
222 archive.getOtherLicenses().remove(license);
223 }
224 }
225 }
226 }
227
228 private void reportDeclaredNotices(final List<Archive> archives)
229 throws IOException {
230
231 for (final Archive archive : archives) {
232 final Set<Notice> undeclared =
233 new HashSet<>(archive.getNotices());
234
235 final File contents = archive.contentsDirectory();
236 final List<File> files =
237 this.fileSystem.noticesDeclaredIn(contents);
238
239 for (final File file : files) {
240
241 final Notice notice = new Notice(this.ioSystem.slurp(file));
242
243 undeclared.remove(notice);
244 }
245
246 archive.getOtherNotices().addAll(undeclared);
247
248 final Set<Notice> declared =
249 new HashSet<>(archive.getNotices());
250 declared.removeAll(undeclared);
251 archive.getDeclaredNotices().addAll(declared);
252
253 for (final Notice notice : undeclared) {
254
255 for (final Notice declare : declared) {
256 if (notice.implies(declare)) {
257 archive.getOtherLicenses().remove(notice);
258 }
259 }
260 }
261
262 this.templates
263 .template("legal/archive-notices.vm")
264 .add("archive", archive)
265 .add("reports", this.reports)
266 .write(new File(this.layout.getOutputDirectory(),
267 this.reports.notices(archive)));
268 }
269 }
270
271 private void reportNotices(final List<Archive> archives) throws IOException {
272 final Map<Notice, Notice> notices = new HashMap<>();
273
274 for (final Archive archive : archives) {
275 final List<File> noticeDocuments =
276 this.fileSystem.noticesOnly(archive.contentsDirectory());
277 for (final File file : noticeDocuments) {
278 final Notice notice = new Notice(this.ioSystem.slurp(file));
279
280 Notice existing = notices.get(notice);
281 if (existing == null) {
282 notices.put(notice, notice);
283 existing = notice;
284 }
285
286 existing.getLocations().add(file);
287 existing.getArchives().add(archive);
288 archive.getNotices().add(existing);
289 }
290 }
291
292 this.templates
293 .template("legal/notices.vm")
294 .add("notices", notices.values())
295 .add("reports", this.reports)
296 .write(new File(this.layout.getOutputDirectory(),
297 "notices.html"));
298 }
299
300 private void unpackContents(final Set<File> files) throws IOException {
301 for (final File file : files) {
302 unpack(file);
303 }
304 }
305
306 private Set<File> mirrorRepositoryFrom(final Configuration configuration)
307 throws IOException {
308 final Set<File> files = new HashSet<>();
309 if (HTTP.isRepositoryFor(configuration)) {
310 final NexusClient client = new NexusClient(this.platform);
311 final Set<URI> resources =
312 client.crawl(configuration.getStagingRepositoryURI());
313
314 for (final URI uri : resources) {
315 if (!uri.getPath().matches(CRAWL_PATTERN)) {
316 continue;
317 }
318 files.add(client.download(uri, mirroredFrom(uri)));
319 }
320 } else if (LOCAL_FILE_SYSTEM.isRepositoryFor(configuration)) {
321 final File file = new File(configuration.getStagingRepositoryURI());
322 final List<File> collect =
323 this.platform.fileSystem().archivesInPath(file,
324 configuration.getFileRepositoryPathNameFilter());
325
326 for (final File f : collect) {
327 files.add(copyToMirror(f));
328 }
329 }
330 return files;
331 }
332
333 private void unpack(final File archive) {
334 log.info("Unpack {}", archive);
335
336 try {
337 final ZipInputStream zip = this.ioSystem.unzip(archive);
338
339 final File contents =
340 new Archive(archive, this.fileSystem, this.layout)
341 .contentsDirectory();
342
343 try {
344 ZipEntry entry = null;
345
346 while ((entry = zip.getNextEntry()) != null) {
347
348 if (entry.isDirectory()) {
349 continue;
350 }
351
352
353 final String path = entry.getName();
354 final Path target = contents.toPath().toAbsolutePath().normalize();
355 final Path resolved = target.resolve(path).normalize();
356
357 if(!resolved.startsWith(target)) {
358 throw new IOException("Invalid archive entry: " + path);
359 }
360
361 final File fileEntry = new File(contents, path);
362 this.fileSystem.mkparent(fileEntry);
363
364
365 this.ioSystem.copy(zip, fileEntry);
366
367 if (fileEntry.getName().endsWith(".jar")) {
368 unpack(fileEntry);
369 }
370 }
371 } finally {
372 this.ioSystem.close(zip);
373 }
374 } catch (final IOException e) {
375 log.error("Not a zip {}", archive);
376 }
377 }
378
379 private File copyToMirror(final File src) throws IOException {
380 final URI uri = src.toURI();
381
382 final File file = mirroredFrom(uri);
383
384 log.info("Copy {}", uri);
385
386 this.fileSystem.mkparent(file);
387
388 this.ioSystem.copy(this.ioSystem.read(src), file);
389
390 return file;
391 }
392
393 private File mirroredFrom(final URI uri) {
394 final String name =
395 uri.toString()
396 .replace(
397 this.configuration.getStagingRepositoryURI()
398 .toString(), "").replaceFirst("^/", "");
399 return new File(this.layout.getRepositoryDirectory(), name);
400 }
401
402 }