1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.rat;
20
21 import java.io.File;
22 import java.io.FilenameFilter;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.PrintWriter;
26 import java.io.Serializable;
27 import java.net.URL;
28 import java.nio.charset.StandardCharsets;
29 import java.nio.file.Files;
30 import java.nio.file.Paths;
31 import java.util.Arrays;
32 import java.util.Comparator;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.Optional;
36 import java.util.TreeMap;
37 import java.util.function.Consumer;
38 import java.util.function.Supplier;
39 import java.util.regex.PatternSyntaxException;
40 import java.util.stream.Collectors;
41
42 import org.apache.commons.cli.CommandLine;
43 import org.apache.commons.cli.Converter;
44 import org.apache.commons.cli.DefaultParser;
45 import org.apache.commons.cli.DeprecatedAttributes;
46 import org.apache.commons.cli.HelpFormatter;
47 import org.apache.commons.cli.Option;
48 import org.apache.commons.cli.OptionGroup;
49 import org.apache.commons.cli.Options;
50 import org.apache.commons.cli.ParseException;
51 import org.apache.commons.io.FileUtils;
52 import org.apache.commons.io.filefilter.FalseFileFilter;
53 import org.apache.commons.io.filefilter.NameFileFilter;
54 import org.apache.commons.io.filefilter.OrFileFilter;
55 import org.apache.commons.io.filefilter.RegexFileFilter;
56 import org.apache.commons.io.filefilter.WildcardFileFilter;
57 import org.apache.commons.io.function.IOSupplier;
58 import org.apache.commons.lang3.StringUtils;
59 import org.apache.commons.text.WordUtils;
60 import org.apache.rat.api.Document;
61 import org.apache.rat.config.AddLicenseHeaders;
62 import org.apache.rat.document.impl.FileDocument;
63 import org.apache.rat.license.LicenseSetFactory.LicenseFilter;
64 import org.apache.rat.report.IReportable;
65 import org.apache.rat.utils.DefaultLog;
66 import org.apache.rat.utils.Log;
67 import org.apache.rat.utils.Log.Level;
68 import org.apache.rat.walker.ArchiveWalker;
69 import org.apache.rat.walker.DirectoryWalker;
70
71 import static java.lang.String.format;
72
73
74
75
76 public final class Report {
77
78 private static final int HELP_WIDTH = 120;
79 private static final int HELP_PADDING = 5;
80
81
82
83
84
85
86 private static final String[] NOTES = {
87 "Rat highlights possible issues.",
88 "Rat reports require interpretation.",
89 "Rat often requires some tuning before it runs well against a project.",
90 "Rat relies on heuristics: it may miss issues"
91 };
92
93 private enum StyleSheets {
94 PLAIN("plain-rat", "The default style"),
95 MISSING_HEADERS("missing-headers", "Produces a report of files that are missing headers"),
96 UNAPPROVED_LICENSES("unapproved-licenses", "Produces a report of the files with unapproved licenses");
97 private final String arg;
98 private final String desc;
99
100 StyleSheets(final String arg, final String description) {
101 this.arg = arg;
102 this.desc = description;
103 }
104
105 public String arg() {
106 return arg;
107 }
108
109 public String desc() {
110 return desc;
111 }
112 }
113
114 private static final Map<String, Supplier<String>> ARGUMENT_TYPES;
115
116 static {
117 ARGUMENT_TYPES = new TreeMap<>();
118 ARGUMENT_TYPES.put("FileOrURI", () -> "A file name or URI");
119 ARGUMENT_TYPES.put("DirOrArchive", () -> "A directory or archive file to scan");
120 ARGUMENT_TYPES.put("Expression", () -> "A wildcard file matching pattern. example: *-test-*.txt");
121 ARGUMENT_TYPES.put("LicenseFilter", () -> format("A defined filter for the licenses to include. Valid values: %s.",
122 asString(LicenseFilter.values())));
123 ARGUMENT_TYPES.put("LogLevel", () -> format("The log level to use. Valid values %s.", asString(Log.Level.values())));
124 ARGUMENT_TYPES.put("ProcessingType", () -> format("Specifies how to process file types. Valid values are: %s",
125 Arrays.stream(ReportConfiguration.Processing.values())
126 .map(v -> format("\t%s: %s", v.name(), v.desc()))
127 .collect(Collectors.joining(""))));
128 ARGUMENT_TYPES.put("StyleSheet", () -> format("Either an external xsl file or maybe one of the internal named sheets. Internal sheets are: %s.",
129 Arrays.stream(StyleSheets.values())
130 .map(v -> format("\t%s: %s", v.arg(), v.desc()))
131 .collect(Collectors.joining(""))));
132 }
133
134
135
136
137
138
139
140 private static final DeprecatedAttributes ADD_ATTRIBUTES = DeprecatedAttributes.builder().setForRemoval(true).setSince("0.17")
141 .setDescription("Use '-A' or '--addLicense' instead.").get();
142 static final OptionGroup ADD = new OptionGroup()
143 .addOption(Option.builder("a").hasArg(false)
144 .desc(format("[%s]", ADD_ATTRIBUTES))
145 .deprecated(ADD_ATTRIBUTES)
146 .build())
147
148 .addOption(new Option("A", "addLicense", false,
149 "Add the default license header to any file with an unknown license that is not in the exclusion list. "
150 + "By default new files will be created with the license header, "
151 + "to force the modification of existing files use the --force option.")
152 );
153
154
155
156
157
158
159 static final Option OUT = Option.builder().option("o").longOpt("out").hasArg()
160 .desc("Define the output file where to write a report to (default is System.out).")
161 .converter(Converter.FILE).build();
162
163
164 static final DeprecatedAttributes DIR_ATTRIBUTES = DeprecatedAttributes.builder().setForRemoval(true).setSince("0.17")
165 .setDescription("Use '--'").get();
166 static final Option DIR = Option.builder().option("d").longOpt("dir").hasArg()
167 .desc(format("[%s] %s", DIR_ATTRIBUTES, "Used to indicate end of list when using --exclude.")).argName("DirOrArchive")
168 .deprecated(DIR_ATTRIBUTES).build();
169
170
171
172
173 static final Option FORCE = new Option("f", "force", false,
174 format("Forces any changes in files to be written directly to the source files (i.e. new files are not created). Only valid with --%s",
175 ADD.getOptions().stream().filter(o -> !o.isDeprecated()).findAny().get().getLongOpt()));
176
177
178
179 static final Option COPYRIGHT = Option.builder().option("c").longOpt("copyright").hasArg()
180 .desc(format("The copyright message to use in the license headers, usually in the form of \"Copyright 2008 Foo\". Only valid with --%s",
181 ADD.getOptions().stream().filter(o -> !o.isDeprecated()).findAny().get().getLongOpt()))
182 .build();
183
184
185
186 static final Option EXCLUDE_CLI = Option.builder("e").longOpt("exclude").hasArgs().argName("Expression")
187 .desc("Excludes files matching wildcard <Expression>. May be followed by multiple arguments. "
188 + "Note that '--' or a following option is required when using this parameter.")
189 .build();
190
191
192
193
194 static final Option EXCLUDE_FILE_CLI = Option.builder("E").longOpt("exclude-file")
195 .argName("FileOrURI")
196 .hasArg().desc("Excludes files matching regular expression in the input file.")
197 .build();
198
199
200
201
202 static final Option STYLESHEET_CLI = Option.builder("s").longOpt("stylesheet").hasArg().argName("StyleSheet")
203 .desc("XSLT stylesheet to use when creating the report. Not compatible with -x. "
204 + "Either an external xsl file may be specified or one of the internal named sheets: plain-rat (default), missing-headers, or unapproved-licenses")
205 .build();
206
207
208
209 static final Option HELP = new Option("h", "help", false, "Print help for the RAT command line interface and exit.");
210
211
212
213
214
215 static final Option LICENSES = Option.builder().longOpt("licenses").hasArgs().argName("FileOrURI")
216 .desc("File names or URLs for license definitions. May be followed by multiple arguments. "
217 + "Note that '--' or a following option is required when using this parameter.")
218 .build();
219
220
221
222
223 static final Option NO_DEFAULTS = new Option(null, "no-default-licenses", false, "Ignore default configuration. By default all approved default licenses are used");
224
225
226
227
228 static final Option SCAN_HIDDEN_DIRECTORIES = new Option(null, "scan-hidden-directories", false, "Scan hidden directories");
229
230
231
232
233
234 static final Option LIST_LICENSES = Option.builder().longOpt("list-licenses").hasArg().argName("LicenseFilter")
235 .desc("List the defined licenses (default is NONE). Valid options are: " + asString(LicenseFilter.values()))
236 .converter(s -> LicenseFilter.valueOf(s.toUpperCase()))
237 .build();
238
239
240
241
242
243 static final Option LIST_FAMILIES = Option.builder().longOpt("list-families").hasArg().argName("LicenseFilter")
244 .desc("List the defined license families (default is NONE). Valid options are: " + asString(LicenseFilter.values()))
245 .converter(s -> LicenseFilter.valueOf(s.toUpperCase()))
246 .build();
247
248
249
250
251
252 static final Option LOG_LEVEL = Option.builder().longOpt("log-level")
253 .hasArg().argName("LogLevel")
254 .desc("sets the log level.")
255 .converter(s -> Log.Level.valueOf(s.toUpperCase()))
256 .build();
257
258
259
260
261
262 static final Option DRY_RUN = Option.builder().longOpt("dry-run")
263 .desc("If set do not update the files but generate the reports.")
264 .build();
265
266
267
268 static final Option XML = new Option("x", "xml", false, "Output the report in raw XML format. Not compatible with -s");
269
270
271
272
273
274 static final Option ARCHIVE = Option.builder().longOpt("archive").hasArg().argName("ProcessingType")
275 .desc(format("Specifies the level of detail in ARCHIVE file reporting. (default is %s)",
276 ReportConfiguration.Processing.NOTIFICATION))
277 .converter(s -> ReportConfiguration.Processing.valueOf(s.toUpperCase()))
278 .build();
279
280
281
282
283 static final Option STANDARD = Option.builder().longOpt("standard").hasArg().argName("ProcessingType")
284 .desc(format("Specifies the level of detail in STANDARD file reporting. (default is %s)",
285 Defaults.STANDARD_PROCESSING))
286 .converter(s -> ReportConfiguration.Processing.valueOf(s.toUpperCase()))
287 .build();
288
289 private static String asString(final Object[] args) {
290 return Arrays.stream(args).map(Object::toString).collect(Collectors.joining(", "));
291 }
292
293
294
295
296
297
298
299
300 public static void main(final String[] args) throws Exception {
301 DefaultLog.getInstance().info(new VersionInfo().toString());
302 ReportConfiguration configuration = parseCommands(args, Report::printUsage);
303 if (configuration != null) {
304 configuration.validate(DefaultLog.getInstance()::error);
305 new Reporter(configuration).output();
306 }
307 }
308
309
310
311
312
313
314
315
316
317 public static ReportConfiguration parseCommands(final String[] args, final Consumer<Options> helpCmd) throws IOException {
318 return parseCommands(args, helpCmd, false);
319 }
320
321
322
323
324
325
326
327
328
329
330 public static ReportConfiguration parseCommands(final String[] args, final Consumer<Options> helpCmd, final boolean noArgs) throws IOException {
331 Options opts = buildOptions();
332 CommandLine cl;
333 Log log = DefaultLog.getInstance();
334 try {
335 cl = DefaultParser.builder().setDeprecatedHandler(DeprecationReporter.getLogReporter(log)).build().parse(opts, args);
336 } catch (ParseException e) {
337 log.error(e.getMessage());
338 log.error("Please use the \"--help\" option to see a list of valid commands and options");
339 System.exit(1);
340 return null;
341
342 }
343
344 if (cl.hasOption(LOG_LEVEL)) {
345 DeprecationReporter.logDeprecated(log, LOG_LEVEL);
346 if (log instanceof DefaultLog) {
347 DefaultLog dLog = (DefaultLog) log;
348 try {
349 dLog.setLevel(cl.getParsedOptionValue(LOG_LEVEL));
350 } catch (ParseException e) {
351 logParseException(log, e, LOG_LEVEL, cl, dLog.getLevel());
352 }
353 } else {
354 log.error("log was not a DefaultLog instance. LogLevel not set.");
355 }
356 }
357 if (cl.hasOption(HELP)) {
358 helpCmd.accept(opts);
359 return null;
360 }
361
362 String[] clArgs;
363 if (!noArgs) {
364 clArgs = cl.getArgs();
365 if (clArgs == null || clArgs.length != 1) {
366 helpCmd.accept(opts);
367 return null;
368 }
369 } else {
370 clArgs = new String[]{null};
371 }
372 return createConfiguration(log, clArgs[0], cl);
373 }
374
375 private static void logParseException(final Log log, final ParseException e, final Option opt, final CommandLine cl, final Object dflt) {
376 log.warn(format("Invalid %s specified: %s ", opt.getOpt(), cl.getOptionValue(opt)));
377 log.warn(format("%s set to: %s", opt.getOpt(), dflt));
378 log.debug(e);
379 }
380
381 static ReportConfiguration createConfiguration(final Log log, final String baseDirectory, final CommandLine cl) throws IOException {
382 final ReportConfiguration configuration = new ReportConfiguration(log);
383
384 if (cl.hasOption(DIR)) {
385 DeprecationReporter.logDeprecated(log, DIR);
386 }
387
388 if (cl.hasOption(DRY_RUN)) {
389 DeprecationReporter.logDeprecated(log, DRY_RUN);
390 configuration.setDryRun(cl.hasOption(DRY_RUN));
391 }
392
393 if (cl.hasOption(LIST_FAMILIES)) {
394 DeprecationReporter.logDeprecated(log, LIST_FAMILIES);
395 try {
396 configuration.listFamilies(cl.getParsedOptionValue(LIST_FAMILIES));
397 } catch (ParseException e) {
398 logParseException(log, e, LIST_FAMILIES, cl, Defaults.LIST_FAMILIES);
399 }
400 }
401
402 if (cl.hasOption(LIST_LICENSES)) {
403 DeprecationReporter.logDeprecated(log, LIST_LICENSES);
404 try {
405 configuration.listLicenses(cl.getParsedOptionValue(LIST_LICENSES));
406 } catch (ParseException e) {
407 logParseException(log, e, LIST_LICENSES, cl, Defaults.LIST_LICENSES);
408 }
409 }
410
411 if (cl.hasOption(ARCHIVE)) {
412 DeprecationReporter.logDeprecated(log, ARCHIVE);
413 try {
414 configuration.setArchiveProcessing(cl.getParsedOptionValue(ARCHIVE));
415 } catch (ParseException e) {
416 logParseException(log, e, ARCHIVE, cl, Defaults.ARCHIVE_PROCESSING);
417 }
418 }
419
420 if (cl.hasOption(STANDARD)) {
421 DeprecationReporter.logDeprecated(log, STANDARD);
422 try {
423 configuration.setStandardProcessing(cl.getParsedOptionValue(STANDARD));
424 } catch (ParseException e) {
425 logParseException(log, e, STANDARD, cl, Defaults.STANDARD_PROCESSING);
426 }
427 }
428
429 if (cl.hasOption(OUT)) {
430 DeprecationReporter.logDeprecated(log, OUT);
431 try {
432 configuration.setOut((File) cl.getParsedOptionValue(OUT));
433 } catch (ParseException e) {
434 logParseException(log, e, OUT, cl, "System.out");
435 }
436 }
437
438 if (cl.hasOption(SCAN_HIDDEN_DIRECTORIES)) {
439 DeprecationReporter.logDeprecated(log, SCAN_HIDDEN_DIRECTORIES);
440 configuration.setDirectoriesToIgnore(FalseFileFilter.FALSE);
441 }
442
443 if (ADD.getSelected() != null) {
444
445 Arrays.stream(cl.getOptions()).filter(o -> o.getOpt().equals("a")).forEach(o -> cl.hasOption(o.getOpt()));
446 if (cl.hasOption(FORCE)) {
447 DeprecationReporter.logDeprecated(log, FORCE);
448 }
449 if (cl.hasOption(COPYRIGHT)) {
450 DeprecationReporter.logDeprecated(log, COPYRIGHT);
451 }
452
453 configuration.setAddLicenseHeaders(cl.hasOption(FORCE) ? AddLicenseHeaders.FORCED : AddLicenseHeaders.TRUE);
454 configuration.setCopyrightMessage(cl.getOptionValue(COPYRIGHT));
455 }
456
457 if (cl.hasOption(EXCLUDE_CLI)) {
458 DeprecationReporter.logDeprecated(log, EXCLUDE_CLI);
459 String[] excludes = cl.getOptionValues(EXCLUDE_CLI);
460 if (excludes != null) {
461 parseExclusions(log, Arrays.asList(excludes)).ifPresent(configuration::setFilesToIgnore);
462 }
463 } else if (cl.hasOption(EXCLUDE_FILE_CLI)) {
464 DeprecationReporter.logDeprecated(log, EXCLUDE_FILE_CLI);
465 String excludeFileName = cl.getOptionValue(EXCLUDE_FILE_CLI);
466 if (excludeFileName != null) {
467 parseExclusions(log,FileUtils.readLines(new File(excludeFileName), StandardCharsets.UTF_8))
468 .ifPresent(configuration::setFilesToIgnore);
469 }
470 }
471
472 if (cl.hasOption(XML)) {
473 DeprecationReporter.logDeprecated(log, XML);
474 configuration.setStyleReport(false);
475 } else {
476 configuration.setStyleReport(true);
477 if (cl.hasOption(STYLESHEET_CLI)) {
478 DeprecationReporter.logDeprecated(log, STYLESHEET_CLI);
479 String[] style = cl.getOptionValues(STYLESHEET_CLI);
480 if (style.length != 1) {
481 log.error("Please specify a single stylesheet");
482 System.exit(1);
483 }
484
485 URL url = Report.class.getClassLoader().getResource(String.format("org/apache/rat/%s.xsl", style[0]));
486 IOSupplier<InputStream> ioSupplier = url == null
487 ? () -> Files.newInputStream(Paths.get(style[0]))
488 : url::openStream;
489 configuration.setStyleSheet(ioSupplier);
490 }
491 }
492
493 Defaults.Builder defaultBuilder = Defaults.builder();
494 if (cl.hasOption(NO_DEFAULTS)) {
495 DeprecationReporter.logDeprecated(log, NO_DEFAULTS);
496 defaultBuilder.noDefault();
497 }
498 if (cl.hasOption(LICENSES)) {
499 DeprecationReporter.logDeprecated(log, LICENSES);
500 for (String fn : cl.getOptionValues(LICENSES)) {
501 defaultBuilder.add(fn);
502 }
503 }
504 Defaults defaults = defaultBuilder.build(log);
505 configuration.setFrom(defaults);
506 if (StringUtils.isNotBlank(baseDirectory)) {
507 configuration.setReportable(getDirectory(baseDirectory, configuration));
508 }
509 return configuration;
510 }
511
512
513
514
515
516
517
518
519 static Optional<FilenameFilter> parseExclusions(final Log log, final List<String> excludes) {
520 final OrFileFilter orFilter = new OrFileFilter();
521 int ignoredLines = 0;
522 for (String exclude : excludes) {
523
524
525 if (exclude.startsWith("#") || StringUtils.isEmpty(exclude)) {
526 ignoredLines++;
527 continue;
528 }
529
530 String exclusion = exclude.trim();
531
532
533 try {
534 orFilter.addFileFilter(new RegexFileFilter(exclusion));
535 } catch (PatternSyntaxException e) {
536
537 }
538 orFilter.addFileFilter(new NameFileFilter(exclusion));
539 if (exclude.contains("?") || exclude.contains("*")) {
540 orFilter.addFileFilter(WildcardFileFilter.builder().setWildcards(exclusion).get());
541 }
542 }
543 if (ignoredLines > 0) {
544 log.info("Ignored " + ignoredLines + " lines in your exclusion files as comments or empty lines.");
545 }
546 return orFilter.getFileFilters().isEmpty() ? Optional.empty() : Optional.of(orFilter.negate());
547 }
548
549 static Options buildOptions() {
550 return new Options()
551 .addOption(ARCHIVE)
552 .addOption(STANDARD)
553 .addOption(DRY_RUN)
554 .addOption(LIST_FAMILIES)
555 .addOption(LIST_LICENSES)
556 .addOption(HELP)
557 .addOption(OUT)
558 .addOption(NO_DEFAULTS)
559 .addOption(LICENSES)
560 .addOption(SCAN_HIDDEN_DIRECTORIES)
561 .addOptionGroup(ADD)
562 .addOption(FORCE)
563 .addOption(COPYRIGHT)
564 .addOption(EXCLUDE_CLI)
565 .addOption(EXCLUDE_FILE_CLI)
566 .addOption(DIR)
567 .addOption(LOG_LEVEL)
568 .addOptionGroup(new OptionGroup()
569 .addOption(XML).addOption(STYLESHEET_CLI));
570 }
571
572 private static void printUsage(final Options opts) {
573 printUsage(new PrintWriter(System.out), opts);
574 }
575
576 private static String createPadding(final int len) {
577 char[] padding = new char[len];
578 Arrays.fill(padding, ' ');
579 return new String(padding);
580 }
581
582 static String header(final String txt) {
583 return String.format("%n====== %s ======%n", WordUtils.capitalizeFully(txt));
584 }
585
586 static void printUsage(final PrintWriter writer, final Options opts) {
587 HelpFormatter helpFormatter = new HelpFormatter.Builder().get();
588 helpFormatter.setWidth(HELP_WIDTH);
589 helpFormatter.setOptionComparator(new OptionComparator());
590 VersionInfo versionInfo = new VersionInfo();
591 String syntax = format("java -jar apache-rat/target/apache-rat-%s.jar [options] [DIR|ARCHIVE]", versionInfo.getVersion());
592 helpFormatter.printHelp(writer, helpFormatter.getWidth(), syntax, header("Available options"), opts,
593 helpFormatter.getLeftPadding(), helpFormatter.getDescPadding(),
594 header("Argument Types"), false);
595
596 String argumentPadding = createPadding(helpFormatter.getLeftPadding() + HELP_PADDING);
597 for (Map.Entry<String, Supplier<String>> argInfo : ARGUMENT_TYPES.entrySet()) {
598 writer.format("%n<%s>%n", argInfo.getKey());
599 helpFormatter.printWrapped(writer, helpFormatter.getWidth(), helpFormatter.getLeftPadding() + HELP_PADDING + HELP_PADDING,
600 argumentPadding + argInfo.getValue().get());
601 }
602 writer.println(header("Notes"));
603
604 int idx = 1;
605 for (String note : NOTES) {
606 writer.format("%d. %s%n", idx++, note);
607 }
608 writer.flush();
609 }
610
611 private Report() {
612
613 }
614
615
616
617
618
619
620
621
622
623 private static IReportable getDirectory(final String baseDirectory, final ReportConfiguration config) {
624 File base = new File(baseDirectory);
625
626 if (!base.exists()) {
627 config.getLog().log(Level.ERROR, "Directory '" + baseDirectory + "' does not exist");
628 return null;
629 }
630
631 Document doc = new FileDocument(base);
632 if (base.isDirectory()) {
633 return new DirectoryWalker(config, doc);
634 }
635
636 return new ArchiveWalker(config, doc);
637 }
638
639
640
641
642 public static class OptionComparator implements Comparator<Option>, Serializable {
643
644 private static final long serialVersionUID = 5305467873966684014L;
645
646 private String getKey(final Option opt) {
647 String key = opt.getOpt();
648 key = key == null ? opt.getLongOpt() : key;
649 return key;
650 }
651
652
653
654
655
656
657
658
659
660
661
662 @Override
663 public int compare(final Option opt1, final Option opt2) {
664 return getKey(opt1).compareToIgnoreCase(getKey(opt2));
665 }
666 }
667 }