1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.creadur.tentacles;
18
19 import org.apache.hc.client5.http.classic.methods.HttpGet;
20 import org.apache.hc.client5.http.classic.methods.HttpHead;
21 import org.apache.hc.client5.http.classic.methods.HttpUriRequest;
22 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
23 import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
24 import org.apache.hc.core5.http.ClassicHttpResponse;
25 import org.apache.hc.core5.http.Header;
26 import org.apache.hc.core5.http.HttpHeaders;
27 import org.apache.logging.log4j.LogManager;
28 import org.apache.logging.log4j.Logger;
29
30 import org.codehaus.swizzle.stream.StreamLexer;
31
32 import java.io.File;
33 import java.io.IOException;
34 import java.io.InputStream;
35 import java.net.URI;
36 import java.util.LinkedHashSet;
37 import java.util.Set;
38
39 public class NexusClient {
40
41 private static final Logger log = LogManager.getLogger(NexusClient.class);
42 private static final String SLASH = "/";
43 private static final String ONE_UP = "../";
44 private static final String USER_AGENT_CONTENTS = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13";
45
46 private final CloseableHttpClient client;
47 private final FileSystem fileSystem;
48 private final IOSystem ioSystem;
49 private final int retries;
50
51 public NexusClient(final Platform platform) {
52
53 System.setProperty("http.keepAlive", "false");
54 System.setProperty("http.maxConnections", "50");
55
56 this.retries = Integer.parseInt(System.getProperty("NexusClient.retries", "5"));
57
58 this.client = HttpClientBuilder.create().disableContentCompression()
59 .build();
60 this.fileSystem = platform.fileSystem();
61 this.ioSystem = platform.ioSystem();
62 }
63
64 public File download(final URI uri, final File file) throws IOException {
65 final long length = getContentLength(uri);
66
67 if (file.exists()) {
68 if (file.length() == length) {
69 log.info("Skipping download as file exists already: {}", uri);
70 return file;
71 } else {
72 log.info("Incomplete - Continue downloading {} ({} bytes)", uri, length);
73 }
74 } else {
75 log.info("Downloading {} bytes from {}", length, uri);
76 }
77
78 try (ClassicHttpResponse response = get(uri); InputStream content = response.getEntity().getContent()) {
79 this.fileSystem.mkparent(file);
80 this.ioSystem.copy(content, file);
81 }
82
83 return file;
84 }
85
86 private Long getContentLength(final URI uri) throws IOException {
87 try (final ClassicHttpResponse head = head(uri)) {
88 final Header[] headers = head.getHeaders(HttpHeaders.CONTENT_LENGTH);
89
90 if (headers != null && headers.length >= 1) {
91 return Long.valueOf(headers[0].getValue());
92 }
93
94 return (long) -1;
95 }
96 }
97
98 private ClassicHttpResponse get(final URI uri) throws IOException {
99 return get(new HttpGet(uri), this.retries);
100 }
101
102 private ClassicHttpResponse head(final URI uri) throws IOException {
103 return get(new HttpHead(uri), this.retries);
104 }
105
106 private ClassicHttpResponse get(final HttpUriRequest request, int tries) throws IOException {
107 try {
108 request.setHeader(HttpHeaders.USER_AGENT, USER_AGENT_CONTENTS);
109 return this.client.execute(request);
110 } catch (final IOException e) {
111 if (tries > 0) {
112 try {
113 Thread.sleep(250);
114 } catch (final InterruptedException ie) {
115 Thread.interrupted();
116 throw new IOException("Interrupted", ie);
117 }
118 return get(request, tries--);
119 } else {
120 throw e;
121 }
122 }
123 }
124
125 public Set<URI> crawl(final URI index) throws IOException {
126 log.info("Crawl {}", index);
127 final Set<URI> resources = new LinkedHashSet<>();
128
129 try (final ClassicHttpResponse response = get(index);
130 final InputStream content = response.getEntity().getContent()) {
131
132 final StreamLexer lexer = new StreamLexer(content);
133 final Set<URI> crawl = new LinkedHashSet<>();
134
135
136
137 while (lexer.readAndMark("<a ", "/a>")) {
138 try {
139 final String link = lexer.peek("href=\"", "\"");
140 final String name = lexer.peek(">", "<");
141
142 final URI uri = index.resolve(link);
143
144 if (name.equals(ONE_UP)) {
145 continue;
146 }
147 if (link.equals(ONE_UP)) {
148 continue;
149 }
150
151 if (name.endsWith(SLASH)) {
152 crawl.add(uri);
153 continue;
154 }
155
156 resources.add(uri);
157
158 } finally {
159 lexer.unmark();
160 }
161 }
162
163 for (final URI uri : crawl) {
164 resources.addAll(crawl(uri));
165 }
166
167 return resources;
168 }
169 }
170 }