1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.index.updater;
20
21 import javax.inject.Inject;
22 import javax.inject.Named;
23 import javax.inject.Singleton;
24
25 import java.io.BufferedInputStream;
26 import java.io.BufferedOutputStream;
27 import java.io.BufferedReader;
28 import java.io.File;
29 import java.io.FileInputStream;
30 import java.io.FileNotFoundException;
31 import java.io.FileOutputStream;
32 import java.io.IOException;
33 import java.io.InputStream;
34 import java.io.InputStreamReader;
35 import java.io.OutputStream;
36 import java.io.OutputStreamWriter;
37 import java.io.Writer;
38 import java.nio.charset.StandardCharsets;
39 import java.nio.file.Files;
40 import java.text.ParseException;
41 import java.text.SimpleDateFormat;
42 import java.util.ArrayList;
43 import java.util.Date;
44 import java.util.List;
45 import java.util.Properties;
46 import java.util.Set;
47 import java.util.TimeZone;
48
49 import org.apache.lucene.document.Document;
50 import org.apache.lucene.index.DirectoryReader;
51 import org.apache.lucene.index.IndexReader;
52 import org.apache.lucene.index.IndexWriter;
53 import org.apache.lucene.index.IndexWriterConfig;
54 import org.apache.lucene.index.MultiBits;
55 import org.apache.lucene.index.StoredFields;
56 import org.apache.lucene.store.Directory;
57 import org.apache.lucene.util.Bits;
58 import org.apache.maven.index.context.DocumentFilter;
59 import org.apache.maven.index.context.IndexUtils;
60 import org.apache.maven.index.context.IndexingContext;
61 import org.apache.maven.index.context.NexusAnalyzer;
62 import org.apache.maven.index.context.NexusIndexWriter;
63 import org.apache.maven.index.fs.Lock;
64 import org.apache.maven.index.fs.Locker;
65 import org.apache.maven.index.incremental.IncrementalHandler;
66 import org.apache.maven.index.updater.IndexDataReader.IndexDataReadResult;
67 import org.codehaus.plexus.util.FileUtils;
68 import org.codehaus.plexus.util.io.RawInputStreamFacade;
69 import org.slf4j.Logger;
70 import org.slf4j.LoggerFactory;
71
72
73
74
75
76
77
78 @Singleton
79 @Named
80 public class DefaultIndexUpdater implements IndexUpdater {
81
82 private final Logger logger = LoggerFactory.getLogger(getClass());
83
84 protected Logger getLogger() {
85 return logger;
86 }
87
88 private final IncrementalHandler incrementalHandler;
89
90 private final List<IndexUpdateSideEffect> sideEffects;
91
92 @Inject
93 public DefaultIndexUpdater(
94 final IncrementalHandler incrementalHandler, final List<IndexUpdateSideEffect> sideEffects) {
95 this.incrementalHandler = incrementalHandler;
96 this.sideEffects = sideEffects;
97 }
98
99 public IndexUpdateResult fetchAndUpdateIndex(final IndexUpdateRequest updateRequest) throws IOException {
100 IndexUpdateResult result = new IndexUpdateResult();
101
102 IndexingContext context = updateRequest.getIndexingContext();
103
104 ResourceFetcher fetcher = null;
105
106 if (!updateRequest.isOffline()) {
107 fetcher = updateRequest.getResourceFetcher();
108
109
110
111 if (fetcher == null) {
112 throw new IOException("Update of the index without provided ResourceFetcher is impossible.");
113 }
114
115 fetcher.connect(context.getId(), context.getIndexUpdateUrl());
116 }
117
118 File cacheDir = updateRequest.getLocalIndexCacheDir();
119 Locker locker = updateRequest.getLocker();
120 Lock lock = locker != null && cacheDir != null ? locker.lock(cacheDir) : null;
121 try {
122 if (cacheDir != null) {
123 LocalCacheIndexAdaptor cache = new LocalCacheIndexAdaptor(cacheDir, result);
124
125 if (!updateRequest.isOffline()) {
126 cacheDir.mkdirs();
127
128 try {
129 if (fetchAndUpdateIndex(updateRequest, fetcher, cache).isSuccessful()) {
130 cache.commit();
131 }
132 } finally {
133 fetcher.disconnect();
134 }
135 }
136
137 fetcher = cache.getFetcher();
138 } else if (updateRequest.isOffline()) {
139 throw new IllegalArgumentException("LocalIndexCacheDir can not be null in offline mode");
140 }
141
142 try {
143 if (!updateRequest.isCacheOnly()) {
144 LuceneIndexAdaptor target = new LuceneIndexAdaptor(updateRequest);
145 result = fetchAndUpdateIndex(updateRequest, fetcher, target);
146
147 if (result.isSuccessful()) {
148 target.commit();
149 }
150 }
151 } finally {
152 fetcher.disconnect();
153 }
154 } finally {
155 if (lock != null) {
156 lock.release();
157 }
158 }
159
160 return result;
161 }
162
163 private Date loadIndexDirectory(
164 final IndexUpdateRequest updateRequest,
165 final ResourceFetcher fetcher,
166 final boolean merge,
167 final String remoteIndexFile)
168 throws IOException {
169 File indexDir;
170 if (updateRequest.getIndexTempDir() != null) {
171 updateRequest.getIndexTempDir().mkdirs();
172 indexDir = Files.createTempDirectory(updateRequest.getIndexTempDir().toPath(), remoteIndexFile + ".dir")
173 .toFile();
174 } else {
175 indexDir = Files.createTempDirectory(remoteIndexFile + ".dir").toFile();
176 }
177 try (BufferedInputStream is = new BufferedInputStream(fetcher.retrieve(remoteIndexFile));
178 Directory directory = updateRequest.getFSDirectoryFactory().open(indexDir)) {
179 Date timestamp;
180
181 Set<String> rootGroups;
182 Set<String> allGroups;
183 if (remoteIndexFile.endsWith(".gz")) {
184 IndexDataReadResult result =
185 unpackIndexData(is, updateRequest, directory, updateRequest.getIndexingContext());
186 timestamp = result.getTimestamp();
187 rootGroups = result.getRootGroups();
188 allGroups = result.getAllGroups();
189 } else {
190
191 throw new IllegalArgumentException(
192 "The legacy format is no longer supported " + "by this version of maven-indexer.");
193 }
194
195 if (updateRequest.getDocumentFilter() != null) {
196 filterDirectory(directory, updateRequest.getDocumentFilter());
197 }
198
199 if (merge) {
200 updateRequest.getIndexingContext().merge(directory, null, allGroups, rootGroups);
201 } else {
202 updateRequest.getIndexingContext().replace(directory, allGroups, rootGroups);
203 }
204 if (sideEffects != null && sideEffects.size() > 0) {
205 getLogger().info(IndexUpdateSideEffect.class.getName() + " extensions found: " + sideEffects.size());
206 for (IndexUpdateSideEffect sideeffect : sideEffects) {
207 sideeffect.updateIndex(directory, updateRequest.getIndexingContext(), merge);
208 }
209 }
210
211 return timestamp;
212 } finally {
213 try {
214 FileUtils.deleteDirectory(indexDir);
215 } catch (IOException ex) {
216
217 }
218 }
219 }
220
221 private static void filterDirectory(final Directory directory, final DocumentFilter filter) throws IOException {
222 IndexReader r = null;
223 IndexWriter w = null;
224 try {
225 r = DirectoryReader.open(directory);
226 w = new NexusIndexWriter(directory, new IndexWriterConfig(new NexusAnalyzer()));
227
228 Bits liveDocs = MultiBits.getLiveDocs(r);
229
230 int numDocs = r.maxDoc();
231 StoredFields storedFields = r.storedFields();
232
233 for (int i = 0; i < numDocs; i++) {
234 if (liveDocs != null && !liveDocs.get(i)) {
235 continue;
236 }
237
238 Document d = storedFields.document(i);
239
240 if (!filter.accept(d)) {
241 boolean success = w.tryDeleteDocument(r, i) != -1;
242
243 }
244 }
245 w.commit();
246 } finally {
247 IndexUtils.close(r);
248 IndexUtils.close(w);
249 }
250
251 w = null;
252 try {
253
254 w = new NexusIndexWriter(directory, new IndexWriterConfig(new NexusAnalyzer()));
255
256 w.commit();
257 } finally {
258 IndexUtils.close(w);
259 }
260 }
261
262 private Properties loadIndexProperties(final File indexDirectoryFile, final String remoteIndexPropertiesName) {
263 File indexProperties = new File(indexDirectoryFile, remoteIndexPropertiesName);
264
265 try (FileInputStream fis = new FileInputStream(indexProperties)) {
266 Properties properties = new Properties();
267
268 properties.load(fis);
269
270 return properties;
271 } catch (IOException e) {
272 getLogger().debug("Unable to read remote properties stored locally", e);
273 }
274 return null;
275 }
276
277 private void storeIndexProperties(final File dir, final String indexPropertiesName, final Properties properties)
278 throws IOException {
279 File file = new File(dir, indexPropertiesName);
280
281 if (properties != null) {
282 try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
283 properties.store(os, null);
284 }
285 } else {
286 file.delete();
287 }
288 }
289
290 private Properties downloadIndexProperties(final ResourceFetcher fetcher) throws IOException {
291 try (InputStream fis = fetcher.retrieve(IndexingContext.INDEX_REMOTE_PROPERTIES_FILE)) {
292 Properties properties = new Properties();
293
294 properties.load(fis);
295
296 return properties;
297 }
298 }
299
300 public Date getTimestamp(final Properties properties, final String key) {
301 String indexTimestamp = properties.getProperty(key);
302
303 if (indexTimestamp != null) {
304 try {
305 SimpleDateFormat df = new SimpleDateFormat(IndexingContext.INDEX_TIME_FORMAT);
306 df.setTimeZone(TimeZone.getTimeZone("GMT"));
307 return df.parse(indexTimestamp);
308 } catch (ParseException ex) {
309 }
310 }
311 return null;
312 }
313
314
315
316
317
318
319
320 public static IndexDataReadResult unpackIndexData(
321 final InputStream is, final int threads, final Directory d, final IndexingContext context)
322 throws IOException {
323 return unpackIndexData(d, new IndexDataReader(is, threads), context);
324 }
325
326
327
328
329
330
331
332 public static IndexDataReadResult unpackIndexData(
333 final InputStream is, final IndexUpdateRequest request, final Directory d, final IndexingContext context)
334 throws IOException {
335 return unpackIndexData(d, new IndexDataReader(is, request), context);
336 }
337
338 private static IndexDataReadResult unpackIndexData(
339 final Directory d, IndexDataReader dr, final IndexingContext context) throws IOException {
340 IndexWriterConfig config = new IndexWriterConfig(new NexusAnalyzer());
341 config.setUseCompoundFile(false);
342 try (NexusIndexWriter w = new NexusIndexWriter(d, config)) {
343 return dr.readIndex(w, context);
344 }
345 }
346
347
348
349
350 public static class FileFetcher implements ResourceFetcher {
351 private final File basedir;
352
353 public FileFetcher(File basedir) {
354 this.basedir = basedir;
355 }
356
357 public void connect(String id, String url) throws IOException {
358
359 }
360
361 public void disconnect() throws IOException {
362
363 }
364
365 public void retrieve(String name, File targetFile) throws IOException, FileNotFoundException {
366 FileUtils.copyFile(getFile(name), targetFile);
367 }
368
369 public InputStream retrieve(String name) throws IOException, FileNotFoundException {
370 return new FileInputStream(getFile(name));
371 }
372
373 private File getFile(String name) {
374 return new File(basedir, name);
375 }
376 }
377
378 private abstract class IndexAdaptor {
379 protected final File dir;
380
381 protected Properties properties;
382
383 protected IndexAdaptor(File dir) {
384 this.dir = dir;
385 }
386
387 public abstract Properties getProperties();
388
389 public abstract void storeProperties() throws IOException;
390
391 public abstract void addIndexChunk(ResourceFetcher source, String filename) throws IOException;
392
393 public abstract Date setIndexFile(ResourceFetcher source, String string) throws IOException;
394
395 public Properties setProperties(ResourceFetcher source) throws IOException {
396 this.properties = downloadIndexProperties(source);
397 return properties;
398 }
399
400 public abstract Date getTimestamp();
401
402 public void commit() throws IOException {
403 storeProperties();
404 }
405 }
406
407 private class LuceneIndexAdaptor extends IndexAdaptor {
408 private final IndexUpdateRequest updateRequest;
409
410 LuceneIndexAdaptor(IndexUpdateRequest updateRequest) {
411 super(updateRequest.getIndexingContext().getIndexDirectoryFile());
412 this.updateRequest = updateRequest;
413 }
414
415 public Properties getProperties() {
416 if (properties == null) {
417 properties = loadIndexProperties(dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE);
418 }
419 return properties;
420 }
421
422 public void storeProperties() throws IOException {
423 storeIndexProperties(dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE, properties);
424 }
425
426 public Date getTimestamp() {
427 return updateRequest.getIndexingContext().getTimestamp();
428 }
429
430 public void addIndexChunk(ResourceFetcher source, String filename) throws IOException {
431 loadIndexDirectory(updateRequest, source, true, filename);
432 }
433
434 public Date setIndexFile(ResourceFetcher source, String filename) throws IOException {
435 return loadIndexDirectory(updateRequest, source, false, filename);
436 }
437
438 public void commit() throws IOException {
439 super.commit();
440
441 updateRequest.getIndexingContext().commit();
442 }
443 }
444
445 private class LocalCacheIndexAdaptor extends IndexAdaptor {
446 private static final String CHUNKS_FILENAME = "chunks.lst";
447
448 private final IndexUpdateResult result;
449
450 private final ArrayList<String> newChunks = new ArrayList<>();
451
452 LocalCacheIndexAdaptor(File dir, IndexUpdateResult result) {
453 super(dir);
454 this.result = result;
455 }
456
457 public Properties getProperties() {
458 if (properties == null) {
459 properties = loadIndexProperties(dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE);
460 }
461 return properties;
462 }
463
464 public void storeProperties() throws IOException {
465 storeIndexProperties(dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE, properties);
466 }
467
468 public Date getTimestamp() {
469 Properties properties = getProperties();
470 if (properties == null) {
471 return null;
472 }
473
474 Date timestamp = DefaultIndexUpdater.this.getTimestamp(properties, IndexingContext.INDEX_TIMESTAMP);
475
476 if (timestamp == null) {
477 timestamp = DefaultIndexUpdater.this.getTimestamp(properties, IndexingContext.INDEX_LEGACY_TIMESTAMP);
478 }
479
480 return timestamp;
481 }
482
483 public void addIndexChunk(ResourceFetcher source, String filename) throws IOException {
484 File chunk = new File(dir, filename);
485 FileUtils.copyStreamToFile(new RawInputStreamFacade(source.retrieve(filename)), chunk);
486 newChunks.add(filename);
487 }
488
489 public Date setIndexFile(ResourceFetcher source, String filename) throws IOException {
490 cleanCacheDirectory(dir);
491
492 result.setFullUpdate(true);
493
494 File target = new File(dir, filename);
495 FileUtils.copyStreamToFile(new RawInputStreamFacade(source.retrieve(filename)), target);
496
497 return null;
498 }
499
500 @Override
501 public void commit() throws IOException {
502 File chunksFile = new File(dir, CHUNKS_FILENAME);
503 try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(chunksFile, true));
504 Writer w = new OutputStreamWriter(os, StandardCharsets.UTF_8)) {
505 for (String filename : newChunks) {
506 w.write(filename + "\n");
507 }
508 w.flush();
509 }
510 super.commit();
511 }
512
513 public List<String> getChunks() throws IOException {
514 ArrayList<String> chunks = new ArrayList<>();
515
516 File chunksFile = new File(dir, CHUNKS_FILENAME);
517 try (BufferedReader r = new BufferedReader(
518 new InputStreamReader(new FileInputStream(chunksFile), StandardCharsets.UTF_8))) {
519 String str;
520 while ((str = r.readLine()) != null) {
521 chunks.add(str);
522 }
523 }
524 return chunks;
525 }
526
527 public ResourceFetcher getFetcher() {
528 return new LocalIndexCacheFetcher(dir) {
529 @Override
530 public List<String> getChunks() throws IOException {
531 return LocalCacheIndexAdaptor.this.getChunks();
532 }
533 };
534 }
535 }
536
537 abstract static class LocalIndexCacheFetcher extends FileFetcher {
538 LocalIndexCacheFetcher(File basedir) {
539 super(basedir);
540 }
541
542 public abstract List<String> getChunks() throws IOException;
543 }
544
545 private IndexUpdateResult fetchAndUpdateIndex(
546 final IndexUpdateRequest updateRequest, ResourceFetcher source, IndexAdaptor target) throws IOException {
547 IndexUpdateResult result = new IndexUpdateResult();
548
549 if (!updateRequest.isForceFullUpdate()) {
550 Properties localProperties = target.getProperties();
551 Date localTimestamp = null;
552
553 if (localProperties != null) {
554 localTimestamp = getTimestamp(localProperties, IndexingContext.INDEX_TIMESTAMP);
555 }
556
557
558
559 Properties remoteProperties = target.setProperties(source);
560
561 Date updateTimestamp = getTimestamp(remoteProperties, IndexingContext.INDEX_TIMESTAMP);
562
563
564 if (updateTimestamp != null) {
565 List<String> filenames = incrementalHandler.loadRemoteIncrementalUpdates(
566 updateRequest, localProperties, remoteProperties);
567
568
569 if (filenames != null) {
570 for (String filename : filenames) {
571 target.addIndexChunk(source, filename);
572 }
573
574 result.setTimestamp(updateTimestamp);
575 result.setSuccessful(true);
576 return result;
577 }
578 } else {
579 updateTimestamp = getTimestamp(remoteProperties, IndexingContext.INDEX_LEGACY_TIMESTAMP);
580 }
581
582
583
584
585 if (localTimestamp != null) {
586
587
588
589 if (updateTimestamp != null && localTimestamp != null && !updateTimestamp.after(localTimestamp)) {
590
591 result.setSuccessful(true);
592 return result;
593 }
594 }
595 } else {
596
597 target.setProperties(source);
598 }
599
600 if (!updateRequest.isIncrementalOnly()) {
601 Date timestamp;
602 try {
603 timestamp = target.setIndexFile(source, IndexingContext.INDEX_FILE_PREFIX + ".gz");
604 if (source instanceof LocalIndexCacheFetcher) {
605
606
607 for (String filename : ((LocalIndexCacheFetcher) source).getChunks()) {
608 target.addIndexChunk(source, filename);
609 }
610 }
611 } catch (IOException ex) {
612
613 try {
614 timestamp = target.setIndexFile(source, IndexingContext.INDEX_FILE_PREFIX + ".zip");
615 } catch (IOException ex2) {
616 getLogger().error("Fallback to *.zip also failed: " + ex2);
617
618 throw ex;
619 }
620 }
621
622 result.setTimestamp(timestamp);
623 result.setSuccessful(true);
624 result.setFullUpdate(true);
625 }
626
627 return result;
628 }
629
630
631
632
633 protected void cleanCacheDirectory(File dir) throws IOException {
634 File[] members = dir.listFiles();
635 if (members == null) {
636 return;
637 }
638
639 for (File member : members) {
640 if (!Locker.LOCK_FILE.equals(member.getName())) {
641 FileUtils.forceDelete(member);
642 }
643 }
644 }
645 }