001// License: GPL. For details, see Readme.txt file.
002package org.openstreetmap.gui.jmapviewer;
003
004import java.io.BufferedReader;
005import java.io.ByteArrayInputStream;
006import java.io.ByteArrayOutputStream;
007import java.io.File;
008import java.io.FileInputStream;
009import java.io.FileNotFoundException;
010import java.io.FileOutputStream;
011import java.io.IOException;
012import java.io.InputStream;
013import java.io.InputStreamReader;
014import java.io.OutputStreamWriter;
015import java.io.PrintWriter;
016import java.net.HttpURLConnection;
017import java.net.URL;
018import java.net.URLConnection;
019import java.nio.charset.Charset;
020import java.util.HashMap;
021import java.util.Map;
022import java.util.Map.Entry;
023import java.util.Random;
024import java.util.logging.Level;
025import java.util.logging.Logger;
026
027import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader;
028import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController;
029import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
030import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
031import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
032import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
033import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate;
034
035/**
036 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and
037 * saves all loaded files in a directory located in the temporary directory.
038 * If a tile is present in this file cache it will not be loaded from OSM again.
039 *
040 * @author Jan Peter Stotz
041 * @author Stefan Zeller
042 */
043public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader {
044
045    private static final Logger log = FeatureAdapter.getLogger(OsmFileCacheTileLoader.class.getName());
046
047    protected static final String TAGS_FILE_EXT = "tags";
048
049    private static final Charset TAGS_CHARSET = Charset.forName("UTF-8");
050
051    public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24;
052    public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7;
053
054    protected String cacheDirBase;
055
056    protected final Map<TileSource, File> sourceCacheDirMap;
057
058    protected long maxCacheFileAge = Long.MAX_VALUE;  // max. age not limited
059    protected long recheckAfter = FILE_AGE_ONE_WEEK;
060
061    public static File getDefaultCacheDir() throws SecurityException {
062        String tempDir = null;
063        String userName = System.getProperty("user.name");
064        try {
065            tempDir = System.getProperty("java.io.tmpdir");
066        } catch (SecurityException e) {
067            log.log(Level.WARNING,
068                    "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: "
069                    + e.toString());
070            throw e; // rethrow
071        }
072        try {
073            if (tempDir == null)
074                throw new IOException("No temp directory set");
075            String subDirName = "JMapViewerTiles";
076            // On Linux/Unix systems we do not have a per user tmp directory.
077            // Therefore we add the user name for getting a unique dir name.
078            if (userName != null && userName.length() > 0) {
079                subDirName += "_" + userName;
080            }
081            File cacheDir = new File(tempDir, subDirName);
082            return cacheDir;
083        } catch (Exception e) {
084        }
085        return null;
086    }
087
088    /**
089     * Create a OSMFileCacheTileLoader with given cache directory.
090     * If cacheDir is not set or invalid, IOException will be thrown.
091     * @param map the listener checking for tile load events (usually the map for display)
092     * @param cacheDir directory to store cached tiles
093     */
094    public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException  {
095        super(map);
096        if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs()))
097            throw new IOException("Cannot access cache directory");
098
099        log.finest("Tile cache directory: " + cacheDir);
100        cacheDirBase = cacheDir.getAbsolutePath();
101        sourceCacheDirMap = new HashMap<>();
102    }
103
104    /**
105     * Create a OSMFileCacheTileLoader with system property temp dir.
106     * If not set an IOException will be thrown.
107     * @param map the listener checking for tile load events (usually the map for display)
108     */
109    public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException {
110        this(map, getDefaultCacheDir());
111    }
112
113    @Override
114    public TileJob createTileLoaderJob(final Tile tile) {
115        return new FileLoadJob(tile);
116    }
117
118    protected File getSourceCacheDir(TileSource source) {
119        File dir = sourceCacheDirMap.get(source);
120        if (dir == null) {
121            dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));
122            if (!dir.exists()) {
123                dir.mkdirs();
124            }
125        }
126        return dir;
127    }
128
129    protected class FileLoadJob implements TileJob {
130        InputStream input = null;
131
132        Tile tile;
133        File tileCacheDir;
134        File tileFile = null;
135        Long fileMtime = null;
136        Long now = null; // current time in milliseconds (keep consistent value for the whole run)
137
138        public FileLoadJob(Tile tile) {
139            this.tile = tile;
140        }
141
142        @Override
143        public Tile getTile() {
144            return tile;
145        }
146
147        @Override
148        public void run() {
149            synchronized (tile) {
150                if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading())
151                    return;
152                tile.loaded = false;
153                tile.error = false;
154                tile.loading = true;
155            }
156            now = System.currentTimeMillis();
157            tileCacheDir = getSourceCacheDir(tile.getSource());
158
159            if (loadTileFromFile(recheckAfter)) {
160                log.log(Level.FINE, "TMS - found in tile cache: {0}", tile);
161                tile.setLoaded(true);
162                listener.tileLoadingFinished(tile, true);
163                return;
164            }
165            TileJob job = new TileJob() {
166
167                @Override
168                public void run() {
169                    if (loadOrUpdateTile()) {
170                        tile.setLoaded(true);
171                        listener.tileLoadingFinished(tile, true);
172                    } else {
173                        // failed to download - use old cache file if available
174                        if (loadTileFromFile(maxCacheFileAge)) {
175                            tile.setLoaded(true);
176                            tile.error = false;
177                            listener.tileLoadingFinished(tile, true);
178                            log.log(Level.FINE, "TMS - found stale tile in cache: {0}", tile);
179                        } else {
180                            // failed completely
181                            tile.setLoaded(true);
182                            listener.tileLoadingFinished(tile, false);
183                        }
184                    }
185                }
186                @Override
187                public Tile getTile() {
188                    return tile;
189                }
190            };
191            JobDispatcher.getInstance().addJob(job);
192        }
193
194        protected boolean loadOrUpdateTile() {
195            try {
196                URLConnection urlConn = loadTileFromOsm(tile);
197                if (fileMtime != null && now - fileMtime <= maxCacheFileAge) {
198                    switch (tile.getSource().getTileUpdate()) {
199                    case IfModifiedSince:
200                        urlConn.setIfModifiedSince(fileMtime);
201                        break;
202                    case LastModified:
203                        if (!isOsmTileNewer(fileMtime)) {
204                            log.log(Level.FINE, "TMS - LastModified test: local version is up to date: {0}", tile);
205                            tileFile.setLastModified(now);
206                            return true;
207                        }
208                        break;
209                    default:
210                        break;
211                    }
212                }
213                if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) {
214                    String fileETag = tile.getValue("etag");
215                    if (fileETag != null) {
216                        switch (tile.getSource().getTileUpdate()) {
217                        case IfNoneMatch:
218                            urlConn.addRequestProperty("If-None-Match", fileETag);
219                            break;
220                        case ETag:
221                            if (hasOsmTileETag(fileETag)) {
222                                log.log(Level.FINE, "TMS - ETag test: local version is up to date: {0}", tile);
223                                tileFile.setLastModified(now);
224                                return true;
225                            }
226                        default:
227                            break;
228                        }
229                    }
230                    tile.putValue("etag", urlConn.getHeaderField("ETag"));
231                }
232                if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {
233                    // If we are isModifiedSince or If-None-Match has been set
234                    // and the server answers with a HTTP 304 = "Not Modified"
235                    switch (tile.getSource().getTileUpdate()) {
236                    case IfModifiedSince:
237                        log.log(Level.FINE, "TMS - IfModifiedSince test: local version is up to date: {0}", tile);
238                        break;
239                    case IfNoneMatch:
240                        log.log(Level.FINE, "TMS - IfNoneMatch test: local version is up to date: {0}", tile);
241                        break;
242                    default:
243                        break;
244                    }
245                    if (loadTileFromFile(maxCacheFileAge)) {
246                        tileFile.setLastModified(now);
247                        return true;
248                    }
249                }
250
251                loadTileMetadata(tile, urlConn);
252                saveTagsToFile();
253
254                if ("no-tile".equals(tile.getValue("tile-info")))
255                {
256                    log.log(Level.FINE, "TMS - No tile: tile-info=no-tile: {0}", tile);
257                    tile.setError("No tile at this zoom level");
258                    return true;
259                } else {
260                    for (int i = 0; i < 5; ++i) {
261                        if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {
262                            Thread.sleep(5000+(new Random()).nextInt(5000));
263                            continue;
264                        }
265                        byte[] buffer = loadTileInBuffer(urlConn);
266                        if (buffer != null) {
267                            tile.loadImage(new ByteArrayInputStream(buffer));
268                            saveTileToFile(buffer);
269                            log.log(Level.FINE, "TMS - downloaded tile from server: {0}", tile.getUrl());
270                            return true;
271                        }
272                    }
273                }
274            } catch (Exception e) {
275                tile.setError(e.getMessage());
276                if (input == null) {
277                    try {
278                        log.log(Level.WARNING, "TMS - Failed downloading {0}: {1}", new Object[]{tile.getUrl(), e.getMessage()});
279                        return false;
280                    } catch(IOException i) {
281                    }
282                }
283            }
284            log.log(Level.WARNING, "TMS - Failed downloading tile: {0}", tile);
285            return false;
286        }
287
288        protected boolean loadTileFromFile(long maxAge) {
289            try {
290                tileFile = getTileFile();
291                if (!tileFile.exists())
292                    return false;
293                loadTagsFromFile();
294
295                fileMtime = tileFile.lastModified();
296                if (now - fileMtime > maxAge)
297                    return false;
298
299                if ("no-tile".equals(tile.getValue("tile-info"))) {
300                    tile.setError("No tile at this zoom level");
301                    if (tileFile.exists()) {
302                        tileFile.delete();
303                    }
304                    tileFile = null;
305                } else {
306                    try (FileInputStream fin = new FileInputStream(tileFile)) {
307                        if (fin.available() == 0)
308                            throw new IOException("File empty");
309                        tile.loadImage(fin);
310                    }
311                }
312                return true;
313
314            } catch (Exception e) {
315                log.log(Level.WARNING, "TMS - Error while loading image from tile cache: {0}; {1}", new Object[]{e.getMessage(), tile});
316                tileFile.delete();
317                tileFile = null;
318                fileMtime = null;
319            }
320            return false;
321        }
322
323        protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException {
324            input = urlConn.getInputStream();
325            try {
326                ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());
327                byte[] buffer = new byte[2048];
328                boolean finished = false;
329                do {
330                    int read = input.read(buffer);
331                    if (read >= 0) {
332                        bout.write(buffer, 0, read);
333                    } else {
334                        finished = true;
335                    }
336                } while (!finished);
337                if (bout.size() == 0)
338                    return null;
339                return bout.toByteArray();
340            } finally {
341                input.close();
342                input = null;
343            }
344        }
345
346        /**
347         * Performs a <code>HEAD</code> request for retrieving the
348         * <code>LastModified</code> header value.
349         *
350         * Note: This does only work with servers providing the
351         * <code>LastModified</code> header:
352         * <ul>
353         * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.CycleMap} - supported</li>
354         * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.Mapnik} - not supported</li>
355         * </ul>
356         *
357         * @param fileAge time of the
358         * @return <code>true</code> if the tile on the server is newer than the
359         *         file
360         * @throws IOException
361         */
362        protected boolean isOsmTileNewer(long fileAge) throws IOException {
363            URL url;
364            url = new URL(tile.getUrl());
365            HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
366            prepareHttpUrlConnection(urlConn);
367            urlConn.setRequestMethod("HEAD");
368            urlConn.setReadTimeout(30000); // 30 seconds read timeout
369            // System.out.println("Tile age: " + new
370            // Date(urlConn.getLastModified()) + " / "
371            // + new Date(fileMtime));
372            long lastModified = urlConn.getLastModified();
373            if (lastModified == 0)
374                return true; // no LastModified time returned
375            return (lastModified > fileAge);
376        }
377
378        protected boolean hasOsmTileETag(String eTag) throws IOException {
379            URL url;
380            url = new URL(tile.getUrl());
381            HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
382            prepareHttpUrlConnection(urlConn);
383            urlConn.setRequestMethod("HEAD");
384            urlConn.setReadTimeout(30000); // 30 seconds read timeout
385            // System.out.println("Tile age: " + new
386            // Date(urlConn.getLastModified()) + " / "
387            // + new Date(fileMtime));
388            String osmETag = urlConn.getHeaderField("ETag");
389            if (osmETag == null)
390                return true;
391            return (osmETag.equals(eTag));
392        }
393
394        protected File getTileFile() {
395            return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."
396                    + tile.getSource().getTileType());
397        }
398
399        protected File getTagsFile() {
400            return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."
401                    + TAGS_FILE_EXT);
402        }
403
404        protected void saveTileToFile(byte[] rawData) {
405            File file = getTileFile();
406            file.getParentFile().mkdirs();
407            try (
408                FileOutputStream f = new FileOutputStream(file)
409            ) {
410                f.write(rawData);
411            } catch (Exception e) {
412                log.log(Level.SEVERE, "Failed to save tile content: {0}", e.getLocalizedMessage());
413            }
414        }
415
416        protected void saveTagsToFile() {
417            File tagsFile = getTagsFile();
418            tagsFile.getParentFile().mkdirs();
419            if (tile.getMetadata() == null) {
420                tagsFile.delete();
421                return;
422            }
423            try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) {
424                for (Entry<String, String> entry : tile.getMetadata().entrySet()) {
425                    f.println(entry.getKey() + "=" + entry.getValue());
426                }
427            } catch (Exception e) {
428                System.err.println("Failed to save tile tags: " + e.getLocalizedMessage());
429            }
430        }
431
432        protected void loadTagsFromFile() {
433            File tagsFile = getTagsFile();
434            try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) {
435                for (String line = f.readLine(); line != null; line = f.readLine()) {
436                    final int i = line.indexOf('=');
437                    if (i == -1 || i == 0) {
438                        System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line);
439                        continue;
440                    }
441                    tile.putValue(line.substring(0,i),line.substring(i+1));
442                }
443            } catch (FileNotFoundException e) {
444            } catch (Exception e) {
445                System.err.println("Failed to load tile tags: " + e.getLocalizedMessage());
446            }
447        }
448    }
449
450    public long getMaxFileAge() {
451        return maxCacheFileAge;
452    }
453
454    /**
455     * Sets the maximum age of the local cached tile in the file system. If a
456     * local tile is older than the specified file age
457     * {@link OsmFileCacheTileLoader} will connect to the tile server and check
458     * if a newer tile is available using the mechanism specified for the
459     * selected tile source/server.
460     *
461     * @param maxFileAge
462     *            maximum age in milliseconds
463     * @see #FILE_AGE_ONE_DAY
464     * @see #FILE_AGE_ONE_WEEK
465     * @see TileSource#getTileUpdate()
466     */
467    public void setCacheMaxFileAge(long maxFileAge) {
468        this.maxCacheFileAge = maxFileAge;
469    }
470
471    public String getCacheDirBase() {
472        return cacheDirBase;
473    }
474
475    public void setTileCacheDir(String tileCacheDir) {
476        File dir = new File(tileCacheDir);
477        dir.mkdirs();
478        this.cacheDirBase = dir.getAbsolutePath();
479    }
480
481    @Override
482    public void clearCache(TileSource source) {
483        clearCache(source, null);
484    }
485
486    @Override
487    public void clearCache(TileSource source, TileClearController controller) {
488        File dir = getSourceCacheDir(source);
489        if (dir != null) {
490            if (controller != null) controller.initClearDir(dir);
491            if (dir.isDirectory()) {
492                File[] files = dir.listFiles();
493                if (controller != null) controller.initClearFiles(files);
494                for (File file : files) {
495                    if (controller != null && controller.cancel()) return;
496                    file.delete();
497                    if (controller != null) controller.fileDeleted(file);
498                }
499            }
500            dir.delete();
501        }
502        if (controller != null) controller.clearFinished();
503    }
504}