summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDennis Nienhüser <nienhueser@kde.org>2016-10-02 09:23:51 (GMT)
committerDennis Nienhüser <nienhueser@kde.org>2016-10-02 09:48:31 (GMT)
commitcc9a2d9ab974e35d9addc1a9e196148dc7e42c85 (patch)
tree6918784145aa5a1a0b310433c72eb6055f16acec
parent8e1d8e36da9e200328204c6b1c3dd25d1682c013 (diff)
Use urls as input and download data as needed. Use bounding polygon.
Creating vector tiles locally is now as easy as running marble-vectorosm-tilecreator http://download.geofabrik.de/europe/germany/baden-wuerttemberg/bremen-latest.osm.pbf Local files can be passed as well. See --help for additional options.
-rw-r--r--tools/vectorosm-tilecreator/TileDirectory.cpp236
-rw-r--r--tools/vectorosm-tilecreator/TileDirectory.h50
-rw-r--r--tools/vectorosm-tilecreator/main.cpp34
3 files changed, 274 insertions, 46 deletions
diff --git a/tools/vectorosm-tilecreator/TileDirectory.cpp b/tools/vectorosm-tilecreator/TileDirectory.cpp
index 8385fa8..2f7a044 100644
--- a/tools/vectorosm-tilecreator/TileDirectory.cpp
+++ b/tools/vectorosm-tilecreator/TileDirectory.cpp
@@ -11,27 +11,55 @@
#include "TileDirectory.h"
#include "TileIterator.h"
#include <GeoDataDocumentWriter.h>
+#include "MarbleZipReader.h"
+#include <GeoDataLatLonAltBox.h>
#include <QFileInfo>
#include <QDebug>
#include <QProcess>
#include <QDir>
+#include <QUrl>
+#include <QNetworkRequest>
+#include <QNetworkReply>
+#include <QThread>
#include <iostream>
+#include <iomanip>
+
+using namespace std;
namespace Marble {
-TileDirectory::TileDirectory(TileType tileType, const QString &baseDir, ParsingRunnerManager &manager, QString const &extension) :
- m_baseDir(baseDir),
+TileDirectory::TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, QString const &extension) :
+ m_cacheDir(cacheDir),
+ m_baseDir(),
m_manager(manager),
- m_zoomLevel(QFileInfo(baseDir).baseName().toInt()),
+ m_zoomLevel(-1),
m_tileX(-1),
m_tileY(-1),
m_tagZoomLevel(-1),
m_extension(extension),
- m_tileType(tileType)
+ m_tileType(tileType),
+ m_landmassFile("land-polygons-split-4326.zip")
{
- // nothing to do
+ if (m_tileType == Landmass) {
+ m_zoomLevel = 7;
+ m_baseDir = QString("%1/landmass/%2").arg(cacheDir).arg(m_zoomLevel);
+ m_inputFile = QString("%1/land-polygons-split-4326/land_polygons.shp").arg(cacheDir);
+ auto const landmassZip = QString("%1/%2").arg(m_cacheDir).arg(m_landmassFile);
+ if (!QFileInfo(landmassZip).exists()) {
+ QString const url = QString("http://data.openstreetmapdata.com/%1").arg(m_landmassFile);
+ download(url, landmassZip);
+
+ MarbleZipReader unzip(landmassZip);
+ if (!unzip.extractAll(m_cacheDir)) {
+ qWarning() << "Failed to extract" << landmassZip << "to" << m_cacheDir;
+ }
+ }
+ } else {
+ m_zoomLevel = 10;
+ m_baseDir = QString("%1/osm/%2").arg(cacheDir).arg(m_zoomLevel);
+ }
}
TileId TileDirectory::tileFor(int zoomLevel, int tileX, int tileY) const
@@ -59,6 +87,30 @@ QSharedPointer<GeoDataDocument> TileDirectory::load(int zoomLevel, int tileX, in
void TileDirectory::setInputFile(const QString &filename)
{
m_inputFile = filename;
+
+ if (m_tileType == OpenStreetMap) {
+ QUrl url = QUrl(filename);
+ if (url.scheme().isEmpty()) {
+ // local file
+ m_boundingBox = boundingBox(m_inputFile);
+ } else {
+ // remote file: check if already downloaded
+ QFileInfo cacheFile = QString("%1/%2").arg(m_cacheDir).arg(url.fileName());
+ if (!cacheFile.exists()) {
+ download(filename, cacheFile.absoluteFilePath());
+ }
+ m_inputFile = cacheFile.absoluteFilePath();
+
+ QString polyFile = QUrl(filename).fileName();
+ polyFile.remove("-latest.osm.pbf").append(".poly");
+ QString poly = QString("%1/%2").arg(url.adjusted(QUrl::RemoveFilename).toString()).arg(polyFile);
+ QString const polyTarget = QString("%1/%2").arg(m_cacheDir).arg(polyFile);
+ if (!QFileInfo(polyTarget).exists()) {
+ download(poly, polyTarget);
+ }
+ setBoundingPolygon(polyTarget);
+ }
+ }
}
GeoDataDocument* TileDirectory::clip(int zoomLevel, int tileX, int tileY)
@@ -68,9 +120,11 @@ GeoDataDocument* TileDirectory::clip(int zoomLevel, int tileX, int tileY)
if (!m_clipper || oldMap != m_landmass || m_tagZoomLevel != zoomLevel) {
setTagZoomLevel(zoomLevel);
GeoDataDocument* input = m_tagsFilter ? m_tagsFilter->accepted() : m_landmass.data();
- m_clipper = QSharedPointer<VectorClipper>(new VectorClipper(input));
+ if (input) {
+ m_clipper = QSharedPointer<VectorClipper>(new VectorClipper(input));
+ }
}
- return m_clipper->clipTo(zoomLevel, tileX, tileY);
+ return m_clipper ? m_clipper->clipTo(zoomLevel, tileX, tileY) : nullptr;
}
QString TileDirectory::name() const
@@ -187,8 +241,8 @@ void TileDirectory::setTagZoomLevel(int zoomLevel)
{
m_tagZoomLevel = zoomLevel;
if (m_tileType == OpenStreetMap) {
- QStringList const tags = tagsFilteredIn(m_tagZoomLevel);
if (m_tagZoomLevel < 17) {
+ QStringList const tags = tagsFilteredIn(m_tagZoomLevel);
m_tagsFilter = QSharedPointer<TagsFilter>(new TagsFilter(m_landmass.data(), tags));
} else {
m_tagsFilter.clear();
@@ -196,6 +250,26 @@ void TileDirectory::setTagZoomLevel(int zoomLevel)
}
}
+void TileDirectory::download(const QString &url, const QString &target)
+{
+ m_download = QSharedPointer<Download>(new Download);
+ m_download->target = target;
+ m_download->reply = m_downloadManager.get(QNetworkRequest(QUrl(url)));
+ connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), m_download.data(), SLOT(updateProgress(qint64,qint64)));
+ connect(m_download->reply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(updateProgress()));
+ QEventLoop loop;
+ connect(m_download->reply, SIGNAL(finished()), &loop, SLOT(quit()));
+ loop.exec();
+ cout << endl;
+}
+
+void TileDirectory::printProgress(double progress, int barWidth) const
+{
+ int const position = barWidth * progress;
+ cout << " [" << string(position, '=') << ">";
+ cout << string(barWidth-position, ' ') << "] " << int(progress * 100.0) << "%";
+}
+
GeoDataLatLonBox TileDirectory::boundingBox() const
{
return m_boundingBox;
@@ -206,12 +280,60 @@ void TileDirectory::setBoundingBox(const GeoDataLatLonBox &boundingBox)
m_boundingBox = boundingBox;
}
+void TileDirectory::setBoundingPolygon(const QString &file)
+{
+ m_boundingPolygon.clear();
+ QFile input(file);
+ QString country = "Unknown";
+ if ( input.open( QFile::ReadOnly ) ) {
+ QTextStream stream( &input );
+ country = stream.readLine();
+ double lat( 0.0 ), lon( 0.0 );
+ GeoDataLinearRing box;
+ while ( !stream.atEnd() ) {
+ bool inside = true;
+ QString line = stream.readLine().trimmed();
+ QStringList entries = line.split( QLatin1Char( ' ' ), QString::SkipEmptyParts );
+ if ( entries.size() == 1 ) {
+ if (entries.first() == QLatin1String("END") && inside) {
+ inside = false;
+ if (!box.isEmpty()) {
+ m_boundingPolygon << box;
+ box = GeoDataLinearRing();
+ }
+ } else if (entries.first() == QLatin1String("END") && !inside) {
+ qDebug() << "END not expected here";
+ } else if ( entries.first().startsWith( QLatin1String( "!" ) ) ) {
+ qDebug() << "Warning: Negative polygons not supported, skipping";
+ } else {
+ //int number = entries.first().toInt();
+ inside = true;
+ }
+ } else if ( entries.size() == 2 ) {
+ lon = entries.first().toDouble();
+ lat = entries.last().toDouble();
+ GeoDataCoordinates point( lon, lat, 0.0, GeoDataCoordinates::Degree );
+ box << point;
+ } else {
+ qDebug() << "Warning: Ignoring line in" << file
+ << "with" << entries.size() << "fields:" << line;
+ }
+ }
+ }
+
+ if (!m_boundingPolygon.isEmpty()) {
+ m_boundingBox = GeoDataLatLonBox::fromLineString(m_boundingPolygon.first());
+ for (int i=1, n=m_boundingPolygon.size(); i<n; ++i) {
+ m_boundingBox |= GeoDataLatLonBox::fromLineString(m_boundingPolygon[i]);
+ }
+ }
+}
+
void TileDirectory::createTiles() const
{
- int const zoomLevel = m_tileType == OpenStreetMap ? 10 : 7;
QSharedPointer<GeoDataDocument> map;
QSharedPointer<VectorClipper> clipper;
- TileIterator iter(m_boundingBox, zoomLevel);
+ TileIterator iter(m_boundingBox, m_zoomLevel);
qint64 count = 0;
foreach(auto const &tileId, iter) {
++count;
@@ -221,15 +343,16 @@ void TileDirectory::createTiles() const
continue;
}
- std::cout << "Creating " << (m_tileType == OpenStreetMap ? "osm" : "landmass");
- std::cout << " cache tile " << count << "/" << iter.total() << " (";
- std::cout << zoomLevel << "/" << tileId.x() << "/" << tileId.y() << ") \r";
- std::cout.flush();
+ printProgress(count / double(iter.total()));
+ cout << " Creating " << (m_tileType == OpenStreetMap ? "osm" : "landmass");
+ cout << " cache tile " << count << "/" << iter.total() << " (";
+ cout << m_zoomLevel << "/" << tileId.x() << "/" << tileId.y() << ") \r";
+ cout.flush();
QDir().mkpath(outputDir);
if (m_tileType == OpenStreetMap) {
QString const output = QString("-o=%1").arg(outputFile);
- int const N = pow(2, zoomLevel);
+ int const N = pow(2, m_zoomLevel);
double const minLon = TileId::tileX2lon(tileId.x(), N) * RAD2DEG;
double const maxLon = TileId::tileX2lon(tileId.x()+1, N) * RAD2DEG;
double const maxLat = TileId::tileY2lat(tileId.y(), N) * RAD2DEG;
@@ -240,7 +363,6 @@ void TileDirectory::createTiles() const
<< "--complete-ways" << "--complex-ways" << bbox << output << m_inputFile);
osmconvert.waitForFinished();
if (osmconvert.exitCode() != 0) {
- qWarning() << bbox;
qWarning() << osmconvert.readAllStandardError();
qWarning() << "osmconvert failed: " << osmconvert.errorString();
}
@@ -249,15 +371,66 @@ void TileDirectory::createTiles() const
map = open(m_inputFile, m_manager);
clipper = QSharedPointer<VectorClipper>(new VectorClipper(map.data()));
}
- auto tile = clipper->clipTo(zoomLevel, tileId.x(), tileId.y());
- if (tile->size() > 0) {
- if (!GeoDataDocumentWriter::write(outputFile, *tile)) {
- qWarning() << "Failed to write tile" << outputFile;
- }
+ auto tile = clipper->clipTo(m_zoomLevel, tileId.x(), tileId.y());
+ if (!GeoDataDocumentWriter::write(outputFile, *tile)) {
+ qWarning() << "Failed to write tile" << outputFile;
+ }
+ }
+ }
+ printProgress(1.0);
+ cout << " " << (m_tileType == OpenStreetMap ? "osm" : "landmass") << " cache tiles complete." << endl;
+}
+
+bool TileDirectory::contains(const TileId &tile) const
+{
+ int const N = pow(2, tile.zoomLevel());
+ double const west = TileId::tileX2lon(tile.x(), N);
+ double const east = TileId::tileX2lon(tile.x()+1, N);
+ double const north = TileId::tileY2lat(tile.y(), N);
+ double const south = TileId::tileY2lat(tile.y()+1, N);
+ QVector<GeoDataCoordinates> bounds;
+ bounds << GeoDataCoordinates(west, north);
+ bounds << GeoDataCoordinates(east, north);
+ bounds << GeoDataCoordinates(east, south);
+ bounds << GeoDataCoordinates(west, south);
+
+ if (m_boundingPolygon.isEmpty()) {
+ foreach(auto const &coordinate, bounds) {
+ if (m_boundingBox.contains(coordinate)) {
+ return true;
}
}
+ return false;
}
+ foreach(auto const &ring, m_boundingPolygon) {
+ foreach(auto const &coordinate, bounds) {
+ if (ring.contains(coordinate)) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+void TileDirectory::updateProgress()
+{
+ double const progress = m_download->total > 0 ? m_download->received / double(m_download->total) : 0.0;
+ printProgress(progress);
+
+ cout << " ";
+ cout << std::fixed << std::setprecision(1) << m_download->received / 1000000.0 << '/';
+ cout << std::fixed << std::setprecision(1) << m_download->total / 1000000.0 << " MB";
+
+ cout << " Downloading " << m_download->reply->url().fileName().toStdString();
+
+ cout << " \r";
+ cout.flush();
+}
+
+void TileDirectory::handleFinishedDownload(const QString &filename, const QString &id)
+{
+ qDebug() << "File " << filename << "(" << id << ") has been downloaded.";
}
GeoDataLatLonBox TileDirectory::boundingBox(const QString &filename) const
@@ -281,4 +454,25 @@ GeoDataLatLonBox TileDirectory::boundingBox(const QString &filename) const
return boundingBox;
}
+void Download::updateProgress(qint64 received_, qint64 total_)
+{
+ received = received_;
+ total = total_;
+
+ QString const tempFile = QString("%1.download").arg(target);
+ if (!m_file.isOpen()) {
+ m_file.setFileName(tempFile);
+ m_file.open(QFile::WriteOnly);
+ }
+ m_file.write(reply->readAll());
+
+ if (reply->isFinished()) {
+ m_file.flush();
+ m_file.close();
+ QFile::rename(tempFile, target);
+ }
}
+
+}
+
+#include "moc_TileDirectory.cpp"
diff --git a/tools/vectorosm-tilecreator/TileDirectory.h b/tools/vectorosm-tilecreator/TileDirectory.h
index 8a43892..9da7cdc 100644
--- a/tools/vectorosm-tilecreator/TileDirectory.h
+++ b/tools/vectorosm-tilecreator/TileDirectory.h
@@ -8,26 +8,55 @@
// Copyright 2016 Dennis Nienhüser <nienhueser@kde.org>
//
+#ifndef MARBLE_TILEDIRECTORY_H
+#define MARBLE_TILEDIRECTORY_H
#include "VectorClipper.h"
#include "TagsFilter.h"
#include <TileId.h>
+#include <GeoDataLinearRing.h>
+#include <ParsingRunnerManager.h>
+#include <QNetworkAccessManager>
#include <QSharedPointer>
-#include <ParsingRunnerManager.h>
+#include <QObject>
+#include <QFile>
+
+class QNetworkReply;
namespace Marble {
-class TileDirectory
+class Download : public QObject
+{
+ Q_OBJECT
+
+public:
+ QString target;
+ QNetworkReply* reply;
+ qint64 received;
+ qint64 total;
+
+public Q_SLOTS:
+ void updateProgress(qint64 received, qint64 total);
+
+private:
+ QFile m_file;
+};
+
+class TileDirectory : public QObject
{
+ Q_OBJECT
+
public:
+
enum TileType
{
Landmass,
OpenStreetMap
};
- TileDirectory(TileType tileType, const QString &baseDir, ParsingRunnerManager &manager, const QString &extension);
+ TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, const QString &extension);
+
QSharedPointer<GeoDataDocument> load(int zoomLevel, int tileX, int tileY);
void setInputFile(const QString &filename);
@@ -39,12 +68,21 @@ public:
GeoDataLatLonBox boundingBox(const QString &filename) const;
GeoDataLatLonBox boundingBox() const;
void setBoundingBox(const GeoDataLatLonBox &boundingBox);
+ void setBoundingPolygon(const QString &filename);
void createTiles() const;
+ bool contains(const TileId &tile) const;
+
+private Q_SLOTS:
+ void updateProgress();
+ void handleFinishedDownload(const QString &filename, const QString &id);
private:
QStringList tagsFilteredIn(int zoomLevel) const;
void setTagZoomLevel(int zoomLevel);
+ void download(const QString &url, const QString &target);
+ void printProgress(double progress, int barWidth=40) const;
+ QString m_cacheDir;
QString m_baseDir;
ParsingRunnerManager &m_manager;
QSharedPointer<GeoDataDocument> m_landmass;
@@ -58,6 +96,12 @@ private:
TileType m_tileType;
QString m_inputFile;
GeoDataLatLonBox m_boundingBox;
+ QVector<GeoDataLinearRing> m_boundingPolygon;
+ QNetworkAccessManager m_downloadManager;
+ QString m_landmassFile;
+ QSharedPointer<Download> m_download;
};
}
+
+#endif
diff --git a/tools/vectorosm-tilecreator/main.cpp b/tools/vectorosm-tilecreator/main.cpp
index ee334b3..b6321f6 100644
--- a/tools/vectorosm-tilecreator/main.cpp
+++ b/tools/vectorosm-tilecreator/main.cpp
@@ -27,6 +27,7 @@
#include <QElapsedTimer>
#include <QSharedPointer>
#include <QFileInfo>
+#include <QUrl>
#include <QMessageLogContext>
#include <QProcess>
@@ -69,9 +70,6 @@ QString tileFileName(const QCommandLineParser &parser, int x, int y, int zoomLev
bool writeTile(GeoDataDocument* tile, const QString &outputFile)
{
- if (tile->size() == 0) {
- return true;
- }
if (!GeoDataDocumentWriter::write(outputFile, *tile)) {
qWarning() << "Could not write the file " << outputFile;
return false;
@@ -96,7 +94,6 @@ int main(int argc, char *argv[])
{{"t", "osmconvert"}, "Tile data using osmconvert."},
{"conflict-resolution", "How to deal with existing tiles: overwrite, skip or merge", "mode", "overwrite"},
{{"c", "cache-directory"}, "Directory for temporary data.", "cache", "cache"},
- {{"l", "landmass"}, "File with landmass polygons (world-wide).", "filename", "landmass.shp"},
{{"z", "zoom-level"}, "Zoom level according to which OSM information has to be processed.", "levels", "11,13,15,17"},
{{"o", "output"}, "Output file or directory", "output", QString("%1/maps/earth/vectorosm").arg(MarbleDirs::localPath())},
{{"e", "extension"}, "Output file type: o5m (default), osm or kml", "file extension", "o5m"}
@@ -124,15 +121,8 @@ int main(int argc, char *argv[])
return 1;
}
- QFileInfo file( inputFileName );
- if ( !file.exists() ) {
- qWarning() << "File " << file.absoluteFilePath() << " does not exist. Exiting.";
- return 2;
- }
-
MarbleModel model;
ParsingRunnerManager manager(model.pluginManager());
-
QString const cacheDirectory = parser.value("cache-directory");
QDir().mkpath(cacheDirectory);
if (!QFileInfo(cacheDirectory).isWritable()) {
@@ -169,16 +159,14 @@ int main(int argc, char *argv[])
}
}
} else {
- TileDirectory loader(TileDirectory::Landmass, QString("%1/landmass/7").arg(cacheDirectory), manager, parser.value("extension"));
- loader.setInputFile(parser.value("landmass"));
- auto const boundingBox = loader.boundingBox(inputFileName);
- loader.setBoundingBox(boundingBox);
- loader.createTiles();
-
- TileDirectory mapTiles(TileDirectory::OpenStreetMap, QString("%1/osm/10").arg(cacheDirectory), manager, parser.value("extension"));
+ TileDirectory mapTiles(TileDirectory::OpenStreetMap, cacheDirectory, manager, parser.value("extension"));
mapTiles.setInputFile(inputFileName);
- mapTiles.setBoundingBox(boundingBox);
mapTiles.createTiles();
+ auto const boundingBox = mapTiles.boundingBox();
+
+ TileDirectory loader(TileDirectory::Landmass, cacheDirectory, manager, parser.value("extension"));
+ loader.setBoundingBox(boundingBox);
+ loader.createTiles();
typedef QMap<QString, QVector<TileId> > Tiles;
Tiles tiles;
@@ -192,9 +180,11 @@ int main(int argc, char *argv[])
total += iter.total();
foreach(auto const &tileId, iter) {
auto const tile = TileId(QString(), zoomLevel, tileId.x(), tileId.y());
- auto const mapTile = mapTiles.tileFor(zoomLevel, tileId.x(), tileId.y());
- auto const name = QString("%1/%2/%3").arg(mapTile.zoomLevel()).arg(mapTile.x()).arg(mapTile.y());
- tiles[name] << tile;
+ if (mapTiles.contains(tile)) {
+ auto const mapTile = mapTiles.tileFor(zoomLevel, tileId.x(), tileId.y());
+ auto const name = QString("%1/%2/%3").arg(mapTile.zoomLevel()).arg(mapTile.x()).arg(mapTile.y());
+ tiles[name] << tile;
+ }
}
}