Merge pull request #418 from OpenHistoricalMap/imposm3

Import admin_areas imposm workflow
OpenHistoricalMap · Dec 3, 2024 · c90b629 · c90b629
2 parents ef23359 + 73a3659
commit c90b629
Show file tree

Hide file tree

Showing 23 changed files with 600 additions and 550 deletions.
diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml
@@ -5,7 +5,7 @@ on:
       - 'main'
       - 'staging'
       - 'development'
-      - 'imposm/tables'
+      - 'imposm3'
 jobs:
   build:
     runs-on: ubuntu-20.04

diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile
@@ -1,7 +1,6 @@
 FROM osgeo/gdal:ubuntu-small-3.2.3
 
-RUN apt-get -y update
-RUN apt-get install -y \
+RUN apt-get -y update && apt-get install -y \
     g++ \
     libboost-dev \
     libboost-system-dev \
@@ -22,46 +21,34 @@ RUN apt-get install -y \
     software-properties-common && \
     rm -rf /var/lib/apt/lists/*
 
-# Install python
 RUN add-apt-repository ppa:deadsnakes/ppa && \
     apt-get update && \
-    apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \
+    apt-get install -y python3.9 python3.9-dev python3-pip && \
     rm -rf /var/lib/apt/lists/* && \
     python3 -m pip install pip --upgrade && \
     python3 -m pip install wheel
 
-# Install postgresql-client
 RUN apt-get update && apt-get install -y postgresql-client && \
     rm -rf /var/lib/apt/lists/*
 
-# Install Go
 RUN wget -c https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz -O - | tar -xz -C /usr/local
 ENV PATH $PATH:/usr/local/go/bin
 
 WORKDIR /go
 ENV GOPATH /go
-
-# Clone the repository directly
 RUN git clone https://github.com/omniscale/imposm3.git $GOPATH/src/github.com/omniscale/imposm3
 
 WORKDIR $GOPATH/src/github.com/omniscale/imposm3
 RUN git checkout v0.14.0
 
-# Fix cache package conflict
+# Apply fixes or patches (if necessary)
 RUN sed -i '/setMaxFileSize/d' cache/ldb_pre_121.go
-
 RUN go install github.com/omniscale/imposm3/cmd/imposm
 ENV PATH $PATH:$GOPATH/bin
 
-# Install awscli and gsutil to get the pbf file
-RUN pip3 install -U setuptools
-RUN pip3 install awscli
-
-ENV IMPOSMDATA /mnt/data
-RUN mkdir -p $IMPOSMDATA && chmod 777 -R $IMPOSMDATA
-WORKDIR $IMPOSMDATA
-VOLUME $IMPOSMDATA
-
+# Install AWS CLI for downloading files
+RUN pip3 install -U setuptools awscli psycopg2
 WORKDIR /osm
 COPY . .
-CMD ./start.sh
+RUN chmod +x ./start.sh
+CMD ["./start.sh"]
diff --git a/images/tiler-imposm/config/layers/admin_areas.json b/images/tiler-imposm/config/layers/admin_areas.json
@@ -7,7 +7,101 @@
       "source:datetime"
     ]
   },
-  "generalized_tables": {},
+  "generalized_tables": {
+    "admin_boundaries_lines_z0_2": {
+      "source": "admin_areas",
+      "tolerance": 5000,
+      "geometry_transform": "ST_Boundary(geometry)",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2)"
+    },
+    "admin_boundaries_centroid_z0_2": {
+      "source": "admin_areas",
+      "tolerance": 5000,
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2) AND name IS NOT NULL AND name <> ''"
+    },
+    "admin_boundaries_lines_z3_5": {
+      "source": "admin_areas",
+      "tolerance": 1000,
+      "geometry_transform": "ST_Boundary(geometry)",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4)"
+    },
+    "admin_boundaries_centroid_z3_5": {
+      "source": "admin_areas",
+      "tolerance": 1000,
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4) AND name IS NOT NULL AND name <> ''"
+    },
+    "admin_boundaries_lines_z6_7": {
+      "source": "admin_areas",
+      "tolerance": 500,
+      "geometry_transform": "ST_Boundary(geometry)",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6)"
+    },
+    "admin_boundaries_centroid_z6_7": {
+      "source": "admin_areas",
+      "tolerance": 500,
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6) AND name IS NOT NULL AND name <> ''"
+    },
+    "admin_boundaries_lines_z8_9": {
+      "source": "admin_areas",
+      "tolerance": 200,
+      "geometry_transform": "ST_Boundary(geometry)",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9)"
+    },
+    "admin_boundaries_centroid_z8_9": {
+      "source": "admin_areas",
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9) AND name IS NOT NULL AND name <> ''"
+    },
+    "admin_boundaries_lines_z10_12": {
+      "source": "admin_areas",
+      "tolerance": 100,
+      "geometry_transform": "ST_Boundary(geometry)",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)"
+    },
+    "admin_boundaries_centroid_z10_12": {
+      "source": "admin_areas",
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''"
+    },
+    "admin_boundaries_lines_z13_15": {
+      "source": "admin_areas",
+      "tolerance": 50,
+      "geometry_transform": "ST_Boundary(geometry)",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)"
+    },
+    "admin_boundaries_centroid_z13_15": {
+      "source": "admin_areas",
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''"
+    },
+    "admin_boundaries_lines_z16_20": {
+      "source": "admin_areas",
+      "geometry_transform": "ST_Boundary(geometry)",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)"
+    },
+    "admin_boundaries_centroid_z16_20": {
+      "source": "admin_areas",
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''"
+    }
+  },
   "tables": {
     "admin_areas": {
       "fields": [
@@ -45,6 +139,16 @@
           "type": "boolint",
           "name": "has_label",
           "key": false
+        },
+        {
+          "type": "string",
+          "name": "start_date",
+          "key": "start_date"
+        },
+        {
+          "type": "string",
+          "name": "end_date",
+          "key": "end_date"
         }
       ],
       "type": "polygon",

diff --git a/images/tiler-imposm/config/layers/admin_lines.json b/images/tiler-imposm/config/layers/admin_lines.json
@@ -8,37 +8,37 @@
     ]
   },
   "generalized_tables": {
-    "admin_lines_zoom_0_2": {
+    "admin_lines_z0_2": {
       "source": "admin_lines",
       "sql_filter": "maritime='yes'",
       "tolerance": 5000
     },
-    "admin_lines_zoom_3_5": {
+    "admin_lines_z3_5": {
       "source": "admin_lines",
       "sql_filter": "maritime='yes'",
       "tolerance": 2000
     },
-    "admin_lines_zoom_6_7": {
+    "admin_lines_z6_7": {
       "source": "admin_lines",
       "sql_filter": "maritime='yes'",
       "tolerance": 1000
     },
-    "admin_lines_zoom_8_9": {
+    "admin_lines_z8_9": {
       "source": "admin_lines",
       "sql_filter": "maritime='yes'",
       "tolerance": 500
     },
-    "admin_lines_zoom_10_12": {
+    "admin_lines_z10_12": {
       "source": "admin_lines",
       "sql_filter": "maritime='yes'",
       "tolerance": 100
     },
-    "admin_lines_zoom_13_15": {
+    "admin_lines_z13_15": {
       "source": "admin_lines",
       "sql_filter": "maritime='yes'",
       "tolerance": 10
     },
-    "admin_lines_zoom_16_20": {
+    "admin_lines_z16_20": {
       "source": "admin_lines",
       "sql_filter": "maritime='yes'"
     }

diff --git a/images/tiler-imposm/config/layers/water_areas.json b/images/tiler-imposm/config/layers/water_areas.json
@@ -1,42 +1,70 @@
 {
-  "tags": {
-    "load_all": true,
-    "exclude": [
-      "created_by",
-      "source",
-      "source:datetime"
-    ]
-  },
   "generalized_tables": {
-    "water_areas_zoom_0_2": {
+    "water_areas_z0_2": {
+      "source": "water_areas",
+      "sql_filter": "type IN ('water', 'riverbank') AND area > 100000000",
+      "tolerance": 5000
+    },
+    "water_areas_centroid_z0_2": {
       "source": "water_areas",
       "sql_filter": "type IN ('water', 'riverbank') AND area > 100000000",
-      "tolerance": 250000
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
+    },
+    "water_areas_z3_5": {
+      "source": "water_areas",
+      "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 50000000",
+      "tolerance": 1000
     },
-    "water_areas_zoom_3_5": {
+    "water_areas_centroid_z3_5": {
       "source": "water_areas",
       "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 50000000",
-      "tolerance": 25000
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
     },
-    "water_areas_zoom_6_7": {
+    "water_areas_z6_7": {
       "source": "water_areas",
       "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 1000000",
-      "tolerance": 5000
+      "tolerance": 500
     },
-    "water_areas_zoom_8_9": {
+    "water_areas_centroid_z6_7": {
+      "source": "water_areas",
+      "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 1000000",
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
+    },
+    "water_areas_z8_9": {
       "source": "water_areas",
       "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 10000",
-      "tolerance": 1000
+      "tolerance": 200
+    },
+    "water_areas_centroid_z8_9": {
+      "source": "water_areas",
+      "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 10000",
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
     },
-    "water_areas_zoom_10_12": {
+    "water_areas_z10_12": {
       "source": "water_areas",
       "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank')",
       "tolerance": 100
     },
-    "water_areas_zoom_13_15": {
+    "water_areas_centroid_z10_12": {
+      "source": "water_areas",
+      "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank')",
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
+    },
+    "water_areas_z13_15": {
+      "source": "water_areas",
+      "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank', 'dock')",
+      "tolerance": 50
+    },
+    "water_areas_centroid_z13_15": {
       "source": "water_areas",
       "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank', 'dock')",
-      "tolerance": 10
+      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')"
     }
   },
   "tables": {

diff --git a/images/tiler-imposm/config/layers/water_lines.json b/images/tiler-imposm/config/layers/water_lines.json
@@ -8,22 +8,22 @@
     ]
   },
   "generalized_tables": {
-    "water_lines_zoom_8_9": {
+    "water_lines_z8_9": {
       "source": "water_lines",
       "sql_filter": "type IN ('river', 'canal')",
       "tolerance": 1000
     },
-    "water_lines_zoom_10_11": {
+    "water_lines_z10_11": {
       "source": "water_lines",
       "sql_filter": "type IN ('river', 'canal', 'cliff', 'dam')",
       "tolerance": 100
     },
-    "water_lines_zoom_12_14": {
+    "water_lines_z12_14": {
       "source": "water_lines",
       "sql_filter": "type IN ('river', 'canal', 'cliff', 'dam', 'stream')",
       "tolerance": 10
     },
-    "water_lines_zoom_15_20": {
+    "water_lines_z15_20": {
       "source": "water_lines",
       "sql_filter": "type IN ('river', 'canal', 'cliff', 'dam', 'stream', 'ditch', 'drain')",
       "tolerance": 0

diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh
@@ -175,7 +175,10 @@ function importData() {
 
     touch $INIT_FILE
 
-    # Update the DB
+    # Update tables
+    python update_tables.py
+
+    # Updata data with minute replication
     updateData
 }