Merge pull request #224 from OpenHistoricalMap/staging

Remove tiler cache manually when container gets too busy.

images/tiler-server/Dockerfile

@@ -51,4 +51,5 @@ COPY ./expire-watcher.sh .
 COPY ./seed-by-diffs.sh .
 COPY ./tile_cache_downloader.sh .
 COPY ./cache_cleaner.sh .
+COPY ./rm_tegola_ps.sh .
 # CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh

images/tiler-server/cache_cleaner.sh

@@ -3,5 +3,5 @@ flag=true
 while "$flag" = true; do
   pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
   flag=false
-  ./tile_cache_downloader.sh & ./expire-watcher.sh
-done
+  ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_tegola_ps.sh
+done

images/tiler-server/rm_tegola_ps.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+if [[ -n "${KILL_PROCESS}" && "${KILL_PROCESS}" == "manually" ]]; then
+    while true; do
+        NUM_PS=$(ps | grep ${PROCESS_NAME} | grep -v grep | wc -l)
+        if [[ $NUM_PS -gt $MAX_NUM_PS ]]; then
+            echo "${PROCESS_NAME} processes"
+            ps aux | grep ${PROCESS_NAME} | grep -v grep
+            # After clearing the S3 cache, terminate all 'tegola' processes.
+            killall ${PROCESS_NAME}
+            # Clean cache manually from s3
+            aws s3 rm s3://${TILER_CACHE_BUCKET}/mnt/data/osm/ --recursive
+        fi
+        sleep 600
+    done
+fi

ohm/requirements.yaml

@@ -1,4 +1,4 @@
 dependencies:
 - name: osm-seed
-  version: '0.1.0-n771.h5ab71b9'
+  version: '0.1.0-n768.h455eb76'
   repository: https://devseed.com/osm-seed-chart/

values.production.template.yaml

@@ -415,11 +415,15 @@ osm-seed:
       limits:
         memory: "10Gi"
         cpu: "2"
+    env:
+      KILL_PROCESS: manually
+      MAX_NUM_PS: 5
+      PROCESS_NAME: tegola
     autoscaling:
-      enabled: false
+      enabled: true
       minReplicas: 1
-      maxReplicas: 2
-      cpuUtilization: 60
+      maxReplicas: 1
+      cpuUtilization: 90
   # ====================================================================================================
   # Variables for tiler-visor
   # ====================================================================================================

values.staging.template.yaml

@@ -442,11 +442,15 @@ osm-seed:
         limits:
           memory: "10Gi"
           cpu: "2"
+      env:
+        KILL_PROCESS: manually
+        MAX_NUM_PS: 4
+        PROCESS_NAME: tegola
       autoscaling:
-        enabled: false
+        enabled: true
         minReplicas: 1
-        maxReplicas: 2
-        cpuUtilization: 60
+        maxReplicas: 1
+        cpuUtilization: 90
     # ====================================================================================================
     # Variables for tiler-visor
     # ====================================================================================================