Add cronjob for seeding the tiler

images/tiler-cache/Dockerfile

@@ -26,8 +26,8 @@ WORKDIR /app
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY tiler-purge .
-COPY tiler-seed .
+COPY tiler-purge ./tiler-purge
+COPY tiler-seed ./tiler-seed
 COPY start.sh .
 
 ENTRYPOINT ["/bin/bash", "-c"]

ohm/templates/tiler-cache-purge/deployment.yaml

@@ -0,0 +1,61 @@
+{{- if .Values.ohm.tilerCachePurge.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-tiler-cache-sqs
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    component: tiler-cache-sqs-deployment
+    release: {{ .Release.Name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ template "osm-seed.name" . }}
+      release: {{ .Release.Name }}
+      run: {{ .Release.Name }}-tiler-cache-sqs
+  template:
+    metadata:
+      labels:
+        app: {{ template "osm-seed.name" . }}
+        release: {{ .Release.Name }}
+        run: {{ .Release.Name }}-tiler-cache-sqs
+    spec:
+      serviceAccountName: job-service-account
+      containers:
+        - name: {{ .Chart.Name }}tiler-cache-sqs
+          image: "{{ .Values.ohm.tilerCache.image.name }}:{{ .Values.ohm.tilerCache.image.tag }}"
+          env:
+            - name: REGION_NAME
+              value: {{ .Values.ohm.tilerCachePurge.env.REGION_NAME | quote }}
+            - name: ENVIRONMENT
+              value: {{ .Values.ohm.tilerCachePurge.env.ENVIRONMENT | quote }}
+            - name: SQS_QUEUE_URL
+              value: {{  .Values.ohm.tilerCachePurge.env.SQS_QUEUE_URL | quote }}
+            - name: DOCKER_IMAGE
+              value: {{ .Values.ohm.tilerCachePurge.env.DOCKER_IMAGE | quote }}
+            - name: NODEGROUP_TYPE
+              value: {{ .Values.ohm.tilerCachePurge.env.NODEGROUP_TYPE | quote }}
+            - name: MAX_ACTIVE_JOBS
+              value: {{ .Values.ohm.tilerCachePurge.env.MAX_ACTIVE_JOBS | quote }}
+            - name: DELETE_OLD_JOBS_AGE
+              value: {{ .Values.ohm.tilerCachePurge.env.DELETE_OLD_JOBS_AGE | quote }}
+            - name: MIN_ZOOM
+              value: {{ .Values.ohm.tilerCachePurge.env.MIN_ZOOM | quote }}
+            - name: MAX_ZOOM
+              value: {{ .Values.ohm.tilerCachePurge.env.MAX_ZOOM | quote }}
+
+          {{- if .Values.ohm.tilerCachePurge.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.ohm.tilerCachePurge.resources.requests.memory }}
+              cpu: {{ .Values.ohm.tilerCachePurge.resources.requests.cpu }}
+            limits:
+              memory: {{ .Values.ohm.tilerCachePurge.resources.limits.memory }}
+              cpu: {{ .Values.ohm.tilerCachePurge.resources.limits.cpu }}
+          {{- end }}
+      {{- if .Values.ohm.tilerCachePurge.nodeSelector.enabled }}
+      nodeSelector:
+        {{ .Values.ohm.tilerCachePurge.nodeSelector.label_key }} : {{ .Values.ohm.tilerCachePurge.nodeSelector.label_value }}
+      {{- end }}
+{{- end }}

ohm/templates/tiler-cache-seed/cronjob.yaml

@@ -0,0 +1,53 @@
+{{- if .Values.ohm.tilerCacheSeed.enabled -}}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Release.Name }}-tiler-cache-seed-job
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    component: tiler-cache-seed-job
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  schedule: {{ quote .Values.ohm.tilerCacheSeed.schedule }}
+  startingDeadlineSeconds: 100
+  successfulJobsHistoryLimit: 2
+  failedJobsHistoryLimit: 2
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          {{- if .Values.ohm.tilerCacheSeed.nodeSelector.enabled }}
+          nodeSelector:
+            {{ .Values.ohm.tilerCacheSeed.nodeSelector.label_key }} : {{ .Values.ohm.tilerCacheSeed.nodeSelector.label_value }}
+          {{- end }}
+          containers:
+          - name: {{ .Release.Name }}-tiler-cache-seed-job
+            image: {{ .Values.ohm.tilerCache.image.name }}:{{ .Values.ohm.tilerCache.image.tag }}
+            command: ['/start.sh']
+            {{- if .Values.ohm.tilerCacheSeed.resources.enabled }}   
+            resources:
+              requests:
+                memory: {{ .Values.ohm.tilerCacheSeed.resources.requests.memory }}
+                cpu: {{ .Values.ohm.tilerCacheSeed.resources.requests.cpu }}
+              limits:
+                memory: {{ .Values.ohm.tilerCacheSeed.resources.limits.memory }}
+                cpu: {{ .Values.ohm.tilerCacheSeed.resources.limits.cpu }}
+            {{- end }}
+            env:
+              - name: GEOJSON_URL
+                value: {{ .Values.ohm.tilerCacheSeed.geojsonUrl | quote }}
+              - name: FEATURE_TYPE
+                value: {{ .Values.ohm.tilerCacheSeed.featureType | quote }}
+              - name: ZOOM_LEVELS
+                value: {{ .Values.ohm.tilerCacheSeed.zoomLevels | quote }}
+              - name: CONCURRENCY
+                value: {{ .Values.ohm.tilerCacheSeed.concurrency | quote }}
+              - name: S3_BUCKET
+                value: {{ .Values.ohm.tilerCacheSeed.s3Bucket | quote }}
+              - name: OUTPUT_FILE
+                value: {{ .Values.ohm.tilerCacheSeed.outputFile | quote }}
+          restartPolicy: OnFailure
+      backoffLimit: 3
+{{- end }}

values.staging.template.yaml

@@ -956,27 +956,50 @@ ohm:
   tilerCache:
     enabled: true
 
-  tilerCacheSqs:
+  tilerCachePurge:
     enabled: true
     env:
       ENVIRONMENT: staging
-      REGION_NAME: "us-east-1"
-      NAMESPACE: "default" # Namespace to run the job
-      DOCKER_IMAGE: "ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1741.h7dadda7" # TODO, this should be automatically updated from tiler server image
+      REGION_NAME: 'us-east-1'
+      NAMESPACE: 'default' # Namespace to run the job
+      DOCKER_IMAGE: 'ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1741.h7dadda7' # TODO, this should be automatically updated from tiler server image
       SQS_QUEUE_URL: {{STAGING_SQS_QUEUE_URL}}
-      NODEGROUP_TYPE: "job_large" # Nodegroup type to run the job
+      NODEGROUP_TYPE: 'job_large' # Nodegroup type to run the job
       MAX_ACTIVE_JOBS: 2 # Maximum number of active jobs in high concurrency queue
       DELETE_OLD_JOBS_AGE: 600 # Age in seconds to delete old jobs
       MIN_ZOOM: 8 # Minimum zoom level to clean and seed cache
       MAX_ZOOM: 16 # Maximum zoom level to clean and seed cache
     resources:
       enabled: false
       requests:
-        memory: "20Gi"
-        cpu: "8"
+        memory: '20Gi'
+        cpu: '8'
       limits:
-        memory: "24Gi"
-        cpu: "10"
+        memory: '24Gi'
+        cpu: '10'
+    nodeSelector:
+      enabled: true
+      label_key: nodegroup_type
+      label_value: web_large
+
+  tilerCacheSeed:
+    enabled: true
+    schedule: '* * * * *'
+    env:
+      GEOJSON_URL: https://gist.githubusercontent.com/Rub21/9aaf349d74d974c0393700af8eeeb43d/raw/9f8fa0dde911705208141a4ff941e2e5b51e245b/eu.geojson
+      FEATURE_TYPE: polygon
+      ZOOM_LEVELS: '0,2,3,4,5,6,7'
+      CONCURRENCY: 32
+      S3_BUCKET: osmseed-dev
+      OUTPUT_FILE: /logs/tiler_benchmark.log
+    resources:
+      enabled: false
+      requests:
+        memory: '20Gi'
+        cpu: '8'
+      limits:
+        memory: '24Gi'
+        cpu: '10'
     nodeSelector:
       enabled: true
       label_key: nodegroup_type