Implement a cache daemon that can move the cache into place quickly o…

…n demand

 - a long running background process we'll deploy as a daemonset on k8s nodes
 - moves the cache into place via hardlinking the golden copy from a shared location
 - operates using the host filesystem, and thus has pretty priviledged access

Contributing.md

@@ -182,3 +182,7 @@ We also need to build the server docker image and push it to Gadget's container
 ```bash
 make upload-container-image version=0.0.x
 ```
+
+### Getting PASETO tokens locally
+
+You can sign PASETO tokens locally with this handy online tool: https://token.dev/paseto/. Ensure you use the V2 algorithm in the public mode, and copy the PASTEO public and private key from the `development` folder.

Dockerfile

@@ -8,6 +8,10 @@ EOF
 
 WORKDIR /app
 
+RUN GRPC_HEALTH_PROBE_VERSION=v0.4.23 \
+    && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} \
+    && chmod +x /bin/grpc_health_probe
+
 COPY flake.nix flake.lock  ./
 COPY development ./development
 
@@ -20,15 +24,11 @@ RUN nix develop -c go mod download
 
 # copy everything else and build the project
 COPY . ./
-RUN nix develop -c make release/server_linux_$TARGETARCH
+RUN nix develop -c make release/server_linux_$TARGETARCH release/cached_linux_$TARGETARCH
 
-FROM buildpack-deps:bullseye AS build-release-stage
+FROM debian:bullseye-slim AS release-stage
 ARG TARGETARCH
 
-RUN GRPC_HEALTH_PROBE_VERSION=v0.4.23 \
-    && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} \
-    && chmod +x /bin/grpc_health_probe
-
 RUN useradd -ms /bin/bash main
 USER main
 WORKDIR /home/main
@@ -37,11 +37,15 @@ RUN mkdir -p /home/main/secrets
 VOLUME /home/main/secrets/tls
 VOLUME /home/main/secrets/paseto
 
+COPY --from=build-stage /bin/grpc_health_probe /bin/grpc_health_probe
+COPY --from=build-stage /app/release/cached_linux_${TARGETARCH} cached
 COPY --from=build-stage /app/release/server_linux_${TARGETARCH} server
+
 COPY migrations migrations
 COPY entrypoint.sh entrypoint.sh
 
-# smoke test -- ensure the server command can run
+# smoke test -- ensure the commands can run
 RUN ./server --help
+RUN ./cached --help
 
 ENTRYPOINT ["./entrypoint.sh"]

Makefile

@@ -8,11 +8,13 @@ DB_USER ?= postgres
 DB_PASS ?= password
 DB_URI := postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl
 
-GRPC_PORT ?= 5051
 GRPC_HOST ?= localhost
+GRPC_PORT ?= 5051
+GRPC_CACHED_PORT ?= 5053
 
-DEV_TOKEN_ADMIN ?= v2.public.eyJzdWIiOiJhZG1pbiIsImlhdCI6IjIwMjEtMTAtMTVUMTE6MjA6MDAuMDM0WiJ9WtEey8KfQQRy21xoHq1C5KQatEevk8RxS47k4bRfMwVCPHumZmVuk6ADcfDHTmSnMtEGfFXdxnYOhRP6Clb_Dw
-DEV_TOKEN_PROJECT_1 ?= v2.public.eyJzdWIiOiIxIiwiaWF0IjoiMjAyMS0xMC0xNVQxMToyMDowMC4wMzVaIn2MQ14RfIGpoEycCuvRu9J3CZp6PppUXf5l5w8uKKydN3C31z6f6GgOEPNcnwODqBnX7Pjarpz4i2uzWEqLgQYD
+DEV_TOKEN_ADMIN ?= v2.public.eyJzdWIiOiJhZG1pbiJ9yt40HNkcyOUtDeFa_WPS6vi0WiE4zWngDGJLh17TuYvssTudCbOdQEkVDRD-mSNTXLgSRDXUkO-AaEr4ZLO4BQ
+DEV_TOKEN_PROJECT_1 ?= v2.public.eyJzdWIiOiIxIn2jV7FOdEXafKDtAnVyDgI4fmIbqU7C1iuhKiL0lDnG1Z5-j6_ObNDd75sZvLZ159-X98_mP4qvwzui0w8pjt8F
+DEV_SHARED_READER_TOKEN ?= v2.public.eyJzdWIiOiJzaGFyZWQtcmVhZGVyIn1CxWdB02s9el0Wt7qReARZ-7JtIb4Zj3D4Oiji1yXHqj0orkpbcVlswVUiekECJC16d1NrHwD2FWSwRORZn8gK
 
 PKG_GO_FILES := $(shell find pkg/ -type f -name '*.go')
 INTERNAL_GO_FILES := $(shell find internal/ -type f -name '*.go')
@@ -63,7 +65,7 @@ development/server.key:
 
 development/server.crt: development/server.key
 
-build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go bin/server bin/client development/server.crt
+build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go internal/pb/cache.pb.go internal/pb/cache_grpc.pb.go bin/server bin/client bin/cached development/server.crt
 
 lint:
 	golangci-lint run
@@ -86,8 +88,9 @@ release/migrations.tar.gz: migrations/*
 	tar -zcf $@ migrations
 
 release: build
-release: release/server_linux_amd64 release/server_macos_amd64 release/server_macos_arm64
-release: release/client_linux_amd64 release/client_macos_amd64 release/client_macos_arm64
+release: release/server_linux_amd64 release/server_macos_amd64 release/server_macos_arm64 release/server_linux_arm64
+release: release/client_linux_amd64 release/client_macos_amd64 release/client_macos_arm64 release/client_linux_arm64
+release: release/cached_linux_amd64 release/cached_macos_amd64 release/cached_macos_arm64 release/cached_linux_arm64
 release: release/migrations.tar.gz
 
 test: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
@@ -121,6 +124,11 @@ server-profile: export DL_ENV=dev
 server-profile: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go
 	go run cmd/server/main.go --dburi $(DB_URI) --port $(GRPC_PORT) --profile cpu.prof --log-level info
 
+cached: export DL_ENV=dev
+cached: export DL_TOKEN=$(DEV_SHARED_READER_TOKEN)
+cached: internal/pb/cache.pb.go internal/pb/cache_grpc.pb.go
+	go run cmd/cached/main.go --upstream-host $(GRPC_HOST) --upstream-port $(GRPC_PORT) --staging-path tmp/cache-stage
+
 client-update: export DL_TOKEN=$(DEV_TOKEN_PROJECT_1)
 client-update: export DL_SKIP_SSL_VERIFICATION=1
 client-update:
@@ -169,6 +177,16 @@ client-getcache: export DL_SKIP_SSL_VERIFICATION=1
 client-getcache:
 	go run cmd/client/main.go getcache --host $(GRPC_HOST) --path input/cache
 
+client-populate-disk-cache: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
+client-populate-disk-cache: export DL_SKIP_SSL_VERIFICATION=1
+client-populate-disk-cache:
+	mkdir -p tmp/pods/test-pod/volumes/example && go run cmd/client/main.go populate-disk-cache --host $(GRPC_HOST) --port $(GRPC_CACHED_PORT) --id test-pod
+
+client-rebuild-with-cached-cache: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
+client-rebuild-with-cached-cache: export DL_SKIP_SSL_VERIFICATION=1
+client-rebuild-with-cached-cache: 
+	go run cmd/client/main.go rebuild --host $(GRPC_HOST) --project 1 --prefix "$(prefix)" --dir tmp/pods/test-pod/workdir --cachedir pods/test-pod/volumes/example
+
 client-gc-contents: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
 client-gc-contents: export DL_SKIP_SSL_VERIFICATION=1
 client-gc-contents:
@@ -197,15 +215,14 @@ else
 	docker push gcr.io/gadget-core-production/dateilager:latest
 endif
 
-upload-prerelease-container-image: release
-	docker build -t gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) .
-	docker push gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT)
+upload-prerelease-container-image:
+	docker build --platform linux/arm64,linux/amd64 --push -t us-central1-docker.pkg.dev/gadget-core-production/core-production/dateilager:pre-$(GIT_COMMIT) .
 
 build-local-container:
-	docker build -t dl-local:latest .
+	docker build --load -t dl-local:dev .
 
 run-container: release build-local-container
-	docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:latest $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl
+	docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:dev $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl
 
 gen-docs:
 	go run cmd/gen-docs/main.go
@@ -244,7 +261,9 @@ else
 endif
 
 js/src/pb: $(PROTO_FILES)
-	cd js && mkdir -p ./src/pb && npx protoc --experimental_allow_proto3_optional --ts_out ./src/pb --ts_opt long_type_bigint,ts_nocheck,eslint_disable,add_pb_suffix --proto_path ../internal/pb/ ../$^
+	cd js && mkdir -p ./src/pb && for proto in $^; do \
+		npx protoc --experimental_allow_proto3_optional --ts_out ./src/pb --ts_opt long_type_bigint,ts_nocheck,eslint_disable,add_pb_suffix --proto_path ../internal/pb/ ../$$proto; \
+	done
 
 js/dist: js/node_modules js/src/pb
 	cd js && npm run build

cmd/cached/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/gadget-inc/dateilager/pkg/cli"
+
+func main() {
+	cli.CacheDaemonExecute()
+}

development/paseto.key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEILTL+0PfTOIQcn2VPkpxMwf6Gbt9n4UEFDjZ4RuUKjd0
+-----END PRIVATE KEY-----

development/paseto.pub

@@ -1,3 +1,3 @@
 -----BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEASKQkA/AxlNCdOHTnp5McesmQ+y756VTtGz8Xrt1G0fs=
------END PUBLIC KEY-----
+MCowBQYDK2VwAyEAHrnbu7wEfAP9cGBOAHHwmH4Wsot1ciXBHwBBXQ4gsaI=
+-----END PUBLIC KEY-----

flake.nix

@@ -54,6 +54,7 @@
 
             postgresql = pkgs.postgresql_14;
 
+            glibcLocales = pkgs.glibcLocales;
             ## DateiLager outputs
 
             dateilager = callPackage ./. {
@@ -72,6 +73,7 @@
               flake.packages.postgresql
               flake.packages.dev
               flake.packages.clean
+              flake.packages.glibcLocales
               git
               protobuf
               protoc-gen-go

internal/key/key.go

@@ -1,6 +1,8 @@
 package key
 
 import (
+	"time"
+
 	"github.com/gadget-inc/dateilager/pkg/stringutil"
 	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
@@ -36,7 +38,9 @@ const (
 	Worker            = IntKey("dl.worker")
 	WorkerCount       = IntKey("dl.worker_count")
 	Ignores           = StringSliceKey("dl.ignores")
+	DurationMS        = DurationKey("dl.duration_ms")
 	CloneToProject    = Int64Key("dl.clone_to_project")
+	CachePath         = StringKey("dl.cache_path")
 )
 
 var (
@@ -148,3 +152,13 @@ func (isk Int64SliceKey) Field(value []int64) zap.Field {
 func (isk Int64SliceKey) Attribute(value []int64) attribute.KeyValue {
 	return attribute.Int64Slice(string(isk), value)
 }
+
+type DurationKey string
+
+func (dk DurationKey) Field(value time.Duration) zap.Field {
+	return zap.Duration(string(dk), value)
+}
+
+func (dk DurationKey) Attribute(value time.Duration) attribute.KeyValue {
+	return attribute.Float64(string(dk), float64(value.Milliseconds()))
+}