Implement a cache daemon that can move the cache into place quickly o…

…n demand

 - a long running background process we'll deploy as a daemonset on k8s nodes
 - moves the cache into place via hardlinking the golden copy from a shared location
 - operates using the host filesystem, and thus has pretty priviledged access

Contributing.md

@@ -188,3 +188,7 @@ We also need to build the server docker image and push it to Gadget's container
 ```bash
 make upload-container-image version=0.0.x
 ```
+
+### Getting PASETO tokens locally
+
+You can sign PASETO tokens locally with this handy online tool: https://token.dev/paseto/. Ensure you use the V2 algorithm in the public mode, and copy the PASTEO public and private key from the `development` folder.

Dockerfile

@@ -27,7 +27,7 @@ RUN go mod download
 
 # copy everything else and build the project
 COPY . ./
-RUN make release/server_linux_$TARGETARCH
+RUN make release/server_linux_$TARGETARCH release/cached_linux_$TARGETARCH
 
 FROM buildpack-deps:bullseye AS build-release-stage
 ARG TARGETARCH
@@ -48,11 +48,14 @@ RUN mkdir -p /home/main/secrets
 VOLUME /home/main/secrets/tls
 VOLUME /home/main/secrets/paseto
 
+COPY --from=build-stage /app/release/cached_linux_${TARGETARCH} cached
 COPY --from=build-stage /app/release/server_linux_${TARGETARCH} server
+
 COPY migrations migrations
 COPY entrypoint.sh entrypoint.sh
 
-# smoke test -- ensure the server command can run
+# smoke test -- ensure the commands can run
 RUN ./server --help
+RUN ./cached --help
 
 ENTRYPOINT ["./entrypoint.sh"]

Makefile

@@ -8,11 +8,13 @@ DB_USER ?= postgres
 DB_PASS ?= password
 DB_URI := postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl
 
-GRPC_PORT ?= 5051
 GRPC_HOST ?= localhost
+GRPC_PORT ?= 5051
+GRPC_CACHED_PORT ?= 5053
 
-DEV_TOKEN_ADMIN ?= v2.public.eyJzdWIiOiJhZG1pbiIsImlhdCI6IjIwMjEtMTAtMTVUMTE6MjA6MDAuMDM0WiJ9WtEey8KfQQRy21xoHq1C5KQatEevk8RxS47k4bRfMwVCPHumZmVuk6ADcfDHTmSnMtEGfFXdxnYOhRP6Clb_Dw
-DEV_TOKEN_PROJECT_1 ?= v2.public.eyJzdWIiOiIxIiwiaWF0IjoiMjAyMS0xMC0xNVQxMToyMDowMC4wMzVaIn2MQ14RfIGpoEycCuvRu9J3CZp6PppUXf5l5w8uKKydN3C31z6f6GgOEPNcnwODqBnX7Pjarpz4i2uzWEqLgQYD
+DEV_TOKEN_ADMIN ?= v2.public.eyJzdWIiOiJhZG1pbiJ9yt40HNkcyOUtDeFa_WPS6vi0WiE4zWngDGJLh17TuYvssTudCbOdQEkVDRD-mSNTXLgSRDXUkO-AaEr4ZLO4BQ
+DEV_TOKEN_PROJECT_1 ?= v2.public.eyJzdWIiOiIxIn2jV7FOdEXafKDtAnVyDgI4fmIbqU7C1iuhKiL0lDnG1Z5-j6_ObNDd75sZvLZ159-X98_mP4qvwzui0w8pjt8F
+DEV_SHARED_READER_TOKEN ?= v2.public.eyJzdWIiOiJzaGFyZWQtcmVhZGVyIn1CxWdB02s9el0Wt7qReARZ-7JtIb4Zj3D4Oiji1yXHqj0orkpbcVlswVUiekECJC16d1NrHwD2FWSwRORZn8gK
 
 PKG_GO_FILES := $(shell find pkg/ -type f -name '*.go')
 INTERNAL_GO_FILES := $(shell find internal/ -type f -name '*.go')
@@ -63,7 +65,7 @@ development/server.key:
 
 development/server.crt: development/server.key
 
-build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go bin/server bin/client development/server.crt
+build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go internal/pb/cache.pb.go internal/pb/cache_grpc.pb.go bin/server bin/client bin/cached development/server.crt
 
 lint:
 	golangci-lint run
@@ -86,8 +88,9 @@ release/migrations.tar.gz: migrations/*
 	tar -zcf $@ migrations
 
 release: build
-release: release/server_linux_amd64 release/server_macos_amd64 release/server_macos_arm64
-release: release/client_linux_amd64 release/client_macos_amd64 release/client_macos_arm64
+release: release/server_linux_amd64 release/server_macos_amd64 release/server_macos_arm64 release/server_linux_arm64
+release: release/client_linux_amd64 release/client_macos_amd64 release/client_macos_arm64 release/client_linux_arm64
+release: release/cached_linux_amd64 release/cached_macos_amd64 release/cached_macos_arm64 release/cached_linux_arm64
 release: release/migrations.tar.gz
 
 test: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
@@ -121,6 +124,11 @@ server-profile: export DL_ENV=dev
 server-profile: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go
 	go run cmd/server/main.go --dburi $(DB_URI) --port $(GRPC_PORT) --profile cpu.prof --log-level info
 
+cached: export DL_ENV=dev
+cached: export DL_TOKEN=$(DEV_SHARED_READER_TOKEN)
+cached: internal/pb/cache.pb.go internal/pb/cache_grpc.pb.go
+	go run cmd/cached/main.go --upstream-host $(GRPC_HOST) --upstream-port $(GRPC_PORT) --port $(GRPC_CACHED_PORT) --staging-path tmp/cache-stage
+
 client-update: export DL_TOKEN=$(DEV_TOKEN_PROJECT_1)
 client-update: export DL_SKIP_SSL_VERIFICATION=1
 client-update:
@@ -169,6 +177,11 @@ client-getcache: export DL_SKIP_SSL_VERIFICATION=1
 client-getcache:
 	go run cmd/client/main.go getcache --host $(GRPC_HOST) --path input/cache
 
+client-getcache-from-daemon: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
+client-getcache-from-daemon: export DL_SKIP_SSL_VERIFICATION=1
+client-getcache-from-daemon:
+	mkdir -p tmp/pods/test-pod/volumes/example && go run cmd/client/main.go getcache-from-daemon --host $(GRPC_HOST) --port $(GRPC_CACHED_PORT) input/cache
+
 client-gc-contents: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
 client-gc-contents: export DL_SKIP_SSL_VERIFICATION=1
 client-gc-contents:
@@ -242,8 +255,8 @@ else
 	cd js && npm install
 endif
 
-js/src/pb: $(PROTO_FILES)
-	cd js && mkdir -p ./src/pb && npx protoc --experimental_allow_proto3_optional --ts_out ./src/pb --ts_opt long_type_bigint,ts_nocheck,eslint_disable,add_pb_suffix --proto_path ../internal/pb/ ../$^
+js/src/pb: internal/pb/fs.proto
+	cd js && mkdir -p ./src/pb && npx protoc --experimental_allow_proto3_optional --ts_out ./src/pb --ts_opt long_type_bigint,ts_nocheck,eslint_disable,add_pb_suffix --proto_path ../internal/pb/ ../internal/pb/fs.proto
 
 js/dist: js/node_modules js/src/pb
 	cd js && npm run build

cmd/cached/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/gadget-inc/dateilager/pkg/cli"
+
+func main() {
+	cli.CacheDaemonExecute()
+}

development/paseto.key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEILTL+0PfTOIQcn2VPkpxMwf6Gbt9n4UEFDjZ4RuUKjd0
+-----END PRIVATE KEY-----

development/paseto.pub

@@ -1,3 +1,3 @@
 -----BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEASKQkA/AxlNCdOHTnp5McesmQ+y756VTtGz8Xrt1G0fs=
------END PUBLIC KEY-----
+MCowBQYDK2VwAyEAHrnbu7wEfAP9cGBOAHHwmH4Wsot1ciXBHwBBXQ4gsaI=
+-----END PUBLIC KEY-----

flake.nix

@@ -54,6 +54,9 @@
 
             postgresql = pkgs.postgresql_14;
 
+            golangci-lint = pkgs.golangci-lint;
+
+            glibcLocales = pkgs.glibcLocales;
             ## DateiLager outputs
 
             dateilager = callPackage ./. {
@@ -72,6 +75,8 @@
               flake.packages.postgresql
               flake.packages.dev
               flake.packages.clean
+              flake.packages.golangci-lint
+              flake.packages.glibcLocales
               git
               protobuf
               protoc-gen-go
@@ -83,6 +88,9 @@
             shellHook = ''
               # prepend the built binaries to the $PATH
               export PATH="./bin":$PATH
+
+              # silence ginko deprecations -- they come from the csi test suite that we don't control
+              export ACK_GINKGO_DEPRECATIONS=1.16.5
           '';
           };
         }

internal/key/key.go

@@ -1,6 +1,8 @@
 package key
 
 import (
+	"time"
+
 	"github.com/gadget-inc/dateilager/pkg/stringutil"
 	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
@@ -36,7 +38,9 @@ const (
 	Worker            = IntKey("dl.worker")
 	WorkerCount       = IntKey("dl.worker_count")
 	Ignores           = StringSliceKey("dl.ignores")
+	DurationMS        = DurationKey("dl.duration_ms")
 	CloneToProject    = Int64Key("dl.clone_to_project")
+	CachePath         = StringKey("dl.cache_path")
 )
 
 var (
@@ -148,3 +152,13 @@ func (isk Int64SliceKey) Field(value []int64) zap.Field {
 func (isk Int64SliceKey) Attribute(value []int64) attribute.KeyValue {
 	return attribute.Int64Slice(string(isk), value)
 }
+
+type DurationKey string
+
+func (dk DurationKey) Field(value time.Duration) zap.Field {
+	return zap.Duration(string(dk), value)
+}
+
+func (dk DurationKey) Attribute(value time.Duration) attribute.KeyValue {
+	return attribute.Float64(string(dk), float64(value.Milliseconds()))
+}