From 75cd8b3b05b7fb6fc16e617a9b8c84174d993f60 Mon Sep 17 00:00:00 2001 From: Harry Brundage Date: Sun, 5 May 2024 09:45:37 -0400 Subject: [PATCH] Implement a CSI for kubernetes that prepopulates a mounted emptyDir with a cache --- Makefile | 5 + flake.nix | 3 + go.mod | 22 +++-- go.sum | 73 +++++++++++--- internal/key/key.go | 2 + pkg/api/cached.go | 5 + pkg/api/cachedcsi.go | 204 ++++++++++++++++++++++++++++++++++++++++ pkg/cached/cachedcsi.go | 88 +++++++++++++++++ pkg/cli/cachedaemon.go | 46 +++++---- test/cached_csi_test.go | 164 ++++++++++++++++++++++++++++++++ 10 files changed, 575 insertions(+), 37 deletions(-) create mode 100644 pkg/api/cachedcsi.go create mode 100644 pkg/cached/cachedcsi.go create mode 100644 test/cached_csi_test.go diff --git a/Makefile b/Makefile index 580cf84c..19a6daa2 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,11 @@ cached: export DL_TOKEN=$(DEV_SHARED_READER_TOKEN) cached: internal/pb/cache.pb.go internal/pb/cache_grpc.pb.go go run cmd/cached/main.go --upstream-host $(GRPC_HOST) --upstream-port $(GRPC_PORT) --port $(GRPC_CACHED_PORT) --staging-path tmp/cache-stage +cached-csi: export DL_ENV=dev +cached-csi: export DL_TOKEN=$(DEV_SHARED_READER_TOKEN) +cached-csi: internal/pb/cache.pb.go internal/pb/cache_grpc.pb.go + go run cmd/cached/main.go --upstream-host $(GRPC_HOST) --upstream-port $(GRPC_PORT) --staging-path tmp/cache-stage --csi-socket unix://tmp/csi.sock + client-update: export DL_TOKEN=$(DEV_TOKEN_PROJECT_1) client-update: export DL_SKIP_SSL_VERIFICATION=1 client-update: diff --git a/flake.nix b/flake.nix index 4324bd72..e52b2ccd 100644 --- a/flake.nix +++ b/flake.nix @@ -85,6 +85,9 @@ shellHook = '' # prepend the built binaries to the $PATH export PATH="./bin":$PATH + + # silence ginko deprecations -- they come from the csi test suite that we don't control + export ACK_GINKGO_DEPRECATIONS=1.16.5 ''; }; } diff --git a/go.mod b/go.mod index 1b8130fe..71045ad0 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,16 @@ module github.com/gadget-inc/dateilager go 1.22 require ( + github.com/container-storage-interface/spec v1.9.0 github.com/dgraph-io/ristretto v0.1.1 github.com/gadget-inc/fsdiff v0.4.4 github.com/gobwas/glob v0.2.3 + github.com/golang/protobuf v1.5.4 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/jackc/pgx/v5 v5.5.0 github.com/jackc/puddle/v2 v2.2.1 github.com/klauspost/compress v1.16.5 + github.com/kubernetes-csi/csi-test v2.2.0+incompatible github.com/minio/sha256-simd v1.0.0 github.com/o1egl/paseto v1.0.0 github.com/spf13/cobra v1.6.0 @@ -23,9 +26,9 @@ require ( go.uber.org/zap v1.23.0 golang.org/x/oauth2 v0.15.0 golang.org/x/sync v0.5.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.19.0 google.golang.org/grpc v1.59.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.33.0 ) require ( @@ -40,16 +43,21 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/glog v1.1.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/klauspost/cpuid/v2 v2.1.2 // indirect github.com/kr/text v0.2.0 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.33.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect @@ -60,12 +68,14 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.16.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5dbccf09..88a3171d 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= +github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -89,6 +91,9 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gadget-inc/fsdiff v0.4.4 h1:hLl/rCcWmW3P27LqMEyT2LVNWDLsREqsncODECH1vNw= github.com/gadget-inc/fsdiff v0.4.4/go.mod h1:GJurAL/F4qq4hp7uLc7nhqu9PbszI4/7fkTAUNa/VIY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -98,11 +103,15 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -137,8 +146,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -150,8 +159,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -161,8 +170,12 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= @@ -173,6 +186,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjd github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -203,10 +217,25 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/csi-test v2.2.0+incompatible h1:ksIV60Q+4mY0Fg8LKvBssjEcvbyxo7nz0eAD6ZLMux0= +github.com/kubernetes-csi/csi-test v2.2.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/o1egl/paseto v1.0.0 h1:bwpvPu2au176w4IBlhbyUv/S5VPptERIA99Oap5qUd0= github.com/o1egl/paseto v1.0.0/go.mod h1:5HxsZPmw/3RI2pAwGo1HhOOwSdvBpcuVzO7uDkm+CLU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= +github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -282,8 +311,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -316,6 +345,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -336,14 +366,15 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -364,6 +395,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -374,7 +406,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -392,12 +427,13 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -451,7 +487,10 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -549,15 +588,21 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/key/key.go b/internal/key/key.go index 70c90506..f0513c27 100644 --- a/internal/key/key.go +++ b/internal/key/key.go @@ -41,6 +41,8 @@ const ( DurationMS = DurationKey("dl.duration_ms") CloneToProject = Int64Key("dl.clone_to_project") CachePath = StringKey("dl.cache_path") + VolumeID = StringKey("dl.volume_id") + TargetPath = StringKey("dl.target_path") ) var ( diff --git a/pkg/api/cached.go b/pkg/api/cached.go index 0c77b777..fdf64ae1 100644 --- a/pkg/api/cached.go +++ b/pkg/api/cached.go @@ -10,6 +10,7 @@ import ( "path" "time" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/gadget-inc/dateilager/internal/files" "github.com/gadget-inc/dateilager/internal/key" "github.com/gadget-inc/dateilager/internal/logger" @@ -20,7 +21,11 @@ import ( type Cached struct { pb.UnimplementedCacheServer + csi.UnimplementedIdentityServer + csi.UnimplementedNodeServer + Client *client.Client + name string StagingPath string // the current directory holding a fully formed downloaded cache currentDir string diff --git a/pkg/api/cachedcsi.go b/pkg/api/cachedcsi.go new file mode 100644 index 00000000..0f45dae8 --- /dev/null +++ b/pkg/api/cachedcsi.go @@ -0,0 +1,204 @@ +package api + +import ( + "context" + "fmt" + "math" + "os" + "path" + "path/filepath" + "syscall" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/gadget-inc/dateilager/internal/key" + "github.com/gadget-inc/dateilager/internal/logger" + "github.com/gadget-inc/dateilager/pkg/version" + "github.com/golang/protobuf/ptypes/wrappers" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + DriverName = "com.gadget.dateilager.cached" +) + +// GetPluginInfo returns metadata of the plugin +func (c *Cached) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + resp := &csi.GetPluginInfoResponse{ + Name: DriverName, + VendorVersion: version.Version, + } + + return resp, nil +} + +// GetPluginCapabilities returns available capabilities of the plugin +func (c *Cached) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + resp := &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{}, + } + + return resp, nil +} + +// Probe returns the health and readiness of the plugin +func (c *Cached) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { + + return &csi.ProbeResponse{ + Ready: &wrappers.BoolValue{ + Value: true, + }, + }, nil +} + +// NodeGetCapabilities returns the supported capabilities of the node server +// this driver has no capabilities like expansion or staging, because we only use it for node local volumes +func (c *Cached) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + nscaps := []*csi.NodeServiceCapability{} + + return &csi.NodeGetCapabilitiesResponse{ + Capabilities: nscaps, + }, nil +} + +// NodeGetInfo returns the supported capabilities of the node server. This +// Usually, a CSI driver would return some interesting stuff about the node here for the controller to use to place volumes, but because we're only supporting node local volumes, we return something very basic +func (c *Cached) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + return &csi.NodeGetInfoResponse{ + NodeId: first(os.Getenv("NODE_NAME"), "dev"), + MaxVolumesPerNode: 110, + }, nil +} + +func (c *Cached) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume ID must be provided") + } + + if req.TargetPath == "" { + return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Target Path must be provided") + } + + if req.VolumeCapability == nil { + return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume Capability must be provided") + } + + targetPath := req.GetTargetPath() + volumeID := req.GetVolumeId() + volumeAttributes := req.GetVolumeContext() + + var cachePath string + var targetPermissions os.FileMode + + if suffix, exists := volumeAttributes["placeCacheAtPath"]; exists { + // running in suffix mode, desired outcome: + // - the mount point is writable by the pod + // - the cache is mounted at the suffix, and is not writable + cachePath = path.Join(targetPath, suffix) + targetPermissions = 0777 + } else { + // running in unsuffixed mode, desired outcome: + // - the mount point *is* the cache, and is not writable by the pod + cachePath = targetPath + targetPermissions = 0755 + } + + if err := os.MkdirAll(targetPath, targetPermissions); err != nil { + return nil, fmt.Errorf("failed to create target directory %s: %s", targetPath, err) + } + + if err := os.Chmod(targetPath, targetPermissions); err != nil { + return nil, fmt.Errorf("failed to change ownership of target directory %s: %s", targetPath, err) + } + + version, err := c.WriteCache(cachePath) + if err != nil { + return nil, err + } + + logger.Info(ctx, "volume published", key.VolumeID.Field(volumeID), key.TargetPath.Field(targetPath), key.Version.Field(version)) + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (s *Cached) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Volume ID must be provided") + } + + if req.TargetPath == "" { + return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Target Path must be provided") + } + + targetPath := req.GetTargetPath() + + // Clean up directory + if err := os.RemoveAll(targetPath); err != nil { + return nil, fmt.Errorf("failed to remove directory %s: %s", targetPath, err) + } + + logger.Info(ctx, "volume unpublished and data removed", key.TargetPath.Field(targetPath)) + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +// NodeGetVolumeStats returns the volume capacity statistics available for the given volume. +func (c *Cached) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats Volume ID must be provided") + } + + volumePath := req.VolumePath + if volumePath == "" { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats Volume Path must be provided") + } + + usedBytes, err := getFolderSize(volumePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to retrieve used size statistics for volume path %q: %s", volumePath, err) + } + + var stat syscall.Statfs_t + err = syscall.Statfs(volumePath, &stat) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to retrieve total size statistics for volume path %q: %s", volumePath, err) + } + + // Calculate free space in bytes + freeBytes := stat.Bavail * uint64(stat.Bsize) + if freeBytes > math.MaxInt64 { + return nil, status.Errorf(codes.Internal, "total size statistics for volume path too big for int64: %d", freeBytes) + } + signedFreeBytes := int64(freeBytes) + + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Available: signedFreeBytes, + Total: signedFreeBytes + usedBytes, + Used: usedBytes, + Unit: csi.VolumeUsage_BYTES, + }, + }, + }, nil +} + +func first(one, two string) string { + if one == "" { + return two + } + return one +} + +func getFolderSize(path string) (int64, error) { + var totalSize int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + totalSize += info.Size() + } + return nil + }) + return totalSize, err +} diff --git a/pkg/cached/cachedcsi.go b/pkg/cached/cachedcsi.go new file mode 100644 index 00000000..3f896feb --- /dev/null +++ b/pkg/cached/cachedcsi.go @@ -0,0 +1,88 @@ +package cached + +import ( + "context" + "fmt" + "net" + "net/url" + "os" + "path" + "path/filepath" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/gadget-inc/dateilager/internal/logger" + "github.com/gadget-inc/dateilager/internal/pb" + "github.com/gadget-inc/dateilager/pkg/api" + "github.com/gadget-inc/dateilager/pkg/client" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +// NewCacheServer returns a CSI plugin that contains the necessary gRPC +// interfaces to interact with Kubernetes over unix domain sockets for +// managaing volumes on behalf of pods +func NewCacheCSIServer(ctx context.Context, client *client.Client, stagingPath string) *CacheServer { + grpcServer := grpc.NewServer( + grpc.UnaryInterceptor( + grpc_middleware.ChainUnaryServer( + grpc_recovery.UnaryServerInterceptor(), + otelgrpc.UnaryServerInterceptor(), + logger.UnaryServerInterceptor(), + ), + ), + ) + + healthServer := health.NewServer() + healthpb.RegisterHealthServer(grpcServer, healthServer) + + cached := &api.Cached{ + Client: client, + StagingPath: stagingPath, + } + pb.RegisterCacheServer(grpcServer, cached) + + csi.RegisterIdentityServer(grpcServer, cached) + csi.RegisterNodeServer(grpcServer, cached) + + server := &CacheServer{ + Grpc: grpcServer, + Health: healthServer, + Cached: cached, + } + + return server +} + +// Run starts the CSI plugin by communication over the given endpoint +func (s *CacheServer) ServeCSI(ctx context.Context, listenSocketPath string) error { + u, err := url.Parse(listenSocketPath) + if err != nil { + return fmt.Errorf("unable to parse address: %q", err) + } + + addr := path.Join(u.Host, filepath.FromSlash(u.Path)) + if u.Host == "" { + addr = filepath.FromSlash(u.Path) + } + + // CSI plugins talk only over UNIX sockets currently + if u.Scheme != "unix" { + return fmt.Errorf("currently only unix domain sockets are supported, have incorrect protocol: %s", u.Scheme) + } else { + // remove the socket if it's already there. This can happen if we deploy a new version and the socket was created from the old running plugin. + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove unix domain socket file %s, error: %s", addr, err) + } + } + + listener, err := net.Listen(u.Scheme, addr) + if err != nil { + return fmt.Errorf("failed to listen: %v", err) + } + + return s.Grpc.Serve(listener) +} diff --git a/pkg/cli/cachedaemon.go b/pkg/cli/cachedaemon.go index e87233c0..2829e83d 100644 --- a/pkg/cli/cachedaemon.go +++ b/pkg/cli/cachedaemon.go @@ -43,6 +43,7 @@ func NewCacheDaemonCommand() *cobra.Command { timeout uint headlessHost string stagingPath string + csiSocket string ) cmd := &cobra.Command{ @@ -89,28 +90,27 @@ func NewCacheDaemonCommand() *cobra.Command { shutdownTelemetry = telemetry.Init(ctx, telemetry.Server) } - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return fmt.Errorf("cannot open TLS cert and key files (%s, %s): %w", certFile, keyFile, err) - } - - pasetoKey, err := parsePublicKey(pasetoFile) - if err != nil { - return fmt.Errorf("cannot parse Paseto public key %s: %w", pasetoFile, err) - } - cl, err := client.NewClient(ctx, upstreamHost, upstreamPort, client.WithheadlessHost(headlessHost)) if err != nil { return err } ctx = client.IntoContext(ctx, cl) - listen, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) - if err != nil { - return fmt.Errorf("failed to listen on TCP port %d: %w", port, err) - } + var s *cached.CacheServer + if csiSocket == "" { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("cannot open TLS cert and key files (%s, %s): %w", certFile, keyFile, err) + } + pasetoKey, err := parsePublicKey(pasetoFile) + if err != nil { + return fmt.Errorf("cannot parse Paseto public key %s: %w", pasetoFile, err) + } - s := cached.NewServer(ctx, cl, &cert, stagingPath, pasetoKey) + s = cached.NewServer(ctx, cl, &cert, stagingPath, pasetoKey) + } else { + s = cached.NewCacheCSIServer(ctx, cl, stagingPath) + } osSignals := make(chan os.Signal, 1) signal.Notify(osSignals, os.Interrupt, syscall.SIGTERM) @@ -124,8 +124,19 @@ func NewCacheDaemonCommand() *cobra.Command { return fmt.Errorf("failed to prepare cache daemon in %s: %w", stagingPath, err) } - logger.Info(ctx, "start server", key.Port.Field(port), key.Environment.Field(env.String())) - return s.Serve(listen) + if csiSocket == "" { + listen, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return fmt.Errorf("failed to listen on TCP port %d: %w", port, err) + } + + logger.Info(ctx, "start server", key.Port.Field(port), key.Environment.Field(env.String())) + return s.Serve(listen) + } else { + logger.Info(ctx, "start CSI server") + return s.ServeCSI(ctx, csiSocket) + } + }, PostRunE: func(cmd *cobra.Command, _ []string) error { if shutdownTelemetry != nil { @@ -156,6 +167,7 @@ func NewCacheDaemonCommand() *cobra.Command { flags.UintVar(&timeout, "timeout", 0, "GRPC client timeout (ms)") flags.IntVar(&port, "port", 5053, "cache API port") flags.StringVar(&stagingPath, "staging-path", "", "path for staging downloaded caches") + flags.StringVar(&csiSocket, "csi-socket", "", "path for running the Kubernetes CSI Driver interface") _ = cmd.MarkPersistentFlagRequired("staging-path") return cmd diff --git a/test/cached_csi_test.go b/test/cached_csi_test.go new file mode 100644 index 00000000..5cf4488b --- /dev/null +++ b/test/cached_csi_test.go @@ -0,0 +1,164 @@ +package test + +import ( + "fmt" + "os" + "path" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/gadget-inc/dateilager/internal/auth" + "github.com/gadget-inc/dateilager/internal/db" + util "github.com/gadget-inc/dateilager/internal/testutil" + "github.com/gadget-inc/dateilager/pkg/cached" + "github.com/kubernetes-csi/csi-test/pkg/sanity" + "github.com/stretchr/testify/require" +) + +func TestCachedCSIDriver(t *testing.T) { + tc := util.NewTestCtx(t, auth.Admin, 1) + defer tc.Close() + + writeProject(tc, 1, 2) + writeObject(tc, 1, 1, i(2), "a", "a v1") + writePackedFiles(tc, 1, 1, nil, "pack/a") + writePackedFiles(tc, 1, 1, nil, "pack/b") + _, err := db.CreateCache(tc.Context(), tc.Connect(), "", 100) + require.NoError(t, err) + + c, _, close := createTestClient(tc) + defer close() + + tmpDir := emptyTmpDir(t) + // defer os.RemoveAll(tmpDir) + + s := cached.NewCacheCSIServer(tc.Context(), c, path.Join(tmpDir, "cached", "staging")) + require.NoError(t, s.Cached.Prepare(tc.Context()), "cached.Prepare must succeed") + defer s.Grpc.GracefulStop() + + socket := path.Join(tmpDir, "csi.sock") + endpoint := "unix://" + socket + go s.ServeCSI(tc.Context(), endpoint) + + sanityPath := path.Join(tmpDir, "csi") + require.NoError(t, os.MkdirAll(sanityPath, 0755), "couldn't make staging path") + + cfg := &sanity.Config{ + StagingPath: path.Join(tmpDir, "staging"), + TargetPath: path.Join(tmpDir, "target"), + Address: endpoint, + } + + sanity.Test(t, cfg) +} + +func TestCachedCSIDriverMountsCache(t *testing.T) { + tc := util.NewTestCtx(t, auth.Admin, 1) + defer tc.Close() + + writeProject(tc, 1, 2) + writeObject(tc, 1, 1, i(2), "a", "a v1") + aHash := writePackedFiles(tc, 1, 1, nil, "pack/a") + bHash := writePackedFiles(tc, 1, 1, nil, "pack/b") + version, err := db.CreateCache(tc.Context(), tc.Connect(), "", 100) + require.NoError(t, err) + + c, _, close := createTestClient(tc) + defer close() + + tmpDir := emptyTmpDir(t) + // defer os.RemoveAll(tmpDir) + + s := cached.NewCacheCSIServer(tc.Context(), c, path.Join(tmpDir, "cached", "staging")) + require.NoError(t, s.Cached.Prepare(tc.Context()), "cached.Prepare must succeed") + defer s.Grpc.GracefulStop() + + targetDir := path.Join(tmpDir, "vol-target") + + _, err = s.Cached.NodePublishVolume(tc.Context(), &csi.NodePublishVolumeRequest{ + VolumeId: "foobar", + StagingTargetPath: path.Join(tmpDir, "vol-staging-target"), + TargetPath: targetDir, + VolumeCapability: &csi.VolumeCapability{}, + }) + require.NoError(t, err) + + verifyDir(t, targetDir, -1, map[string]expectedFile{ + fmt.Sprintf("objects/%v/pack/a/1", aHash): {content: "pack/a/1 v1"}, + fmt.Sprintf("objects/%v/pack/a/2", aHash): {content: "pack/a/2 v1"}, + fmt.Sprintf("objects/%v/pack/b/1", bHash): {content: "pack/b/1 v1"}, + fmt.Sprintf("objects/%v/pack/b/2", bHash): {content: "pack/b/2 v1"}, + "versions": {content: fmt.Sprintf("%v\n", version)}, + }) + + fileInfo, err := os.Stat(targetDir) + require.NoError(t, err) + + // the target dir should not be world writable -- only by the user the CSI driver is running as (which will be root) + require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(fileInfo.Mode()&os.ModePerm)) + + // files inside cache dir should also *not* be writable -- it's managed by the CSI and must remain pristine + cacheFileInfo, err := os.Stat(path.Join(targetDir, "objects/c4d899b73f3d61e159d3194f4d96f1c4cd451f6ef5668dbc4ff5c2334407bbcd/pack/a/1")) + require.NoError(t, err) + require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(cacheFileInfo.Mode()&os.ModePerm)) +} + +func TestCachedCSIDriverMountsCacheAtSuffix(t *testing.T) { + tc := util.NewTestCtx(t, auth.Admin, 1) + defer tc.Close() + + writeProject(tc, 1, 2) + writeObject(tc, 1, 1, i(2), "a", "a v1") + aHash := writePackedFiles(tc, 1, 1, nil, "pack/a") + bHash := writePackedFiles(tc, 1, 1, nil, "pack/b") + version, err := db.CreateCache(tc.Context(), tc.Connect(), "", 100) + require.NoError(t, err) + + c, _, close := createTestClient(tc) + defer close() + + tmpDir := emptyTmpDir(t) + // defer os.RemoveAll(tmpDir) + + s := cached.NewCacheCSIServer(tc.Context(), c, path.Join(tmpDir, "cached", "staging")) + require.NoError(t, s.Cached.Prepare(tc.Context()), "cached.Prepare must succeed") + defer s.Grpc.GracefulStop() + + targetDir := path.Join(tmpDir, "vol-target") + _, err = s.Cached.NodePublishVolume(tc.Context(), &csi.NodePublishVolumeRequest{ + VolumeId: "foobar", + StagingTargetPath: path.Join(tmpDir, "vol-staging-target"), + TargetPath: targetDir, + VolumeCapability: &csi.VolumeCapability{}, + VolumeContext: map[string]string{"placeCacheAtPath": "inner_mount"}, + }) + require.NoError(t, err) + + verifyDir(t, path.Join(tmpDir, "vol-target"), -1, map[string]expectedFile{ + fmt.Sprintf("inner_mount/objects/%v/pack/a/1", aHash): {content: "pack/a/1 v1"}, + fmt.Sprintf("inner_mount/objects/%v/pack/a/2", aHash): {content: "pack/a/2 v1"}, + fmt.Sprintf("inner_mount/objects/%v/pack/b/1", bHash): {content: "pack/b/1 v1"}, + fmt.Sprintf("inner_mount/objects/%v/pack/b/2", bHash): {content: "pack/b/2 v1"}, + "inner_mount/versions": {content: fmt.Sprintf("%v\n", version)}, + }) + + fileInfo, err := os.Stat(targetDir) + require.NoError(t, err) + + // the target dir *should* be world writable -- we're going to use it as a scratch space to do useful stuff with the cache + require.Equal(t, formatFileMode(os.FileMode(0777)), formatFileMode(fileInfo.Mode()&os.ModePerm)) + + // the cache dir should *not* be writable -- it's managed by the CSI and must remain pristine + cacheFileInfo, err := os.Stat(path.Join(targetDir, "inner_mount")) + require.NoError(t, err) + require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(cacheFileInfo.Mode()&os.ModePerm)) + + // files inside cache dir should *not* be writable -- it's managed by the CSI and must remain pristine + cacheFileInfo, err = os.Stat(path.Join(targetDir, "inner_mount/objects/c4d899b73f3d61e159d3194f4d96f1c4cd451f6ef5668dbc4ff5c2334407bbcd/pack/a/1")) + require.NoError(t, err) + require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(cacheFileInfo.Mode())) +} + +func formatFileMode(mode os.FileMode) string { + return fmt.Sprintf("%#o", mode) +}