diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3d32f950 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.direnv +.vscode +Dockerfile +js +test +tmp +bin +release +docs +input +node_modules +development +!development/nix \ No newline at end of file diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index bd073f12..dc7c1ace 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -46,8 +46,4 @@ runs: - name: Add Go binaries to PATH run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - shell: bash -l {0} - - - name: Install Go binaries - run: make install - shell: bash -l {0} + shell: bash -l {0} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2847fe02..6f93e691 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,45 @@ -FROM --platform=linux/amd64 registry.fedoraproject.org/fedora-minimal:40 +FROM nixos/nix:2.22.0 AS build-stage +ARG TARGETARCH -RUN microdnf install -y curl findutils gzip iputils less postgresql procps shadow-utils tar time which \ - && microdnf clean all +RUN tee -a /etc/nix/nix.conf < /bin/nix-env-shell \ + && echo 'exec nix develop --command bash -c "$@"' >> /bin/nix-env-shell \ + && chmod +x /bin/nix-env-shell + +SHELL ["/bin/nix-env-shell"] + +# copy the go modules and download em +COPY go.mod go.sum ./ +RUN go mod download + +# copy everything else and build the project +COPY . ./ +RUN make release/server_linux_$TARGETARCH + +FROM buildpack-deps:bullseye AS build-release-stage +ARG TARGETARCH + +RUN apt-get update && \ + apt-get install -y curl findutils gzip net-tools less postgresql procps tar time && \ + rm -rf /var/cache/apt/archives /var/lib/apt/lists/* RUN GRPC_HEALTH_PROBE_VERSION=v0.4.23 \ - && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 \ + && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} \ && chmod +x /bin/grpc_health_probe -RUN GO_MIGRATE_VERSION=v4.16.2 \ - && curl -Lfso /tmp/migrate.tar.gz https://github.com/golang-migrate/migrate/releases/download/${GO_MIGRATE_VERSION}/migrate.linux-amd64.tar.gz \ - && tar -xzf /tmp/migrate.tar.gz -C /bin \ - && chmod +x /bin/migrate - RUN useradd -ms /bin/bash main USER main WORKDIR /home/main @@ -20,8 +48,11 @@ RUN mkdir -p /home/main/secrets VOLUME /home/main/secrets/tls VOLUME /home/main/secrets/paseto -COPY release/server_linux_amd64 server +COPY --from=build-stage /app/release/server_linux_${TARGETARCH} server COPY migrations migrations COPY entrypoint.sh entrypoint.sh +# smoke test -- ensure the server command can run +RUN ./server --help + ENTRYPOINT ["./entrypoint.sh"] diff --git a/Makefile b/Makefile index 52fc95c3..4978a14a 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto') MIGRATE_DIR := ./migrations SERVICE := $(PROJECT).server -.PHONY: install migrate migrate-create clean build lint release +.PHONY: migrate migrate-create clean build lint release .PHONY: test test-one test-fuzz test-js lint-js build-js .PHONY: reset-db setup-local server server-profile install-js .PHONY: client-update client-large-update client-get client-rebuild client-rebuild-with-cache @@ -29,16 +29,6 @@ SERVICE := $(PROJECT).server .PHONY: health upload-container-image run-container gen-docs .PHONY: load-test-new load-test-get load-test-update -install: - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - go install github.com/grpc-ecosystem/grpc-health-probe@v0.4 - go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.15 - go install github.com/bojand/ghz/cmd/ghz@v0.110.0 - go install github.com/gadget-inc/fsdiff/cmd/fsdiff@v0.4 - go install github.com/stamblerre/gocode@latest - go install golang.org/x/tools/cmd/goimports@latest - migrate: migrate -database $(DB_URI)?sslmode=disable -path $(MIGRATE_DIR) up @@ -78,9 +68,14 @@ build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go bin/server bin/client deve lint: golangci-lint run + + release/%_linux_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $< +release/%_linux_arm64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(BUILD_FLAGS) -o $@ $< + release/%_macos_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $< @@ -193,22 +188,23 @@ health: grpc-health-probe -addr $(GRPC_SERVER) grpc-health-probe -addr $(GRPC_SERVER) -service $(SERVICE) -upload-container-image: release +upload-container-image: ifndef version $(error version variable must be set) else - docker build -t gcr.io/gadget-core-production/dateilager:$(version) -t gcr.io/gadget-core-production/dateilager:latest . + docker build --platform linux/amd64 -t gcr.io/gadget-core-production/dateilager:$(version) -t gcr.io/gadget-core-production/dateilager:latest . docker push gcr.io/gadget-core-production/dateilager:$(version) docker push gcr.io/gadget-core-production/dateilager:latest endif -upload-prerelease-container-image: release - docker build -t gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) . - docker push gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) +upload-prerelease-container-image: + docker build --platform linux/arm64,linux/amd64 --push -t us-central1-docker.pkg.dev/gadget-core-production/core-production/dateilager:pre-$(GIT_COMMIT) . + +build-local-container: + docker build --load -t dl-local:dev . -run-container: release - docker build -t dl-local:latest . - docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:latest $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl +run-container: release build-local-container + docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:dev $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl gen-docs: go run cmd/gen-docs/main.go diff --git a/Readme.md b/Readme.md index 6d0d6611..b70d517d 100644 --- a/Readme.md +++ b/Readme.md @@ -2,11 +2,11 @@ Dateilager is a content-addressed, networked filesystem for keeping large, similar directories among a wide variety of hosts. It has a few key properties: - - file contents and versions are stored in by a central server, with readers and writers as clients - - files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access - - readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity) - - incremental updates by readers are O(changes) and optimized to be very fast - - one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy) +- file contents and versions are stored in by a central server, with readers and writers as clients +- files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access +- readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity) +- incremental updates by readers are O(changes) and optimized to be very fast +- one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy) Dateilager is used in production to power https://gadget.dev to sync the filesystems for all the apps on the Gadget platform. Dateilager shines at syncing the (often very large) `node_modules` folder shared between the many node.js applications, as well as the comparatively small `.js` files comprising the actual business logic of one particular app. @@ -37,10 +37,6 @@ Ensure that you have a working [Go development environment](https://golang.org/d You will also require `npm`. -```bash -$ make install -``` - ### Setup VSCode (Optional) We recommend using VSCode for development, and there's an example settings file at `.vscode/settings.example.json` to get started with: diff --git a/flake.nix b/flake.nix index f56fe58f..07a0d818 100644 --- a/flake.nix +++ b/flake.nix @@ -11,6 +11,7 @@ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" + "aarch64-linux" ] (system: nixpkgs.lib.fix (flake: let @@ -36,7 +37,7 @@ callPackage = pkgs.newScope (flake.packages // { inherit lib callPackage; }); in { - packages = { + packages = { ## DateiLager development scripts clean = callPackage ./development/nix/scripts/clean.nix { }; @@ -63,6 +64,27 @@ }; defaultPackage = flake.packages.dateilager; + + devShell = pkgs.mkShell { + buildInputs = with pkgs; [ + flake.packages.go + flake.packages.nodejs + flake.packages.postgresql + flake.packages.dev + flake.packages.clean + git + protobuf + protoc-gen-go + protoc-gen-go-grpc + go-migrate + mkcert + ]; + + shellHook = '' + # prepend the built binaries to the $PATH + export PATH="./bin":$PATH + ''; + }; } ))); } diff --git a/internal/pb/fs.pb.go b/internal/pb/fs.pb.go index e9c8be82..7fdfce45 100644 --- a/internal/pb/fs.pb.go +++ b/internal/pb/fs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.29.1 -// protoc v3.21.12 +// protoc-gen-go v1.33.0 +// protoc v4.24.4 // source: internal/pb/fs.proto package pb diff --git a/internal/pb/fs_grpc.pb.go b/internal/pb/fs_grpc.pb.go index 8fc90881..944329bd 100644 --- a/internal/pb/fs_grpc.pb.go +++ b/internal/pb/fs_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.12 +// - protoc v4.24.4 // source: internal/pb/fs.proto package pb