diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..752e9901 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.direnv +Dockerfile +js +test +tmp +bin +input +node_modules \ No newline at end of file diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index bd073f12..dc7c1ace 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -46,8 +46,4 @@ runs: - name: Add Go binaries to PATH run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - shell: bash -l {0} - - - name: Install Go binaries - run: make install - shell: bash -l {0} + shell: bash -l {0} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2847fe02..d5a79be2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,30 @@ -FROM --platform=linux/amd64 registry.fedoraproject.org/fedora-minimal:40 +FROM nixos/nix:2.22.0 AS build-stage +ARG TARGETARCH -RUN microdnf install -y curl findutils gzip iputils less postgresql procps shadow-utils tar time which \ - && microdnf clean all +RUN echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf +WORKDIR /app + +COPY flake.nix flake.lock ./ +COPY development ./development + +# build the nix environment once as an early docker layer +RUN nix develop -c true + +# copy the go modules and download em +COPY go.mod go.sum ./ +RUN nix develop -c go mod download + +# copy everything else and build the project +COPY . ./ +RUN nix develop -c make release/server_linux_$TARGETARCH + +FROM buildpack-deps:bullseye AS build-release-stage +ARG TARGETARCH RUN GRPC_HEALTH_PROBE_VERSION=v0.4.23 \ - && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 \ + && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} \ && chmod +x /bin/grpc_health_probe -RUN GO_MIGRATE_VERSION=v4.16.2 \ - && curl -Lfso /tmp/migrate.tar.gz https://github.com/golang-migrate/migrate/releases/download/${GO_MIGRATE_VERSION}/migrate.linux-amd64.tar.gz \ - && tar -xzf /tmp/migrate.tar.gz -C /bin \ - && chmod +x /bin/migrate - RUN useradd -ms /bin/bash main USER main WORKDIR /home/main @@ -20,8 +33,11 @@ RUN mkdir -p /home/main/secrets VOLUME /home/main/secrets/tls VOLUME /home/main/secrets/paseto -COPY release/server_linux_amd64 server +COPY --from=build-stage /app/release/server_linux_${TARGETARCH} server COPY migrations migrations COPY entrypoint.sh entrypoint.sh +# smoke test -- ensure the server command can run +RUN ./server --help + ENTRYPOINT ["./entrypoint.sh"] diff --git a/Makefile b/Makefile index 52fc95c3..4655a4c3 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto') MIGRATE_DIR := ./migrations SERVICE := $(PROJECT).server -.PHONY: install migrate migrate-create clean build lint release +.PHONY: migrate migrate-create clean build lint release .PHONY: test test-one test-fuzz test-js lint-js build-js .PHONY: reset-db setup-local server server-profile install-js .PHONY: client-update client-large-update client-get client-rebuild client-rebuild-with-cache @@ -29,16 +29,6 @@ SERVICE := $(PROJECT).server .PHONY: health upload-container-image run-container gen-docs .PHONY: load-test-new load-test-get load-test-update -install: - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - go install github.com/grpc-ecosystem/grpc-health-probe@v0.4 - go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.15 - go install github.com/bojand/ghz/cmd/ghz@v0.110.0 - go install github.com/gadget-inc/fsdiff/cmd/fsdiff@v0.4 - go install github.com/stamblerre/gocode@latest - go install golang.org/x/tools/cmd/goimports@latest - migrate: migrate -database $(DB_URI)?sslmode=disable -path $(MIGRATE_DIR) up @@ -78,9 +68,14 @@ build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go bin/server bin/client deve lint: golangci-lint run + + release/%_linux_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $< +release/%_linux_arm64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(BUILD_FLAGS) -o $@ $< + release/%_macos_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $< @@ -193,7 +188,7 @@ health: grpc-health-probe -addr $(GRPC_SERVER) grpc-health-probe -addr $(GRPC_SERVER) -service $(SERVICE) -upload-container-image: release +upload-container-image: ifndef version $(error version variable must be set) else @@ -206,8 +201,10 @@ upload-prerelease-container-image: release docker build -t gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) . docker push gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) -run-container: release +build-local-container: docker build -t dl-local:latest . + +run-container: release build-local-container docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:latest $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl gen-docs: diff --git a/Readme.md b/Readme.md index 6d0d6611..b70d517d 100644 --- a/Readme.md +++ b/Readme.md @@ -2,11 +2,11 @@ Dateilager is a content-addressed, networked filesystem for keeping large, similar directories among a wide variety of hosts. It has a few key properties: - - file contents and versions are stored in by a central server, with readers and writers as clients - - files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access - - readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity) - - incremental updates by readers are O(changes) and optimized to be very fast - - one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy) +- file contents and versions are stored in by a central server, with readers and writers as clients +- files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access +- readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity) +- incremental updates by readers are O(changes) and optimized to be very fast +- one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy) Dateilager is used in production to power https://gadget.dev to sync the filesystems for all the apps on the Gadget platform. Dateilager shines at syncing the (often very large) `node_modules` folder shared between the many node.js applications, as well as the comparatively small `.js` files comprising the actual business logic of one particular app. @@ -37,10 +37,6 @@ Ensure that you have a working [Go development environment](https://golang.org/d You will also require `npm`. -```bash -$ make install -``` - ### Setup VSCode (Optional) We recommend using VSCode for development, and there's an example settings file at `.vscode/settings.example.json` to get started with: diff --git a/flake.nix b/flake.nix index f56fe58f..07a0d818 100644 --- a/flake.nix +++ b/flake.nix @@ -11,6 +11,7 @@ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" + "aarch64-linux" ] (system: nixpkgs.lib.fix (flake: let @@ -36,7 +37,7 @@ callPackage = pkgs.newScope (flake.packages // { inherit lib callPackage; }); in { - packages = { + packages = { ## DateiLager development scripts clean = callPackage ./development/nix/scripts/clean.nix { }; @@ -63,6 +64,27 @@ }; defaultPackage = flake.packages.dateilager; + + devShell = pkgs.mkShell { + buildInputs = with pkgs; [ + flake.packages.go + flake.packages.nodejs + flake.packages.postgresql + flake.packages.dev + flake.packages.clean + git + protobuf + protoc-gen-go + protoc-gen-go-grpc + go-migrate + mkcert + ]; + + shellHook = '' + # prepend the built binaries to the $PATH + export PATH="./bin":$PATH + ''; + }; } ))); } diff --git a/internal/pb/fs.pb.go b/internal/pb/fs.pb.go index e9c8be82..7fdfce45 100644 --- a/internal/pb/fs.pb.go +++ b/internal/pb/fs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.29.1 -// protoc v3.21.12 +// protoc-gen-go v1.33.0 +// protoc v4.24.4 // source: internal/pb/fs.proto package pb diff --git a/internal/pb/fs_grpc.pb.go b/internal/pb/fs_grpc.pb.go index 8fc90881..944329bd 100644 --- a/internal/pb/fs_grpc.pb.go +++ b/internal/pb/fs_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.12 +// - protoc v4.24.4 // source: internal/pb/fs.proto package pb