Skip to content

Commit

Permalink
Build dateilager within the dockerfile using the nix environment
Browse files Browse the repository at this point in the history
This switches the build process to:
 - build DL within the dockerfile for repeatable builds, instead of copying in from outside
 - use the nix environment to setup the env for the build, so it's the same go version / protoc version etc etc in and outside of docker
 - use a multistage docker build so we have slim containers with just the binary at the end

I did all this because I was fighting to get a docker image built for linux_arm64, which is what orbstack needs to run the image locally in a k8s setup! It was annoying. This also adds linux_arm64 as a build target to the makefile, and teaches the dockerfile how to build the binary for the target architecture that the dockerfile is building for.

Yeesh.
  • Loading branch information
airhorns committed May 5, 2024
1 parent cc8f6bb commit 42a6646
Show file tree
Hide file tree
Showing 8 changed files with 77 additions and 41 deletions.
9 changes: 9 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
.git
.direnv
Dockerfile
js
test
tmp
bin
input
node_modules
6 changes: 1 addition & 5 deletions .github/actions/setup-env/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,4 @@ runs:

- name: Add Go binaries to PATH
run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
shell: bash -l {0}

- name: Install Go binaries
run: make install
shell: bash -l {0}
shell: bash -l {0}
36 changes: 26 additions & 10 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,17 +1,30 @@
FROM --platform=linux/amd64 registry.fedoraproject.org/fedora-minimal:40
FROM nixos/nix:2.22.0 AS build-stage
ARG TARGETARCH

RUN microdnf install -y curl findutils gzip iputils less postgresql procps shadow-utils tar time which \
&& microdnf clean all
RUN echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
WORKDIR /app

COPY flake.nix flake.lock ./
COPY development ./development

# build the nix environment once as an early docker layer
RUN nix develop -c true

# copy the go modules and download em
COPY go.mod go.sum ./
RUN nix develop -c go mod download

# copy everything else and build the project
COPY . ./
RUN nix develop -c make release/server_linux_$TARGETARCH

FROM buildpack-deps:bullseye AS build-release-stage
ARG TARGETARCH

RUN GRPC_HEALTH_PROBE_VERSION=v0.4.23 \
&& curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 \
&& curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} \
&& chmod +x /bin/grpc_health_probe

RUN GO_MIGRATE_VERSION=v4.16.2 \
&& curl -Lfso /tmp/migrate.tar.gz https://github.com/golang-migrate/migrate/releases/download/${GO_MIGRATE_VERSION}/migrate.linux-amd64.tar.gz \
&& tar -xzf /tmp/migrate.tar.gz -C /bin \
&& chmod +x /bin/migrate

RUN useradd -ms /bin/bash main
USER main
WORKDIR /home/main
Expand All @@ -20,8 +33,11 @@ RUN mkdir -p /home/main/secrets
VOLUME /home/main/secrets/tls
VOLUME /home/main/secrets/paseto

COPY release/server_linux_amd64 server
COPY --from=build-stage /app/release/server_linux_${TARGETARCH} server
COPY migrations migrations
COPY entrypoint.sh entrypoint.sh

# smoke test -- ensure the server command can run
RUN ./server --help

ENTRYPOINT ["./entrypoint.sh"]
23 changes: 10 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,14 @@ PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto')
MIGRATE_DIR := ./migrations
SERVICE := $(PROJECT).server

.PHONY: install migrate migrate-create clean build lint release
.PHONY: migrate migrate-create clean build lint release
.PHONY: test test-one test-fuzz test-js lint-js build-js
.PHONY: reset-db setup-local server server-profile install-js
.PHONY: client-update client-large-update client-get client-rebuild client-rebuild-with-cache
.PHONY: client-getcache client-gc-contents client-gc-project client-gc-random-projects
.PHONY: health upload-container-image run-container gen-docs
.PHONY: load-test-new load-test-get load-test-update

install:
go install google.golang.org/protobuf/cmd/[email protected]
go install google.golang.org/grpc/cmd/[email protected]
go install github.com/grpc-ecosystem/[email protected]
go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/[email protected]
go install github.com/bojand/ghz/cmd/[email protected]
go install github.com/gadget-inc/fsdiff/cmd/[email protected]
go install github.com/stamblerre/gocode@latest
go install golang.org/x/tools/cmd/goimports@latest

migrate:
migrate -database $(DB_URI)?sslmode=disable -path $(MIGRATE_DIR) up

Expand Down Expand Up @@ -78,9 +68,14 @@ build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go bin/server bin/client deve
lint:
golangci-lint run



release/%_linux_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $<

release/%_linux_arm64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(BUILD_FLAGS) -o $@ $<

release/%_macos_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $<

Expand Down Expand Up @@ -193,7 +188,7 @@ health:
grpc-health-probe -addr $(GRPC_SERVER)
grpc-health-probe -addr $(GRPC_SERVER) -service $(SERVICE)

upload-container-image: release
upload-container-image:
ifndef version
$(error version variable must be set)
else
Expand All @@ -206,8 +201,10 @@ upload-prerelease-container-image: release
docker build -t gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) .
docker push gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT)

run-container: release
build-local-container:
docker build -t dl-local:latest .

run-container: release build-local-container
docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:latest $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl

gen-docs:
Expand Down
14 changes: 5 additions & 9 deletions Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

Dateilager is a content-addressed, networked filesystem for keeping large, similar directories among a wide variety of hosts. It has a few key properties:

- file contents and versions are stored in by a central server, with readers and writers as clients
- files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access
- readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity)
- incremental updates by readers are O(changes) and optimized to be very fast
- one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy)
- file contents and versions are stored in by a central server, with readers and writers as clients
- files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access
- readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity)
- incremental updates by readers are O(changes) and optimized to be very fast
- one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy)

Dateilager is used in production to power https://gadget.dev to sync the filesystems for all the apps on the Gadget platform. Dateilager shines at syncing the (often very large) `node_modules` folder shared between the many node.js applications, as well as the comparatively small `.js` files comprising the actual business logic of one particular app.

Expand Down Expand Up @@ -37,10 +37,6 @@ Ensure that you have a working [Go development environment](https://golang.org/d

You will also require `npm`.

```bash
$ make install
```

### Setup VSCode (Optional)

We recommend using VSCode for development, and there's an example settings file at `.vscode/settings.example.json` to get started with:
Expand Down
24 changes: 23 additions & 1 deletion flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"x86_64-linux"
"x86_64-darwin"
"aarch64-darwin"
"aarch64-linux"
]
(system: nixpkgs.lib.fix (flake:
let
Expand All @@ -36,7 +37,7 @@
callPackage = pkgs.newScope (flake.packages // { inherit lib callPackage; });
in
{
packages = {
packages = {
## DateiLager development scripts

clean = callPackage ./development/nix/scripts/clean.nix { };
Expand All @@ -63,6 +64,27 @@
};

defaultPackage = flake.packages.dateilager;

devShell = pkgs.mkShell {
buildInputs = with pkgs; [
flake.packages.go
flake.packages.nodejs
flake.packages.postgresql
flake.packages.dev
flake.packages.clean
git
protobuf
protoc-gen-go
protoc-gen-go-grpc
go-migrate
mkcert
];

shellHook = ''
# prepend the built binaries to the $PATH
export PATH="./bin":$PATH
'';
};
}
)));
}
4 changes: 2 additions & 2 deletions internal/pb/fs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion internal/pb/fs_grpc.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 42a6646

Please sign in to comment.