Merge pull request #30 from peter-mount/kernelUpdate

Kernel update

.dockerignore

@@ -0,0 +1,3 @@
+# Exclude absolutely everything then negate rules on what we want to be sent to docker for the build
+*
+!builds

.gitignore

@@ -18,7 +18,6 @@ docs
 Dockerfile.*
 
 *.iml
-Dockerfile.*
 config.yaml
 .idea
 go.sum

Docker.include

@@ -0,0 +1,115 @@
+# Makefile extensions for docker containers
+
+DOCKER_IMAGE 	?= test:latest
+DOCKER_BUILDER	?= builder
+
+BUILDX			= docker buildx
+
+DOCKER-RM	 	= $(call cmd,"BLDX RM",$(DOCKER_BUILDER)); $(BUILDX) rm $(DOCKER_BUILDER) || true
+
+DOCKER-CREATE 	= $(call cmd,"BLDX CREATE",$(DOCKER_BUILDER));\
+				  $(BUILDX) inspect $(DOCKER_BUILDER) >/dev/null 2>&1 || $(BUILDX) create --name $(DOCKER_BUILDER) --driver docker-container --bootstrap
+
+DOCKER-SWITCH 	= $(call cmd,"BLDX USE",$(DOCKER_BUILDER));\
+ 				  $(BUILDX) use $(DOCKER_BUILDER)
+
+DOCKER-BUILDX	= $(call cmd,"BLDX BUILD",$1);\
+				  $(BUILDX) build --platform $2 -t $1 --push .
+
+targets-init += docker-init
+.PHONY: docker-init
+docker-init:
+	$(DOCKER-CREATE)
+	$(DOCKER-SWITCH)
+
+targets-real-clean += docker-real-clean
+.PHONY: docker-real-clean
+docker-real-clean:
+	$(DOCKER-RM)
+
+targets-dist += docker-dist
+.PHONY: docker-tools
+docker-dist: init
+	$(call DOCKER-BUILDX,$(DOCKER-TAG),"$(shell $(BUILDX) inspect $(DOCKER_BUILDER) | grep Platforms | cut -f2- -d ':'|sed -e "s/ //g")",".")
+
+docker-version = $(shell printf '%02d' $(shell echo "$1" | tr . ' ' | sed -e "s/ 0*/ /g") 2>/dev/null)
+
+resolve-platforms = resolve-docker-platforms
+.PHONY: resolve-docker-platforms
+resolve-docker-platforms:
+ifeq ("$(PLATFORMS)","")
+	$(call cmd,"BLDX INSPCT",$(DOCKER_BUILDER))
+	$(eval DISC_PLATFORMS=)
+	$(eval export PLATFORMS=$(foreach PLATFORM,\
+		$(shell $(BUILDX) inspect $(DOCKER_BUILDER) | grep Platforms | cut -f2- -d ':'|sed -e "s/ //g" -e "s/,/ /g"),\
+		$(eval GOOS=$(word 1,$(subst /, ,$(PLATFORM))))\
+		$(eval GOARCH=$(word 2,$(subst /, ,$(PLATFORM))))\
+		$(eval GOARM=$(shell echo -n $(word 3,$(subst /, ,$(PLATFORM))) | sed -e "s/v2//" -e "s/v//" ))\
+		$(eval DISC_PLATFORMS=$(DISC_PLATFORMS) $(GOOS):$(GOARCH):$(GOARM)) \
+	))
+	$(eval export PLATFORMS=$(DISC_PLATFORMS))
+endif
+
+# Verify qemu is correct
+targets-validate += validate-docker-version
+.PHONY: validate-docker-version
+validate-docker-version:
+	$(call cmd,"CHECK-VERSN","docker")
+	@if ! command -v docker >/dev/null 2>&1; then \
+		$(ECHO) "Can't find docker. Install with 'sudo apt-get install docker-ce' or docker.io.";\
+		exit 1;\
+	fi
+
+	$(eval docker_version="$(shell docker --version | cut -d' ' -f3 | tr -cd '0-9.')")
+	@if [ $(call docker-version,$(docker_version)) -lt "19" ]; then \
+		$(ECHO) "docker $(docker_version) too old. Need >= 19.03";\
+		exit 1;\
+	fi
+
+	@if [ $(shell docker version | grep Experimental: | grep -c true) -eq 0 ]; then\
+		$(ECHO) "docker experimental flag not enabled: Set with 'export DOCKER_CLI_EXPERIMENTAL=enabled'";\
+		exit 1; \
+	fi
+
+	$(call cmd,"CHECK-VERSN","kernel")
+	$(eval kernel_version="$(shell uname -r|cut -f1 -d'-')")
+	@if [ $(call docker-version,$(kernel_version)) -lt $(call docker-version,'4.8') ]]; then\
+		$(ECHO) "Kernel $(kernel_version) too old - need >= 4.8. Install a newer kernel.";\
+		exit 1; \
+	fi
+
+	$(call cmd,"CHECK-VERSN","binfmt_misc")
+	@if [ $(shell mount | grep -c /proc/sys/fs/binfmt_misc) -eq 0 ]; then\
+		$(ECHO) "/proc/sys/fs/binfmt_misc not mounted. Mount with 'sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc'";\
+		exit 1; \
+	fi
+
+	@if ! command -v update-binfmts >/dev/null 2>&1; then\
+		$(ECHO) "Can't find update-binfmts. Install with 'sudo apt-get install binfmt-support'.";\
+		exit 1; \
+	fi
+
+	$(eval binfmt_version="$(shell update-binfmts --version | cut -f2 -d' ')")
+	@if [ $(call docker-version,$(binfmt_version)) -lt $(call docker-version,"2.2") ]; then\
+		$(ECHO) "update-binfmts $(binfmt_version) too old. Need >= 2.1.7";\
+		exit 1; \
+	fi
+
+	$(call cmd,"CHECK-VERSN","qemu-aarch64")
+	@if [ ! -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]; then\
+		if [ ! -e /usr/bin/qemu-aarch64-static ]; then\
+			$(ECHO) "Missing QEMU. Install with 'sudo apt-get install qemu-user-static'.";\
+			exit 1;\
+		fi;\
+	fi
+
+	@if [ ! -e '/proc/sys/fs/binfmt_misc/qemu-aarch64' ]; then\
+		$(ECHO) 'QEMU not registered in binfmt_misc.';\
+		exit 1;\
+	fi
+
+	$(eval qemu_flags="$(shell grep flags: /proc/sys/fs/binfmt_misc/qemu-aarch64 2>/dev/null | cut -d' ' -f2)")
+	@if [ $(shell echo $(qemu_flags) | grep -c F) -eq 0 ]; then\
+		$(ECHO) 'QEMU not registered in binfmt_misc with fix-binary (F) flag.';\
+		exit 1;\
+	fi

Dockerfile

@@ -1,90 +1,5 @@
-ARG arch=amd64
-ARG goos=linux
-
-# ============================================================
-# Build container containing our pre-pulled libraries.
-# As this changes rarely it means we can use the cache between
-# building each microservice.
-FROM golang:alpine as build
-
-# The golang alpine image is missing git so ensure we have additional tools
-RUN apk add --no-cache \
-      curl \
-      git \
-      tzdata \
-      zip
-
-WORKDIR /work
-COPY go.mod .
-RUN go mod download
-
-# ============================================================
-# source container contains the source as it exists within the
-# repository.
-FROM build as source
-WORKDIR /work
-ADD . .
-
-# ============================================================
-# Run all tests in a new container so any output won't affect
-# the final build.
-FROM source as test
-RUN CGO_ENABLED=0 go test -v \
-      ./util \
-      ./darwinref \
-      ./darwind3 \
-      ./ldb \
-      ./issues
-
-# ============================================================
-# Compile the source.
-FROM source as compiler
-ARG module=
-ARG arch
-ARG goos
-ARG goarch
-ARG goarm
-WORKDIR /work
-
-# NB: CGO_ENABLED=0 forces a static build
-RUN PACKAGE=${module};\
-    if [ "$PACKAGE" = "darwintt" ];\
-    then\
-      PACKAGE="darwintimetable";\
-    fi;\
-    echo "Building ${module} as ${PACKAGE}";\
-    CGO_ENABLED=0 \
-    GOOS=${goos} \
-    GOARCH=${goarch} \
-    GOARM=${goarm} \
-    go build \
-      -o /dest/${module} \
-      ./${PACKAGE}/bin
-
-# ============================================================
-# Optional stage, upload the binaries as a tar file
-FROM compiler AS upload
-ARG uploadPath=
-ARG uploadCred=
-ARG uploadName=
-RUN if [ -n "${uploadCred}" -a -n "${uploadPath}" -a -n "${uploadName}" ] ;\
-    then \
-      cd /dest; \
-      tar cvzpf /tmp/${uploadName}.tgz * && \
-      zip /tmp/${uploadName}.zip * && \
-      curl -u ${uploadCred} --upload-file /tmp/${uploadName}.tgz ${uploadPath}/ && \
-      curl -u ${uploadCred} --upload-file /tmp/${uploadName}.zip ${uploadPath}/; \
-    fi
-
-# ============================================================
-# Finally build the final runtime container for the specific
-# microservice
-FROM alpine
-RUN apk add --no-cache \
-      curl \
-      tzdata
-
-COPY --from=compiler /dest/ /usr/bin/
-
-ENTRYPOINT ["@@entrypoint@@"]
-CMD [ "-c", "/config.yaml"]
+# syntax=docker/dockerfile:1
+FROM --platform=$BUILDPLATFORM golang:alpine AS build
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM" >/log

Go.include

@@ -0,0 +1,134 @@
+
+# Extract args from platform definition.
+# Here PLATFORM is os:arch:arm, usually arm is "" unless arch=="arm" when it
+# is then one of "5","6" or "7"
+GO-OS		=$(word 1,$(subst :, ,$(PLATFORM)))
+GO-ARCH		=$(word 2,$(subst :, ,$(PLATFORM)))
+GO-ARM		=$(word 3,$(subst :, ,$(PLATFORM)))
+GO-ARCH-DIR	=$(call GO-OS,$1)/$(call GO-ARCH,$1)$(call GO-ARM,$1)
+
+# $(call GO-BUILD,platform,destination,src)
+GO-BUILD	= $(call cmd,"GO BUILD","$(subst /, ,$(call GO-ARCH-DIR,$1)) $(shell basename $2)");\
+			  mkdir -p $(shell dirname $2);\
+			  CGO_ENABLED=0 GOOS=$(call GO-OS,$1) GOARCH=$(call GO-ARCH,$1) GOARM=$(GO-ARM,$1) \
+			  	go build \
+			  		-ldflags="-X '$(PACKAGE_PREFIX).Version=$(shell basename $2) ($(VERSION) $(subst /, ,$(call GO-ARCH-DIR,$1)) $(shell id -u -n) $(shell date))'" \
+			  		-o $2 \
+			  		$3
+
+GO-CLEAN	= $(call cmd,"GO CLEAN",$1);go clean $1
+GO-MOD		= $(call cmd,"GO MOD",$1);go mod $1
+
+# Append -test.v to GO_TEST to show status of each test.
+# Without it, only shows total time per module if they pass
+GO-TEST		= $(GO_TEST)$(call cmd,"GO TEST",$1);(cd $1;go test ./...)
+
+targets-clean += go-clean
+.PHONY: go-clean
+go-clean:
+	$(call GO-CLEAN,-testcache)
+
+# Init goInit
+targets-init += go-init
+.PHONY: go-init
+go-init:
+	$(call GO-MOD,download)
+
+# Target to run all tests, results into builds directory
+targets-test += go-test
+.PHONY: go-test
+go-test:
+	$(MKDIR) $(BUILDS)
+	$(call cmd,"GO TEST",$(BUILDS)/go-test.txt);go test ./... >$(BUILDS)/go-test.txt 2>&1 || cat $(BUILDS)/go-test.txt
+
+targets-tools += go-tools
+.PHONY: go-tools
+go-tools: $(subst /bin/main.go,,$(subst tools,$(BUILDS),$(shell ls tools/*/bin/main.go)))
+
+# Rule to build a go application
+# For this to work: the main function is in tools/<toolName>/bin/main.go
+# The compiled binary will be placed in $(BUILDS)/<os>/<arch>/
+
+# This takes precedence of the main one, it allows us to ignore a tool if the .donotbuild file exists
+$(BUILDS)/%: tools/%/bin/main.go tools/%/.donotbuild
+	$(call cmd,"IGNORE",$(shell basename $@))
+
+$(BUILDS)/%: tools/%/bin/main.go
+	$(foreach PLATFORM,$(PLATFORMS),\
+		$(call GO-BUILD,$(PLATFORM),$(BUILDS)/$(call GO-ARCH-DIR,$(PLATFORM))/$(BINDIR)$(shell basename $@),$<)${\n}\
+	)
+
+# Validates the installed version of go against the version declared in go.mod
+MINIMUM_SUPPORTED_GO_MAJOR_VERSION	= $(shell grep "^go" go.mod | cut -f2 -d' ' | cut -f1 -d'.')
+MINIMUM_SUPPORTED_GO_MINOR_VERSION	= $(shell grep "^go" go.mod | cut -f2 -d' ' | cut -f2 -d'.')
+GO_MAJOR_VERSION = $(shell go version | cut -f3 -d' ' | cut -c 3- | cut -f1 -d' ' | cut -f1 -d'.')
+GO_MINOR_VERSION = $(shell go version | cut -f3 -d' ' | cut -c 3- | cut -f1 -d' ' | cut -f2 -d'.')
+GO_VERSION_VALIDATION_ERR_MSG = Golang version $(GO_MAJOR_VERSION).$(GO_MINOR_VERSION) is not supported, please update to at least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
+targets-validate += validate-go-version
+.PHONY: validate-go-version
+validate-go-version:
+	$(call cmd,"CHECK_VERSN","go")
+	@if [ $(GO_MAJOR_VERSION) -gt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
+		exit 0 ;\
+	elif [ $(GO_MAJOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
+		$(ECHO) '$(GO_VERSION_VALIDATION_ERR_MSG)';\
+		exit 1; \
+	elif [ $(GO_MINOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MINOR_VERSION) ] ; then \
+		$(ECHO) '$(GO_VERSION_VALIDATION_ERR_MSG)';\
+		exit 1; \
+	fi
+
+# This discovers all platforms supported by the locally installed go compiler.
+# This will only expand then if the PLATFORMS environment variable was not set
+# when invoking make
+#
+# For now filter out various platforms due to:
+# incompatibilities with bbolt (missing syscalls)
+resolve-platforms = resolve-go-platforms
+.PHONY: resolve-go-platforms
+resolve-go-platforms:
+ifeq ("$(PLATFORMS)","")
+	$(eval DISC_PLATFORMS=)
+	$(foreach DISC_PLATFORM,$(shell go tool dist list), \
+		$(eval GOOS=$(word 1,$(subst /, ,$(DISC_PLATFORM)))) \
+		$(if $(filter android,$(GOOS)),,\
+			$(if $(filter ios,$(GOOS)),,\
+				$(eval GOARCH=$(word 2,$(subst /, ,$(DISC_PLATFORM)))) \
+				$(if $(filter loong64,$(GOARCH)),,\
+					$(if $(filter aix,$(GOOS)),,\
+						$(if $(filter js,$(GOOS)),,\
+							$(if $(filter plan9,$(GOOS)),,\
+								$(foreach GOARM, \
+									$(if $(filter arm,$(GOARCH)),6 7,:), \
+									$(eval DISC_PLATFORMS=$(DISC_PLATFORMS) $(GOOS):$(GOARCH):$(GOARM)) \
+								)\
+							)\
+						)\
+					)\
+				) \
+			)\
+		)\
+	)
+	$(eval export PLATFORMS=$(DISC_PLATFORMS))
+endif
+
+# Generates platforms.md based on the local go installation.
+# This does nothing other than keep that page in sync with what is currently
+# supported by go and the build system.
+platforms.md: resolve-platforms
+	$(shell ( \
+		echo "# Supported Platforms"; \
+		echo; \
+		echo "The following platforms are supported by virtue of how the build system works:"; \
+		echo; \
+		echo "| Operating System | CPU Architectures |"; \
+		echo "| ---------------- | ----------------- |"; \
+		$(foreach OS, $(shell ls $(BUILDS)), echo "| $(OS) | $(foreach ARCH,$(shell ls $(BUILDS)/$(OS)),$(ARCH)) |"; ) \
+		echo; \
+		echo "Operating Systems: $(shell ls $(BUILDS)|wc -l) CPU Architectures: $(shell ls -d $(BUILDS)/*/*| cut -f3 -d'/' | sort |uniq | wc -l)"; \
+		echo; \
+		echo "This is all non-mobile platforms supported by go version \`$(GO_MAJOR_VERSION).$(GO_MINOR_VERSION)\`" ;\
+		echo; \
+		echo "This page is automatically generated from the output of \`go tool dist list\`"; \
+	  ) >$@ \
+	)