From 77a02d8a2bb17c77af333119433ac0d8279b06e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=8D=9A=E4=BB=81=28Buo-ren=20Lin=29?= <buo.ren.lin@gmail.com> Date: Fri, 27 Sep 2024 13:54:20 +0800 Subject: [PATCH] refactor: Incorporate 3.0.2 version of the subject template(CI/CD implementation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 林博仁(Buo-ren Lin) <buo.ren.lin@gmail.com> --- .editorconfig | 7 +- .gitattributes | 8 +- .../workflows/check-potential-problems.yml | 19 +- .github/workflows/release.yml | 26 ++- .gitignore | 2 +- .gitlab-ci.yml | 2 + .gitmodules | 2 +- .pre-commit-config.yaml | 23 +- .reuse/dep5 | 11 - README.md | 103 ++++----- REUSE.toml | 19 ++ _config.yml | 1 - .../do-static-analysis.install-system-deps.sh | 206 ++++++++++++++++-- continuous-integration/do-static-analysis.sh | 6 + ...ate-build-artifacts.install-system-deps.sh | 6 +- .../generate-release-description.sh | 58 +++-- ...tch-github-actions-sudo-security-policy.sh | 143 ++++++++++++ ...low_github_actions_default_envvars.sudoers | 20 ++ continuous-integration/sudoers.d/README.md | 8 + .../upload-gitlab-generic-packages.sh | 2 +- 20 files changed, 570 insertions(+), 102 deletions(-) delete mode 100644 .reuse/dep5 create mode 100644 REUSE.toml delete mode 100644 _config.yml create mode 100755 continuous-integration/patch-github-actions-sudo-security-policy.sh create mode 100644 continuous-integration/sudoers.d/90_allow_github_actions_default_envvars.sudoers create mode 100644 continuous-integration/sudoers.d/README.md diff --git a/.editorconfig b/.editorconfig index d94dd8b..4e41e32 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,7 +20,7 @@ insert_final_newline = true trim_trailing_whitespace = true # Git configuration files uses tabs as indentation units -[.git*] +[/.git{modules,config}] indent_style = tab # Avoid git patch fail to apply due to stripped unmodified lines that contains only spaces @@ -59,3 +59,8 @@ indent_size = 2 [.*.{yml,yaml}] indent_size = 2 + +# Keep the indentation style of the license text verbatim +[/LICENSES/*] +indent_size = unset +indent_style = unset diff --git a/.gitattributes b/.gitattributes index d072302..f5ddf8d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,7 +12,13 @@ # Avoid exporting development files to release archive /.* export-ignore -/continuous-integration/ export-ignore +/continuous-integration export-ignore # Keep editorconfig for ease of editing of product files /.editorconfig -export-ignore + +# Keep REUSE DEP5 declaration file in the release archive for legal +# conformance +/.reuse/ -export-ignore +/.reuse/* export-ignore +/.reuse/dep5 -export-ignore diff --git a/.github/workflows/check-potential-problems.yml b/.github/workflows/check-potential-problems.yml index c85e79f..018c486 100644 --- a/.github/workflows/check-potential-problems.yml +++ b/.github/workflows/check-potential-problems.yml @@ -9,7 +9,9 @@ # SPDX-License-Identifier: CC-BY-SA-4.0 name: 檢查專案中的潛在問題 on: - - push + push: + branches: + - '**' jobs: check-using-precommit: name: 使用 pre-commit 檢查專案中的潛在問題 @@ -17,6 +19,7 @@ jobs: env: PIP_CACHE_DIR: ${{ github.workspace }}/.cache/pip PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit + SHELLCHECK_DIR: ${{ github.workspace }}/.cache/shellcheck-stable steps: - name: 自版控庫取出內容 uses: actions/checkout@v4 @@ -34,7 +37,19 @@ jobs: key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} path: ${{ env.PRE_COMMIT_HOME }} - - name: Running static analysis program + - name: >- + Configure pre-built ShellCheck cache to speed up continuous integration + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-${{ runner.arch }}-shellcheck + path: ${{ env.SHELLCHECK_DIR }} + + - name: >- + Patch the sudo security policy so that programs run via sudo + will recognize environment variables predefined by GitHub + run: sudo ./continuous-integration/patch-github-actions-sudo-security-policy.sh + + - name: Run the static analysis programs run: | sudo ./continuous-integration/do-static-analysis.install-system-deps.sh ./continuous-integration/do-static-analysis.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe95e42..48c3f2b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,8 +18,27 @@ jobs: name: Release product and their build aritfacts runs-on: ubuntu-20.04 steps: - - name: Checkout content from the Git repository + - name: Check out content from the Git repository uses: actions/checkout@v4 + with: + # Increase fetch depth if you may have more than this amount + # of revisions between releases + fetch-depth: 100 + + # Fetch tags as well to generate detailed changes between two releases + # WORKAROUND: Adding this option triggers actions/checkout#1467 + #fetch-tags: true + + - name: >- + WORKAROUND: Fetch tags that points to the revisions + checked-out(actions/checkout#1467) + run: |- + git fetch \ + --prune \ + --prune-tags \ + --force \ + --depth=100 \ + --no-recurse-submodules - name: Determine the project identifier run: printf "project_id=${GITHUB_REPOSITORY##*/}\\n" >> $GITHUB_ENV @@ -33,6 +52,11 @@ jobs: - name: Determine the release identifier run: printf "release_id=${project_id}-${release_version}\\n" >> $GITHUB_ENV + - name: >- + Patch the sudo security policy so that programs run via sudo + will recognize environment variables predefined by GitHub + run: sudo ./continuous-integration/patch-github-actions-sudo-security-policy.sh + - name: Generate the release archive run: |- sudo ./continuous-integration/generate-build-artifacts.install-system-deps.sh diff --git a/.gitignore b/.gitignore index 4647fc3..60348c7 100644 --- a/.gitignore +++ b/.gitignore @@ -86,5 +86,5 @@ # Don't track GNU gettext machine-readable message catalogs *.mo -# Don't track continuous intregration virtual environments +# Don't track continuous integration virtual environments /continuous-integration/venv/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a0e67e..3291a62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,12 +16,14 @@ do-static-analysis: variables: PIP_CACHE_DIR: ${CI_PROJECT_DIR}/.cache/pip PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit + SHELLCHECK_DIR: ${CI_PROJECT_DIR}/.cache/shellcheck-stable cache: # Enable per-job and per-branch caching key: $CI_JOB_NAME-$CI_COMMIT_REF_SLUG paths: - ${PIP_CACHE_DIR} - ${PRE_COMMIT_HOME} + - ${SHELLCHECK_DIR} script: - ./continuous-integration/do-static-analysis.install-system-deps.sh diff --git a/.gitmodules b/.gitmodules index e2e3f4c..5b18932 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ # * Git - gitmodules Documentation # https://git-scm.com/docs/gitmodules # -# Copyright 2021 林博仁(Buo-ren, Lin) <Buo.Ren.Lin@gmail.com> +# Copyright 2024 林博仁(Buo-ren Lin) <buo.ren.lin@gmail.com> # SPDX-License-Identifier: CC-BY-SA-4.0 [submodule "Docker"] path = Docker diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 266efa5..741f6db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,13 +42,34 @@ repos: # Check REUSE compliance # https://reuse.software/ - repo: https://github.com/fsfe/reuse-tool - rev: v1.0.0 + rev: v4.0.3 hooks: - id: reuse + # Check shell scripts with ShellCheck + # NOTE: ShellCheck must be available in the command search PATHs + # https://www.shellcheck.net/ + # https://github.com/jumanjihouse/pre-commit-hooks#shellcheck + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: shellcheck + # Check YAML files # https://github.com/adrienverge/yamllint - repo: https://github.com/adrienverge/yamllint rev: v1.30.0 hooks: - id: yamllint + + # Check EditorConfig style compliance + # https://github.com/editorconfig-checker/editorconfig-checker.python + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: 2.7.3 + hooks: + - id: editorconfig-checker + alias: ec + exclude: | + (?ix)^( + LICENSES/.* + )$ diff --git a/.reuse/dep5 b/.reuse/dep5 deleted file mode 100644 index 778dd19..0000000 --- a/.reuse/dep5 +++ /dev/null @@ -1,11 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: 《自由知識協作平台》「虛擬化」主題 -Upstream-Contact: 議題 · 自由知識協作平台 Libre Knowledge Collaboration Platform / 虛擬化 Virtualization · GitLab <https://gitlab.com/libre-knowledge/virtualization/-/issues> -Source: https://gitlab.com/libre-knowledge/virtualization - -Files: - *README.md - */README.md - _config.yml -Copyright: 2023 自由知識協作平台貢獻者 <https://gitlab.com/libre-knowledge/libre-knowledge/-/issues> -License: CC-BY-SA-4.0 diff --git a/README.md b/README.md index 3840aa7..ac50abd 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ -# 虛擬化 Virtualization +# 虛擬化<br>Virtualization 將一個或多個實體資源,如硬體、作業系統、應用程式或資料儲存,透過軟體技術的手段,將其抽象化、隔離化、統合化及自動化,形成一個或多個虛擬的資源,讓使用者可以彈性地使用和管理這些虛擬化的資源,並且可以在不影響其他虛擬化資源的情況下進行管理和配置 <https://gitlab.com/libre-knowledge/virtualization> -[![GitLab CI 持續整合流程狀態標章](https://gitlab.com/libre-knowledge/virtualization/badges/main/pipeline.svg?ignore_skipped=true "點擊查看 GitLab CI 持續整合流程的運行狀態")](https://gitlab.com/libre-knowledge/virtualization/-/commits/main) [![「檢查專案中的潛在問題」GitHub Actions 作業流程狀態標章](https://github.com/libre-knowledge/virtualization/actions/workflows/check-potential-problems.yml/badge.svg "本專案使用 GitHub Actions 自動化檢查專案中的潛在問題")](https://github.com/libre-knowledge/virtualization/actions/workflows/check-potential-problems.yml) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white "本專案使用 pre-commit 檢查專案中的潛在問題")](https://github.com/pre-commit/pre-commit) [![REUSE 規範遵從狀態標章](https://api.reuse.software/badge/github.com/libre-knowledge/virtualization "本專案遵從 REUSE 規範降低軟體授權合規成本")](https://api.reuse.software/info/github.com/libre-knowledge/virtualization) +[![GitLab CI 持續整合流程狀態標章](https://gitlab.com/libre-knowledge/virtualization/badges/main/pipeline.svg?ignore_skipped=true "點擊查看 GitLab CI 持續整合流程的運行狀態")](https://gitlab.com/libre-knowledge/virtualization/-/commits/main) [![「檢查專案中的潛在問題」GitHub Actions 作業流程狀態標章](https://github.com/libre-knowledge/virtualization/actions/workflows/check-potential-problems.yml/badge.svg "本專案使用 GitHub Actions 自動化檢查專案中的潛在問題")](https://github.com/libre-knowledge/virtualization/actions/workflows/check-potential-problems.yml) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white "本專案使用 pre-commit 檢查專案中的潛在問題")](https://github.com/pre-commit/pre-commit) [![REUSE 規範遵從狀態標章](https://api.reuse.software/badge/gitlab.com/libre-knowledge/virtualization "本專案遵從 REUSE 規範降低軟體授權合規成本")](https://api.reuse.software/info/gitlab.com/libre-knowledge/virtualization) ## 基本概念 +以下列舉本主題相關的基本概念說明資源: + ### 主端<br>Host 運行虛擬化解決方案的主機,負責提供運算資源給[客端](#客端-guest)使用,一個虛擬化主端系統可以運行多個[客端](#客端-guest)環境 @@ -15,17 +17,17 @@ 於[主端](#主端-host)上運行的虛擬運算環境,與主端系統有一定程度的隔離,可以存取的資源受限 -### 虛擬化監管器<br>Hypervisor / Virtual Machine Monitor(VMM) +### 虛擬機監管器<br>Virtual Machine Hypervisor / Virtual Machine Monitor(VMM) 於[主端](#主端-host)運算環境運行,管理並分配硬體資源給[客端](#客端-guest)的系統或應用 -### 原生/裸機型虛擬化監管器(第一類虛擬化監管器)<br>Native/Bare-metal hypervisor(TYPE-1 Hypervisor) +### 原生/裸機型虛擬機監管器(第一類虛擬機監管器)<br>Native/Bare-metal hypervisor(TYPE-1 Hypervisor) -直接運行於主機硬體上(而非一般用途作業系統上)的虛擬化監管器,運行效能相對第二類虛擬化監管器較好 +直接運行於主機硬體上(而非一般用途作業系統上)的虛擬機監管器,運行效能相對第二類虛擬機監管器較好 -### 託管型虛擬化監管器(第二類虛擬化監管器)<br>Hosted hypervisor(TYPE-2 Hypervisor) +### 託管型虛擬機監管器(第二類虛擬機監管器)<br>Hosted hypervisor(TYPE-2 Hypervisor) -以應用軟體的方式運行於一般用途作業系統上的[虛擬化監管器](#虛擬化監管器-hypervisor-virtual-machine-monitor-vmm),運行效能較第一類虛擬化監管器差 +以應用軟體的方式運行於一般用途作業系統上的[虛擬機監管器](#虛擬機監管器-virtual-machine-hypervisor-virtual-machine-monitor-vmm),運行效能較第一類虛擬機監管器差 ### 硬體虛擬化<br>Hardware virtualization @@ -37,75 +39,66 @@ 由於必須要翻譯所有的客端請求為主端指令故效能較差 -相關的解決方案: - -* QEMU(未使用等[伴虛擬化](#伴虛擬化-para-virtualization)支援的情況下) -* Oracle VirtualBox(6.1 版之前非 64位元與多執行緒的客體系統虛擬化) - -### 伴虛擬化<rp>(</rp><rt>Para-virtualization</rt><rp>) +### 伴虛擬化<br>Para-virtualization 透過與[客端](#客端-guest)作業系統特別提供的界面交互來減少需要大量運算或時間才能處理的[客端](#客端-guest)操作,藉以改善虛擬化效能的技術 -需要[主端](#主端-host)[虛擬化監管器](#虛擬化監管器-hypervisor-virtual-machine-monitor-vmm)與[客端](#客端-guest)作業系統支援對應的 API 才能夠使用(如 Linux) - -相關方案: - -* VMware ESXi -* Hyper-V -* QEMU(搭配 KVM) -* Xen +需要[主端](#主端-host)[虛擬機監管器](#虛擬機監管器-virtual-machine-hypervisor-virtual-machine-monitor-vmm)與[客端](#客端-guest)作業系統支援對應的 API 才能夠使用(如 Linux) ### 硬體輔助虛擬化<br>Hardware-assisted virtualization 由中央處理器、晶片組等硬體提供,將虛擬機器的部份工作取代以提高虛擬機器運行效能的虛擬化解決方案 -須硬體、韌體與[虛擬化監管器](#虛擬化監管器-hypervisor-virtual-machine-monitor-vmm)支援 +須硬體、韌體與[虛擬機監管器](#虛擬機監管器-virtual-machine-hypervisor-virtual-machine-monitor-vmm)支援 -相關硬體解決方案: +### 作業系統層級虛擬化<br>Operating system level virtualization -* VT-x/AMD-V -* VT-d -* VT-c -* AMD-Si -* Extended Page Tables(Second Level Address Translation (SLAT)) +透過抽象化作業系統的功能與命名空間等隔離技術使多個客端作業系統可以與主端作業系統共用作業系統核心並同行運行的技術 -相關虛擬化監管器解決方案: +## 解決方案 -* KVM -* QEMU(搭配 KVM) -* XEN(搭配 KVM) -* Oracle VirtualBox +以下列舉本主題相關的解決方案: -### 作業系統層級虛擬化<br>Operating system level virtualization +### 虛擬機監管器 -透過抽象化作業系統的功能與命名空間等隔離技術使多個客端作業系統可以與主端作業系統共用作業系統核心並同行運行的技術 +* ESXi + 由 VMware 推出之支援[伴虛擬化](#伴虛擬化-para-virtualization)的[原生/裸機型虛擬機監管器](#原生裸機型虛擬機監管器第一類虛擬機監管器nativebare-metal-hypervisortype-1-hypervisor)解決方案 +* Hyper-V + 由微軟所推出之支援[伴虛擬化](#伴虛擬化-para-virtualization)的[原生/裸機型虛擬機監管器](#原生裸機型虛擬機監管器第一類虛擬機監管器nativebare-metal-hypervisortype-1-hypervisor)解決方案 +* QEMU + 支援[全虛擬化](#全虛擬化-full-virtualization)與(搭配 KVM)[伴虛擬化](#伴虛擬化-para-virtualization)的[託管型虛擬機監管器](#託管型虛擬機監管器第二類虛擬機監管器hosted-hypervisortype-2-hypervisor)解決方案 +* VirtualBox + 由<ruby>甲骨文<rp>(</rp><rt>Oracle</rt><rp>)</rp></ruby>公司推出之具備使用者友善的圖形化操作界面、支援[伴虛擬化](#伴虛擬化-para-virtualization)的[託管型虛擬機監管器](#託管型虛擬機監管器第二類虛擬機監管器hosted-hypervisortype-2-hypervisor)解決方案 +* Xen + 支援[全虛擬化](#全虛擬化-full-virtualization)與(搭配 KVM)[伴虛擬化](#伴虛擬化-para-virtualization)的開放來源碼[原生/裸機型虛擬機監管器](#原生裸機型虛擬機監管器第一類虛擬機監管器nativebare-metal-hypervisortype-1-hypervisor)解決方案 -相關方案: +### 作業系統層級虛擬化 -* Docker +* [Docker](https://gitlab.com/libre-knowledge/docker) + 主流的容器實現之一 * Kubernetes * Podman * LXC * LXD -## 解決方案 +### 硬體輔助虛擬化 -* [Docker](https://gitlab.com/libre-knowledge/docker) - 主流的容器實現之一 -* LXC -* LXD -* Oracle VirtualBox -* Proxmox VE -* QEMU -* Xen +* VT-x/AMD-V +* VT-d +* VT-c +* AMD-Si +* Extended Page Tables(Second Level Address Translation (SLAT)) -### 開發方案 +### 虛擬化資源編排工具 +* Docker Compose * Vagrant 開發環境建置與統合工具 快速建置相同的開發環境,避免團隊成員間環境不一致造成開發阻礙 ## 參考資料 +以下列舉撰寫本主題內容時所參考的第三方資源: + * [虛擬化 - 維基百科,自由的百科全書](https://zh.wikipedia.org/zh-tw/%E8%99%9B%E6%93%AC%E5%8C%96) [Virtualization - Wikipedia](https://en.wikipedia.org/wiki/Virtualization) 維基百科條目 @@ -119,9 +112,21 @@ * [QEMU - Wikipedia](https://en.wikipedia.org/wiki/QEMU) * [Hardware virtualization - Wikipedia](https://en.wikipedia.org/wiki/Hardware_virtualization) * [3.3. Hardware vs. Software Virtualization](https://docs.oracle.com/en/virtualization/virtualbox/6.0/admin/hwvirt.html) +* [Understanding the Virtualization Spectrum - Xen](https://wiki.xenproject.org/wiki/Understanding_the_Virtualization_Spectrum) + 說明全虛擬化與伴虛擬化的差異,與 Xen 所支援的實作方式 + +<!-- +(待補) + +## 子主題 + +以下列舉本主題相關的主題: + +(待補) +--> --- -本主題為[自由知識協作平台](https://libre-knowledge.github.io/)的一部分,除部份特別標註之經合理使用(fair use)原則使用的內容外允許公眾於授權範圍內自由使用 +本主題為[自由知識協作平台](https://gitlab.com/libre-knowledge/libre-knowledge)的一部分,除部份特別標註之經合理使用(fair use)原則使用的內容外允許公眾於授權範圍內自由使用 -如有任何問題,歡迎於本主題的[議題追蹤系統](https://github.com/libre-knowledge/virtualization/-/issues)創建新議題反饋 +如有任何問題,歡迎於本主題的[議題追蹤系統](https://gitlab.com/libre-knowledge/virtualization/-/issues)創建新議題反饋 diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 0000000..557d181 --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,19 @@ +# REUSE licensing information association file +# +# References: +# +# * REUSE.toml | REUSE Specification – Version 3.2 | REUSE +# https://reuse.software/spec-3.2/#reusetoml +# +# Copyright 2024 林博仁(Buo-ren Lin) <buo.ren.lin@gmail.com> +# SPDX-License-Identifier: CC-BY-SA-4.0+ + +version = 1 + +[[annotations]] +path = [ + "*.README.md", + "**/README.md" +] +SPDX-FileCopyrightText = '2024 自由知識協作平台貢獻者 <https://gitlab.com/libre-knowledge/libre-knowledge/-/issues>' +SPDX-License-Identifier = 'CC-BY-SA-4.0+' diff --git a/_config.yml b/_config.yml deleted file mode 100644 index f980e76..0000000 --- a/_config.yml +++ /dev/null @@ -1 +0,0 @@ -theme: jekyll-theme-slate diff --git a/continuous-integration/do-static-analysis.install-system-deps.sh b/continuous-integration/do-static-analysis.install-system-deps.sh index 3dce3c3..d633082 100755 --- a/continuous-integration/do-static-analysis.install-system-deps.sh +++ b/continuous-integration/do-static-analysis.install-system-deps.sh @@ -7,6 +7,29 @@ set \ -o errexit \ -o nounset +if test -v BASH_SOURCE; then + # Convenience variables + # shellcheck disable=SC2034 + { + script="$( + realpath \ + --strip \ + "${BASH_SOURCE[0]}" + )" + script_dir="${script%/*}" + script_filename="${script##*/}" + script_name="${script_filename%%.*}" + } +fi + +trap_exit(){ + if test -v temp_dir \ + && test -e "${temp_dir}"; then + rm -rf "${temp_dir}" + fi +} +trap trap_exit EXIT + if test "${EUID}" -ne 0; then printf \ 'Error: This program should be run as the superuser(root) user.\n' \ @@ -14,6 +37,29 @@ if test "${EUID}" -ne 0; then exit 1 fi +project_dir="$(dirname "${script_dir}")" +cache_dir="${project_dir}/.cache" + +if ! test -e "${cache_dir}"; then + install_opts=( + --directory + ) + if test -v SUDO_USER; then + # Configure same user as the running environment to avoid access + # problems afterwards + install_opts+=( + --owner "${SUDO_USER}" + --group "${SUDO_GID}" + ) + fi + if ! install "${install_opts[@]}" "${cache_dir}"; then + printf \ + 'Error: Unable to create the cache directory.\n' \ + 1>&2 + exit 2 + fi +fi + apt_archive_cache_mtime_epoch="$( stat \ --format=%Y \ @@ -35,24 +81,24 @@ fi # Silence warnings regarding unavailable debconf frontends export DEBIAN_FRONTEND=noninteractive -if ! test -v CI; then - base_runtime_dependency_pkgs=( - wget - ) - if ! dpkg -s "${base_runtime_dependency_pkgs[@]}" &>/dev/null; then +base_runtime_dependency_pkgs=( + wget +) +if ! dpkg -s "${base_runtime_dependency_pkgs[@]}" &>/dev/null; then + printf \ + 'Info: Installing base runtime dependency packages...\n' + if ! \ + apt-get install \ + -y \ + "${base_runtime_dependency_pkgs[@]}"; then printf \ - 'Info: Installing base runtime dependency packages...\n' - if ! \ - apt-get install \ - -y \ - "${base_runtime_dependency_pkgs[@]}"; then - printf \ - 'Error: Unable to install the base runtime dependency packages.\n' \ - 1>&2 - exit 2 - fi + 'Error: Unable to install the base runtime dependency packages.\n' \ + 1>&2 + exit 2 fi +fi +if ! test -v CI; then printf \ 'Info: Detecting local region code...\n' wget_opts=( @@ -136,11 +182,18 @@ if ! test -v CI; then fi runtime_dependency_pkgs=( + # For matching the ShellCheck version string + grep + git python3-minimal python3-pip python3-venv + + # For extracting prebuilt ShellCheck software archive + tar + xz-utils ) if ! dpkg -s "${runtime_dependency_pkgs[@]}" &>/dev/null; then printf \ @@ -154,5 +207,128 @@ if ! dpkg -s "${runtime_dependency_pkgs[@]}" &>/dev/null; then fi fi +shellcheck_dir="${cache_dir}/shellcheck-stable" + +if ! test -e "${shellcheck_dir}/shellcheck"; then + printf \ + "Info: Determining the host machine's hardware architecture...\\n" + if ! arch="$(arch)"; then + printf \ + "Error: Unable to determine the host machine's hardware architecture.\\n" \ + 1>&2 + exit 1 + fi + + printf \ + 'Info: Checking ShellCheck architecure availability...\n' + case "${arch}" in + x86_64|armv6hf|aarch64) + # Assuming the ShellCheck architecture is the same, which + # is probably incorrect... + shellcheck_arch="${arch}" + ;; + *) + printf \ + 'Error: Unsupported ShellCheck architecture "%s".\n' \ + "${arch}" \ + 1>&2 + exit 1 + ;; + esac + + printf \ + 'Info: Determining the ShellCheck prebuilt archive details...\n' + prebuilt_shellcheck_archive_url="https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.${shellcheck_arch}.tar.xz" + prebuilt_shellcheck_archive_filename="${prebuilt_shellcheck_archive_url##*/}" + + printf \ + 'Info: Creating the temporary directory for storing downloaded files...\n' + mktemp_opts=( + --directory + --tmpdir + ) + if ! temp_dir="$( + mktemp \ + "${mktemp_opts[@]}" \ + "${script_name}.XXXXXX" + )"; then + printf \ + 'Error: Unable to create the temporary directory for storing downloaded files.\n' \ + 1>&2 + exit 2 + fi + + printf \ + 'Info: Downloading the prebuilt ShellCheck software archive...\n' + downloaded_prebuilt_shellcheck_archive="${temp_dir}/${prebuilt_shellcheck_archive_filename}" + wget_opts=( + --output-document "${downloaded_prebuilt_shellcheck_archive}" + ) + if ! \ + wget \ + "${wget_opts[@]}" \ + "${prebuilt_shellcheck_archive_url}"; then + printf \ + 'Error: Unable to download the prebuilt ShellCheck software archive...\n' \ + 1>&2 + exit 2 + fi + + printf \ + 'Info: Extracting the prebuilt ShellCheck software archive...\n' + tar_opts=( + --extract + --verbose + --directory="${cache_dir}" + --file="${downloaded_prebuilt_shellcheck_archive}" + ) + if test -v SUDO_USER; then + # Configure same user as the running environment to avoid access + # problems afterwards + tar_opts+=( + --owner="${SUDO_USER}" + --group="${SUDO_GID}" + ) + fi + if ! tar "${tar_opts[@]}"; then + printf \ + 'Error: Unable to extract the prebuilt ShellCheck software archive.\n' \ + 1>&2 + exit 2 + fi +fi + +printf \ + 'Info: Setting up the command search PATHs so that the locally installed shellcheck command can be located...\n' +PATH="${shellcheck_dir}:${PATH}" + +printf \ + 'Info: Querying the ShellCheck version...\n' +if ! shellcheck_version_raw="$(shellcheck --version)"; then + printf \ + 'Error: Unable to query the ShellCheck version.\n' \ + 1>&2 + exit 2 +fi + +grep_opts=( + --perl-regexp + --only-matching +) +if ! shellcheck_version="$( + grep \ + "${grep_opts[@]}" \ + '(?<=version: ).*' \ + <<<"${shellcheck_version_raw}" + )"; then + printf \ + 'Error: Unable to parse out the ShellCheck version string.\n' \ + 1>&2 +fi + +printf \ + 'Info: ShellCheck version is "%s".\n' \ + "${shellcheck_version}" + printf \ 'Info: Operation completed without errors.\n' diff --git a/continuous-integration/do-static-analysis.sh b/continuous-integration/do-static-analysis.sh index 3424661..6055e74 100755 --- a/continuous-integration/do-static-analysis.sh +++ b/continuous-integration/do-static-analysis.sh @@ -19,6 +19,8 @@ if ! script="$( fi script_dir="${script%/*}" +project_dir="$(dirname "${script_dir}")" +cache_dir="${project_dir}/.cache" if ! test -e "${script_dir}/venv"; then printf \ @@ -53,6 +55,10 @@ if ! pip show pre-commit &>/dev/null; then fi fi +printf \ + 'Info: Setting up the command search PATHs so that the installed shellcheck command can be located...\n' +PATH="${cache_dir}/shellcheck-stable:${PATH}" + printf \ 'Info: Running pre-commit...\n' if ! \ diff --git a/continuous-integration/generate-build-artifacts.install-system-deps.sh b/continuous-integration/generate-build-artifacts.install-system-deps.sh index 9122d7f..e7ac480 100755 --- a/continuous-integration/generate-build-artifacts.install-system-deps.sh +++ b/continuous-integration/generate-build-artifacts.install-system-deps.sh @@ -14,12 +14,12 @@ required_commands=( realpath ) flag_dependency_check_failed=false -for required_command in "${required_commands[@]}"; do - if ! command -v "${required_command}" >/dev/null; then +for command in "${required_commands[@]}"; do + if ! command -v "${command}" >/dev/null; then flag_dependency_check_failed=true printf \ 'Error: Unable to locate the "%s" command in the command search PATHs.\n' \ - "${required_command}" \ + "${command}" \ 1>&2 fi done diff --git a/continuous-integration/generate-release-description.sh b/continuous-integration/generate-release-description.sh index 32fc7d7..1b670b4 100755 --- a/continuous-integration/generate-release-description.sh +++ b/continuous-integration/generate-release-description.sh @@ -21,69 +21,99 @@ fi script_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)" project_dir="${script_dir%/*}" -printf \ - 'Info: Determining release description...\n' -git_tag_list="$(git tag --list)" -git_tag_count="$(wc -l <<<"${git_tag_list}")" +printf 'Info: Querying the list of the release tag(s)...\n' +if ! git_tag_list="$(git tag --list 'v*')"; then + printf \ + 'Error: Unable to query the list of the release tag(s).\n' \ + 1>&2 + exit 2 +fi + +printf 'Info: Counting the release tags...\n' +if ! git_tag_count="$(wc -l <<<"${git_tag_list}")"; then + printf \ + 'Error: Unable to count the release tags.\n' \ + 1>&2 + exit 2 +fi detailed_changes_markup="## Detailed changes"$'\n\n' -git_log_opts=( - --format='format:* %s (%h) - %an' -) if test -v CI_COMMIT_TAG; then release_tag="${CI_COMMIT_TAG}" fi if test "${git_tag_count}" -eq 1; then + printf \ + 'Info: Only one release tag was detected, generating the release description text from the very beginning to the "%s" release tag...\n' \ + "${release_tag}" if ! detailed_changes_markup+="$( git log \ "${git_log_opts[@]}" \ "${release_tag}" )"; then printf \ - 'Error: Unable to generate the commit list from Git.\n' \ + 'Error: Unable to generate the release description text from Git.\n' \ 1>&2 exit 2 fi else + printf \ + 'Info: Multiple release tags were detected, determining the previous release tag...\n' + printf \ + 'Info: Version-sorting the release tag list...\n' if ! sorted_git_tag_list="$( sort \ -V \ <<<"${git_tag_list}" )"; then printf \ - 'Error: Unable to version-sort the Git tag list.\n' \ + 'Error: Unable to version-sort the release tag list.\n' \ 1>&2 exit 2 fi + + printf \ + 'Info: Filtering out the two latest release tags from the release tag list...\n' if ! latest_two_git_tags="$( tail \ -n 2 \ <<<"${sorted_git_tag_list}" )"; then printf \ - 'Error: Unable to filter out the two latest tags from the Git tag list.\n' \ + 'Error: Unable to filter out the two latest release tags from the release tag list.\n' \ 1>&2 exit 2 fi + + printf \ + 'Info: Filtering out the previous release tag from the two latest release tags...\n' if ! previous_git_tag="$( head \ -n 1 \ <<<"${latest_two_git_tags}" )"; then printf \ - 'Error: Unable to filter out the previous release tag from the two latest Git tags.\n' \ + 'Error: Unable to filter out the previous release tag from the two latest release tags.\n' \ 1>&2 exit 2 fi + + printf \ + 'Info: Generating the release description text from the previous release tag(%s) to the current release tag(%s)...\n' \ + "${previous_git_tag}" \ + "${release_tag}" \ + 1>&2 + git_log_opts=( + --format='format:* %s (%h) - %an' + ) if ! detailed_changes_markup+="$( git log \ "${git_log_opts[@]}" \ "${previous_git_tag}..${release_tag}" )"; then printf \ - 'Error: Unable to generate the Git commit list between the "%s" tag and the "%s" tag.\n' \ + 'Error: Unable to generate the release description text from the previous release tag(%s) to the current release tag(%s).\n' \ "${previous_git_tag}" \ "${release_tag}" \ 1>&2 @@ -97,9 +127,9 @@ printf \ "${detailed_changes_file}" if ! \ printf \ - '%s' \ + '%s\n' \ "${detailed_changes_markup}" \ - >"${detailed_changes_file}"; then + | tee "${detailed_changes_file}"; then printf \ 'Error: Unable to write the detailed changes markup to the "%s" file.\n' \ "${detailed_changes_file}" \ diff --git a/continuous-integration/patch-github-actions-sudo-security-policy.sh b/continuous-integration/patch-github-actions-sudo-security-policy.sh new file mode 100755 index 0000000..1cbf30b --- /dev/null +++ b/continuous-integration/patch-github-actions-sudo-security-policy.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Patch the sudo security policy so that the environment variables +# defined in the GitHub Actions CI environment would be inherited in +# processes run using sudo +# +# References: +# +# * Including other files from within sudoers | Sudoers Manual | Sudo +# https://www.sudo.ws/docs/man/sudoers.man/#Including_other_files_from_within_sudoers +# +# Copyright 2023 林博仁(Buo-ren, Lin) <buo.ren.lin@gmail.com> +# SPDX-License-Identifier: CC-BY-SA-4.0 + +# Configure the interpreter behavior to bail out during problematic +# situations +set \ + -o errexit \ + -o nounset + +required_commands=( + install + realpath + + # For checking the validity of the sudoers file + visudo +) +flag_dependency_check_failed=false +for required_command in "${required_commands[@]}"; do + if ! command -v "${required_command}" >/dev/null; then + flag_dependency_check_failed=true + printf \ + 'Error: Unable to locate the "%s" command in the command search PATHs.\n' \ + "${required_command}" \ + 1>&2 + fi +done +if test "${flag_dependency_check_failed}" == true; then + printf \ + 'Error: Dependency check failed, please check your installation.\n' \ + 1>&2 + exit 1 +fi + +if test "${EUID}" -ne 0; then + printf \ + 'Error: This program is required to be run as the superuser(root).\n' \ + 1>&2 + exit 1 +fi + +if test -v BASH_SOURCE; then + # Convenience variables + # shellcheck disable=SC2034 + { + script="$( + realpath \ + --strip \ + "${BASH_SOURCE[0]}" + )" + script_dir="${script%/*}" + } +fi + +ci_dir="${script_dir}" +sudoers_dropin_dir="${ci_dir}/sudoers.d" + +sudoers_dropin_dir_system=/etc/sudoers.d +if ! test -e "${sudoers_dropin_dir_system}"; then + printf \ + 'Info: Creating the sudoers drop-in directory...\n' + install_opts=( + --directory + --owner=root + --group=root + --mode=0755 + --verbose + ) + if ! \ + install \ + "${install_opts[@]}" \ + "${sudoers_dropin_dir_system}"; then + printf \ + 'Error: Unable to create the sudoers drop-in directory.\n' \ + 1>&2 + exit 2 + fi +fi + +for dropin_file in "${sudoers_dropin_dir}/"*.sudoers; do + dropin_filename="${dropin_file##*/}" + + printf \ + 'Info: Validating the "%s" sudoers drop-in file...\n' \ + "${dropin_filename}" + visudo_opts=( + # Enable check-only mode + --check + + # Specify an alternate sudoers file location + --file="${dropin_file}" + + # NOTE: We don't use --quiet as it will also inhibit the syntax + # error messages, dump the stdout stream instead + #--quiet + ) + if ! visudo "${visudo_opts[@]}" >/dev/null; then + printf \ + 'Error: Syntax validation failed for the "%s" sudoers drop-in file.\n' \ + "${dropin_filename}" \ + 1>&2 + exit 2 + fi + + printf \ + 'Info: Installing the "%s" sudoers drop-in file...\n' \ + "${dropin_filename}" + + # sudo will not accept filename with the period symbol in the + # filename, strip the convenicence filename suffix + dropin_filename_without_suffix="${dropin_filename%.sudoers}" + + installed_dropin_file="${sudoers_dropin_dir_system}/${dropin_filename_without_suffix}" + install_opts=( + --owner=root + --group=root + --mode=0644 + --verbose + ) + if ! \ + install \ + "${install_opts[@]}" \ + "${dropin_file}" \ + "${installed_dropin_file}"; then + printf \ + 'Error: Unable to install the sudoers drop-in configuration file "%s".\n' \ + "${dropin_filename}" \ + 1>&2 + exit 2 + fi +done + +printf \ + 'Info: Operation completed without errors.\n' diff --git a/continuous-integration/sudoers.d/90_allow_github_actions_default_envvars.sudoers b/continuous-integration/sudoers.d/90_allow_github_actions_default_envvars.sudoers new file mode 100644 index 0000000..626dd1f --- /dev/null +++ b/continuous-integration/sudoers.d/90_allow_github_actions_default_envvars.sudoers @@ -0,0 +1,20 @@ +# Allow exposing GitHub Actions default environment variables to the invoked superuser processes +# +# References: +# +# * Command environment | Sudoers Manual | Sudo +# https://www.sudo.ws/docs/man/sudoers.man/#Command_environment +# * Default environment - variablesVariables - GitHub Docs +# https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables +# +# Copyright 2023 林博仁(Buo-ren, Lin) <Buo.Ren.Lin@gmail.com> +# SPDX-License-Identifier: CC-BY-SA-4.0 + +# Whether we are in an CI environment +Defaults env_keep += "CI" + +# Variables defined by GitHub +Defaults env_keep += "GITHUB_*" + +# Variables defined by the GitHub Action runners +Defaults env_keep += "RUNNER_*" diff --git a/continuous-integration/sudoers.d/README.md b/continuous-integration/sudoers.d/README.md new file mode 100644 index 0000000..87dffa0 --- /dev/null +++ b/continuous-integration/sudoers.d/README.md @@ -0,0 +1,8 @@ +# sudoers.d + +The drop-in configuration files for [the Sudoers configuration file](https://www.sudo.ws/docs/man/sudoers.man/) + +## References + +* [Sudoers Manual | Sudo](https://www.sudo.ws/docs/man/sudoers.man/) +* [Including other files from within sudoers | Sudoers Manual | Sudo](https://www.sudo.ws/docs/man/sudoers.man/#Including_other_files_from_within_sudoers) diff --git a/continuous-integration/upload-gitlab-generic-packages.sh b/continuous-integration/upload-gitlab-generic-packages.sh index 97e7639..2e7268b 100755 --- a/continuous-integration/upload-gitlab-generic-packages.sh +++ b/continuous-integration/upload-gitlab-generic-packages.sh @@ -25,7 +25,7 @@ script_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)" project_dir="${script_dir%/*}" for file in "${project_dir}/${CI_PROJECT_NAME}-"*; do - if test "${file}" == "${project_dir}/${CI_PROJECT_NAME}-*"; then + if test "${file}" = "${project_dir}/${CI_PROJECT_NAME}-*"; then # No release packages are found, avoid missing file error break fi