diff --git a/hack/cache-common-inc.sh b/hack/cache-common-inc.sh index 74fdd059764..08460f05a5e 100755 --- a/hack/cache-common-inc.sh +++ b/hack/cache-common-inc.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +# print the error message and exit with status 1 +function error_exit() { + echo "Error: $*" >&2 + exit 1 +} + # e.g. # ```console # $ download_template_if_needed templates/default.yaml @@ -97,7 +103,7 @@ function size_from_location() { ) } -# Check the remote location and return the http code and size. +# Check the remote location and print the http code and size. # If GITHUB_ACTIONS is true, the result is not cached. # e.g. # ```console @@ -113,7 +119,7 @@ function check_location() { fi } -# Check the remote location and return the http code and size. +# Check the remote location and print the http code and size. # The result is cached in .check_location-response-cache.yaml # e.g. # ```console @@ -209,8 +215,7 @@ function location_to_sha256() { elif command -v shasum >/dev/null; then sha256="$(echo -n "${location}" | shasum -a 256 | cut -d' ' -f1)" else - echo "sha256sum or shasum not found" >&2 - exit 1 + error_exit "sha256sum or shasum not found" fi echo "${sha256}" ) @@ -351,16 +356,32 @@ function hash_file() { # /Users/user/Library/Caches/lima/download/by-url-sha256/346ee1ff9e381b78ba08e2a29445960b5cd31c51f896fc346b82e26e345a5b9a/data # on macOS # /home/user/.cache/lima/download/by-url-sha256/346ee1ff9e381b78ba08e2a29445960b5cd31c51f896fc346b82e26e345a5b9a/data # on others function download_to_cache() { - local code_time_type_url - code_time_type_url=$( - curl -sSLI -w "%{http_code}\t%header{Last-Modified}\t%header{Content-Type}\t%{url_effective}" "$1" -o /dev/null - ) + local cache_path + cache_path=$(location_to_cache_path "$1") + # before checking remote location, check if the data file is already downloaded and the time file is updated within 10 minutes + if [[ -f ${cache_path}/data && -n "$(find "${cache_path}/time" -mmin -10 || true)" ]]; then + echo "${cache_path}/data" + return + fi + + # check the remote location + local curl_info_json write_out + write_out='{ + "http_code":%{http_code}, + "last_modified":"%header{Last-Modified}", + "content_type":"%{content_type}", + "url":"%{url_effective}", + "filename":"%{filename_effective}" + }' + curl_info_json=$(curl -sSLI -w "${write_out}" "$1" -o /dev/null) local code time type url - IFS=$'\t' read -r code time type url filename <<<"${code_time_type_url}" - [[ ${code} == 200 ]] || exit 1 + code=$(jq -r '.http_code' <<<"${curl_info_json}") + time=$(jq -r '.last_modified' <<<"${curl_info_json}") + type=$(jq -r '.content_type' <<<"${curl_info_json}") + url=$(jq -r '.url' <<<"${curl_info_json}") + [[ ${code} == 200 ]] || error_exit "Failed to download $1" - local cache_path cache_path=$(location_to_cache_path "${url}") [[ -d ${cache_path} ]] || mkdir -p "${cache_path}" @@ -369,18 +390,23 @@ function download_to_cache() { [[ -f ${cache_path}/time && "$(<"${cache_path}/time")" == "${time}" ]] || needs_download=1 [[ -f ${cache_path}/type && "$(<"${cache_path}/type")" == "${type}" ]] || needs_download=1 if [[ ${needs_download} -eq 1 ]]; then - local code_time_type_url_filename - code_time_type_url_filename=$( + curl_info_json=$( echo "downloading ${url}" >&2 - curl -SL -w "%{http_code}\t%header{Last-Modified}\t%header{Content-Type}\t%{url_effective}\t%{filename_effective}" --no-clobber -o "${cache_path}/data" "${url}" + curl -SL -w "${write_out}" --no-clobber -o "${cache_path}/data" "${url}" ) local filename - IFS=$'\t' read -r code time type url filename <<<"${code_time_type_url_filename}" - [[ ${code} == 200 ]] || exit 1 + code=$(jq -r '.http_code' <<<"${curl_info_json}") + time=$(jq -r '.last_modified' <<<"${curl_info_json}") + type=$(jq -r '.content_type' <<<"${curl_info_json}") + url=$(jq -r '.url' <<<"${curl_info_json}") + filename=$(jq -r '.filename' <<<"${curl_info_json}") + [[ ${code} == 200 ]] || error_exit "Failed to download ${url}" [[ "${cache_path}/data" == "${filename}" ]] || mv "${filename}" "${cache_path}/data" # sha256.digest seems existing if expected digest is available. so, not creating it here. # sha256sum "${cache_path}/data" | awk '{print "sha256:"$1}' >"${cache_path}/sha256.digest" echo -n "${time}" >"${cache_path}/time" + else + touch "${cache_path}/time" fi [[ -f ${cache_path}/type ]] || echo -n "${type}" >"${cache_path}/type" [[ -f ${cache_path}/url ]] || echo -n "${url}" >"${cache_path}/url" diff --git a/hack/update-template-debian.sh b/hack/update-template-debian.sh new file mode 100755 index 00000000000..d9bd39ef9c2 --- /dev/null +++ b/hack/update-template-debian.sh @@ -0,0 +1,437 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +# Functions in this script assume error handling with 'set -e'. +# To ensure 'set -e' works correctly: +# - Use 'set +e' before assignments and '$(set -e; )' to capture output without exiting on errors. +# - Avoid calling functions directly in conditions to prevent disabling 'set -e'. +# - Use 'shopt -s inherit_errexit' (Bash 4.4+) to avoid repeated 'set -e' in all '$(...)'. +shopt -s inherit_errexit || error_exit "inherit_errexit not supported. Please use bash 4.4 or later." + +function debian_print_help() { + cat <]] [--daily[=]] [--timestamped[=]] [--version ] ... + +Description: + This script updates the Debian image location in the specified templates. + If the image location in the template contains a release date in the URL, the script replaces it with the latest available date. + If no flags are specified, the script uses the version from the image location basename in the template. + + Image location basename format: debian-[-backports]-genericcloud-[-daily][-].qcow2 + + Published Debian image information is fetched from the following URLs: + + https://cloud.debian.org/images/cloud/[-backports]/[daily/](latest|)/debian-[-backports]-genericcloud-[-daily][-].json + + The downloaded JSON file will be cached in the Lima cache directory. + +Examples: + Update the Debian image location in templates/**.yaml: + $ $(basename "${BASH_SOURCE[0]}") templates/**.yaml + + Update the Debian image location in ~/.lima/debian/lima.yaml: + $ $(basename "${BASH_SOURCE[0]}") ~/.lima/debian/lima.yaml + $ limactl factory-reset debian + + Update the Debian image location to debian-13-genericcloud-.qcow2 in ~/.lima/debian/lima.yaml: + $ $(basename "${BASH_SOURCE[0]}") --version trixie ~/.lima/debian/lima.yaml + $ limactl factory-reset debian + +Flags: + --backports[=] Use the backports image + The boolean value can be true, false, 1, or 0 + --daily[=] Use the daily image + --timestamped[=] Use the timestamped image + --version Use the specified version + The version can be a codename, version number, or alias (testing, stable, oldstable) + -h, --help Print this help message +HELP +} + +readonly debian_base_url=https://cloud.debian.org/images/cloud/ + +readonly debian_target_vendor=genericcloud + +readonly -A debian_version_to_codename=( + [10]=buster + [11]=bullseye + [12]=bookworm + [13]=trixie + [14]=forky +) + +declare -A debian_codename_to_version +function debian_setup_codename_to_version() { + local version codename + for version in "${!debian_version_to_codename[@]}"; do + codename=${debian_version_to_codename[${version}]} + debian_codename_to_version[${codename}]="${version}" + done + readonly -A debian_codename_to_version +} +debian_setup_codename_to_version + +readonly -A debian_alias_to_codename=( + [testing]=trixie + [stable]=bookworm + [oldstable]=bullseye +) + +# debian_downloaded_json downloads the JSON file for the given url_spec(JSON) and caches it +# e.g. +# ```console +# debian_downloaded_json '{"backports":false,"daily":false,"version":12,"arch":"amd64","file_extension":"qcow2"}' +# +# ``` +function debian_downloaded_json() { + local url_spec=$1 json_url_spec json_url + json_url_spec=$(jq -r '. | del(.timestamp) | .file_extension = "json"' <<<"${url_spec}") || error_exit "Failed to create JSON URL spec" + json_url=$(debian_location_from_url_spec "${json_url_spec}") + download_to_cache "${json_url}" +} + +function debian_digest_from_upload_entry() { + local upload_entry=$1 debian_digest digest + debian_digest=$(jq -e -r '.metadata.annotations."cloud.debian.org/digest"' <<<"${upload_entry}") || + error_exit "Failed to get the digest from ${upload_entry}" + case "${debian_digest%:*}" in + sha512) digest=$(echo "${debian_digest#*:}==" | base64 -d | xxd -p -c -) || + error_exit "Failed to decode the digest from ${debian_digest}" ;; + *) error_exit "Unsupported digest type: ${debian_digest%:*}" ;; + esac + echo "${debian_digest/:*/:}${digest}" +} + +# debian_image_url_timestamped prints the latest image URL and its digest for the given flavor, version, arch, and path suffix. +function debian_image_url_timestamped() { + local url_spec=$1 debian_downloaded_json jq_filter upload_entry timestamp timestamped_url_spec location arch digest + debian_downloaded_json=$(debian_downloaded_json "${url_spec}") + # shellcheck disable=SC2016 + jq_filter=' + [.items[]|select(.kind == "Upload")| + select(.metadata.labels."upload.cloud.debian.org/image-format" == $ARGS.named.url_spec.image_format)]|first + ' + upload_entry=$(jq -e -r --argjson url_spec "${url_spec}" "${jq_filter}" "${debian_downloaded_json}") || + error_exit "Failed to find the upload entry from ${debian_downloaded_json}" + timestamp=$(jq -e -r '.metadata.labels."cloud.debian.org/version"' <<<"${upload_entry}") || + error_exit "Failed to get the timestamp from ${upload_entry}" + timestamped_url_spec=$(json_vars timestamp <<<"${url_spec}") + location=$(debian_location_from_url_spec "${timestamped_url_spec}") + location=$(validate_url_without_redirect "${location}") + arch=$(jq -e -r '.arch' <<<"${url_spec}") || error_exit "missing arch in ${url_spec}" + arch=$(limayaml_arch "${arch}") + digest=$(debian_digest_from_upload_entry "${upload_entry}") + json_vars location arch digest +} + +# debian_image_url_not_timestamped prints the release image URL for the given url_spec(JSON) +function debian_image_url_not_timestamped() { + local url_spec=$1 location arch + location=$(debian_location_from_url_spec "${url_spec}") + location=$(validate_url_without_redirect "${location}") + arch=$(jq -e -r '.arch' <<<"${url_spec}") || error_exit "missing arch in ${url_spec}" + arch=$(limayaml_arch "${arch}") + json_vars location arch +} + +# debian_version_resolve_aliases resolves the version aliases. +# e.g. +# ```console +# debian_version_resolve_aliases testing +# 13 +# debian_version_resolve_aliases stable +# 12 +# debian_version_resolve_aliases oldstable +# 11 +# debian_version_resolve_aliases bookworm +# 12 +# debian_version_resolve_aliases 10 +# 10 +# debian_version_resolve_aliases '' +# +# ``` +function debian_version_resolve_aliases() { + local version=$1 + [[ -v debian_alias_to_codename[${version}] ]] && version=${debian_alias_to_codename[${version}]} + [[ -v debian_codename_to_version[${version}] ]] && version=${debian_codename_to_version[${version}]} + [[ -v debian_version_to_codename[${version}] ]] || error_exit "Unsupported version: ${version}" + [[ -z ${version} ]] || echo "${version}" +} + +function debian_arch_from_location_basename() { + local location=$1 location_basename arch + location_basename=$(basename "${location}") + location_basename=${location_basename/-backports/} + arch=$(echo "${location_basename}" | cut -d- -f4 | cut -d. -f1) + [[ -n ${arch} ]] || error_exit "Failed to get arch from ${location}" + echo "${arch}" +} + +function debian_file_extension_from_location_basename() { + local location=$1 location_basename file_extension + location_basename=$(basename "${location}") + file_extension=$(echo "${location_basename}" | cut -d. -f2-) # remove the first field + [[ -n ${file_extension} ]] || error_exit "Failed to get file extension from ${location}" + echo "${file_extension}" +} + +function debian_image_format_from_file_extension() { + local file_extension=$1 + case "${file_extension}" in + json) echo "json" ;; + qcow2) echo "qcow2" ;; + raw) echo "raw" ;; + tar.xz) echo "internal" ;; + *) error_exit "Unsupported file extension: ${file_extension}" ;; + esac +} + +# debian_url_spec_from_location returns the URL spec for the given location. +# If the location is not supported, it returns 1. +# e.g. +# ```console +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2 +# {"backports":false,"daily":false,"version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm/20241004-1890/debian-12-generic-amd64-20241004-1890.qcow2 +# {"backports":false,"daily":false,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-genericcloud-amd64-daily.qcow2 +# {"backports":false,"daily":true,"version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm/daily/20241019-1905/debian-12-genericcloud-amd64-daily-20241019-1905.qcow2 +# {"backports":false,"daily":true,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm-backports/latest/debian-12-backports-genericcloud-amd64.qcow2 +# {"backports":true,"daily":false,"version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm-backports/20241004-1890/debian-12-backports-genericcloud-amd64-20241004-1890.qcow2 +# {"backports":true,"daily":false,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm-backports/daily/latest/debian-12-backports-genericcloud-amd64-daily.qcow2 +# {"backports":true,"daily":true,"version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# debian_url_spec_from_location https://cloud.debian.org/images/cloud/bookworm-backports/daily/20241019-1905/debian-12-backports-genericcloud-amd64-daily-20241019-1905.qcow2 +# {"backports":true,"daily":true,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2","image_format":"qcow2"} +# ``` +# shellcheck disable=SC2034 +function debian_url_spec_from_location() { + local location=$1 backports=false daily=false timestamp='' codename version='' arch file_extension image_format + local -r timestamp_pattern='[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' + case "${location}" in + ${debian_base_url}*-backports/*) backports=true ;;& + ${debian_base_url}*/daily/*) daily=true ;;& + ${debian_base_url}*/${timestamp_pattern}/*) [[ ${location} =~ ${timestamp_pattern} ]] && timestamp=${BASH_REMATCH[0]} ;; + ${debian_base_url}*/latest/*) timestamp='' ;; + *) + # echo "Unsupported image location: ${location}" >&2 + return 1 + ;; + esac + codename=$(echo "${location#"${debian_base_url}"}" | cut -d/ -f1 | cut -d- -f1) + [[ -v debian_codename_to_version[${codename}] ]] || error_exit "Unknown codename: ${codename}" + version=${debian_codename_to_version[${codename}]} + arch=$(debian_arch_from_location_basename "${location}") + file_extension=$(debian_file_extension_from_location_basename "${location}") + image_format=$(debian_image_format_from_file_extension "${file_extension}") + json_vars backports daily timestamp version arch file_extension image_format +} + +# debian_location_from_url_spec returns the location for the given URL spec. +# e.g. +# ```console +# debian_location_from_url_spec '{"backports":false,"daily":false,"version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2 +# debian_location_from_url_spec '{"backports":false,"daily":false,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm/20241019-1905/debian-12-generic-amd64-20241019-1905.qcow2 +# debian_location_from_url_spec '{"backports":false,"daily":true,"version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-genericcloud-amd64-daily.qcow2 +# debian_location_from_url_spec '{"backports":false,"daily":true,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm/daily/20241019-1905/debian-12-generic-amd64-daily-20241019-1905.qcow2 +# debian_location_from_url_spec '{"backports":true,"daily":false,"version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm-backports/latest/debian-12-backports-genericcloud-amd64.qcow2 +# debian_location_from_url_spec '{"backports":true,"daily":false,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm-backports/20241019-1905/debian-12-backports-genericcloud-amd64-20241019-1905.qcow2 +# debian_location_from_url_spec '{"backports":true,"daily":true,"version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm-backports/daily/latest/debian-12-backports-genericcloud-amd64-daily.qcow2 +# debian_location_from_url_spec '{"backports":true,"daily":true,"timestamp":"20241019-1905","version":12,"arch":"amd64","file_extension":"qcow2"}' +# https://cloud.debian.org/images/cloud/bookworm-backports/daily/20241019-1905/debian-12-backports-genericcloud-amd64-daily-20241019-1905.qcow2 +# ``` +function debian_location_from_url_spec() { + local url_spec=$1 base_url version backports daily timestamp arch file_extension + base_url=${debian_base_url} + version=$(jq -e -r '.version' <<<"${url_spec}") + [[ -v debian_version_to_codename[${version}] ]] || error_exit "Unsupported version: ${version}" + base_url+=${debian_version_to_codename[${version}]} + backports=$(jq -r 'if .backports then "-backports" else empty end' <<<"${url_spec}") + base_url+=${backports}/ + daily=$(jq -r 'if .daily then "daily" else empty end' <<<"${url_spec}") + base_url+=${daily:+${daily}/} + timestamp=$(jq -r 'if .timestamp then .timestamp else empty end' <<<"${url_spec}") + base_url+=${timestamp:-latest}/ + arch=$(jq -e -r '.arch' <<<"${url_spec}") + file_extension=$(jq -e -r '.file_extension' <<<"${url_spec}") + base_url+=debian-${version}${backports}-${debian_target_vendor}-${arch}${daily:+-${daily}}${timestamp:+-${timestamp}}.${file_extension} + echo "${base_url}" +} + +# debian_cache_key_for_image_kernel_overriding returns the cache key for the given location, kernel_location, flavor, and version. +# If the image location is not supported, it returns 1. +# kernel_location is not validated. +# e.g. +# ```console +# debian_cache_key_for_image_kernel_overriding https://cloud-images.debian.com/minimal/releases/24.04/release-20210914/debian-24.04-minimal-cloudimg-amd64.img +# debian_latest_24.04-minimal-amd64-release-.img +# debian_cache_key_for_image_kernel_overriding https://cloud-images.debian.com/minimal/releases/24.04/release-20210914/debian-24.04-minimal-cloudimg-amd64.img https://... +# debian_latest_with_kernel_24.04-minimal-amd64-release-.img +# debian_cache_key_for_image_kernel_overriding https://cloud-images.debian.com/releases/24.04/release/debian-24.04-server-cloudimg-amd64.img null +# debian_release_24.04-server-amd64-.img +# ``` +function debian_cache_key_for_image_kernel_overriding() { + local location=$1 kernel_location=${2:-null} overriding=${3:-"{}"} url_spec with_kernel='' version backports arch daily timestamped file_extension + url_spec=$(debian_url_spec_from_location "${location}" | jq -r ". + ${overriding}") + [[ ${kernel_location} != "null" ]] && with_kernel=_with_kernel + version=$(jq -r '.version|if . then "-\(.)" else empty end' <<<"${url_spec}") + backports=$(jq -r 'if .backports then "-backports" else empty end' <<<"${url_spec}") + arch=$(jq -e -r '.arch' <<<"${url_spec}") + daily=$(jq -r 'if .daily then "-daily" else empty end' <<<"${url_spec}") + timestamped=$(jq -r 'if .timestamp then "-timestamped" else empty end' <<<"${url_spec}") + file_extension=$(jq -e -r '.file_extension' <<<"${url_spec}") + echo "debian${with_kernel}${version}${backports}-${debian_target_vendor}-${arch}${daily}${timestamped}.${file_extension}" +} + +function debian_image_entry_for_image_kernel_overriding() { + local location=$1 kernel_location=$2 overriding=${3:-"{}"} url_spec timestamped + [[ ${kernel_location} == "null" ]] || error_exit "Updating image with kernel is not supported" + url_spec=$(debian_url_spec_from_location "${location}" | jq -r ". + ${overriding}") + timestamped=$(jq -r 'if .timestamp then "timestamped" else "not_timestamped" end' <<<"${url_spec}") + + local image_entry + image_entry=$(debian_image_url_"${timestamped}" "${url_spec}") + if [[ -z ${image_entry} ]]; then + error_exit "Failed to get the ${url_spec} image location for ${location}" + elif jq -e ".location == \"${location}\"" <<<"${image_entry}" >/dev/null; then + echo "Image location is up-to-date: ${location}" >&2 + else + echo "${image_entry}" + fi +} + +# check if the script is executed or sourced +# shellcheck disable=SC1091 +if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then + scriptdir=$(dirname "${BASH_SOURCE[0]}") + # shellcheck source=./cache-common-inc.sh + . "${scriptdir}/cache-common-inc.sh" + + # shellcheck source=/dev/null # avoid shellcheck hangs on source looping + . "${scriptdir}/update-template.sh" +else + # this script is sourced + if [[ -v SUPPORTED_DISTRIBUTIONS ]]; then + SUPPORTED_DISTRIBUTIONS+=("debian") + else + declare -a SUPPORTED_DISTRIBUTIONS=("debian") + fi + # required functions for Debian + function debian_cache_key_for_image_kernel() { debian_cache_key_for_image_kernel_overriding "$@"; } + function debian_image_entry_for_image_kernel() { debian_image_entry_for_image_kernel_overriding "$@"; } + + return 0 +fi + +declare -a templates=() +declare overriding="{}" +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + debian_print_help + exit 0 + ;; + -d | --debug) set -x ;; + --backports | --daily | --timestamped) + overriding=$(json "${1#--}" true <<<"${overriding}") + ;; + --backports=* | --daily=* | --timestamped=*) + overriding=$( + key=${1#--} value=$(validate_boolean "${1#*=}") + json "${key%%=*}" "${value}" <<<"${overriding}" + ) + ;; + --version) + if [[ -n $2 && $2 != -* ]]; then + overriding=$( + version=$(debian_version_resolve_aliases "$2") + json_vars version <<<"${overriding}" + ) + shift + else + error_exit "--version requires a value" + fi + ;; + --version=*) + overriding=$( + version=$(debian_version_resolve_aliases "${1#*=}") + json_vars version <<<"${overriding}" + ) + ;; + *.yaml) templates+=("$1") ;; + *) + error_exit "Unknown argument: $1" + ;; + esac + shift + [[ -z ${overriding} ]] && overriding="{}" +done + +if [[ ${#templates[@]} -eq 0 ]]; then + debian_print_help + exit 0 +fi + +declare -A image_entry_cache=() + +for template in "${templates[@]}"; do + echo "Processing ${template}" + # 1. extract location by parsing template using arch + yq_filter=" + .images[] | [.location, .kernel.location, .kernel.cmdline] | @tsv + " + parsed=$(yq eval "${yq_filter}" "${template}") + + # 3. get the image location + arr=() + while IFS= read -r line; do arr+=("${line}"); done <<<"${parsed}" + locations=("${arr[@]}") + for ((index = 0; index < ${#locations[@]}; index++)); do + [[ ${locations[index]} != "null" ]] || continue + set -e + IFS=$'\t' read -r location kernel_location kernel_cmdline <<<"${locations[index]}" + set +e # Disable 'set -e' to avoid exiting on error for the next assignment. + cache_key=$( + set -e # Enable 'set -e' for the next command. + debian_cache_key_for_image_kernel_overriding "${location}" "${kernel_location}" "${overriding}" + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || continue + image_entry=$( + set -e # Enable 'set -e' for the next command. + if [[ -v image_entry_cache[${cache_key}] ]]; then + echo "${image_entry_cache[${cache_key}]}" + else + debian_image_entry_for_image_kernel_overriding "${location}" "${kernel_location}" "${overriding}" + fi + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || continue + set -e + image_entry_cache[${cache_key}]="${image_entry}" + if [[ -n ${image_entry} ]]; then + [[ ${kernel_cmdline} != "null" ]] && + jq -e 'has("kernel")' <<<"${image_entry}" >/dev/null && + image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}") + echo "${image_entry}" | jq + limactl edit --log-level error --set " + .images[${index}] = ${image_entry}| + (.images[${index}] | ..) style = \"double\" + " "${template}" + fi + done +done diff --git a/hack/update-template-ubuntu.sh b/hack/update-template-ubuntu.sh index 082c1169618..1550d7e47ec 100755 --- a/hack/update-template-ubuntu.sh +++ b/hack/update-template-ubuntu.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash -function print_help() { +set -eu -o pipefail + +# Functions in this script assume error handling with 'set -e'. +# To ensure 'set -e' works correctly: +# - Use 'set +e' before assignments and '$(set -e; )' to capture output without exiting on errors. +# - Avoid calling functions directly in conditions to prevent disabling 'set -e'. +# - Use 'shopt -s inherit_errexit' (Bash 4.4+) to avoid repeated 'set -e' in all '$(...)'. +shopt -s inherit_errexit || error_exit "inherit_errexit not supported. Please use bash 4.4 or later." + +function ubuntu_print_help() { cat <.img in ~/.lima/docker/lima.yaml: $ $(basename "${BASH_SOURCE[0]}") --minimal --version 24.04 ~/.lima/docker/lima.yaml + $ limactl factory-reset docker Flags: --flavor Use the specified flavor image --server Shortcut for --flavor server --minimal Shortcut for --flavor minimal --version Use the specified version + The version can be an alias: latest, latest_lts, or lts. -h, --help Print this help message HELP } -scriptdir=$(dirname "${BASH_SOURCE[0]}") -# shellcheck source=./cache-common-inc.sh -# shellcheck disable=SC1091 -. "${scriptdir}/cache-common-inc.sh" - -set -eu -o pipefail - -readonly -A base_urls=( +readonly -A ubuntu_base_urls=( [minimal]=https://cloud-images.ubuntu.com/minimal/releases/ [server]=https://cloud-images.ubuntu.com/releases/ ) -# validate_url checks if the URL is valid and returns the location if it is. -# If the URL is redirected, it returns the redirected location. -# e.g. -# ```console -# validate_url https://cloud-images.ubuntu.com/server/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img -# https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img -# ``` -function validate_url() { - local url=$1 - code_location=$(curl -sSL -o /dev/null -I -w "%{http_code}\t%{url_effective}" "${url}") - read -r code location <<<"${code_location}" - [[ ${code} -eq 200 ]] && echo "${location}" -} - -# ubuntu_base_url returns the base URL for the given flavor. +# ubuntu_base_url prints the base URL for the given flavor. # e.g. # ```console # ubuntu_base_url minimal # https://cloud-images.ubuntu.com/minimal/releases/ # ``` function ubuntu_base_url() { - # shellcheck disable=SC2015 - [[ -v base_urls[$1] ]] && echo "${base_urls[$1]}" || ( - echo "Unsupported flavor: $1" >&2 - exit 1 - ) + [[ -v ubuntu_base_urls[$1] ]] || error_exit "Unsupported flavor: $1" + echo "${ubuntu_base_urls[$1]}" } -# downloaded_json downloads the JSON file for the given flavor and returns the path. +# ubuntu_downloaded_json downloads the JSON file for the given flavor and prints the path. # e.g. # ```console -# downloaded_json server +# ubuntu_downloaded_json server # /Users/user/Library/Caches/lima/download/by-url-sha256/255f982f5bbda07f5377369093e21c506d7240f5ba901479bdadfa205ddafb01/data # ``` -function downloaded_json() { +function ubuntu_downloaded_json() { local flavor=$1 base_url json_url json_url=$(ubuntu_base_url "${flavor}")streams/v1/com.ubuntu.cloud:released:download.json download_to_cache "${json_url}" } - # ubuntu_image_url_try_replace_release_with_version tries to replace the release with the version in the URL. -# If the URL is valid, it returns the URL with the version. +# If the URL is valid, it prints the URL with the version. function ubuntu_image_url_try_replace_release_with_version() { local location=$1 release=$2 version=$3 location_using_version - # shellcheck disable=SC2310 - if location_using_version=$(validate_url "${location/\/${release}\//\/${version}\/}"); then + set +e # Disable 'set -e' to avoid exiting on error for the next assignment. + location_using_version=$( + set -e + validate_url "${location/\/${release}\//\/${version}\/}" 2>/dev/null + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + if [[ $? -eq 0 ]]; then echo "${location_using_version}" else echo "${location}" fi + set -e } -# ubuntu_image_url_latest returns the latest image URL and its digest for the given version, flavor, arch, and path suffix. +# ubuntu_image_url_latest prints the latest image URL and its digest for the given flavor, version, arch, and path suffix. function ubuntu_image_url_latest() { - local version=$1 flavor=$2 arch=$3 path_suffix=$4 base_url downloaded_json jq_filter location_digest_release + local flavor=$1 version=$2 arch=$3 path_suffix=$4 base_url ubuntu_downloaded_json jq_filter location_digest_release base_url=$(ubuntu_base_url "${flavor}") - # shellcheck disable=SC2310 - downloaded_json=$(downloaded_json "${flavor}") || return 0 + ubuntu_downloaded_json=$(ubuntu_downloaded_json "${flavor}") jq_filter=" [ .products[\"com.ubuntu.cloud:${flavor}:${version}:${arch}\"] | @@ -118,22 +110,23 @@ function ubuntu_image_url_latest() { [\"${base_url}\"+.path, \"sha256:\"+.sha256, \$release] | @tsv ] | last " - location_digest_release=$(jq -e -r "${jq_filter}" "${downloaded_json}") || return 0 + location_digest_release=$(jq -r "${jq_filter}" "${ubuntu_downloaded_json}") + [[ ${location_digest_release} != "null" ]] || + error_exit "The URL for ubuntu-${version}-${flavor}-cloudimg-${arch}${path_suffix} is not provided at ${ubuntu_base_urls[${flavor}]}." local location digest release location_using_version read -r location digest release <<<"${location_digest_release}" - # shellcheck disable=SC2310 - location=$(validate_url "${location}") || return 0 + location=$(validate_url "${location}") location=$(ubuntu_image_url_try_replace_release_with_version "${location}" "${release}" "${version}") - echo -e "${location}\t${digest}" + arch=$(limayaml_arch "${arch}") + json_vars location arch digest } -# ubuntu_image_url_release returns the release image URL for the given version, flavor, arch, and path suffix. +# ubuntu_image_url_release prints the release image URL for the given flavor, version, arch, and path suffix. function ubuntu_image_url_release() { - local version=$1 flavor=$2 arch=$3 path_suffix=$4 base_url + local flavor=$1 version=$2 arch=$3 path_suffix=$4 base_url base_url=$(ubuntu_base_url "${flavor}") - # shellcheck disable=SC2310 - downloaded_json=$(downloaded_json "${flavor}") || return 0 - local location release location_using_version + ubuntu_downloaded_json=$(ubuntu_downloaded_json "${flavor}") + local jq_filter release location jq_filter=" [ .products | to_entries[] as \$product_entry | @@ -141,100 +134,277 @@ function ubuntu_image_url_release() { .release ] | first " - release=$(jq -e -r "${jq_filter}" "${downloaded_json}") || return 0 - # shellcheck disable=SC2310 - location=$(validate_url "${base_url}${release}/release/ubuntu-${version}-${flavor}-cloudimg-${arch}${path_suffix}") || return 0 - ubuntu_image_url_try_replace_release_with_version "${location}" "${release}" "${version}" + release=$(jq -r "${jq_filter}" "${ubuntu_downloaded_json}") + [[ ${release} != "null" ]] || + error_exit "The URL for ubuntu-${version}-${flavor}-cloudimg-${arch}${path_suffix} is not provided at ${ubuntu_base_urls[${flavor}]}." + location=$(validate_url "${base_url}${release}/release/ubuntu-${version}-${flavor}-cloudimg-${arch}${path_suffix}") + location=$(ubuntu_image_url_try_replace_release_with_version "${location}" "${release}" "${version}") + arch=$(limayaml_arch "${arch}") + json_vars location arch +} + +function ubuntu_file_info() { + local location=$1 location_dirname sha256sums location_basename digest + location=$(validate_url "${location}") + location_dirname=$(dirname "${location}") + sha256sums=$(download_to_cache "${location_dirname}/SHA256SUMS") + location_basename=$(basename "${location}") + # shellcheck disable=SC2034 + digest=${location+$(awk "/${location_basename}/{print \"sha256:\"\$1}" "${sha256sums}")} + json_vars location digest } -# ubuntu_kernel_info_for_image_url returns the kernel and initrd location and digest for the given location. -function ubuntu_kernel_info_for_image_url() { - local location=$1 location_dirname sha256sums location_basename +# ubuntu_image_entry_with_kernel_info prints image entry with kernel and initrd info. +# $1: image_entry +# e.g. +# ```console +# ubuntu_image_entry_with_kernel_info '{"location":"https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img"}' +# {"location":"https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img","kernel":{"location":"https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-vmlinuz-generic","digest":"sha256:..."}} +# ``` +# shellcheck disable=SC2034 +function ubuntu_image_entry_with_kernel_info() { + local image_entry=$1 location + location=$(jq -e -r '.location' <<<"${image_entry}") + local location_dirname location_basename location_prefix location_dirname=$(dirname "${location}")/unpacked - sha256sums=$(curl -sSLf "${location_dirname}/SHA256SUMS") location_basename="$(basename "${location}" | cut -d- -f1-5 | cut -d. -f1-2)" + location_prefix="${location_dirname}/${location_basename}" + local kernel initrd + set +e # Disable 'set -e' to avoid exiting on error for the next assignment. + kernel=$( + set -e + ubuntu_file_info "${location_prefix}-vmlinuz-generic" + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || error_exit "kernel image not found at ${location_prefix}-vmlinuz-generic" + initrd=$( + set -e + ubuntu_file_info "${location_prefix}-initrd-generic" 2>/dev/null + ) # may not exist + set -e + json_vars kernel initrd <<<"${image_entry}" +} - # kernel - local kernel_basename kernel_location kernel_digest - kernel_basename="${location_basename}-vmlinuz-generic" - # shellcheck disable=SC2310 - kernel_location=$(validate_url "${location_dirname}/${kernel_basename}") || return 0 - kernel_digest=${kernel_location+$(awk "/${kernel_basename}/{print \"sha256:\"\$1}" <<<"${sha256sums}")} +function ubuntu_flavor_from_location_basename() { + local location=$1 location_basename flavor + location_basename=$(basename "${location}") + flavor=$(echo "${location_basename}" | cut -d- -f3) + [[ -n ${flavor} ]] || error_exit "Failed to get flavor from ${location}" + echo "${flavor}" +} - # initrd - local initrd_basename initrd_location initrd_digest - initrd_basename="${location_basename}-initrd-generic" - initrd_location=$(validate_url "${location_dirname}/${initrd_basename}") - initrd_digest=${initrd_location+$(awk "/${initrd_basename}/{print \"sha256:\"\$1}" <<<"${sha256sums}")} +function ubuntu_version_from_location_basename() { + local location=$1 location_basename version + location_basename=$(basename "${location}") + version=$(echo "${location_basename}" | cut -d- -f2) + [[ -n ${version} ]] || error_exit "Failed to get version from ${location}" + echo "${version}" +} - echo -e "${kernel_location}\t${kernel_digest}\t${initrd_location}\t${initrd_digest}" +# ubuntu_version_latest_lts prints the latest LTS version for the given flavor. +# e.g. +# ```console +# ubuntu_version_latest_lts minimal +# 24.04 +# ``` +function ubuntu_version_latest_lts() { + local flavor=${1:-server} + ubuntu_downloaded_json=$(ubuntu_downloaded_json "${flavor}") + jq -e -r '[.products[]|.version|select(endswith(".04"))]|last // empty' "${ubuntu_downloaded_json}" } -# limayaml_arch returns the arch in the lima.yaml format -function limayaml_arch() { - local arch=$1 - arch=${arch/amd64/x86_64} - arch=${arch/arm64/aarch64} - arch=${arch/armhf/armv7l} +# ubuntu_version_latest prints the latest version for the given flavor. +# e.g. +# ```console +# ubuntu_version_latest minimal +# 24.10 +# ``` +function ubuntu_version_latest() { + local flavor=${1:-server} + ubuntu_downloaded_json=$(ubuntu_downloaded_json "${flavor}") + jq -e -r '[.products[]|.version]|last // empty' "${ubuntu_downloaded_json}" +} + +# ubuntu_version_resolve_aliases resolves the version aliases. +# e.g. +# ```console +# ubuntu_version_resolve_aliases https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img minimal latest +# 24.10 +# ubuntu_version_resolve_aliases https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img minimal latest_lts +# 24.04 +# ubuntu_version_resolve_aliases https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img +# +# ``` +function ubuntu_version_resolve_aliases() { + local location=$1 flavor version + flavor=${2:-$(ubuntu_flavor_from_location_basename "${location}")} + version=${3:-} + case "${version}" in + latest_lts | lts) ubuntu_version_latest_lts "${flavor}" ;; + latest) ubuntu_version_latest "${flavor}" ;; + *) echo "${version}" ;; + esac +} + +function ubuntu_arch_from_location_basename() { + local location=$1 location_basename arch + location_basename=$(basename "${location}") + arch=$(echo "${location_basename}" | cut -d- -f5 | cut -d. -f1) + [[ -n ${arch} ]] || error_exit "Failed to get arch from ${location}" echo "${arch}" } -declare -a templates=() +function ubuntu_path_suffix_from_location_basename() { + local location=$1 arch path_suffix + arch=$(ubuntu_arch_from_location_basename "${location}") + path_suffix="${location##*"${arch}"}" + [[ -n ${path_suffix} ]] || error_exit "Failed to get path suffix from ${location}" + echo "${path_suffix}" +} + +# ubuntu_location_url_spec prints the URL spec for the given location. +# If the location is not supported, it returns 1. +# e.g. +# ```console +# ubuntu_location_url_spec https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img +# latest +# ubuntu_location_url_spec https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img +# release +# ``` +function ubuntu_location_url_spec() { + local location=$1 url_spec + case "${location}" in + https://cloud-images.ubuntu.com/minimal/releases/*/release/*) url_spec=release ;; + https://cloud-images.ubuntu.com/minimal/releases/*/release-*/*) url_spec=latest ;; + https://cloud-images.ubuntu.com/releases/*/release/*) url_spec=release ;; + https://cloud-images.ubuntu.com/releases/*/release-*/*) url_spec=latest ;; + *) + # echo "Unsupported image location: ${location}" >&2 + return 1 + ;; + esac + echo "${url_spec}" +} + +# ubuntu_cache_key_for_image_kernel_flavor_version prints the cache key for the given location, kernel_location, flavor, and version. +# If the image location is not supported, it returns 1. +# kernel_location is not validated. +# e.g. +# ```console +# ubuntu_cache_key_for_image_kernel_flavor_version https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img +# ubuntu_latest_24.04-minimal-amd64-release-.img +# ubuntu_cache_key_for_image_kernel_flavor_version https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img https://... +# ubuntu_latest_with_kernel_24.04-minimal-amd64-release-.img +# ubuntu_cache_key_for_image_kernel_flavor_version https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img null +# ubuntu_release_24.04-server-amd64-.img +# ``` +function ubuntu_cache_key_for_image_kernel_flavor_version() { + local location=$1 kernel_location=${2:-null} url_spec with_kernel='' flavor version arch path_suffix + url_spec=$(ubuntu_location_url_spec "${location}") + [[ ${kernel_location} != "null" ]] && with_kernel=_with_kernel + flavor=${3:-$(ubuntu_flavor_from_location_basename "${location}")} + version=${4:-$(ubuntu_version_from_location_basename "${location}")} + arch=$(ubuntu_arch_from_location_basename "${location}") + path_suffix=$(ubuntu_path_suffix_from_location_basename "${location}") + echo "ubuntu_${url_spec}${with_kernel}_${version}-${flavor}-${arch}-${path_suffix}" +} +function ubuntu_image_entry_for_image_kernel_flavor_version() { + local location=$1 kernel_location=$2 url_spec + url_spec=$(ubuntu_location_url_spec "${location}") + + local flavor version arch path_suffix + flavor=${3:-$(ubuntu_flavor_from_location_basename "${location}")} + version=${4:-$(ubuntu_version_from_location_basename "${location}")} + arch=$(ubuntu_arch_from_location_basename "${location}") + path_suffix=$(ubuntu_path_suffix_from_location_basename "${location}") + + local image_entry + image_entry=$(ubuntu_image_url_"${url_spec}" "${flavor}" "${version}" "${arch}" "${path_suffix}") + if [[ -z ${image_entry} ]]; then + error_exit "Failed to get the ${url_spec} image location for ${location}" + elif jq -e ".location == \"${location}\"" <<<"${image_entry}" >/dev/null; then + echo "Image location is up-to-date: ${location}" >&2 + elif [[ ${kernel_location} != "null" ]]; then + ubuntu_image_entry_with_kernel_info "${image_entry}" + else + echo "${image_entry}" + fi +} + +# check if the script is executed or sourced +# shellcheck disable=SC1091 +if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then + scriptdir=$(dirname "${BASH_SOURCE[0]}") + # shellcheck source=./cache-common-inc.sh + . "${scriptdir}/cache-common-inc.sh" + + # shellcheck source=/dev/null # avoid shellcheck hangs on source looping + . "${scriptdir}/update-template.sh" +else + # this script is sourced + if [[ -v SUPPORTED_DISTRIBUTIONS ]]; then + SUPPORTED_DISTRIBUTIONS+=("ubuntu") + else + declare -a SUPPORTED_DISTRIBUTIONS=("ubuntu") + fi + # required functions for Ubuntu + function ubuntu_cache_key_for_image_kernel() { ubuntu_cache_key_for_image_kernel_flavor_version "$@"; } + function ubuntu_image_entry_for_image_kernel() { ubuntu_image_entry_for_image_kernel_flavor_version "$@"; } + + return 0 +fi + +declare -a templates=() +declare overriding_flavor= +declare overriding_version= while [[ $# -gt 0 ]]; do case "$1" in -h | --help) - print_help + ubuntu_print_help exit 0 ;; + -d | --debug) set -x ;; --flavor) if [[ -n $2 && $2 != -* ]]; then - flavor="$2" + overriding_flavor="$2" shift else - echo "Error: --flavor requires a value" >&2 - exit 1 + error_exit "--flavor requires a value" fi ;; - --flavor=*) flavor="${1#*=}" ;; - --minimal) flavor="minimal" ;; - --server) flavor="server" ;; + --flavor=*) overriding_flavor="${1#*=}" ;; + --minimal) overriding_flavor="minimal" ;; + --server) overriding_flavor="server" ;; --version) if [[ -n $2 && $2 != -* ]]; then - version="$2" + overriding_version="$2" shift else - echo "Error: --version requires a value" >&2 - exit 1 + error_exit "--version requires a value" fi ;; - --version=*) version="${1#*=}" ;; + --version=*) overriding_version="${1#*=}" ;; *.yaml) templates+=("$1") ;; *) - echo "Unknown argument: $1" >&2 - exit 1 + error_exit "Unknown argument: $1" ;; esac shift done if [[ ${#templates[@]} -eq 0 ]]; then - print_help + ubuntu_print_help exit 0 fi -flavor=${flavor:-server} -downloaded_json=$(downloaded_json "${flavor}") -version="${version:-$(jq -r '[.products[]|.version|select(endswith(".04"))]|last' "${downloaded_json}")}" - -declare -A ubuntu_image_url_latest_cache=() -declare -A ubuntu_image_url_release_cache=() +declare -A image_entry_cache=() for template in "${templates[@]}"; do echo "Processing ${template}" # 1. extract location by parsing template using arch yq_filter=" - .images[] | [.location, .kernel.location, .kernel.cmdline, .initrd.location] | @tsv + .images[] | [.location, .kernel.location, .kernel.cmdline] | @tsv " parsed=$(yq eval "${yq_filter}" "${template}") @@ -244,92 +414,42 @@ for template in "${templates[@]}"; do locations=("${arr[@]}") for ((index = 0; index < ${#locations[@]}; index++)); do [[ ${locations[index]} != "null" ]] || continue - IFS=$'\t' read -r location kernel_location kernel_cmdline initrd_location <<<"${locations[index]}" - location_before="${location}" - - case "${location}" in - https://cloud-images.ubuntu.com/minimal/releases/*/release/*) use_latest=0 ;;& - https://cloud-images.ubuntu.com/minimal/releases/*/release-*/*) use_latest=1 ;;& - https://cloud-images.ubuntu.com/minimal/releases/*) flavor=${flavor:-minimal} ;; - https://cloud-images.ubuntu.com/releases/*/release/*) use_latest=0 ;;& - https://cloud-images.ubuntu.com/releases/*/release-*/*) use_latest=1 ;;& - https://cloud-images.ubuntu.com/releases/*) flavor=${flavor:-server} ;; - *) - # echo "Unsupported image location: ${location}" >&2 - continue - ;; - esac - - location_basename=$(basename "${location}") - version=${version:-$(echo "${location_basename}" | cut -d- -f2)} - flavor=${flavor:-$(echo "${location_basename}" | cut -d- -f3)} - arch=$(echo "${location_basename}" | cut -d- -f5 | cut -d. -f1) - path_suffix="${location_basename##*"${arch}"}" - limayaml_arch=$(limayaml_arch "${arch}") - if [[ ${use_latest} -eq 1 ]]; then - latest_cache_key=${version}-${flavor}-${arch}-${path_suffix} - location_digest=$( - # shellcheck disable=SC2015 - [[ -v ubuntu_image_url_latest_cache[${latest_cache_key}] ]] && echo "${ubuntu_image_url_latest_cache[${latest_cache_key}]}" || - ubuntu_image_url_latest "${version}" "${flavor}" "${arch}" "${path_suffix}" - ) - ubuntu_image_url_latest_cache[${latest_cache_key}]="${location_digest}" - read -r location digest <<<"${location_digest}" - if [[ -z ${location} ]]; then - echo "Failed to get the latest image location for ${location_basename}" >&2 - continue - elif [[ ${location} == "${location_before}" ]]; then - continue - fi - image_entry="{\"location\": \"${location}\", \"arch\": \"${limayaml_arch}\", \"digest\": \"${digest}\"}" - echo -e "${location}\n${digest}" - if [[ ${kernel_location} != "null" ]]; then - kernel_info=$(ubuntu_kernel_info_for_image_url "${location}") - IFS=$'\t' read -r kernel_location kernel_digest initrd_location initrd_digest <<<"${kernel_info}" - if [[ -n ${kernel_location} ]]; then - image_entry=$(jq ". + {kernel: {location: \"${kernel_location}\", digest: \"${kernel_digest}\"}}" <<<"${image_entry}") - [[ ${kernel_cmdline} != "null" ]] && image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}") - echo -e "${kernel_location}\n${kernel_digest}" - fi - if [[ -n ${initrd_location} ]]; then - image_entry=$(jq ". + {initrd: {location: \"${initrd_location}\", digest: \"${initrd_digest}\"}}" <<<"${image_entry}") - echo -e "${initrd_location}\n${initrd_digest}" - fi - fi - else - release_cache_key=${version}-${flavor}-${arch}-${path_suffix} - location=$( - # shellcheck disable=SC2015 - [[ -v ubuntu_image_url_release_cache[${release_cache_key}] ]] && echo "${ubuntu_image_url_release_cache[${release_cache_key}]}" || - ubuntu_image_url_release "${version}" "${flavor}" "${arch}" "${path_suffix}" - ) - ubuntu_image_url_release_cache[${release_cache_key}]="${location}" - if [[ -z ${location} ]]; then - echo "Failed to get the release image location for ${location_basename}" >&2 - continue - elif [[ ${location} == "${location_before}" ]]; then - continue - fi - image_entry="{\"location\": \"${location}\", \"arch\": \"${limayaml_arch}\"}" - echo "${location}" - if [[ ${kernel_location} != "null" ]]; then - kernel_info=$(ubuntu_kernel_info_for_image_url "${location}") - IFS=$'\t' read -r kernel_location kernel_digest initrd_location initrd_digest <<<"${kernel_info}" - if [[ -n ${kernel_location} ]]; then - image_entry=$(jq ". + {kernel: {location: \"${kernel_location}\"}}" <<<"${image_entry}") - [[ ${kernel_cmdline} != "null" ]] && image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}") - echo "${kernel_location}" - fi - if [[ -n ${initrd_location} ]]; then - image_entry=$(jq ". + {initrd: {location: \"${initrd_location}\"}}" <<<"${image_entry}") - echo "${initrd_location}" - fi + set -e + IFS=$'\t' read -r location kernel_location kernel_cmdline <<<"${locations[index]}" + set +e # Disable 'set -e' to avoid exiting on error for the next assignment. + overriding_version=$( + set -e # Enable 'set -e' for the next command. + ubuntu_version_resolve_aliases "${location}" "${overriding_flavor}" "${overriding_version}" + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || continue + cache_key=$( + set -e # Enable 'set -e' for the next command. + ubuntu_cache_key_for_image_kernel_flavor_version "${location}" "${kernel_location}" "${overriding_flavor}" "${overriding_version}" + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || continue + image_entry=$( + set -e # Enable 'set -e' for the next command. + if [[ -v image_entry_cache[${cache_key}] ]]; then + echo "${image_entry_cache[${cache_key}]}" + else + ubuntu_image_entry_for_image_kernel_flavor_version "${location}" "${kernel_location}" "${overriding_flavor}" "${overriding_version}" fi + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || continue + set -e + image_entry_cache[${cache_key}]="${image_entry}" + if [[ -n ${image_entry} ]]; then + [[ ${kernel_cmdline} != "null" ]] && + jq -e 'has("kernel")' <<<"${image_entry}" >/dev/null && + image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}") + echo "${image_entry}" | jq + limactl edit --log-level error --set " + .images[${index}] = ${image_entry}| + (.images[${index}] | ..) style = \"double\" + " "${template}" fi - limactl edit --log-level error --set " - [(.images.[] | path)].[${index}] as \$path| - setpath(\$path; ${image_entry}) - .images[${index}].[] style = \"double\" - " "${template}" done done diff --git a/hack/update-template.sh b/hack/update-template.sh new file mode 100755 index 00000000000..c9b3237a290 --- /dev/null +++ b/hack/update-template.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +# Functions in this script assume error handling with 'set -e'. +# To ensure 'set -e' works correctly: +# - Use 'set +e' before assignments and '$(set -e; )' to capture output without exiting on errors. +# - Avoid calling functions directly in conditions to prevent disabling 'set -e'. +# - Use 'shopt -s inherit_errexit' (Bash 4.4+) to avoid repeated 'set -e' in all '$(...)'. +shopt -s inherit_errexit || error_exit "inherit_errexit not supported. Please use bash 4.4 or later." + +function print_help() { + cat <... + +Description: + This script updates the image location in the specified templates. + If the image location in the template contains a release date in the URL, the script replaces it with the latest available date. + +Examples: + Update the Ubuntu image location in templates/**.yaml: + $ $(basename "${BASH_SOURCE[0]}") templates/**.yaml + + Update the Ubuntu image location in ~/.lima/ubuntu/lima.yaml: + $ $(basename "${BASH_SOURCE[0]}") ~/.lima/ubuntu/lima.yaml + $ limactl factory-reset ubuntu + +Flags: + -h, --help Print this help message +HELP +} + +# json prints the JSON object with the given arguments. +# json [key value ...] +# if the value is empty, both key and value are omitted. +# e.g. +# ```console +# json location https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img arch amd64 digest sha256:... +# {"location":"https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img","arch":"amd64","digest":"sha256:..."} +# ``` +function json() { + local args=() pattern='^(\[.*\]|\{.*\}|true|false|[0-9]+)$' value + [[ ! -p /dev/stdin ]] && args+=(--null-input) + while [[ $# -gt 0 ]]; do + value="${2-}" + if [[ ${value} =~ ${pattern} ]]; then + args+=(--argjson "${1}" "${value}") + elif [[ -n ${value} ]]; then + args+=(--arg "${1}" "${value}") + fi # omit empty values + shift + shift # shift 2 does not work when $# is 1 + done + jq -c "${args[@]}" '. + $ARGS.named | if . == {} then empty else . end' +} + +# json_vars prints the JSON object with the given variable names. +# e.g. +# ```console +# location=https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img +# arch=amd64 +# digest=sha256:... +# json_vars location arch digest +# {"location":"https://cloud-images.ubuntu.com/minimal/releases/24.04/release-20210914/ubuntu-24.04-minimal-cloudimg-amd64.img","arch":"amd64","digest":"sha256:..."} +# ``` +function json_vars() { + local args=() var + for var in "$@"; do + [[ -v ${var} ]] || error_exit "${var} is not set" + args+=("${var}" "${!var}") + done + json "${args[@]}" +} + +# limayaml_arch prints the arch in the lima.yaml format +function limayaml_arch() { + local arch=$1 + arch=${arch/amd64/x86_64} + arch=${arch/arm64/aarch64} + arch=${arch/armhf/armv7l} + echo "${arch}" +} + +function validate_boolean() { + local value=$1 + case "${value}" in + '') ;; + true | 1) echo true ;; + false | 0) echo false ;; + *) error_exit "Invalid boolean value: ${value}" ;; + esac +} + +# validate_url checks if the URL is valid and prints the location if it is. +# If the URL is redirected, it prints the redirected location. +# e.g. +# ```console +# validate_url https://cloud-images.ubuntu.com/server/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img +# https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img +# ``` +function validate_url() { + local url=$1 + code_location=$(curl -sSL -o /dev/null -I -w "%{http_code}\t%{url_effective}" "${url}") + read -r code location <<<"${code_location}" + [[ ${code} -eq 200 ]] || error_exit "[${code}]: The image is not available for download from ${location}" + echo "${location}" +} + +# validate_url_without_redirect checks if the URL is valid and prints the location if it is. +# If the URL is redirected, it prints the URL before the redirection. +# e.g. +# ```console +# validate_url_without_redirect https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.qcow2 +# https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.qcow2 +# ``` +# cloud.debian.org may be redirected to other domains(e.g. chuangtzu.ftp.acc.umu.se), but we want to use the original URL. +function validate_url_without_redirect() { + local url=$1 location + location=$(validate_url "${url}") + [[ -n ${location} ]] || error_exit "The image is not available for download from ${url}" + echo "${url}" +} + +# check if the script is executed or sourced +# shellcheck disable=SC1091 +if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then + scriptdir=$(dirname "${BASH_SOURCE[0]}") + # shellcheck source=./cache-common-inc.sh + . "${scriptdir}/cache-common-inc.sh" + + # Scripts for each distribution are expected to: + # - Add their identifier to the SUPPORTED_DISTRIBUTIONS array. + # - Register the following functions: + # - ${distribution}_cache_key_for_image_kernel + # - Arguments: location, kernel_location + # - Returns: cache_key (string) + # - Exits with an error if the image location is not supported. + # - ${distribution}_image_entry_for_image_kernel + # - Arguments: location, kernel_location + # - Returns: image_entry (JSON object) + # - Exits with an error if the image location is not supported. + declare -a SUPPORTED_DISTRIBUTIONS=() + + # shellcheck source=./update-template-ubuntu.sh + . "${scriptdir}/update-template-ubuntu.sh" + # shellcheck source=./update-template-debian.sh + . "${scriptdir}/update-template-debian.sh" +else + # this script is sourced + return 0 +fi + +declare -a templates=() +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + print_help + exit 0 + ;; + -d | --debug) set -x ;; + *.yaml) templates+=("$1") ;; + *) + error_exit "Unknown argument: $1" + ;; + esac + shift +done + +if [[ ${#templates[@]} -eq 0 ]]; then + print_help + exit 0 +fi + +declare -a distributions=() +# Check if the distribution has the required functions +for distribution in "${SUPPORTED_DISTRIBUTIONS[@]}"; do + if declare -f "${distribution}_cache_key_for_image_kernel" >/dev/null && + declare -f "${distribution}_image_entry_for_image_kernel" >/dev/null; then + distributions+=("${distribution}") + fi +done +[[ ${#distributions[@]} -gt 0 ]] || error_exit "No supported distributions found" + +declare -A image_entry_cache=() + +for template in "${templates[@]}"; do + echo "Processing ${template}" + # 1. extract location by parsing template using arch + yq_filter=" + .images[] | [.location, .kernel.location, .kernel.cmdline] | @tsv + " + parsed=$(yq eval "${yq_filter}" "${template}") + + # 3. get the image location + arr=() + while IFS= read -r line; do arr+=("${line}"); done <<<"${parsed}" + locations=("${arr[@]}") + for ((index = 0; index < ${#locations[@]}; index++)); do + [[ ${locations[index]} != "null" ]] || continue + set -e + IFS=$'\t' read -r location kernel_location kernel_cmdline <<<"${locations[index]}" + for distribution in "${distributions[@]}"; do + set +e # Disable 'set -e' to avoid exiting on error for the next assignment. + cache_key=$( + set -e # Enable 'set -e' for the next command. + "${distribution}_cache_key_for_image_kernel" "${location}" "${kernel_location}" + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || continue + image_entry=$( + set -e # Enable 'set -e' for the next command. + if [[ -v image_entry_cache[${cache_key}] ]]; then + echo "${image_entry_cache[${cache_key}]}" + else + "${distribution}_image_entry_for_image_kernel" "${location}" "${kernel_location}" + fi + ) # Check exit status separately to prevent disabling 'set -e' by using the function call in the condition. + # shellcheck disable=2181 + [[ $? -eq 0 ]] || continue + set -e + image_entry_cache[${cache_key}]="${image_entry}" + if [[ -n ${image_entry} ]]; then + [[ ${kernel_cmdline} != "null" ]] && + jq -e 'has("kernel")' <<<"${image_entry}" >/dev/null && + image_entry=$(jq ".kernel.cmdline = \"${kernel_cmdline}\"" <<<"${image_entry}") + echo "${image_entry}" | jq + limactl edit --log-level error --set " + .images[${index}] = ${image_entry}| + (.images[${index}] | ..) style = \"double\" + " "${template}" + fi + done + done +done