From abf377b921831e3ded2dd89c90a45f766693989a Mon Sep 17 00:00:00 2001 From: marboledacci Date: Fri, 13 Dec 2024 12:54:46 -0500 Subject: [PATCH] Support cache for browser installation (#119) * Trying to implement caching: chrome test 1 * Add fixes for test 1 * fix lint * add test with cache * Fix * Validate file existenceW * Cleanup cache after process * Fix typo * Add debu * Verify cache is breaking checkout * Adding log * Add more debug * Add more debug * Add more debug * update cache management * Update sudo * Updat test * Add link * Add debug * Add extra links * Update tests to have checkout * Rever jq orb update * Use another jq orb version * Add cache from macos * Update save_cache step * Save cache before delete * Update process-cache * fix * Add cache for firefox * Fix linting * fix cache usage * Update cache key for firefox test * update cache key construction * Update use_cache parameter description --- .circleci/test-deploy.yml | 38 ++++++++++++++++++++++++++++++++ src/commands/install-chrome.yml | 33 ++++++++++++++++++++++++++- src/commands/install-firefox.yml | 27 +++++++++++++++++++++++ src/scripts/install-chrome.sh | 35 ++++++++++++++++++++++++----- src/scripts/install-firefox.sh | 18 +++++++++++---- src/scripts/process-cache.sh | 19 ++++++++++++++++ 6 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 src/scripts/process-cache.sh diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml index 91f3f87..4eb329b 100644 --- a/.circleci/test-deploy.yml +++ b/.circleci/test-deploy.yml @@ -30,6 +30,7 @@ jobs: firefox-version: <> replace-existing-chrome: <> chrome-version: <> + - checkout int-tests-chrome: parameters: executor: @@ -65,6 +66,7 @@ jobs: - jq/install - browser-tools/install-chrome - browser-tools/install-chromedriver + - checkout int-test-firefox: parameters: executor: @@ -86,9 +88,45 @@ jobs: - jq/install - browser-tools/install-firefox - browser-tools/install-geckodriver + - checkout + int-test-chrome-cache: + parameters: + executor: + type: executor + executor: <> + steps: + - jq/install + - browser-tools/install-chrome: + use_cache: true + cache_key: v2 + - browser-tools/install-chromedriver + - checkout + int-test-firefox-cache: + parameters: + executor: + type: executor + executor: <> + steps: + - jq/install + - browser-tools/install-firefox: + use_cache: true + cache_key: v2 + - checkout workflows: test-deploy: jobs: + - int-test-chrome-cache: + name: test-chrome-cache-<> + matrix: + parameters: + executor: [cimg-node, macos] + filters: *filters + - int-test-firefox-cache: + name: test-firefox-cache-<> + matrix: + parameters: + executor: [cimg-node, macos] + filters: *filters - int-test-all: name: test-cimg-base-all executor: cimg-base diff --git a/src/commands/install-chrome.yml b/src/commands/install-chrome.yml index fd77bf0..54c53e8 100644 --- a/src/commands/install-chrome.yml +++ b/src/commands/install-chrome.yml @@ -26,12 +26,43 @@ parameters: type: enum enum: [ "stable", "beta" ] default: "stable" - + use_cache: + description: | + Install chrome using a cached version. + Using this option will increase costs and does not improve times considerable, use only if having problems with download. + Defaults to false. + default: false + type: boolean + cache_key: + description: | + Cache key to save chrome. + Defaults to v1. + default: v1 + type: string steps: + - when: + condition: <> + steps: + - restore_cache: + keys: + - chrome-<>-<>-{{ arch }} + - run: + name: Process cache + command: <> + environment: + ORB_PARAM_CHANNEL: << parameters.channel >> - run: name: Install Google Chrome environment: ORB_PARAM_CHROME_VERSION: <> ORB_PARAM_REPLACE_EXISTING: <> ORB_PARAM_CHANNEL: << parameters.channel >> + ORB_PARAM_SAVE_CACHE: <> command: <> + - when: + condition: <> + steps: + - save_cache: + key: chrome-<>-<>-{{ arch }} + paths: + - /tmp/chrome.tar.gz diff --git a/src/commands/install-firefox.yml b/src/commands/install-firefox.yml index 90a8df5..a4349fe 100644 --- a/src/commands/install-firefox.yml +++ b/src/commands/install-firefox.yml @@ -17,10 +17,37 @@ parameters: description: > Directory in which to install Firefox + use_cache: + description: | + Install firefox using a cached version of the installer. + Using this option will increase costs and does not improve times considerable, use only if having problems with download. + Defaults to false. + default: false + type: boolean + cache_key: + description: | + Cache key to save chrome. + Defaults to v1. + default: v1 + type: string steps: + - when: + condition: <> + steps: + - restore_cache: + keys: + - firefox-<>-<>-{{ arch }} - run: name: Install Firefox environment: ORB_PARAM_FIREFOX_INSTALL_DIR: <> ORB_PARAM_FIREFOX_VERSION: <> + ORB_PARAM_SAVE_CACHE: <> command: <> + - when: + condition: <> + steps: + - save_cache: + key: firefox-<>-<>-{{ arch }} + paths: + - /tmp/firefox diff --git a/src/scripts/install-chrome.sh b/src/scripts/install-chrome.sh index ea77f85..ef85c2b 100644 --- a/src/scripts/install-chrome.sh +++ b/src/scripts/install-chrome.sh @@ -4,11 +4,22 @@ if [[ $EUID == 0 ]]; then export SUDO=""; else export SUDO="sudo"; fi # process ORB_PARAM_CHROME_VERSION PROCESSED_CHROME_VERSION=$(circleci env subst "$ORB_PARAM_CHROME_VERSION") +save_cache() { + echo "Saving cache" + if uname -a | grep Darwin >/dev/null 2>&1; then + $SUDO tar -czf /tmp/chrome.tar.gz -C "$CHROME_TEMP_DIR" googlechrome.pkg + elif command -v apt-get >/dev/null 2>&1; then + $SUDO tar -czf /tmp/chrome.tar.gz -C /opt/google/chrome . + else + echo "This system doesn't support cache for chrome" + fi +} + # installation check if uname -a | grep Darwin >/dev/null 2>&1; then if ls /Applications/*Google\ Chrome* >/dev/null 2>&1; then - LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Mac' | jq -r ' .[0] | .version ')" if [[ "$PROCESSED_CHROME_VERSION" == "latest" ]]; then + LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Mac' | jq -r ' .[0] | .version ')" target_version="$LATEST_VERSION" else target_version="$PROCESSED_CHROME_VERSION" @@ -28,8 +39,8 @@ if uname -a | grep Darwin >/dev/null 2>&1; then fi elif grep Alpine /etc/issue >/dev/null 2>&1; then if command -v chromium-browser >/dev/null 2>&1; then - LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')" if [[ "$PROCESSED_CHROME_VERSION" == "latest" ]]; then + LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')" target_version="$LATEST_VERSION" else target_version="$PROCESSED_CHROME_VERSION" @@ -49,8 +60,8 @@ elif grep Alpine /etc/issue >/dev/null 2>&1; then fi elif command -v yum >/dev/null 2>&1; then if command -v google-chrome >/dev/null 2>&1; then - LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')" if [[ "$PROCESSED_CHROME_VERSION" == "latest" ]]; then + LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')" target_version="$LATEST_VERSION" else target_version="$PROCESSED_CHROME_VERSION" @@ -87,10 +98,18 @@ fi if uname -a | grep Darwin >/dev/null 2>&1; then echo "Preparing Chrome installation for MacOS-based systems" # Universal MacOS .pkg with license pre-accepted: https://support.google.com/chrome/a/answer/9915669?hl=en - CHROME_MAC_URL="https://dl.google.com/chrome/mac/${ORB_PARAM_CHANNEL}/accept_tos%3Dhttps%253A%252F%252Fwww.google.com%252Fintl%252Fen_ph%252Fchrome%252Fterms%252F%26_and_accept_tos%3Dhttps%253A%252F%252Fpolicies.google.com%252Fterms/googlechrome.pkg" CHROME_TEMP_DIR="$(mktemp -d)" - curl -L -o "$CHROME_TEMP_DIR/googlechrome.pkg" "$CHROME_MAC_URL" + if [ "$ORB_PARAM_SAVE_CACHE" = 1 ] && [ -f "/tmp/googlechrome.pkg" ]; then + echo "Cache found." + cp /tmp/googlechrome.pkg "$CHROME_TEMP_DIR" + else + CHROME_MAC_URL="https://dl.google.com/chrome/mac/${ORB_PARAM_CHANNEL}/accept_tos%3Dhttps%253A%252F%252Fwww.google.com%252Fintl%252Fen_ph%252Fchrome%252Fterms%252F%26_and_accept_tos%3Dhttps%253A%252F%252Fpolicies.google.com%252Fterms/googlechrome.pkg" + curl -L -o "$CHROME_TEMP_DIR/googlechrome.pkg" "$CHROME_MAC_URL" + fi sudo /usr/sbin/installer -pkg "$CHROME_TEMP_DIR/googlechrome.pkg" -target / + if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then + save_cache + fi sudo rm -rf "$CHROME_TEMP_DIR" echo '#!/usr/bin/env bash' >> google-chrome-$ORB_PARAM_CHANNEL if [[ $ORB_PARAM_CHANNEL == "beta" ]]; then @@ -129,6 +148,9 @@ elif command -v yum >/dev/null 2>&1; then >/dev/null 2>&1 $SUDO yum localinstall -y google-chrome.rpm \ >/dev/null 2>&1 + if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then + save_cache + fi rm -rf google-chrome.rpm liberation-fonts.rpm else # download chrome @@ -152,6 +174,9 @@ else && $SUDO apt-get install -y apt-utils && $SUDO apt-get install -y /tmp/chrome.deb \ && rm /tmp/chrome.deb fi + if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then + save_cache + fi fi TESTING_CHROME_VERSION=${PROCESSED_CHROME_VERSION::-2} diff --git a/src/scripts/install-firefox.sh b/src/scripts/install-firefox.sh index 3ad3760..258be82 100644 --- a/src/scripts/install-firefox.sh +++ b/src/scripts/install-firefox.sh @@ -94,10 +94,20 @@ FIREFOX_FILE_LOCATION="$PLATFORM/en-US/$FIREFOX_FILE" FIREFOX_FILE_NAME="$PLATFORM-en-US-$FIREFOX_FILE" -# download firefox -curl --silent --show-error --location --fail --retry 3 \ - --output "$FIREFOX_FILE_NAME.$FILE_EXT" \ - "$FIREFOX_URL_BASE/$FIREFOX_FILE_LOCATION.$FILE_EXT" +if [ "$ORB_PARAM_SAVE_CACHE" = 1 ] && [ -f "/tmp/firefox" ]; then + echo "Cache found." + mv /tmp/firefox "$FIREFOX_FILE_NAME.$FILE_EXT" +else + # download firefox + echo "Downloading firefox" + curl --silent --show-error --location --fail --retry 3 \ + --output "$FIREFOX_FILE_NAME.$FILE_EXT" \ + "$FIREFOX_URL_BASE/$FIREFOX_FILE_LOCATION.$FILE_EXT" +fi + +if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then + cp "$FIREFOX_FILE_NAME.$FILE_EXT" /tmp/firefox +fi if uname -a | grep Darwin >/dev/null 2>&1; then echo "No PGP data for macOS Firefox releases; skipping PGP verification" diff --git a/src/scripts/process-cache.sh b/src/scripts/process-cache.sh new file mode 100644 index 0000000..ba8d9d1 --- /dev/null +++ b/src/scripts/process-cache.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -x +if [[ $EUID == 0 ]]; then export SUDO=""; else export SUDO="sudo"; fi +if [ -f "/tmp/chrome.tar.gz" ]; then + if uname -a | grep Darwin >/dev/null 2>&1; then + $SUDO tar -xzf /tmp/chrome.tar.gz -C /tmp + elif command -v apt-get >/dev/null 2>&1; then + $SUDO mkdir -p /opt/google/chrome + $SUDO tar -xzf /tmp/chrome.tar.gz -C /opt/google/chrome + $SUDO rm -rf /tmp/chrome.tar.gz + $SUDO ln -s /opt/google/chrome/google-chrome "/usr/bin/google-chrome-$ORB_PARAM_CHANNEL" + $SUDO ln -s "/usr/bin/google-chrome-$ORB_PARAM_CHANNEL" "/etc/alternatives/google-chrome" + $SUDO ln -s "/etc/alternatives/google-chrome" "/usr/bin/google-chrome" + else + echo "This system doesn't support cache for chrome" + $SUDO rm -rf /tmp/chrome.tar.gz + fi +fi +set +x