Support cache for browser installation (#119)

* Trying to implement caching: chrome test 1 * Add fixes for test 1 * fix lint * add test with cache * Fix * Validate file existenceW * Cleanup cache after process * Fix typo * Add debu * Verify cache is breaking checkout * Adding log * Add more debug * Add more debug * Add more debug * update cache management * Update sudo * Updat test * Add link * Add debug * Add extra links * Update tests to have checkout * Rever jq orb update * Use another jq orb version * Add cache from macos * Update save_cache step * Save cache before delete * Update process-cache * fix * Add cache for firefox * Fix linting * fix cache usage * Update cache key for firefox test * update cache key construction * Update use_cache parameter description
CircleCI-Public · Dec 13, 2024 · abf377b · abf377b
1 parent 22701d0
commit abf377b
Show file tree

Hide file tree

Showing 6 changed files with 160 additions and 10 deletions.
diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml
@@ -30,6 +30,7 @@ jobs:
           firefox-version: <<parameters.firefox-version>>
           replace-existing-chrome: <<parameters.replace-existing-chrome>>
           chrome-version: <<parameters.chrome-version>>
+      - checkout
   int-tests-chrome:
     parameters:
       executor:
@@ -65,6 +66,7 @@ jobs:
       - jq/install
       - browser-tools/install-chrome
       - browser-tools/install-chromedriver
+      - checkout
   int-test-firefox:
     parameters:
       executor:
@@ -86,9 +88,45 @@ jobs:
       - jq/install
       - browser-tools/install-firefox
       - browser-tools/install-geckodriver
+      - checkout
+  int-test-chrome-cache:
+    parameters:
+      executor:
+        type: executor
+    executor: <<parameters.executor>>
+    steps:
+      - jq/install
+      - browser-tools/install-chrome:
+          use_cache: true
+          cache_key: v2
+      - browser-tools/install-chromedriver
+      - checkout
+  int-test-firefox-cache:
+    parameters:
+      executor:
+        type: executor
+    executor: <<parameters.executor>>
+    steps:
+      - jq/install
+      - browser-tools/install-firefox:
+          use_cache: true
+          cache_key: v2
+      - checkout
 workflows:
   test-deploy:
     jobs:
+      - int-test-chrome-cache:
+          name: test-chrome-cache-<<matrix.executor>>
+          matrix:
+            parameters:
+              executor: [cimg-node, macos]
+          filters: *filters
+      - int-test-firefox-cache:
+          name: test-firefox-cache-<<matrix.executor>>
+          matrix:
+            parameters:
+              executor: [cimg-node, macos]
+          filters: *filters
       - int-test-all:
           name: test-cimg-base-all
           executor: cimg-base

diff --git a/src/commands/install-chrome.yml b/src/commands/install-chrome.yml
@@ -26,12 +26,43 @@ parameters:
     type: enum
     enum: [ "stable", "beta" ]
     default: "stable"
-
+  use_cache:
+    description: |
+      Install chrome using a cached version.
+      Using this option will increase costs and does not improve times considerable, use only if having problems with download.
+      Defaults to false.
+    default: false
+    type: boolean
+  cache_key:
+    description: |
+      Cache key to save chrome.
+      Defaults to v1.
+    default: v1
+    type: string
 steps:
+  - when:
+      condition: <<parameters.use_cache>>
+      steps:
+      - restore_cache:
+          keys:
+          - chrome-<<parameters.cache_key>>-<<parameters.chrome-version>>-{{ arch }}
+      - run:
+          name: Process cache
+          command: <<include(scripts/process-cache.sh)>>
+          environment:
+            ORB_PARAM_CHANNEL: << parameters.channel >>
   - run:
       name: Install Google Chrome
       environment:
         ORB_PARAM_CHROME_VERSION: <<parameters.chrome-version>>
         ORB_PARAM_REPLACE_EXISTING: <<parameters.replace-existing>>
         ORB_PARAM_CHANNEL: << parameters.channel >>
+        ORB_PARAM_SAVE_CACHE: <<parameters.use_cache>>
       command: <<include(scripts/install-chrome.sh)>>
+  - when:
+      condition: <<parameters.use_cache>>
+      steps:
+      - save_cache:
+          key: chrome-<<parameters.cache_key>>-<<parameters.chrome-version>>-{{ arch }}
+          paths:
+          - /tmp/chrome.tar.gz
diff --git a/src/commands/install-firefox.yml b/src/commands/install-firefox.yml
@@ -17,10 +17,37 @@ parameters:
     description: >
       Directory in which to install Firefox
 
+  use_cache:
+    description: |
+      Install firefox using a cached version of the installer.
+      Using this option will increase costs and does not improve times considerable, use only if having problems with download.
+      Defaults to false.
+    default: false
+    type: boolean
+  cache_key:
+    description: |
+      Cache key to save chrome.
+      Defaults to v1.
+    default: v1
+    type: string
 steps:
+  - when:
+      condition: <<parameters.use_cache>>
+      steps:
+      - restore_cache:
+          keys:
+          - firefox-<<parameters.cache_key>>-<<parameters.version>>-{{ arch }}
   - run:
       name: Install Firefox
       environment:
         ORB_PARAM_FIREFOX_INSTALL_DIR: <<parameters.install-dir>>
         ORB_PARAM_FIREFOX_VERSION: <<parameters.version>>
+        ORB_PARAM_SAVE_CACHE: <<parameters.use_cache>>
       command: <<include(scripts/install-firefox.sh)>>
+  - when:
+      condition: <<parameters.use_cache>>
+      steps:
+      - save_cache:
+          key: firefox-<<parameters.cache_key>>-<<parameters.version>>-{{ arch }}
+          paths:
+          - /tmp/firefox
diff --git a/src/scripts/install-chrome.sh b/src/scripts/install-chrome.sh
@@ -4,11 +4,22 @@ if [[ $EUID == 0 ]]; then export SUDO=""; else export SUDO="sudo"; fi
 # process ORB_PARAM_CHROME_VERSION
 PROCESSED_CHROME_VERSION=$(circleci env subst "$ORB_PARAM_CHROME_VERSION")
 
+save_cache() {
+  echo "Saving cache"
+  if uname -a | grep Darwin >/dev/null 2>&1; then
+    $SUDO tar -czf /tmp/chrome.tar.gz -C "$CHROME_TEMP_DIR" googlechrome.pkg
+  elif command -v apt-get >/dev/null 2>&1; then
+    $SUDO tar -czf /tmp/chrome.tar.gz -C /opt/google/chrome .
+  else
+    echo "This system doesn't support cache for chrome"
+  fi
+}
+
 # installation check
 if uname -a | grep Darwin >/dev/null 2>&1; then
   if ls /Applications/*Google\ Chrome* >/dev/null 2>&1; then
-    LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Mac' | jq -r ' .[0] | .version ')"
     if [[ "$PROCESSED_CHROME_VERSION" == "latest" ]]; then
+      LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Mac' | jq -r ' .[0] | .version ')"
       target_version="$LATEST_VERSION"
     else
       target_version="$PROCESSED_CHROME_VERSION"
@@ -28,8 +39,8 @@ if uname -a | grep Darwin >/dev/null 2>&1; then
   fi
 elif grep Alpine /etc/issue >/dev/null 2>&1; then
   if command -v chromium-browser >/dev/null 2>&1; then
-    LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')"
     if [[ "$PROCESSED_CHROME_VERSION" == "latest" ]]; then
+      LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')"
       target_version="$LATEST_VERSION"
     else
       target_version="$PROCESSED_CHROME_VERSION"
@@ -49,8 +60,8 @@ elif grep Alpine /etc/issue >/dev/null 2>&1; then
   fi
 elif command -v yum >/dev/null 2>&1; then
   if command -v google-chrome >/dev/null 2>&1; then
-    LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')"
     if [[ "$PROCESSED_CHROME_VERSION" == "latest" ]]; then
+      LATEST_VERSION="$(curl -s 'https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=Linux' | jq -r ' .[0] | .version ')"
       target_version="$LATEST_VERSION"
     else
       target_version="$PROCESSED_CHROME_VERSION"
@@ -87,10 +98,18 @@ fi
 if uname -a | grep Darwin >/dev/null 2>&1; then
   echo "Preparing Chrome installation for MacOS-based systems"
   # Universal MacOS .pkg with license pre-accepted: https://support.google.com/chrome/a/answer/9915669?hl=en
-  CHROME_MAC_URL="https://dl.google.com/chrome/mac/${ORB_PARAM_CHANNEL}/accept_tos%3Dhttps%253A%252F%252Fwww.google.com%252Fintl%252Fen_ph%252Fchrome%252Fterms%252F%26_and_accept_tos%3Dhttps%253A%252F%252Fpolicies.google.com%252Fterms/googlechrome.pkg"
   CHROME_TEMP_DIR="$(mktemp -d)"
-  curl -L -o "$CHROME_TEMP_DIR/googlechrome.pkg" "$CHROME_MAC_URL"
+  if [ "$ORB_PARAM_SAVE_CACHE" = 1 ] && [ -f "/tmp/googlechrome.pkg" ]; then
+    echo "Cache found."
+    cp /tmp/googlechrome.pkg "$CHROME_TEMP_DIR"
+  else
+    CHROME_MAC_URL="https://dl.google.com/chrome/mac/${ORB_PARAM_CHANNEL}/accept_tos%3Dhttps%253A%252F%252Fwww.google.com%252Fintl%252Fen_ph%252Fchrome%252Fterms%252F%26_and_accept_tos%3Dhttps%253A%252F%252Fpolicies.google.com%252Fterms/googlechrome.pkg"
+    curl -L -o "$CHROME_TEMP_DIR/googlechrome.pkg" "$CHROME_MAC_URL"
+  fi
   sudo /usr/sbin/installer -pkg "$CHROME_TEMP_DIR/googlechrome.pkg" -target /
+  if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then
+    save_cache
+  fi
   sudo rm -rf "$CHROME_TEMP_DIR"
   echo '#!/usr/bin/env bash' >> google-chrome-$ORB_PARAM_CHANNEL
   if [[ $ORB_PARAM_CHANNEL == "beta" ]]; then
@@ -129,6 +148,9 @@ elif command -v yum >/dev/null 2>&1; then
     >/dev/null 2>&1
   $SUDO yum localinstall -y google-chrome.rpm \
     >/dev/null 2>&1
+  if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then
+    save_cache
+  fi
   rm -rf google-chrome.rpm liberation-fonts.rpm
 else
   # download chrome
@@ -152,6 +174,9 @@ else
       && $SUDO apt-get install -y apt-utils && $SUDO apt-get install -y /tmp/chrome.deb \
       && rm /tmp/chrome.deb
   fi
+  if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then
+    save_cache
+  fi
 fi
 
 TESTING_CHROME_VERSION=${PROCESSED_CHROME_VERSION::-2}

diff --git a/src/scripts/install-firefox.sh b/src/scripts/install-firefox.sh
@@ -94,10 +94,20 @@ FIREFOX_FILE_LOCATION="$PLATFORM/en-US/$FIREFOX_FILE"
 
 FIREFOX_FILE_NAME="$PLATFORM-en-US-$FIREFOX_FILE"
 
-# download firefox
-curl --silent --show-error --location --fail --retry 3 \
-  --output "$FIREFOX_FILE_NAME.$FILE_EXT" \
-  "$FIREFOX_URL_BASE/$FIREFOX_FILE_LOCATION.$FILE_EXT"
+if [ "$ORB_PARAM_SAVE_CACHE" = 1 ] && [ -f "/tmp/firefox" ]; then
+  echo "Cache found."
+  mv /tmp/firefox "$FIREFOX_FILE_NAME.$FILE_EXT"
+else
+  # download firefox
+  echo "Downloading firefox"
+  curl --silent --show-error --location --fail --retry 3 \
+    --output "$FIREFOX_FILE_NAME.$FILE_EXT" \
+    "$FIREFOX_URL_BASE/$FIREFOX_FILE_LOCATION.$FILE_EXT"
+fi
+
+if [ "$ORB_PARAM_SAVE_CACHE" = 1 ]; then
+  cp "$FIREFOX_FILE_NAME.$FILE_EXT" /tmp/firefox
+fi
 
 if uname -a | grep Darwin >/dev/null 2>&1; then
   echo "No PGP data for macOS Firefox releases; skipping PGP verification"

diff --git a/src/scripts/process-cache.sh b/src/scripts/process-cache.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -x
+if [[ $EUID == 0 ]]; then export SUDO=""; else export SUDO="sudo"; fi
+if [ -f "/tmp/chrome.tar.gz" ]; then
+    if uname -a | grep Darwin >/dev/null 2>&1; then
+        $SUDO tar -xzf /tmp/chrome.tar.gz -C /tmp
+    elif command -v apt-get >/dev/null 2>&1; then
+        $SUDO mkdir -p /opt/google/chrome
+        $SUDO tar -xzf /tmp/chrome.tar.gz -C /opt/google/chrome
+        $SUDO rm -rf /tmp/chrome.tar.gz
+        $SUDO ln -s /opt/google/chrome/google-chrome "/usr/bin/google-chrome-$ORB_PARAM_CHANNEL"
+        $SUDO ln -s "/usr/bin/google-chrome-$ORB_PARAM_CHANNEL" "/etc/alternatives/google-chrome"
+        $SUDO ln -s "/etc/alternatives/google-chrome" "/usr/bin/google-chrome"
+    else
+        echo "This system doesn't support cache for chrome"
+        $SUDO rm -rf /tmp/chrome.tar.gz
+    fi
+fi
+set +x