[DRAFT] CI experiments

NVIDIA · Apr 21, 2024 · e875e4c · e875e4c
1 parent 29a24f4
commit e875e4c
Show file tree

Hide file tree

Showing 8 changed files with 900 additions and 225 deletions.
diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh
@@ -13,7 +13,7 @@ explode_std_versions() {
 }
 
 explode_libs() {
-  jq -cr 'map(. as $o | {lib: $o.lib[]} + del($o.lib))'
+  jq -cr 'map(. as $o | {project: $o.project[]} + del($o.project))'
 }
 
 # Filter out the libraries that are dirty
@@ -34,7 +34,10 @@ filter_libs() {
   # echo "Dirty libraries: ${dirty_libs[@]}" >> /dev/stderr
 
   # Construct a regex to filter out the dirty libraries
-  dirty_lib_regex=$(IFS="|"; echo "${dirty_libs[*]}")
+  dirty_lib_regex=$(
+    IFS="|"
+    echo "${dirty_libs[*]}"
+  )
   dirty_lib_regex="^(${dirty_lib_regex})\$"
   jq_filter="map(select(.lib | test(\"$dirty_lib_regex\")))"
   jq -cr "$jq_filter"
@@ -44,39 +47,88 @@ extract_matrix() {
   local file="$1"
   local type="$2"
   local matrix=$(yq -o=json "$file" | jq -cr ".$type")
-  write_output "DEVCONTAINER_VERSION" "$(yq -o json "$file" | jq -cr '.devcontainer_version')"
+  local devcontainer_version=$(yq -o json "$file" | jq -cr '.devcontainer_version')
 
-  local nvcc_full_matrix="$(echo "$matrix" | jq -cr '.nvcc' | explode_std_versions )"
-  local per_cuda_compiler_matrix="$(echo "$nvcc_full_matrix" | jq -cr ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')"
-  write_output "PER_CUDA_COMPILER_MATRIX"  "$per_cuda_compiler_matrix"
-  write_output "PER_CUDA_COMPILER_KEYS" "$(echo "$per_cuda_compiler_matrix" | jq -r 'keys | @json')"
+  write_output "DEVCONTAINER_VERSION" "$devcontainer_version"
 
-  write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)"
+  local full_matrix="$(echo "$matrix" | explode_std_versions | explode_libs)" # | filter_libs)"
 
-  local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs | filter_libs)"
-  write_output "CLANG_CUDA_MATRIX" "$clang_cuda_matrix"
-  write_output "CCCL_INFRA_MATRIX" "$(echo "$matrix" | jq -cr '.["cccl-infra"]' )"
+  create_dispatch_job() {
+    local input_object="$1"
+    local output_object=$(.github/actions/compute-matrix/generate-ci-dispatch-job.sh "$devcontainer_version" "$input_object")
+    echo "$output_object"
+  }
+
+  # For each json object in the nvcc matrix array, accumulate the output json object from `generate-ci-dispatch-job.sh`
+  # into a single json object. If duplicate keys are found, merge the matching value arrays.
+  local wf_json="{}"
+  local input_objects=$(echo "$full_matrix" | jq -cr '.[]')
+  for input_object in $input_objects; do
+    local dispatch_job=$(create_dispatch_job "$input_object")
+    wf_json=$(
+      jq --argjson accum "$wf_json" --argjson new "$dispatch_job" '
+        $new | . as $input |
+        reduce keys[] as $key (
+            $accum;
+            if .[$key] then
+                .[$key] |= (
+                    reduce ($input[$key] | keys_unsorted[]) as $nestedKey (.;
+                        .[$nestedKey] += $input[$key][$nestedKey]
+                    )
+                )
+            else
+                .[$key] = $input[$key]
+            end
+        )
+    ' <<<'{}')
+  done
+  wf_keys=$(echo "$wf_json" | jq -r 'keys | @json')
+
+  echo "wf_json:" >&2
+  echo "$wf_json" | jq '.' >&2
+
+  write_output "WORKFLOW" "$(echo "$wf_json" | jq -c '.')"
+  write_output "WORKFLOW_KEYS" "$wf_keys"
 }
 
+# local per_cuda_compiler_matrix="$(echo "$nvcc_full_matrix" | jq -cr ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')"
+# write_output "PER_CUDA_COMPILER_MATRIX"  "$per_cuda_compiler_matrix"
+# write_output "PER_CUDA_COMPILER_KEYS" "$(echo "$per_cuda_compiler_matrix" | jq -r 'keys | @json')"
+
+# write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)"
+
+# local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs | filter_libs)"
+# write_output "CLANG_CUDA_MATRIX" "$clang_cuda_matrix"
+# write_output "CCCL_INFRA_MATRIX" "$(echo "$matrix" | jq -cr '.["cccl-infra"]' )"
+# }
+
 main() {
   if [ "$1" == "-v" ]; then
     set -x
     shift
   fi
 
-  if [ $# -ne 2 ] || [ "$2" != "pull_request" ]; then
+  if [ $# -ne 2 ]; then
     echo "Usage: $0 [-v] MATRIX_FILE MATRIX_TYPE"
     echo "  -v            : Enable verbose output"
     echo "  MATRIX_FILE   : The path to the matrix file."
-    echo "  MATRIX_TYPE   : The desired matrix. Supported values: 'pull_request'"
+    echo "  MATRIX_TYPE   : The desired matrix."
+    exit 1
+  fi
+
+  local matrix_file="$1"
+  local matrix_type="$2"
+
+  if [[ ! -f "$matrix_file" ]]; then
+    echo "Error: The matrix file '$matrix_file' does not exist."
     exit 1
   fi
 
   echo "Input matrix file:" >&2
-  cat "$1" >&2
-  echo "Matrix Type: $2" >&2
+  cat "$matrix_file" >&2
+  echo "Matrix Type: $matrix_type" >&2
 
-  extract_matrix "$1" "$2"
+  extract_matrix "$matrix_file" "$matrix_type"
 }
 
 main "$@"