Update CI

openpipelines-bio · Jul 4, 2024 · db8e8dc · db8e8dc
1 parent 0ead6c6
commit db8e8dc
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 77 deletions.
diff --git a/.github/workflows/create-documentation-pr.yml b/.github/workflows/create-documentation-pr.yml
@@ -13,55 +13,8 @@ on:
       - '*'
 
 jobs:
-  list:
-    runs-on: ubuntu-latest
-
-    outputs:
-      component_matrix: ${{ steps.set_matrix.outputs.components }}
-      workflow_matrix: ${{ steps.set_matrix.outputs.workflows }}
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - id: ns_list_components
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        platform: docker
-        format: json
-        query_namespace: ^(?!workflows)
-
-    - id: ns_list_workflows
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        format: json
-        query_namespace: ^workflows
-
-    - id: set_matrix
-      run: |
-        echo "components=$(jq -c '[ .[] | 
-          { 
-            "name": .functionality.name,
-            "namespace_separator": .platforms | map(select(.type == "docker"))[0].namespace_separator,
-            "namespace": .functionality.namespace,
-            "config": .info.config,
-            "dir": .info.config | capture("^(?<dir>.*\/)").dir
-          }
-        ]' ${{ steps.ns_list_components.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
-        echo "workflows=$(jq -c '[ .[] | 
-          { 
-            "name": $config.functionality.name,
-            "namespace": $config.functionality.namespace,
-            "main_script": ((.info.config | capture("^(?<dir>.*\/)").dir) + "/" + (.functionality.test_resources | map(select(.type == "nextflow_script")) | unique | .[].path )),
-            "config": .info.config,
-            "entry": .functionality.test_resources | map(select(.type == "nextflow_script")) | .[].entrypoint
-          }
-        ]' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
   push_documentation:
     runs-on: ubuntu-latest
-    needs: [ list ]
 
     steps:
       - uses: actions/checkout@v4
@@ -74,7 +27,7 @@ jobs:
           path: website
           token: ${{ secrets.GTHB_PAT }}
 
-      - uses: viash-io/viash-actions/setup@v5
+      - uses: viash-io/viash-actions/setup@v6
 
       - name: Get tag to use
         id: get_tag
@@ -83,12 +36,25 @@ jobs:
           TAG_OR_BRANCH_NAME=${INPUT_TAG:-"${{ github.ref_name }}"}
           echo "tag=$TAG_OR_BRANCH_NAME" >> $GITHUB_OUTPUT
 
-      - uses: viash-io/viash-actions/pro/generate-documentation-qmd@v5
+      - uses: viash-io/viash-actions/pro/generate-documentation-qmd@v6
+        with:
+          project_directory: openpipelines
+          src: ./
+          query: '^(?!workflows)'
+          output_dir: ../website/components/modules/
+          dest_path: "{namespace}/{name}.qmd"
+          viash_pro_token: ${{ secrets.GTHB_PAT }}
+          tools_version: main_build
+          ref: "${{ steps.get_tag.outputs.tag }}"
+
+
+      - uses: viash-io/viash-actions/pro/generate-documentation-qmd@v6
         with:
           project_directory: openpipelines
           src: ./
-          output_dir: ../website/components/
-          dest_path: "{type}s/{namespace}/{name}.qmd"
+          query: '^workflows'
+          output_dir: ../website/components/workflows/
+          dest_path: "{namespace}/{name}.qmd"
           viash_pro_token: ${{ secrets.GTHB_PAT }}
           tools_version: main_build
           ref: "${{ steps.get_tag.outputs.tag }}"

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -35,7 +35,7 @@ jobs:
       cache_key: ${{ steps.cache.outputs.cache_key }}
 
     steps:
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
       id: cache 
       with:
         s3_bucket: $s3_bucket
@@ -65,7 +65,7 @@ jobs:
       with:
         ref: 'integration_build'
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - uses: nf-core/[email protected]
 

diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml
@@ -131,41 +131,39 @@ jobs:
         git fetch origin ${{github.ref_name}} 
         git checkout -f --no-overlay origin/${{github.ref_name}} -- '.'
     
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - name: Remove target folder from .gitignore
       run: |
         # allow publishing the target folder
         sed -i '/^\/target\/$/d' .gitignore
 
-    - uses: viash-io/viash-actions/ns-build@v5
+    - uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: |
           .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
           .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}'
         parallel: true
         query: ^(?!workflows)
 
-    - uses: viash-io/viash-actions/ns-build@v5
+    - uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
         parallel: true
         query: ^workflows
 
     - name: Build nextflow schemas
-      uses: viash-io/viash-actions/pro/build-nextflow-schemas@v5
+      uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6
       with:
-        components: src
-        workflows: src
+        target_dir: target/nextflow
         viash_pro_token: ${{ secrets.GTHB_PAT }}
         tools_version: 'main_build'
         enable_dataset_input: true
 
     - name: Build parameter files
-      uses: viash-io/viash-actions/pro/build-nextflow-params@v5
+      uses: viash-io/viash-actions/pro/build-nextflow-params@v6
       with:
-        workflows: src
-        components: src
+        target_dir: target/nextflow
         viash_pro_token: ${{ secrets.GTHB_PAT }}
         tools_version: 'main_build'
 
@@ -178,7 +176,7 @@ jobs:
         branch:  ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
     - name: "List components"
       id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
+      uses: viash-io/viash-actions/ns-list@v6
       with:
         platform: docker
         src: src
@@ -187,7 +185,7 @@ jobs:
 
     - name: "List workflows"
       id: ns_list_workflows
-      uses: viash-io/viash-actions/ns-list@v5
+      uses: viash-io/viash-actions/ns-list@v6
       with:
         src: src
         format: json
@@ -235,10 +233,10 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - name: Build container
-      uses: viash-io/viash-actions/ns-build@v5
+      uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: |
           .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
@@ -255,7 +253,7 @@ jobs:
         password: ${{ secrets.GTHB_PAT }}
 
     - name: Push container
-      uses: viash-io/viash-actions/ns-build@v5
+      uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
         platform: docker

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
@@ -28,7 +28,7 @@ jobs:
       cache_key: ${{ steps.cache.outputs.cache_key }}
 
     steps:
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
       id: cache 
       with:
         s3_bucket: $s3_bucket
@@ -58,7 +58,7 @@ jobs:
       with:
         ref: 'integration_build'
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - uses: nf-core/[email protected]
 
@@ -123,7 +123,7 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     # use cache
     - name: Cache resources data

diff --git a/.github/workflows/viash-test.yml b/.github/workflows/viash-test.yml
@@ -41,28 +41,29 @@ jobs:
       with:
         fetch-depth: 0 
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - name: Check if all config can be parsed if there is no unicode support
       run: |
         LANG=C viash ns list > /dev/null
+        JAVA_TOOL_OPTIONS=-Dfile.encoding=ascii viash ns build --parallel --setup meh > /dev/null
 
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
       id: cache
       with:
         s3_bucket: $s3_bucket
         dest_path: resources_test
         cache_key_prefix: resources_test__
 
     - id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
+      uses: viash-io/viash-actions/ns-list@v6
       with:
         platform: docker
         format: json
         query_namespace: ^(?!workflows)
 
     - id: ns_list_filtered
-      uses: viash-io/viash-actions/project/detect-changed-components@v5
+      uses: viash-io/viash-actions/project/detect-changed-components@v6
       with:
         input_file: "${{ steps.ns_list.outputs.output_file }}"
 
@@ -95,7 +96,7 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     # use cache
     - name: Cache resources data
@@ -163,10 +164,10 @@ jobs:
         password: ${{ env.NVIDIA_PASSWORD }}
 
     - name: Run test
-      timeout-minutes: 30
+      timeout-minutes: 40
       run: |
         viash test \
           "${{ matrix.component.config }}" \
           --cpus 4 \
-          --memory "12gb"
+          --memory "14gb"