From 29c1b96c84872abae3aae2836b768e75f26e8ff8 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 22 Nov 2024 10:08:18 +0100 Subject: [PATCH 1/3] add disk labels --- src/workflows/utils/labels.config | 5 +++++ src/workflows/utils/labels_ci.config | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/src/workflows/utils/labels.config b/src/workflows/utils/labels.config index cc79568fdcd..e7e86d2c5e3 100644 --- a/src/workflows/utils/labels.config +++ b/src/workflows/utils/labels.config @@ -18,6 +18,11 @@ process { withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + + withLabel: lowdisk { disk = { 20.GB * task.attempt } } + withLabel: middisk { disk = { 50.GB * task.attempt } } + withLabel: highdisk { disk = { 100.GB * task.attempt } } + withLabel: veryhighdisk { disk = { 200.GB * task.attempt } } } def get_memory(to_compare) { diff --git a/src/workflows/utils/labels_ci.config b/src/workflows/utils/labels_ci.config index 6e1213d70d3..87cedbbf157 100644 --- a/src/workflows/utils/labels_ci.config +++ b/src/workflows/utils/labels_ci.config @@ -6,6 +6,10 @@ process { withLabel: highmem { memory = 14.Gb } withLabel: highcpu { cpus = 4 } withLabel: veryhighmem { memory = 14.Gb } + withLabel: lowdisk { disk = { 14.GB * task.attempt } } + withLabel: middisk { disk = { 14.GB * task.attempt } } + withLabel: highdisk { disk = { 14.GB * task.attempt } } + withLabel: veryhighdisk { disk = { 14.GB * task.attempt } } } env.NUMBA_CACHE_DIR = '/tmp' From 7d01ebac558fd1e7491bae50e592d97958470a45 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 22 Nov 2024 10:08:44 +0100 Subject: [PATCH 2/3] uniformise ci labels --- src/workflows/utils/labels_ci.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/workflows/utils/labels_ci.config b/src/workflows/utils/labels_ci.config index 87cedbbf157..fa62f5357f7 100644 --- a/src/workflows/utils/labels_ci.config +++ b/src/workflows/utils/labels_ci.config @@ -1,11 +1,11 @@ process { - withLabel: lowmem { memory = 14.Gb } + withLabel: lowmem { memory = 14.GB } withLabel: lowcpu { cpus = 4 } - withLabel: midmem { memory = 14.Gb } + withLabel: midmem { memory = 14.GB } withLabel: midcpu { cpus = 4 } - withLabel: highmem { memory = 14.Gb } + withLabel: highmem { memory = 14.GB } withLabel: highcpu { cpus = 4 } - withLabel: veryhighmem { memory = 14.Gb } + withLabel: veryhighmem { memory = 14.GB } withLabel: lowdisk { disk = { 14.GB * task.attempt } } withLabel: middisk { disk = { 14.GB * task.attempt } } withLabel: highdisk { disk = { 14.GB * task.attempt } } From 74b0e69fc090a3875f400156c6cd3492247c3eb0 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 29 Nov 2024 09:37:51 +0100 Subject: [PATCH 3/3] update labels --- src/workflows/utils/labels.config | 21 ++++++++++++++++----- src/workflows/utils/labels_ci.config | 8 ++++---- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/workflows/utils/labels.config b/src/workflows/utils/labels.config index e7e86d2c5e3..e76b7c71786 100644 --- a/src/workflows/utils/labels.config +++ b/src/workflows/utils/labels.config @@ -8,21 +8,32 @@ process { maxRetries = 3 maxMemory = null - // Resource labels + // CPU resources withLabel: singlecpu { cpus = 1 } withLabel: lowcpu { cpus = 4 } withLabel: midcpu { cpus = 10 } withLabel: highcpu { cpus = 20 } + // Memory resources withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - withLabel: lowdisk { disk = { 20.GB * task.attempt } } - withLabel: middisk { disk = { 50.GB * task.attempt } } - withLabel: highdisk { disk = { 100.GB * task.attempt } } - withLabel: veryhighdisk { disk = { 200.GB * task.attempt } } + // Disk space + withLabel: lowdisk { } + withLabel: middisk { } + withLabel: highdisk { } + withLabel: veryhighdisk { } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } def get_memory(to_compare) { diff --git a/src/workflows/utils/labels_ci.config b/src/workflows/utils/labels_ci.config index fa62f5357f7..db4795ab94c 100644 --- a/src/workflows/utils/labels_ci.config +++ b/src/workflows/utils/labels_ci.config @@ -6,10 +6,10 @@ process { withLabel: highmem { memory = 14.GB } withLabel: highcpu { cpus = 4 } withLabel: veryhighmem { memory = 14.GB } - withLabel: lowdisk { disk = { 14.GB * task.attempt } } - withLabel: middisk { disk = { 14.GB * task.attempt } } - withLabel: highdisk { disk = { 14.GB * task.attempt } } - withLabel: veryhighdisk { disk = { 14.GB * task.attempt } } + withLabel: lowdisk { } + withLabel: middisk { } + withLabel: highdisk { } + withLabel: veryhighdisk { } } env.NUMBA_CACHE_DIR = '/tmp'