Replace method_ids with methods/metrics include and exclude

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 <!-- * Restructured `src` directory (PR #3). -->
 
+* `run_benchmark`: replaced `--method_ids` with `--methods_include`/`--methods_exclude` and added `--metrics_include`/`--metrics_exclude` (PR #20).
+
 ## NEW FUNCTIONALITY
 
 * Added `control_methods/true_labels` component (PR #5).

src/workflows/run_benchmark/config.vsh.yaml

@@ -47,19 +47,50 @@ argument_groups:
         required: true
         direction: output
         default: task_info.yaml
-  - name: Methods
+  - name: Method filtering
+    description: |
+      Use these arguments to filter methods by name. By default, all methods are
+      run. If `--methods_include` is defined, only those methods are run. If
+      `--methods_exclude` is defined, all methods except those specified are run.
+      These arguments are mutually exclusive, so only `--methods_include` OR
+      `--methods_exclude` can set but not both.
     arguments:
-      - name: "--method_ids"
+      - name: "--methods_include"
         type: string
         multiple: true
-        description: A list of method ids to run. If not specified, all methods will be run.
+        description: |
+          A list of method ids to include. If specified, only these methods will be run.
+      - name: "--methods_exclude"
+        type: string
+        multiple: true
+        description: |
+          A list of method ids to exclude. If specified, all methods except the ones listed will be run.
+  - name: Metric filtering
+    description: |
+      Use these arguments to filter metrics by name. By default, all metrics are
+      run. If `--metrics_include` is defined, only those metrics are run. If
+      `--metrics_exclude` is defined, all metrics except those specified are run.
+      These arguments are mutually exclusive, so only `--metrics_include` OR
+      `--metrics_exclude` can set but not both.
+    arguments:
+      - name: "--metrics_include"
+        type: string
+        multiple: true
+        description: |
+          A list of metric ids to include. If specified, only these metrics will be run.
+      - name: "--metrics_exclude"
+        type: string
+        multiple: true
+        description: |
+          A list of metric ids to exclude. If specified, all metrics except the ones listed will be run.
 
 resources:
   - type: nextflow_script
     path: main.nf
     entrypoint: run_wf
   - type: file
     path: /_viash.yaml
+  - path: /common/nextflow_helpers/helper.nf
 
 dependencies:
   - name: utils/extract_uns_metadata

src/workflows/run_benchmark/main.nf

@@ -1,3 +1,5 @@
+include { checkItemAllowed } from "${meta.resources_dir}/helper.nf"
+
 workflow auto {
   findStates(params, meta.config)
     | meta.workflow.run(
@@ -57,7 +59,13 @@ workflow run_wf {
         // if the preferred normalisation is none at all,
         // we can pass whichever dataset we want
         def norm_check = (norm == "log_cp10k" && pref == "counts") || norm == pref
-        def method_check = !state.method_ids || state.method_ids.contains(comp.config.name)
+        def method_check = checkItemAllowed(
+          comp.config.name,
+          state.methods_include,
+          state.methods_exclude,
+          "methods_include",
+          "methods_exclude"
+        )
 
         method_check && norm_check
       },
@@ -91,6 +99,18 @@ workflow run_wf {
     // run all metrics
     | runEach(
       components: metrics,
+
+      // use the 'filter' argument to only run the metrics the user asked for
+      filter: { id, state, comp ->
+        checkItemAllowed(
+          comp.config.name,
+          state.metrics_include,
+          state.metrics_exclude,
+          "metrics_include",
+          "metrics_exclude"
+        )
+      },
+
       id: { id, state, comp ->
         id + "." + comp.config.name
       },