diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b2b8d..0c3041e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ * Added `metrics/accuracy` component (PR #5). +* Added `control_methods/random_labels` component (PR #22). + ## MAJOR CHANGES * Updated `api` files (PR #5). @@ -43,3 +45,7 @@ ## BUGFIXES +* `process_dataset`: also seed `np.random` when `--seed` is set (PR #22). + +* `accuracy`: write `metric_values` as a list to match `metric_ids` (PR #22). + diff --git a/src/control_methods/random_labels/config.vsh.yaml b/src/control_methods/random_labels/config.vsh.yaml new file mode 100644 index 0000000..7218ec5 --- /dev/null +++ b/src/control_methods/random_labels/config.vsh.yaml @@ -0,0 +1,45 @@ +# The API specifies which type of component this is. +# It contains specifications for: +# - The input/output files +# - Common parameters +# - A unit test +__merge__: ../../api/comp_control_method.yaml + +# A unique identifier for your component (required). +# Can contain only lowercase letters or underscores. +name: random_labels + +# A relatively short label, used when rendering visualisations (required) +label: Random Labels +# A one sentence summary of how this method works (required). Used when +# rendering summary tables. +summary: "a negative control, where the labels are randomly predicted." +# A multi-line description of how this component works (required). Used +# when rendering reference documentation. +description: | + A negative control, where the labels are randomly predicted based on the + label distribution of the training data, without looking at the input data. + +# Metadata for your component +info: + # Which normalisation method this component prefers to use (required). + preferred_normalization: counts + +# Resources required to run the component +resources: + # The script of your component (required) + - type: python_script + path: script.py + +engines: + # Specifications for the Docker image for this component. + - type: docker + image: openproblems/base_python:1 + +runners: + # This platform allows running the component natively + - type: executable + # Allows turning the component into a Nextflow module / pipeline. + - type: nextflow + directives: + label: [midtime, lowmem, lowcpu] diff --git a/src/control_methods/random_labels/script.py b/src/control_methods/random_labels/script.py new file mode 100644 index 0000000..3e64303 --- /dev/null +++ b/src/control_methods/random_labels/script.py @@ -0,0 +1,47 @@ +import anndata as ad +import numpy as np + +## VIASH START +# Note: this section is auto-generated by viash at runtime. To edit it, make changes +# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. +par = { + 'input_train': 'resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad', + 'input_test': 'resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad', + 'input_solution': 'resources_test/task_template/cxg_mouse_pancreas_atlas/solution.h5ad', + 'output': 'output.h5ad' +} +meta = { + 'name': 'random_labels' +} +## VIASH END + +print('Reading input files', flush=True) +input_train = ad.read_h5ad(par['input_train']) +input_test = ad.read_h5ad(par['input_test']) + +print('Compute label distribution', flush=True) +label_distribution = input_train.obs["label"].value_counts() +label_distribution = label_distribution / label_distribution.sum() + +print('Generate predictions', flush=True) +obs_label_pred = np.random.choice( + label_distribution.index, + size=input_test.n_obs, + replace=True, + p=label_distribution +) + +print("Write output AnnData to file", flush=True) +output = ad.AnnData( + uns={ + 'dataset_id': input_train.uns['dataset_id'], + 'normalization_id': input_train.uns['normalization_id'], + 'method_id': meta['name'] + }, + obs={ + 'label_pred': obs_label_pred + } +) +output.obs_names = input_test.obs_names + +output.write_h5ad(par['output'], compression='gzip') diff --git a/src/data_processors/process_dataset/script.py b/src/data_processors/process_dataset/script.py index 3eb56c2..0274c2d 100644 --- a/src/data_processors/process_dataset/script.py +++ b/src/data_processors/process_dataset/script.py @@ -28,9 +28,10 @@ config = op.project.read_viash_config(meta["config"]) # set seed if need be -if par["seed"]: +if par["seed"] is not None: print(f">> Setting seed to {par['seed']}") random.seed(par["seed"]) + np.random.seed(par["seed"]) print(">> Load data", flush=True) adata = ad.read_h5ad(par["input"]) diff --git a/src/metrics/accuracy/script.py b/src/metrics/accuracy/script.py index 054e809..6260ae7 100644 --- a/src/metrics/accuracy/script.py +++ b/src/metrics/accuracy/script.py @@ -32,7 +32,7 @@ # metric_ids and metric_values can have length > 1 # but should be of equal length uns_metric_ids = [ 'accuracy' ] -uns_metric_values = np.mean(input_solution.obs["label"] == input_prediction.obs["label_pred"]) +uns_metric_values = [ np.mean(input_solution.obs["label"] == input_prediction.obs["label_pred"]) ] print("Write output AnnData to file", flush=True) output = ad.AnnData( diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index 6533bcd..ef85e36 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -96,6 +96,7 @@ dependencies: - name: utils/extract_uns_metadata repository: openproblems - name: control_methods/true_labels + - name: control_methods/random_labels - name: methods/logistic_regression - name: metrics/accuracy diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index dc0991d..de2c94e 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -10,6 +10,7 @@ workflow auto { // construct list of methods and control methods methods = [ true_labels, + random_labels, logistic_regression ]