Consolidate bulk/single-cell TCR analysis into BULKTCR_ANALYSIS; remove dead cellranger/pseudobulk code

.cirro/process-form.json → .cirro/bulk_analysis/process-form.json

@@ -57,36 +57,6 @@
                 "description": "minimum p-value for determining cluster significance (GLIPH2)",
                 "title": "local_min_pvalue",
                 "type": "string"
-            },
-            "p_depth": {
-                "default": "1000",
-                "description": "p_depth (GLIPH2)",
-                "title": "p_depth",
-                "type": "string"
-            },
-            "samplechart_x_col": {
-                "default": "timepoint",
-                "description": "Metadata column for x axis of sample-level plots for Sample workflow notebook",
-                "title": "Sample plot X axis column",
-                "type": "string"
-            },
-            "samplechart_color_col": {
-                "default": "origin",
-                "description": "Metadata column for legend color of sample-level plots for Sample workflow notebook",
-                "title": "Sample plot X axis column",
-                "type": "string"
-            },
-            "vgene_subject_col": {
-                "default": "patient",
-                "description": "Metadata column for grouping of samples for V gene plots for Sample workflow notebook",
-                "title": "V gene plot subject column",
-                "type": "string"
-            },
-            "vgene_x_cols": {
-                "default": "timepoint,origin",
-                "description": "Comma-separated list of metadata columns for x axis of V gene plot (eg. timepoint,origin or timepoint)",
-                "title": "V gene plot X axis columns",
-                "type": "string"
             }
         }
     },

.cirro/process-input.json → .cirro/bulk_analysis/process-input.json

@@ -10,10 +10,5 @@
     "kmer_min_depth": "$.params.dataset.paramJson.kmer_min_depth",
     "local_min_OVE": "$.params.dataset.paramJson.local_min_OVE",
     "local_min_pvalue": "$.params.dataset.paramJson.local_min_pvalue",
-    "outdir": "$.params.dataset.s3|/data/",
-    "p_depth": "$.params.dataset.paramJson.p_depth",
-    "samplechart_x_col": "$.params.dataset.paramJson.samplechart_x_col",
-    "samplechart_color_col": "$.params.dataset.paramJson.samplechart_color_col",
-    "vgene_subject_col": "$.params.dataset.paramJson.vgene_subject_col",
-    "vgene_x_cols": "$.params.dataset.paramJson.vgene_x_cols"
+    "outdir": "$.params.dataset.s3|/data/"
 }

.cirro/singlecell_full_gex/preprocess.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+from cirro.helpers.preprocess_dataset import PreprocessDataset
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+# TCRtoolkit single-cell mode needs a 'sample_sheet.csv' with a 'path' column
+# pointing at each sample's Cell Ranger VDJ outs/ DIRECTORY - see
+# modules/scratch/VDJ_QC/VDJ_QC_analysis.qmd's column contract
+# (sample_sheet_sample_col="sample", sample_sheet_path_col="path").
+#
+# Cirro's dataset-file abstraction doesn't stage whole directories, only
+# individual files (confirmed against WangLab-ComputationalBiology/SCRATCH-QC's
+# own .cirro/qc/preprocess.py, a sibling pipeline solving the same problem: it
+# matches specific files by name via ds.files rather than referencing a
+# directory). So this profile finds the 3 specific files VDJ_QC actually reads
+# out of each sample's outs/ directory (see VDJ_QC_analysis.qmd's
+# detect_vdj_file()) and exposes them as separate, sample-ordered pipeline
+# params. modules/scratch/VDJ_QC/main.nf stages them and reconstructs a local
+# outs/ directory per sample, rewriting sample_sheet.csv's path column to
+# point at the staged copy before VDJ_QC's R code (unchanged) reads it.
+SAMPLESHEET_REQUIRED_COLUMNS = ("sample", "path")
+
+# Maps each file VDJ_QC's detect_vdj_file() looks for to the Nextflow param
+# modules/scratch/VDJ_QC/main.nf reads it back from.
+VDJ_FILE_PARAMS = {
+    "filtered_contig_annotations.csv": "contigs_files",
+    "clonotypes.csv": "clonotypes_files",
+    "metrics_summary.csv": "metrics_files",
+}
+
+
+def samplesheet_from_files(ds):
+    files = ds.files
+    ds.logger.info(f"found files in ds.files: {files.to_dict()}")
+
+    # Cirro's per-file listing gives one row per file under each sample's
+    # directory; the sample's directory is that file's parent. Path()
+    # collapses "s3://" to "s3:/", so restore the double slash afterward.
+    files = files.copy()
+    files['path'] = files['file'].apply(lambda x: str(Path(x).parent).replace('s3:/', 's3://'))
+    files = files[['sample', 'path']].drop_duplicates()
+
+    return pd.merge(ds.samplesheet, files, on='sample', how='left')
+
+
+def samplesheet_from_params(ds):
+    # Fallback for datasets registered as one directory-shaped input per
+    # sample (no flat per-file listing in ds.files). Not paired with
+    # prepare_vdj_file_params() - a genuine directory-shaped registration
+    # gives one real, already-resolvable path per sample directly.
+    return pd.DataFrame({
+        'sample': [x['name'] for x in ds.metadata['inputs']],
+        'path': [x['dataPath'] for x in ds.metadata['inputs']],
+    })
+
+
+def prepare_vdj_file_params(ds, samples):
+    """
+    Match each sample to its filtered_contig_annotations.csv / clonotypes.csv /
+    metrics_summary.csv via ds.files's flat per-file listing. Sets one
+    Nextflow param per filename (VDJ_FILE_PARAMS), each a comma-separated,
+    sample-ordered list of S3 paths - empty string where a sample has no
+    match, so list position always lines up 1:1 with `samples` even when a
+    file is missing for one sample. Uses split(',', -1) semantics on the
+    Nextflow side, so trailing empty entries can't be silently dropped.
+    """
+    files = ds.files
+
+    for fname, param_name in VDJ_FILE_PARAMS.items():
+        hits = files[files['file'].str.endswith(fname)]
+        per_sample = []
+        for sample in samples:
+            sample_hits = sorted(hits.loc[hits['sample'] == sample, 'file'].tolist())
+            if not sample_hits:
+                ds.logger.warning(
+                    f"No {fname} found for sample '{sample}' via ds.files - "
+                    "VDJ_QC will fall back to sample_sheet's own path column for this sample."
+                )
+                per_sample.append('')
+            else:
+                if len(sample_hits) > 1:
+                    ds.logger.warning(f"Multiple {fname} matches for sample '{sample}', using the first: {sample_hits}")
+                per_sample.append(sample_hits[0])
+        ds.add_param(param_name, ','.join(per_sample))
+
+
+def prepare_sample_sheet(ds):
+    sample_sheet = samplesheet_from_files(ds)
+
+    # A left merge against an empty ds.files still returns every ds.samplesheet
+    # row - just with 'path' as NaN - so sample_sheet.empty alone can't detect
+    # "no per-file listing to merge against". Check the column directly.
+    if sample_sheet.empty or sample_sheet['path'].isna().all():
+        ds.logger.warning("No files found in dataset. Preparing sample_sheet from params.")
+        sample_sheet = samplesheet_from_params(ds)
+        if sample_sheet.empty:
+            raise ValueError("No files found in dataset and unable to prepare sample_sheet from params.")
+    else:
+        prepare_vdj_file_params(ds, sample_sheet['sample'].tolist())
+
+    for colname in SAMPLESHEET_REQUIRED_COLUMNS:
+        if colname not in sample_sheet.columns:
+            ds.logger.warning(f"sample_sheet is missing required column '{colname}'. Populating with NaN.")
+            sample_sheet[colname] = np.nan
+
+    sample_sheet.to_csv('sample_sheet.csv', index=None)
+    ds.add_param('sample_sheet', 'sample_sheet.csv')
+    ds.logger.info(sample_sheet.to_dict())
+
+
+def main():
+    ds = PreprocessDataset.from_running()
+    ds.logger.info("List of starting params")
+    ds.logger.info(ds.params)
+
+    prepare_sample_sheet(ds)
+
+    ds.logger.info(ds.params)
+
+
+if __name__ == "__main__":
+    main()

.cirro/singlecell_full_gex/process-form.json

@@ -0,0 +1,73 @@
+{
+    "form": {
+        "title": "TCRtoolkit Single-Cell Analysis (Full SC, GEX supplied)",
+        "description": "Runs the full single-cell TCR pipeline with a GEX/Seurat object: integrates VDJ contigs onto the annotated cells, pseudobulks per-sample clonotypes through the shared bulk-TCR engine (sample/patient/compare), then maps clusters back onto cells for CoNGA/consensus clustering and cell-level repertoire/summary reports. Provide a samplesheet with 'sample' and 'path' columns (path pointing to each sample's Cell Ranger VDJ outs/ directory), plus optional 'patient_id', 'condition', and 'timepoint' metadata columns.",
+        "properties": {
+            "input_annotated_object": {
+                "default": "",
+                "description": "S3 path to the annotated GEX Seurat object (.rds) to integrate VDJ contigs onto.",
+                "title": "Annotated GEX object (Seurat RDS)",
+                "type": "string"
+            },
+            "project_name": {
+                "default": "",
+                "description": "Name used to label outputs and reports for this run.",
+                "title": "Project name",
+                "type": "string"
+            },
+            "pseudobulk_by_phenotype": {
+                "default": false,
+                "description": "Additionally pseudobulk per cell-type/phenotype (sample__phenotype units) alongside the default per-sample pooling.",
+                "title": "Pseudobulk by phenotype",
+                "type": "boolean"
+            },
+            "pseudobulk_qc_min_clones": {
+                "default": 25,
+                "description": "Minimum number of clones a pseudobulked sample must have to pass the QC gate.",
+                "title": "Pseudobulk QC: min clones",
+                "type": "integer"
+            },
+            "pseudobulk_qc_min_cells": {
+                "default": 50,
+                "description": "Minimum number of cells a pseudobulked sample must have to pass the QC gate.",
+                "title": "Pseudobulk QC: min cells",
+                "type": "integer"
+            },
+            "pseudobulk_qc_mode": {
+                "default": "drop",
+                "description": "'drop' skips samples that fail the QC gate; 'hard_stop' aborts the whole run if any sample fails.",
+                "title": "Pseudobulk QC mode",
+                "type": "string",
+                "enum": [
+                    "drop",
+                    "hard_stop"
+                ]
+            },
+            "run_conga": {
+                "default": true,
+                "description": "Run CoNGA (GEX + TCR joint clustering). Requires the GEX object.",
+                "title": "Run CoNGA",
+                "type": "boolean"
+            },
+            "run_consensus": {
+                "default": true,
+                "description": "Run consensus clustering across clustering methods.",
+                "title": "Run consensus clustering",
+                "type": "boolean"
+            },
+            "run_repertoire": {
+                "default": true,
+                "description": "Render the cell-level repertoire report.",
+                "title": "Run repertoire report",
+                "type": "boolean"
+            },
+            "run_master_summary": {
+                "default": true,
+                "description": "Render the master summary report.",
+                "title": "Run master summary report",
+                "type": "boolean"
+            }
+        }
+    },
+    "ui": {}
+}

.cirro/singlecell_full_gex/process-input.json

@@ -0,0 +1,19 @@
+{
+    "mode": "singlecell",
+    "input_vdj_contigs": "$.params.dataset.s3",
+    "sample_sheet": "sample_sheet.csv",
+    "contigs_files": "",
+    "clonotypes_files": "",
+    "metrics_files": "",
+    "input_annotated_object": "$.params.dataset.paramJson.input_annotated_object",
+    "project_name": "$.params.dataset.paramJson.project_name",
+    "pseudobulk_by_phenotype": "$.params.dataset.paramJson.pseudobulk_by_phenotype",
+    "pseudobulk_qc_min_clones": "$.params.dataset.paramJson.pseudobulk_qc_min_clones",
+    "pseudobulk_qc_min_cells": "$.params.dataset.paramJson.pseudobulk_qc_min_cells",
+    "pseudobulk_qc_mode": "$.params.dataset.paramJson.pseudobulk_qc_mode",
+    "run_conga": "$.params.dataset.paramJson.run_conga",
+    "run_consensus": "$.params.dataset.paramJson.run_consensus",
+    "run_repertoire": "$.params.dataset.paramJson.run_repertoire",
+    "run_master_summary": "$.params.dataset.paramJson.run_master_summary",
+    "outdir": "$.params.dataset.s3|/data/"
+}

.cirro/singlecell_vdjonly/preprocess.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+from cirro.helpers.preprocess_dataset import PreprocessDataset
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+# TCRtoolkit single-cell mode needs a 'sample_sheet.csv' with a 'path' column
+# pointing at each sample's Cell Ranger VDJ outs/ DIRECTORY - see
+# modules/scratch/VDJ_QC/VDJ_QC_analysis.qmd's column contract
+# (sample_sheet_sample_col="sample", sample_sheet_path_col="path").
+#
+# Cirro's dataset-file abstraction doesn't stage whole directories, only
+# individual files (confirmed against WangLab-ComputationalBiology/SCRATCH-QC's
+# own .cirro/qc/preprocess.py, a sibling pipeline solving the same problem: it
+# matches specific files by name via ds.files rather than referencing a
+# directory). So this profile finds the 3 specific files VDJ_QC actually reads
+# out of each sample's outs/ directory (see VDJ_QC_analysis.qmd's
+# detect_vdj_file()) and exposes them as separate, sample-ordered pipeline
+# params. modules/scratch/VDJ_QC/main.nf stages them and reconstructs a local
+# outs/ directory per sample, rewriting sample_sheet.csv's path column to
+# point at the staged copy before VDJ_QC's R code (unchanged) reads it.
+SAMPLESHEET_REQUIRED_COLUMNS = ("sample", "path")
+
+# Maps each file VDJ_QC's detect_vdj_file() looks for to the Nextflow param
+# modules/scratch/VDJ_QC/main.nf reads it back from.
+VDJ_FILE_PARAMS = {
+    "filtered_contig_annotations.csv": "contigs_files",
+    "clonotypes.csv": "clonotypes_files",
+    "metrics_summary.csv": "metrics_files",
+}
+
+
+def samplesheet_from_files(ds):
+    files = ds.files
+    ds.logger.info(f"found files in ds.files: {files.to_dict()}")
+
+    # Cirro's per-file listing gives one row per file under each sample's
+    # directory; the sample's directory is that file's parent. Path()
+    # collapses "s3://" to "s3:/", so restore the double slash afterward.
+    files = files.copy()
+    files['path'] = files['file'].apply(lambda x: str(Path(x).parent).replace('s3:/', 's3://'))
+    files = files[['sample', 'path']].drop_duplicates()
+
+    return pd.merge(ds.samplesheet, files, on='sample', how='left')
+
+
+def samplesheet_from_params(ds):
+    # Fallback for datasets registered as one directory-shaped input per
+    # sample (no flat per-file listing in ds.files). Not paired with
+    # prepare_vdj_file_params() - a genuine directory-shaped registration
+    # gives one real, already-resolvable path per sample directly.
+    return pd.DataFrame({
+        'sample': [x['name'] for x in ds.metadata['inputs']],
+        'path': [x['dataPath'] for x in ds.metadata['inputs']],
+    })
+
+
+def prepare_vdj_file_params(ds, samples):
+    """
+    Match each sample to its filtered_contig_annotations.csv / clonotypes.csv /
+    metrics_summary.csv via ds.files's flat per-file listing. Sets one
+    Nextflow param per filename (VDJ_FILE_PARAMS), each a comma-separated,
+    sample-ordered list of S3 paths - empty string where a sample has no
+    match, so list position always lines up 1:1 with `samples` even when a
+    file is missing for one sample. Uses split(',', -1) semantics on the
+    Nextflow side, so trailing empty entries can't be silently dropped.
+    """
+    files = ds.files
+
+    for fname, param_name in VDJ_FILE_PARAMS.items():
+        hits = files[files['file'].str.endswith(fname)]
+        per_sample = []
+        for sample in samples:
+            sample_hits = sorted(hits.loc[hits['sample'] == sample, 'file'].tolist())
+            if not sample_hits:
+                ds.logger.warning(
+                    f"No {fname} found for sample '{sample}' via ds.files - "
+                    "VDJ_QC will fall back to sample_sheet's own path column for this sample."
+                )
+                per_sample.append('')
+            else:
+                if len(sample_hits) > 1:
+                    ds.logger.warning(f"Multiple {fname} matches for sample '{sample}', using the first: {sample_hits}")
+                per_sample.append(sample_hits[0])
+        ds.add_param(param_name, ','.join(per_sample))
+
+
+def prepare_sample_sheet(ds):
+    sample_sheet = samplesheet_from_files(ds)
+
+    # A left merge against an empty ds.files still returns every ds.samplesheet
+    # row - just with 'path' as NaN - so sample_sheet.empty alone can't detect
+    # "no per-file listing to merge against". Check the column directly.
+    if sample_sheet.empty or sample_sheet['path'].isna().all():
+        ds.logger.warning("No files found in dataset. Preparing sample_sheet from params.")
+        sample_sheet = samplesheet_from_params(ds)
+        if sample_sheet.empty:
+            raise ValueError("No files found in dataset and unable to prepare sample_sheet from params.")
+    else:
+        prepare_vdj_file_params(ds, sample_sheet['sample'].tolist())
+
+    for colname in SAMPLESHEET_REQUIRED_COLUMNS:
+        if colname not in sample_sheet.columns:
+            ds.logger.warning(f"sample_sheet is missing required column '{colname}'. Populating with NaN.")
+            sample_sheet[colname] = np.nan
+
+    sample_sheet.to_csv('sample_sheet.csv', index=None)
+    ds.add_param('sample_sheet', 'sample_sheet.csv')
+    ds.logger.info(sample_sheet.to_dict())
+
+
+def main():
+    ds = PreprocessDataset.from_running()
+    ds.logger.info("List of starting params")
+    ds.logger.info(ds.params)
+
+    prepare_sample_sheet(ds)
+
+    ds.logger.info(ds.params)
+
+
+if __name__ == "__main__":
+    main()