From 7b82afae980b04aa24e6357fda804d00d3f2f806 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 31 Aug 2026 16:53:59 +0000 Subject: [PATCH] Add K-Dense scientific agent skills Install the K-Dense-AI/scientific-agent-skills bundle via `npx skills add`: - 163 skills under .agents/skills/ (scientific computing, bioinformatics, ML frameworks, data formats, lab/analysis workflows) - .claude/skills/ symlinks so Claude Code picks them up - skills-lock.json pinning each skill's source and content hash Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Wo5A9xH35XbnbV1PF9vbNQ --- .agents/skills/adaptyv/SKILL.md | 240 + .../adaptyv/references/api-endpoints.md | 690 + .agents/skills/aeon/SKILL.md | 402 + .../aeon/references/anomaly_detection.md | 154 + .../skills/aeon/references/classification.md | 144 + .agents/skills/aeon/references/clustering.md | 123 + .../aeon/references/datasets_benchmarking.md | 392 + .agents/skills/aeon/references/distances.md | 256 + .agents/skills/aeon/references/forecasting.md | 109 + .agents/skills/aeon/references/networks.md | 289 + .agents/skills/aeon/references/regression.md | 118 + .../skills/aeon/references/segmentation.md | 163 + .../aeon/references/similarity_search.md | 187 + .../skills/aeon/references/transformations.md | 246 + .../analytical-method-validation/SKILL.md | 299 + .../assets/validation-protocol-template.md | 106 + .../assets/validation-report-template.md | 116 + .../references/compendial-and-clsi.md | 96 + .../references/framework-selection.md | 87 + .../references/ich-m10-bioanalytical.md | 123 + .../references/ich-q2r2.md | 229 + .../references/source-ledger.md | 125 + .../references/statistics.md | 209 + .../scripts/_catalog.py | 499 + .../scripts/_common.py | 955 + .../scripts/check_accuracy_precision.py | 328 + .../scripts/check_bioanalytical_run.py | 293 + .../scripts/check_detection_limits.py | 310 + .../scripts/check_response.py | 243 + .../scripts/compare_methods.py | 215 + .../scripts/plan_validation.py | 381 + .agents/skills/anndata/SKILL.md | 431 + .../anndata/references/best_practices.md | 532 + .../anndata/references/concatenation.md | 399 + .../anndata/references/data_structure.md | 314 + .../anndata/references/io_operations.md | 466 + .../skills/anndata/references/manipulation.md | 516 + .agents/skills/arbor/SKILL.md | 152 + .../skills/arbor/references/arbor-upstream.md | 91 + .../skills/arbor/references/executor-brief.md | 68 + .../arbor/references/htr-methodology.md | 177 + .../arbor/references/report-template.md | 39 + .agents/skills/arbor/scripts/tree.py | 564 + .agents/skills/arboreto/SKILL.md | 267 + .../skills/arboreto/references/algorithms.md | 152 + .../arboreto/references/basic_inference.md | 181 + .../references/distributed_computing.md | 242 + .../arboreto/scripts/basic_grn_inference.py | 107 + .agents/skills/astropy/SKILL.md | 353 + .../skills/astropy/references/coordinates.md | 280 + .../skills/astropy/references/cosmology.md | 309 + .agents/skills/astropy/references/fits.md | 398 + .agents/skills/astropy/references/tables.md | 495 + .agents/skills/astropy/references/time.md | 412 + .agents/skills/astropy/references/units.md | 178 + .../references/wcs_and_other_modules.md | 377 + .agents/skills/autoskill/.gitignore | 3 + .agents/skills/autoskill/SKILL.md | 233 + .agents/skills/autoskill/config.yaml | 53 + .../autoskill/references/https-proxy.md | 62 + .../references/screenpipe-config.yaml | 61 + .agents/skills/autoskill/scripts/autoskill.py | 35 + .agents/skills/autoskill/scripts/backends.py | 116 + .agents/skills/autoskill/scripts/cluster.py | 54 + .agents/skills/autoskill/scripts/doctor.py | 108 + .../skills/autoskill/scripts/fetch_window.py | 33 + .../skills/autoskill/scripts/match_skills.py | 46 + .agents/skills/autoskill/scripts/promote.py | 58 + .agents/skills/autoskill/scripts/redact.py | 40 + .agents/skills/autoskill/scripts/run.py | 194 + .../skills/autoskill/scripts/synthesize.py | 72 + .agents/skills/benchling-integration/SKILL.md | 229 + .../references/api_endpoints.md | 883 + .../references/authentication.md | 390 + .../references/core_capabilities.md | 355 + .../references/eventbridge.md | 255 + .../references/sdk_reference.md | 772 + .agents/skills/bgpt-paper-search/SKILL.md | 75 + .agents/skills/bids/SKILL.md | 237 + .agents/skills/bids/references/beps.yml | 637 + .../skills/bids/references/bids_schema.json | 21015 +++++++ .../bids/references/bids_specification.md | 165 + .../bids/references/conversion_tools.md | 475 + .../skills/bids/references/core_workflows.md | 552 + .../skills/bids/references/metadata_fields.md | 365 + .agents/skills/bids/scripts/update_schema.py | 89 + .agents/skills/biopython/SKILL.md | 472 + .../skills/biopython/references/advanced.md | 580 + .../skills/biopython/references/alignment.md | 377 + .agents/skills/biopython/references/blast.md | 463 + .../skills/biopython/references/databases.md | 492 + .../biopython/references/phylogenetics.md | 566 + .../biopython/references/sequence_io.md | 289 + .../skills/biopython/references/structure.md | 564 + .agents/skills/bioservices/SKILL.md | 399 + .../references/identifier_mapping.md | 685 + .../references/services_reference.md | 638 + .../references/workflow_patterns.md | 813 + .../bioservices/scripts/batch_id_converter.py | 347 + .../scripts/compound_cross_reference.py | 387 + .../bioservices/scripts/pathway_analysis.py | 309 + .../scripts/protein_analysis_workflow.py | 441 + .agents/skills/bulk-rnaseq/SKILL.md | 198 + .../references/counts-and-handoff.md | 91 + .../bulk-rnaseq/references/design-and-qc.md | 68 + .../bulk-rnaseq/references/upstream-manual.md | 128 + .../bulk-rnaseq/references/upstream-nfcore.md | 108 + .../scripts/build_counts_matrix.py | 203 + .../scripts/validate_samplesheet.py | 206 + .agents/skills/cellxgene-census/SKILL.md | 283 + .../references/census_schema.md | 218 + .../references/common_patterns.md | 368 + .../references/core_workflow_patterns.md | 297 + .agents/skills/cirq/SKILL.md | 370 + .agents/skills/cirq/references/building.md | 307 + .agents/skills/cirq/references/experiments.md | 572 + .agents/skills/cirq/references/hardware.md | 527 + .agents/skills/cirq/references/noise.md | 514 + .agents/skills/cirq/references/simulation.md | 353 + .../skills/cirq/references/transformation.md | 416 + .agents/skills/citation-management/SKILL.md | 329 + .../assets/bibtex_template.bib | 264 + .../assets/citation_checklist.md | 386 + .../references/best_practices.md | 91 + .../references/bibtex_formatting.md | 908 + .../references/citation_validation.md | 835 + .../references/core_workflow.md | 569 + .../references/example_workflows.md | 126 + .../references/google_scholar_search.md | 732 + .../references/metadata_extraction.md | 870 + .../references/pubmed_search.md | 839 + .../references/script_reference.md | 250 + .../references/search_strategies.md | 110 + .../citation-management/scripts/_common.py | 331 + .../scripts/doi_to_bibtex.py | 204 + .../scripts/extract_metadata.py | 690 + .../scripts/format_bibtex.py | 356 + .../scripts/search_google_scholar.py | 268 + .../scripts/search_openalex.py | 297 + .../scripts/search_pubmed.py | 419 + .../scripts/validate_citations.py | 688 + .../skills/clinical-decision-support/SKILL.md | 238 + .../aggregate_cohort_table_template.json | 114 + .../aggregate_model_evaluation_template.json | 115 + .../artifact_intended_use_template.json | 81 + .../decision_logic_traceability_template.json | 108 + .../deidentification_checklist_template.json | 176 + .../assets/evidence_profile_template.json | 139 + .../survival_analysis_plan_template.json | 134 + .../references/README.md | 62 + .../references/cohort_evaluation.md | 142 + .../references/decision_logic_traceability.md | 146 + .../references/evidence_profiles.md | 153 + .../references/model_biomarker_evaluation.md | 152 + .../references/privacy_and_disclosure.md | 144 + .../references/regulatory_and_governance.md | 104 + .../references/safety_and_scope.md | 103 + .../references/security_validation.md | 60 + .../references/sources.md | 119 + .../references/study_reporting.md | 134 + .../references/survival_analysis.md | 156 + .../scripts/_common.py | 223 + .../scripts/cohort_table_generator.py | 288 + .../scripts/decision_logic_traceability.py | 313 + .../scripts/deidentification_checklist.py | 260 + .../scripts/evidence_profile_check.py | 259 + .../scripts/model_biomarker_evaluation.py | 345 + .../scripts/survival_plan_validator.py | 294 + .../scripts/validate_cds_artifact.py | 269 + .agents/skills/clinical-reports/SKILL.md | 248 + ...adverse_event_aggregate_input_template.csv | 1 + .../assets/case_report_template.json | 43 + .../assets/clinical_trial_csr_template.json | 51 + .../clinical_trial_results_template.json | 66 + ...nical_trial_safety_aggregate_template.json | 49 + .../assets/consistency_manifest_template.json | 15 + .../deidentification_process_checklist.json | 56 + .../assets/lab_report_template.json | 36 + .../assets/pathology_report_template.json | 36 + .../assets/provenance_manifest_template.json | 17 + .../assets/quality_review_checklist.json | 34 + .../assets/radiology_report_template.json | 34 + .../assets/research_summary_template.json | 36 + .../assets/terminology_manifest_template.json | 11 + .../trial_protocol_reporting_checklist.json | 70 + .../clinical-reports/references/README.md | 34 + .../references/case_report_guidelines.md | 72 + .../references/clinical_trial_reporting.md | 97 + .../references/data_presentation.md | 92 + .../diagnostic_reports_standards.md | 76 + .../references/medical_terminology.md | 84 + .../privacy_and_deidentification.md | 84 + .../references/professional_review.md | 78 + .../references/report_type_routing.md | 57 + .../references/safety_reporting.md | 109 + .../clinical-reports/references/sources.md | 75 + .../clinical-reports/scripts/_common.py | 263 + .../scripts/check_deidentification.py | 298 + .../scripts/consistency_checker.py | 390 + .../scripts/format_adverse_events.py | 455 + .../scripts/generate_report_template.py | 115 + .../scripts/provenance_validator.py | 263 + .../scripts/terminology_validator.py | 277 + .../scripts/validate_case_report.py | 278 + .../scripts/validate_trial_report.py | 534 + .agents/skills/cobrapy/SKILL.md | 496 + .../cobrapy/references/api_quick_reference.md | 665 + .../skills/cobrapy/references/workflows.md | 600 + .agents/skills/consciousness-council/SKILL.md | 151 + .../references/advanced-configurations.md | 96 + .agents/skills/dask/SKILL.md | 482 + .agents/skills/dask/references/arrays.md | 495 + .agents/skills/dask/references/bags.md | 468 + .../skills/dask/references/best-practices.md | 277 + .agents/skills/dask/references/dataframes.md | 370 + .agents/skills/dask/references/futures.md | 541 + .agents/skills/dask/references/schedulers.md | 517 + .agents/skills/database-lookup/SKILL.md | 386 + .../database-lookup/references/addgene.md | 38 + .../database-lookup/references/alphafold.md | 52 + .../references/alphavantage.md | 261 + .../skills/database-lookup/references/bea.md | 409 + .../database-lookup/references/bindingdb.md | 85 + .../database-lookup/references/biogrid.md | 110 + .../skills/database-lookup/references/bls.md | 235 + .../database-lookup/references/brenda.md | 71 + .../database-lookup/references/cbioportal.md | 206 + .../database-lookup/references/census.md | 251 + .../database-lookup/references/chebi.md | 103 + .../database-lookup/references/chembl.md | 80 + .../references/clinicaltrials.md | 86 + .../database-lookup/references/clinpgx.md | 64 + .../database-lookup/references/clinvar.md | 91 + .../skills/database-lookup/references/cod.md | 121 + .../database-lookup/references/cosmic.md | 59 + .../database-lookup/references/dailymed.md | 65 + .../references/database_selection_guide.md | 166 + .../database-lookup/references/datacommons.md | 237 + .../database-lookup/references/dbsnp.md | 143 + .../database-lookup/references/disgenet.md | 52 + .../database-lookup/references/drugbank.md | 54 + .../skills/database-lookup/references/ecb.md | 191 + .../skills/database-lookup/references/emdb.md | 37 + .../skills/database-lookup/references/ena.md | 372 + .../database-lookup/references/encode.md | 47 + .../database-lookup/references/ensembl.md | 539 + .../skills/database-lookup/references/epa.md | 232 + .../database-lookup/references/eurostat.md | 237 + .../skills/database-lookup/references/fda.md | 64 + .../references/federal-reserve.md | 216 + .../skills/database-lookup/references/fred.md | 297 + .../references/gene-ontology.md | 147 + .../skills/database-lookup/references/geo.md | 130 + .../database-lookup/references/gnomad.md | 93 + .../skills/database-lookup/references/gtex.md | 136 + .../references/gwas-catalog.md | 46 + .../skills/database-lookup/references/hca.md | 35 + .../skills/database-lookup/references/hpo.md | 48 + .../references/human-protein-atlas.md | 57 + .../database-lookup/references/interpro.md | 120 + .../database-lookup/references/jaspar.md | 50 + .../skills/database-lookup/references/kegg.md | 78 + .../database-lookup/references/lincs-l1000.md | 68 + .../references/materials-project.md | 123 + .../references/metabolomics-workbench.md | 98 + .../database-lookup/references/monarch.md | 46 + .../database-lookup/references/mousemine.md | 40 + .../references/nasa-exoplanet-archive.md | 112 + .../skills/database-lookup/references/nasa.md | 121 + .../database-lookup/references/ncbi-gene.md | 64 + .../references/ncbi-protein.md | 104 + .../references/ncbi-taxonomy.md | 121 + .../skills/database-lookup/references/nist.md | 105 + .../skills/database-lookup/references/noaa.md | 199 + .../skills/database-lookup/references/omim.md | 114 + .../database-lookup/references/opentargets.md | 459 + .../references/openweathermap.md | 255 + .../skills/database-lookup/references/pdb.md | 121 + .../database-lookup/references/pride.md | 74 + .../database-lookup/references/pubchem.md | 145 + .../database-lookup/references/quickgo.md | 45 + .../database-lookup/references/reactome.md | 140 + .../references/retrieval-contract.md | 123 + .../database-lookup/references/rummageo.md | 32 + .../skills/database-lookup/references/sdss.md | 130 + .../database-lookup/references/sec-edgar.md | 315 + .../database-lookup/references/simbad.md | 405 + .../skills/database-lookup/references/sra.md | 149 + .../database-lookup/references/string.md | 283 + .../database-lookup/references/tcga-gdc.md | 58 + .../database-lookup/references/treasury.md | 215 + .../database-lookup/references/ucsc-genome.md | 135 + .../database-lookup/references/uniprot.md | 283 + .../skills/database-lookup/references/usgs.md | 260 + .../database-lookup/references/uspto.md | 130 + .../skills/database-lookup/references/who.md | 283 + .../database-lookup/references/worldbank.md | 239 + .../skills/database-lookup/references/zinc.md | 202 + .agents/skills/datamol/SKILL.md | 200 + .../datamol/references/conformers_module.md | 131 + .agents/skills/datamol/references/core_api.md | 136 + .../datamol/references/core_workflows.md | 451 + .../datamol/references/descriptors_viz.md | 195 + .../datamol/references/fragments_scaffolds.md | 174 + .../skills/datamol/references/io_module.md | 112 + .../datamol/references/reactions_data.md | 218 + .../datamol/references/workflow_patterns.md | 104 + .agents/skills/deepchem/SKILL.md | 244 + .../deepchem/references/api_reference.md | 305 + .../deepchem/references/core_capabilities.md | 276 + .../deepchem/references/typical_workflows.md | 109 + .../skills/deepchem/references/workflows.md | 491 + .../deepchem/scripts/graph_neural_network.py | 350 + .../deepchem/scripts/predict_solubility.py | 223 + .../deepchem/scripts/transfer_learning.py | 443 + .agents/skills/deepspot-m/SKILL.md | 175 + .agents/skills/deepspot-m/references/api.md | 186 + .../deepspot-m/references/whole_slide.md | 174 + .agents/skills/deeptools/SKILL.md | 412 + .../deeptools/assets/quick_reference.md | 65 + .../deeptools/references/core_workflows.md | 134 + .../references/effective_genome_sizes.md | 118 + .../references/normalization_methods.md | 424 + .../deeptools/references/tools_reference.md | 569 + .../skills/deeptools/references/workflows.md | 476 + .../deeptools/scripts/validate_files.py | 195 + .../deeptools/scripts/workflow_generator.py | 520 + .agents/skills/depmap/SKILL.md | 301 + .../depmap/references/dependency_analysis.md | 178 + .agents/skills/dhdna-profiler/SKILL.md | 184 + .../references/advanced-profiling.md | 72 + .agents/skills/diffdock/SKILL.md | 488 + .../skills/diffdock/assets/batch_template.csv | 4 + .../assets/custom_inference_config.yaml | 94 + .../references/confidence_and_limitations.md | 182 + .../references/parameters_reference.md | 173 + .../diffdock/references/workflows_examples.md | 401 + .../diffdock/scripts/analyze_results.py | 346 + .../diffdock/scripts/prepare_batch_csv.py | 257 + .../skills/diffdock/scripts/setup_check.py | 283 + .agents/skills/dnanexus-integration/SKILL.md | 325 + .../references/app-development.md | 371 + .../references/authentication.md | 226 + .../references/configuration.md | 444 + .../references/data-operations.md | 474 + .../references/job-execution.md | 482 + .../operations-and-troubleshooting.md | 506 + .../references/python-sdk.md | 548 + .../references/sources.md | 168 + .../references/workflow-languages.md | 292 + .../scripts/inspect_dxpy.py | 366 + .../scripts/validate_dxapp.py | 796 + .agents/skills/docx/LICENSE.txt | 30 + .agents/skills/docx/SKILL.md | 99 + .agents/skills/docx/scripts/__init__.py | 1 + .agents/skills/docx/scripts/accept_changes.py | 135 + .agents/skills/docx/scripts/comment.py | 368 + .agents/skills/docx/scripts/merge_runs.py | 310 + .../docx/scripts/office/helpers/__init__.py | 111 + .../docx/scripts/office/helpers/pptx_chart.py | 170 + .../docx/scripts/office/helpers/pptx_slide.py | 60 + .../docx/scripts/office/helpers/pptx_theme.py | 114 + .../schemas/ISO-IEC29500-4_2016/dml-chart.xsd | 1499 + .../ISO-IEC29500-4_2016/dml-chartDrawing.xsd | 146 + .../ISO-IEC29500-4_2016/dml-diagram.xsd | 1085 + .../ISO-IEC29500-4_2016/dml-lockedCanvas.xsd | 11 + .../schemas/ISO-IEC29500-4_2016/dml-main.xsd | 3081 + .../ISO-IEC29500-4_2016/dml-picture.xsd | 23 + .../dml-spreadsheetDrawing.xsd | 185 + .../dml-wordprocessingDrawing.xsd | 287 + .../schemas/ISO-IEC29500-4_2016/pml.xsd | 1676 + .../shared-additionalCharacteristics.xsd | 28 + .../shared-bibliography.xsd | 144 + .../shared-commonSimpleTypes.xsd | 174 + .../shared-customXmlDataProperties.xsd | 25 + .../shared-customXmlSchemaProperties.xsd | 18 + .../shared-documentPropertiesCustom.xsd | 59 + .../shared-documentPropertiesExtended.xsd | 56 + .../shared-documentPropertiesVariantTypes.xsd | 195 + .../ISO-IEC29500-4_2016/shared-math.xsd | 582 + .../shared-relationshipReference.xsd | 25 + .../schemas/ISO-IEC29500-4_2016/sml.xsd | 4439 ++ .../schemas/ISO-IEC29500-4_2016/vml-main.xsd | 570 + .../ISO-IEC29500-4_2016/vml-officeDrawing.xsd | 509 + .../vml-presentationDrawing.xsd | 12 + .../vml-spreadsheetDrawing.xsd | 108 + .../vml-wordprocessingDrawing.xsd | 96 + .../schemas/ISO-IEC29500-4_2016/wml.xsd | 3646 ++ .../schemas/ISO-IEC29500-4_2016/xml.xsd | 116 + .../ecma/fouth-edition/opc-contentTypes.xsd | 42 + .../ecma/fouth-edition/opc-coreProperties.xsd | 50 + .../schemas/ecma/fouth-edition/opc-digSig.xsd | 49 + .../ecma/fouth-edition/opc-relationships.xsd | 33 + .../docx/scripts/office/schemas/mce/mc.xsd | 75 + .../office/schemas/microsoft/wml-2010.xsd | 560 + .../office/schemas/microsoft/wml-2012.xsd | 67 + .../office/schemas/microsoft/wml-2018.xsd | 14 + .../office/schemas/microsoft/wml-cex-2018.xsd | 20 + .../office/schemas/microsoft/wml-cid-2016.xsd | 13 + .../microsoft/wml-sdtdatahash-2020.xsd | 4 + .../schemas/microsoft/wml-symex-2015.xsd | 8 + .agents/skills/docx/scripts/office/soffice.py | 232 + .../skills/docx/scripts/office/validate.py | 173 + .../scripts/office/validators/__init__.py | 15 + .../docx/scripts/office/validators/base.py | 875 + .../docx/scripts/office/validators/docx.py | 466 + .../docx/scripts/office/validators/pptx.py | 441 + .../scripts/office/validators/redlining.py | 299 + .../docx/scripts/templates/comments.xml | 3 + .../scripts/templates/commentsExtended.xml | 3 + .../scripts/templates/commentsExtensible.xml | 3 + .../docx/scripts/templates/commentsIds.xml | 3 + .../skills/docx/scripts/templates/people.xml | 3 + .agents/skills/esm/SKILL.md | 334 + .../skills/esm/references/biohub-platform.md | 111 + .agents/skills/esm/references/esm-c-api.md | 609 + .agents/skills/esm/references/esm3-api.md | 462 + .agents/skills/esm/references/forge-api.md | 675 + .agents/skills/esm/references/workflows.md | 685 + .agents/skills/etetoolkit/SKILL.md | 327 + .../etetoolkit/references/api_reference.md | 546 + .../references/migration-ete3-to-ete4.md | 579 + .../skills/etetoolkit/references/taxonomy.md | 362 + .../etetoolkit/references/visualization.md | 516 + .../skills/etetoolkit/references/workflows.md | 537 + .../etetoolkit/scripts/quick_visualize.py | 455 + .../etetoolkit/scripts/tree_operations.py | 446 + .agents/skills/exa-search/SKILL.md | 102 + .../exa-search/references/web-extract.md | 53 + .../exa-search/references/web-search.md | 119 + .../skills/exa-search/scripts/exa_extract.py | 117 + .../skills/exa-search/scripts/exa_search.py | 179 + .agents/skills/experimental-design/SKILL.md | 234 + .../references/design_types.md | 129 + .../references/factorial_and_doe.md | 130 + .../references/randomization_and_blocking.md | 116 + .../references/sequential_and_adaptive.md | 97 + .../scripts/doe_designs.py | 183 + .../scripts/randomization.py | 171 + .../skills/exploratory-data-analysis/SKILL.md | 280 + .../assets/report_template.md | 202 + .../bioinformatics_genomics_formats.md | 192 + .../references/chemistry_molecular_formats.md | 183 + .../references/general_scientific_formats.md | 259 + .../references/microscopy_imaging_formats.md | 189 + .../proteomics_metabolomics_formats.md | 217 + .../spectroscopy_analytical_formats.md | 191 + .../scripts/__init__.py | 1 + .../scripts/_capabilities.py | 576 + .../scripts/_common.py | 460 + .../scripts/_structured.py | 391 + .../scripts/_tabular.py | 905 + .../scripts/capability_manifest.py | 184 + .../scripts/distribution_sensitivity.py | 117 + .../scripts/eda_analyzer.py | 345 + .../scripts/image_inspector.py | 214 + .../scripts/missingness_leakage_audit.py | 130 + .../scripts/report_scaffold.py | 143 + .../scripts/sequence_inspector.py | 255 + .../scripts/tabular_profile.py | 109 + .agents/skills/flowio/SKILL.md | 310 + .../skills/flowio/references/api_reference.md | 355 + .../skills/flowio/references/fcs_semantics.md | 315 + .agents/skills/flowio/references/sources.md | 89 + .../flowio/references/troubleshooting.md | 399 + .agents/skills/flowio/references/workflows.md | 368 + .agents/skills/flowio/scripts/inspect_fcs.py | 439 + .agents/skills/fluidsim/SKILL.md | 279 + .../fluidsim/references/advanced_features.md | 347 + .../fluidsim/references/installation.md | 263 + .../fluidsim/references/output_analysis.md | 314 + .../skills/fluidsim/references/parameters.md | 322 + .../references/simulation_workflow.md | 329 + .agents/skills/fluidsim/references/solvers.md | 191 + .agents/skills/fluidsim/scripts/__init__.py | 1 + .agents/skills/fluidsim/scripts/_common.py | 491 + .agents/skills/fluidsim/scripts/_schema.py | 872 + .../skills/fluidsim/scripts/budget_summary.py | 396 + .../scripts/grid_resource_estimator.py | 286 + .../fluidsim/scripts/output_inventory.py | 353 + .../fluidsim/scripts/restart_compatibility.py | 424 + .../fluidsim/scripts/simulation_dry_run.py | 246 + .../scripts/solver_config_validator.py | 70 + .agents/skills/generate-image/SKILL.md | 304 + .../generate-image/references/models.md | 173 + .../generate-image/scripts/generate_image.py | 752 + .agents/skills/geniml/SKILL.md | 310 + .agents/skills/geniml/references/bedspace.md | 267 + .../geniml/references/consensus_peaks.md | 334 + .../skills/geniml/references/region2vec.md | 289 + .agents/skills/geniml/references/scembed.md | 307 + .agents/skills/geniml/references/utilities.md | 385 + .agents/skills/geniml/scripts/__init__.py | 1 + .agents/skills/geniml/scripts/_common.py | 399 + .../skills/geniml/scripts/bed_validator.py | 363 + .../skills/geniml/scripts/consensus_plan.py | 416 + .../skills/geniml/scripts/corpus_auditor.py | 304 + .../skills/geniml/scripts/embedding_plan.py | 476 + .../scripts/model_artifact_inspector.py | 358 + .../geniml/scripts/tokenizer_compatibility.py | 321 + .agents/skills/genomic-coordinates/SKILL.md | 189 + .../references/format-conventions.md | 205 + .../references/reference-builds.md | 154 + .../references/transcript-coordinates.md | 141 + .../references/variant-representation.md | 155 + .../genomic-coordinates/scripts/_common.py | 335 + .../scripts/audit_intervals.py | 511 + .../scripts/check_contigs.py | 382 + .../scripts/convert_coords.py | 180 + .../scripts/normalize_variant.py | 290 + .agents/skills/genomic-intelligence/SKILL.md | 243 + .../references/api-and-auth.md | 45 + .../genomic-intelligence/references/mcp.md | 94 + .../references/sequence-acquisition.md | 52 + .../genomic-intelligence/references/tasks.md | 75 + .agents/skills/geomaster/README.md | 105 + .agents/skills/geomaster/SKILL.md | 366 + .../geomaster/references/advanced-gis.md | 376 + .../skills/geomaster/references/big-data.md | 363 + .../geomaster/references/code-examples.md | 531 + .../references/coordinate-systems.md | 364 + .../geomaster/references/core-libraries.md | 273 + .../geomaster/references/data-sources.md | 330 + .../geomaster/references/gis-software.md | 369 + .../references/industry-applications.md | 420 + .../geomaster/references/machine-learning.md | 462 + .../references/programming-languages.md | 456 + .../geomaster/references/remote-sensing.md | 370 + .../references/scientific-domains.md | 416 + .../references/specialized-topics.md | 428 + .../geomaster/references/troubleshooting.md | 439 + .agents/skills/geopandas/SKILL.md | 250 + .../geopandas/references/crs-management.md | 231 + .../skills/geopandas/references/data-io.md | 323 + .../geopandas/references/data-structures.md | 207 + .../references/geometric-operations.md | 262 + .../geopandas/references/spatial-analysis.md | 294 + .../geopandas/references/visualization.md | 230 + .agents/skills/geopandas/scripts/_common.py | 605 + .../scripts/crs_reprojection_plan.py | 210 + .../skills/geopandas/scripts/export_plan.py | 305 + .../scripts/geometry_validity_report.py | 227 + .../sensitive_coordinates_checklist.py | 230 + .../geopandas/scripts/spatial_join_audit.py | 368 + .../geopandas/scripts/vector_inventory.py | 140 + .../skills/get-available-resources/SKILL.md | 260 + .../references/resource_semantics.md | 206 + .../references/snapshot_schema.md | 172 + .../references/sources.md | 124 + .../scripts/_common.py | 190 + .../scripts/accelerator_diagnostics.py | 151 + .../scripts/detect_resources.py | 1767 + .../scripts/plan_workload.py | 311 + .../scripts/snapshot_tools.py | 486 + .agents/skills/gget/SKILL.md | 153 + .../gget/references/common_workflows.md | 120 + .../skills/gget/references/database_info.md | 336 + .../skills/gget/references/module_catalog.md | 733 + .../gget/references/module_reference.md | 526 + .agents/skills/gget/references/workflows.md | 815 + .../gget/scripts/batch_sequence_analysis.py | 192 + .../gget/scripts/enrichment_pipeline.py | 235 + .agents/skills/gget/scripts/gene_analysis.py | 175 + .agents/skills/ginkgo-cloud-lab/SKILL.md | 106 + .../cell-free-protein-expression-hibit.md | 53 + ...ll-free-protein-expression-optimization.md | 85 + ...cell-free-protein-expression-validation.md | 71 + ...-expression-purification-quantification.md | 60 + .../cfps-strep-purification-thermal-shift.md | 63 + .../cfps-strep-tag-purification-a280.md | 55 + .../references/echo-ms-cfps-detection.md | 49 + .../references/echo-ms-method-onboarding.md | 56 + ...-expression-purification-quantification.md | 49 + ...ecoli-minibinder-expression-histag-a280.md | 62 + .../ecoli-protein-expression-hibit.md | 44 + .../ecoli-protein-expression-histag-a280.md | 47 + .../fluorescent-pixel-art-generation.md | 73 + .../references/ivt-rna-synthesis-qpcr.md | 67 + .../references/minibinder-strep-tag-a280.md | 58 + .../pichia-protein-expression-labchip.md | 43 + .../references/spr-target-onboarding.md | 58 + .agents/skills/glycoengineering/SKILL.md | 339 + .../references/glycan_databases.md | 165 + .agents/skills/gtars/SKILL.md | 282 + .agents/skills/gtars/references/cli.md | 334 + .agents/skills/gtars/references/coverage.md | 224 + .agents/skills/gtars/references/overlap.md | 220 + .agents/skills/gtars/references/python-api.md | 280 + .agents/skills/gtars/references/refget.md | 318 + .agents/skills/gtars/references/tokenizers.md | 256 + .agents/skills/gtars/scripts/__init__.py | 1 + .agents/skills/gtars/scripts/_common.py | 461 + .../gtars/scripts/artifact_inspector.py | 328 + .agents/skills/gtars/scripts/bed_validator.py | 183 + .../gtars/scripts/coverage_preflight.py | 256 + .../skills/gtars/scripts/execution_plan.py | 365 + .../gtars/scripts/refget_digest_plan.py | 311 + .../gtars/scripts/tokenizer_manifest.py | 238 + .agents/skills/histolab/SKILL.md | 243 + .../histolab/references/core_capabilities.md | 305 + .../references/filters_preprocessing.md | 537 + .../histolab/references/slide_management.md | 184 + .../histolab/references/tile_extraction.md | 421 + .../histolab/references/tissue_masks.md | 251 + .../histolab/references/typical_workflows.md | 196 + .../histolab/references/visualization.md | 548 + .agents/skills/hugging-science/SKILL.md | 132 + .../references/flagship-resources.md | 81 + .../references/topics-and-slugs.md | 82 + .../references/using-datasets.md | 107 + .../references/using-models.md | 122 + .../references/using-spaces.md | 119 + .../hugging-science/scripts/fetch_catalog.py | 358 + .agents/skills/hypogenic/SKILL.md | 290 + .../assets/dataset_manifest.example.json | 30 + .../hypogenic/assets/result.example.json | 18 + .../hypogenic/assets/run_config.example.json | 46 + .../hypogenic/assets/task_config.example.yaml | 38 + .../hypogenic/references/configuration.md | 136 + .../skills/hypogenic/references/datasets.md | 146 + .../skills/hypogenic/references/evaluation.md | 155 + .../skills/hypogenic/references/security.md | 167 + .../skills/hypogenic/references/sources.md | 113 + .../skills/hypogenic/references/upstream.md | 188 + .agents/skills/hypogenic/scripts/__init__.py | 1 + .agents/skills/hypogenic/scripts/_common.py | 1312 + .../skills/hypogenic/scripts/audit_dataset.py | 410 + .../hypogenic/scripts/evaluate_local.py | 250 + .../hypogenic/scripts/inspect_outputs.py | 166 + .agents/skills/hypogenic/scripts/plan_run.py | 247 + .../hypogenic/scripts/validate_config.py | 192 + .agents/skills/hypothesis-generation/SKILL.md | 264 + .../assets/evidence_ledger_template.csv | 2 + .../falsification_controls_template.json | 116 + .../assets/hypothesis_record_template.json | 331 + .../assets/operationalization_template.json | 56 + .../prediction_rival_matrix_template.csv | 3 + .../preregistration_scaffold_template.md | 137 + .../assets/search_boundary_template.json | 23 + .../assets/source_ledger.csv | 37 + .../references/causal_inference_and_claims.md | 190 + .../references/concepts_and_workflow.md | 173 + .../references/ethics_safety_and_ai.md | 216 + .../experimental_design_patterns.md | 301 + .../references/hypothesis_quality_criteria.md | 203 + .../literature_search_strategies.md | 208 + .../preregistration_and_open_science.md | 205 + .../references/security_validation.md | 74 + .../references/source_ledger.md | 116 + .../references/tool_reference.md | 246 + .../hypothesis-generation/scripts/_common.py | 412 + .../scripts/audit_evidence_ledger.py | 337 + .../scripts/check_falsification_controls.py | 455 + .../scripts/check_operationalization.py | 237 + .../generate_preregistration_scaffold.py | 384 + .../scripts/lint_causal_claims.py | 189 + .../scripts/validate_hypothesis_schema.py | 1078 + .../scripts/validate_prediction_matrix.py | 286 + .agents/skills/imaging-data-commons/SKILL.md | 496 + .../references/bigquery_guide.md | 858 + .../references/cli_guide.md | 287 + .../references/clinical_data_guide.md | 328 + .../references/cloud_storage_guide.md | 333 + .../references/dicomweb_guide.md | 399 + .../references/digital_pathology_guide.md | 403 + .../references/index_tables_guide.md | 203 + .../references/licensing_and_citation.md | 230 + .../references/mcp_guide.md | 181 + .../references/parquet_access_guide.md | 200 + .../references/rest_api_guide.md | 612 + .../references/sql_patterns.md | 462 + .../references/use_cases.md | 277 + .../scripts/check_version.py | 132 + .agents/skills/infographics/SKILL.md | 315 + .../infographics/references/color_palettes.md | 496 + .../references/design_principles.md | 636 + .../references/infographic_type_catalog.md | 158 + .../references/infographic_types.md | 907 + .../references/iterative_refinement.md | 119 + .../scripts/generate_infographic.py | 291 + .../scripts/generate_infographic_ai.py | 1446 + .../skills/iso-standards-readiness/SKILL.md | 352 + .../templates/capa-record-template.json | 99 + .../templates/document-register-template.json | 75 + .../templates/evidence-manifest-template.json | 65 + .../laboratory-scope-intake-template.json | 111 + ...ical-laboratory-scope-intake-template.json | 111 + .../procedures/CAPA-procedure-template.md | 148 + .../document-control-procedure-template.md | 132 + .../templates/qmsr-transition-template.json | 181 + .../templates/quality-manual-template.md | 182 + .../templates/scope-intake-template.json | 109 + .../templates/supplier-controls-template.json | 119 + .../traceability-matrix-template.json | 154 + .../references/assurance-lanes.md | 116 + .../references/evidence-architecture.md | 282 + .../references/gap-analysis-checklist.md | 346 + .../references/iso-13485.md | 236 + .../references/iso-14971.md | 172 + .../references/iso-15189.md | 219 + .../references/iso-17025.md | 208 + .../references/quality-manual-guide.md | 282 + .../references/source-ledger.md | 357 + .../scripts/_catalog.py | 248 + .../scripts/_common.py | 558 + .../scripts/audit_document_records.py | 148 + .../scripts/check_capa.py | 240 + .../scripts/check_qmsr_transition.py | 164 + .../scripts/check_supplier_controls.py | 166 + .../scripts/check_traceability.py | 171 + .../scripts/gap_analyzer.py | 167 + .../scripts/validate_evidence_manifest.py | 246 + .../scripts/validate_scope_intake.py | 209 + .agents/skills/lab-hardware-cad/SKILL.md | 372 + .../lab-hardware-cad/assets/standards.json | 198 + .../references/behavior-rigs.md | 136 + .../references/build123d-patterns.md | 363 + .../references/fabrication-limits.md | 156 + .../references/labware-adapters.md | 190 + .../references/microfluidics.md | 157 + .../references/optomechanics.md | 148 + .../lab-hardware-cad/references/validation.md | 132 + .../lab-hardware-cad/scripts/_common.py | 650 + .../skills/lab-hardware-cad/scripts/check.py | 645 + .../skills/lab-hardware-cad/scripts/gen.py | 264 + .../lab-hardware-cad/scripts/snapshot.py | 278 + .../skills/labarchive-integration/SKILL.md | 216 + .../references/api_reference.md | 250 + .../references/authentication_guide.md | 191 + .../references/integrations.md | 162 + .../references/sources.md | 213 + .../scripts/entry_operations.py | 381 + .../scripts/notebook_operations.py | 451 + .../scripts/setup_config.py | 258 + .agents/skills/lamindb/SKILL.md | 408 + .../references/annotation-validation.md | 510 + .../lamindb/references/core-concepts.md | 383 + .../lamindb/references/data-management.md | 432 + .../skills/lamindb/references/integrations.md | 663 + .../skills/lamindb/references/ontologies.md | 498 + .../lamindb/references/setup-deployment.md | 755 + .agents/skills/latchbio-integration/SKILL.md | 227 + .../references/data-management.md | 257 + .../references/latch-mcp.md | 158 + .../references/nextflow-snakemake.md | 258 + .../references/operations-and-debugging.md | 320 + .../references/registry.md | 275 + .../references/resource-configuration.md | 274 + .../references/ui-and-automation.md | 355 + .../references/verified-workflows.md | 226 + .../references/workflow-creation.md | 275 + .../scripts/inspect_latch_sdk.py | 290 + .agents/skills/latex-posters/SKILL.md | 369 + .../assets/baposter_template.tex | 257 + .../assets/beamerposter_template.tex | 244 + .../assets/poster_quality_checklist.md | 358 + .../assets/tikzposter_template.tex | 251 + .../skills/latex-posters/references/README.md | 439 + .../references/ai_graphics_for_posters.md | 524 + .../compilation_and_quality_control.md | 467 + .../references/latex_poster_packages.md | 745 + .../references/latex_poster_reference.md | 241 + .../references/poster_content_guide.md | 748 + .../references/poster_design_principles.md | 806 + .../references/poster_layout_design.md | 900 + .../poster_patterns_and_presentation.md | 81 + .../scripts/generate_schematic.py | 198 + .../scripts/generate_schematic_ai.py | 950 + .../latex-posters/scripts/review_poster.sh | 214 + .agents/skills/liteparse/SKILL.md | 295 + .../liteparse/references/api_reference.md | 169 + .../liteparse/references/choosing_a_parser.md | 70 + .../liteparse/references/cli_reference.md | 118 + .../liteparse/references/ocr_and_formats.md | 143 + .../liteparse/references/output_formats.md | 146 + .../liteparse/scripts/batch_parse_dir.py | 163 + .agents/skills/literature-review/SKILL.md | 263 + .../assets/review_template.md | 412 + .../references/citation_styles.md | 166 + .../references/core_workflow.md | 260 + .../references/database_strategies.md | 455 + .../references/example_workflow.md | 68 + .../references/search_and_citation.md | 157 + .../literature-review/scripts/generate_pdf.py | 176 + .../scripts/generate_schematic.py | 198 + .../scripts/generate_schematic_ai.py | 950 + .../scripts/search_databases.py | 303 + .../scripts/verify_citations.py | 222 + .../skills/markdown-mermaid-writing/SKILL.md | 322 + .../examples/example-research-report.md | 221 + .../references/diagrams/architecture.md | 108 + .../references/diagrams/block.md | 177 + .../references/diagrams/c4.md | 136 + .../references/diagrams/class.md | 246 + .../references/diagrams/complex_examples.md | 384 + .../references/diagrams/er.md | 222 + .../references/diagrams/flowchart.md | 177 + .../references/diagrams/gantt.md | 138 + .../references/diagrams/git_graph.md | 74 + .../references/diagrams/kanban.md | 107 + .../references/diagrams/mindmap.md | 74 + .../references/diagrams/packet.md | 55 + .../references/diagrams/pie.md | 52 + .../references/diagrams/quadrant.md | 66 + .../references/diagrams/radar.md | 59 + .../references/diagrams/requirement.md | 88 + .../references/diagrams/sankey.md | 71 + .../references/diagrams/sequence.md | 174 + .../references/diagrams/state.md | 150 + .../references/diagrams/timeline.md | 96 + .../references/diagrams/treemap.md | 66 + .../references/diagrams/user_journey.md | 108 + .../references/diagrams/xy_chart.md | 53 + .../references/diagrams/zenuml.md | 71 + .../references/markdown_style_guide.md | 733 + .../references/mermaid_style_guide.md | 458 + .../templates/decision_record.md | 211 + .../templates/how_to_guide.md | 275 + .../templates/issue.md | 303 + .../templates/kanban.md | 223 + .../templates/presentation.md | 312 + .../templates/project_documentation.md | 412 + .../templates/pull_request.md | 319 + .../templates/research_paper.md | 304 + .../templates/status_report.md | 185 + .../skills/market-research-reports/SKILL.md | 337 + .../assets/FORMATTING_GUIDE.md | 149 + .../assets/claims_ledger_template.csv | 4 + .../competitor_feature_matrix_template.csv | 5 + .../assets/consistency_check_template.csv | 3 + .../assets/forecast_sensitivity_template.json | 90 + .../assets/market_report_template.tex | 279 + .../assets/market_research.sty | 241 + .../market_sizing_scenarios_template.json | 129 + .../assets/report_manifest_template.json | 27 + .../assets/source_ledger_template.csv | 4 + .../references/data_analysis_patterns.md | 290 + .../references/evidence_model.md | 148 + .../references/methods_and_ethics.md | 162 + .../references/official_data_sources.md | 196 + .../references/report_structure_guide.md | 283 + .../references/sources.md | 73 + .../references/visual_generation_guide.md | 158 + .../scripts/_common.py | 312 + .../scripts/audit_claim_citations.py | 326 + .../scripts/calculate_market_sizing.py | 389 + .../scripts/check_unit_consistency.py | 217 + .../scripts/forecast_sensitivity.py | 326 + .../scripts/generate_report_scaffold.py | 444 + .../scripts/validate_competitor_matrix.py | 223 + .../scripts/validate_evidence_ledger.py | 291 + .agents/skills/markitdown/SKILL.md | 264 + .../markitdown/references/api_reference.md | 418 + .../markitdown/references/cloud_and_ocr.md | 320 + .../markitdown/references/file_formats.md | 281 + .../markitdown/references/mcp_and_plugins.md | 243 + .../skills/markitdown/references/migration.md | 356 + .../skills/markitdown/references/security.md | 246 + .../skills/markitdown/references/workflows.md | 309 + .../markitdown/scripts/batch_convert.py | 354 + .../markitdown/scripts/convert_literature.py | 405 + .../scripts/inspect_installation.py | 162 + .agents/skills/matchms/SKILL.md | 276 + .../skills/matchms/references/filtering.md | 299 + .../matchms/references/importing_exporting.md | 323 + .../skills/matchms/references/migration.md | 387 + .../skills/matchms/references/similarity.md | 413 + .agents/skills/matchms/references/sources.md | 113 + .../skills/matchms/references/workflows.md | 451 + .../skills/matchms/scripts/library_search.py | 593 + .agents/skills/matlab/SKILL.md | 274 + .../assets/project_manifest_template.json | 33 + .../assets/python_compatibility_r2026a.json | 27 + .../reproducibility_manifest_template.json | 32 + .../matlab/references/data-import-export.md | 221 + .../matlab/references/executing-scripts.md | 213 + .../references/graphics-visualization.md | 181 + .../skills/matlab/references/mathematics.md | 208 + .../matlab/references/matrices-arrays.md | 228 + .../matlab/references/octave-compatibility.md | 212 + .../skills/matlab/references/programming.md | 225 + .../matlab/references/python-integration.md | 248 + .agents/skills/matlab/scripts/_common.py | 263 + .../scripts/generate_function_scaffold.py | 165 + .../matlab/scripts/inventory_mat_file.py | 351 + .../matlab/scripts/plan_batch_command.py | 257 + .../scripts/plan_python_compatibility.py | 176 + .../matlab/scripts/reproducibility_report.py | 233 + .agents/skills/matlab/scripts/scan_m_code.py | 433 + .../scripts/validate_project_manifest.py | 348 + .agents/skills/matplotlib/SKILL.md | 378 + .../matplotlib/references/api_reference.md | 409 + .../matplotlib/references/common_issues.md | 562 + .../matplotlib/references/plot_types.md | 469 + .../matplotlib/references/styling_guide.md | 600 + .../matplotlib/scripts/plot_template.py | 406 + .../matplotlib/scripts/style_configurator.py | 412 + .agents/skills/medchem/SKILL.md | 321 + .../skills/medchem/references/api_guide.md | 331 + .../medchem/references/rules_catalog.md | 328 + .../medchem/scripts/filter_molecules.py | 302 + .agents/skills/modal/SKILL.md | 468 + .../skills/modal/references/api_reference.md | 225 + .agents/skills/modal/references/examples.md | 276 + .agents/skills/modal/references/functions.md | 260 + .../modal/references/getting-started.md | 171 + .agents/skills/modal/references/gpu.md | 177 + .agents/skills/modal/references/images.md | 266 + .agents/skills/modal/references/resources.md | 117 + .agents/skills/modal/references/scaling.md | 173 + .../skills/modal/references/scheduled-jobs.md | 147 + .agents/skills/modal/references/secrets.md | 119 + .agents/skills/modal/references/volumes.md | 247 + .../skills/modal/references/web-endpoints.md | 259 + .agents/skills/molecular-dynamics/SKILL.md | 458 + .../references/mdanalysis_analysis.md | 208 + .agents/skills/molfeat/SKILL.md | 348 + .../molfeat/references/api_reference.md | 429 + .../references/available_featurizers.md | 335 + .../references/choosing_a_featurizer.md | 192 + .agents/skills/molfeat/references/examples.md | 720 + .agents/skills/ncats-arax/SKILL.md | 178 + .../ncats-arax/references/output-schema.md | 186 + .../ncats-arax/references/query-contract.md | 140 + .../skills/ncats-arax/scripts/arax_client.py | 2087 + .agents/skills/networkx/SKILL.md | 440 + .../skills/networkx/references/algorithms.md | 384 + .../skills/networkx/references/generators.md | 385 + .../networkx/references/graph-basics.md | 284 + .agents/skills/networkx/references/io.md | 457 + .../networkx/references/visualization.md | 531 + .agents/skills/neurokit2/SKILL.md | 323 + .../skills/neurokit2/references/bio_module.md | 244 + .../skills/neurokit2/references/complexity.md | 212 + .../neurokit2/references/ecg_cardiac.md | 193 + .agents/skills/neurokit2/references/eda.md | 185 + .agents/skills/neurokit2/references/eeg.md | 204 + .agents/skills/neurokit2/references/emg.md | 157 + .agents/skills/neurokit2/references/eog.md | 154 + .../neurokit2/references/epochs_events.md | 199 + .agents/skills/neurokit2/references/hrv.md | 205 + .agents/skills/neurokit2/references/ppg.md | 191 + .agents/skills/neurokit2/references/rsp.md | 212 + .../neurokit2/references/signal_processing.md | 160 + .agents/skills/neurokit2/scripts/_common.py | 567 + .../neurokit2/scripts/ecg_hrv_pipeline.py | 303 + .../skills/neurokit2/scripts/eda_pipeline.py | 288 + .../neurokit2/scripts/generate_synthetic.py | 221 + .../neurokit2/scripts/inspect_signal.py | 362 + .../skills/neurokit2/scripts/plan_epochs.py | 281 + .../neurokit2/scripts/validate_multimodal.py | 350 + .agents/skills/neuropixels-analysis/SKILL.md | 412 + .../assets/analysis_template.py | 271 + .../references/AI_CURATION.md | 164 + .../references/ANALYSIS.md | 392 + .../references/AUTOMATED_CURATION.md | 435 + .../references/MOTION_CORRECTION.md | 323 + .../references/PREPROCESSING.md | 273 + .../references/QUALITY_METRICS.md | 359 + .../references/SPIKE_SORTING.md | 339 + .../references/api_reference.md | 229 + .../references/plotting_guide.md | 454 + .../references/standard_workflow.md | 305 + .../scripts/compute_metrics.py | 182 + .../scripts/explore_recording.py | 168 + .../scripts/export_to_phy.py | 79 + .../scripts/neuropixels_pipeline.py | 442 + .../scripts/preprocess_recording.py | 122 + .../scripts/run_sorting.py | 98 + .agents/skills/nextflow/SKILL.md | 195 + .../nextflow/references/configuration.md | 276 + .../skills/nextflow/references/containers.md | 92 + .../skills/nextflow/references/developing.md | 301 + .../skills/nextflow/references/language.md | 327 + .../nextflow/references/nf-core-tools.md | 130 + .../nextflow/references/running-pipelines.md | 131 + .agents/skills/nextflow/references/testing.md | 189 + .agents/skills/omero-integration/SKILL.md | 222 + .../omero-integration/references/advanced.md | 273 + .../references/connection.md | 293 + .../references/data_access.md | 359 + .../references/image_processing.md | 286 + .../omero-integration/references/metadata.md | 313 + .../omero-integration/references/rois.md | 291 + .../omero-integration/references/scripts.md | 304 + .../omero-integration/references/sources.md | 194 + .../omero-integration/references/tables.md | 269 + .../scripts/export_image_metadata.py | 560 + .../omero-integration/scripts/inventory.py | 302 + .../omero-integration/scripts/omero_common.py | 490 + .../scripts/plan_transfer.py | 393 + .../scripts/validate_config.py | 140 + .agents/skills/onekgpd/SKILL.md | 371 + .agents/skills/onekgpd/assets/kgpe.json | 48032 ++++++++++++++++ .../references/annotation_vocabularies.md | 187 + .../onekgpd/references/onekgpd_commands.md | 296 + .agents/skills/onekgpd/scripts/onekgpd_api.py | 794 + .../skills/onekgpd/scripts/onekgpd_meta.py | 485 + .../skills/ontology-term-resolution/SKILL.md | 147 + .../references/curation-rules.md | 110 + .../references/ols4-api.md | 135 + .../references/ontology-registry.md | 110 + .../scripts/ols_client.py | 341 + .../scripts/resolve_terms.py | 255 + .../scripts/validate_terms.py | 297 + .agents/skills/open-notebook/SKILL.md | 297 + .../open-notebook/references/api_reference.md | 715 + .../open-notebook/references/architecture.md | 163 + .../open-notebook/references/configuration.md | 226 + .../open-notebook/references/examples.md | 290 + .../open-notebook/scripts/chat_interaction.py | 190 + .../scripts/notebook_management.py | 142 + .../open-notebook/scripts/source_ingestion.py | 160 + .agents/skills/openpiv/SKILL.md | 469 + .../openpiv/references/advanced_algorithms.md | 233 + .agents/skills/openpiv/scripts/__init__.py | 1 + .agents/skills/openpiv/scripts/analyze.py | 143 + .agents/skills/openpiv/scripts/run_example.py | 78 + .agents/skills/openpiv/scripts/runner.py | 214 + .agents/skills/opentrons-integration/SKILL.md | 322 + .../references/api_reference.md | 382 + .../references/liquid_handling.md | 387 + .../references/migration-api-2-19-to-2-29.md | 328 + .../references/modules_and_deck.md | 409 + .../references/protocol_authoring.md | 352 + .../references/sources.md | 151 + .../references/validation_and_operations.md | 314 + .../requirements-flex.txt | 1 + .../requirements-ot2.txt | 1 + .../scripts/absorbance_reader_template.py | 82 + .../scripts/basic_protocol_template.py | 68 + .../scripts/ot2_basic_protocol_template.py | 63 + .../scripts/pcr_setup_template.py | 146 + .../scripts/runtime_parameters_template.py | 110 + .../scripts/serial_dilution_template.py | 113 + .agents/skills/optimize-for-gpu/SKILL.md | 176 + .../code_transformation_patterns.md | 301 + .../optimize-for-gpu/references/cucim.md | 679 + .../optimize-for-gpu/references/cudf.md | 762 + .../optimize-for-gpu/references/cugraph.md | 733 + .../optimize-for-gpu/references/cuml.md | 710 + .../optimize-for-gpu/references/cupy.md | 668 + .../optimize-for-gpu/references/cuspatial.md | 420 + .../optimize-for-gpu/references/cuvs.md | 671 + .../optimize-for-gpu/references/cuxfilter.md | 600 + .../references/decision_framework.md | 234 + .../references/installation.md | 121 + .../optimize-for-gpu/references/kvikio.md | 612 + .../optimize-for-gpu/references/numba.md | 808 + .../optimize-for-gpu/references/raft.md | 312 + .../optimize-for-gpu/references/warp.md | 623 + .agents/skills/pacsomatic/LICENSE | 21 + .agents/skills/pacsomatic/SKILL.md | 150 + .agents/skills/pacsomatic/config.yaml | 42 + .../pacsomatic/references/agent-playbook.md | 73 + .../references/config-and-output.md | 100 + .../pacsomatic/references/pacsomatic_guide.md | 254 + .../pacsomatic/scripts/run_pacsomatic.py | 794 + .agents/skills/paper-lookup/SKILL.md | 263 + .../skills/paper-lookup/references/arxiv.md | 275 + .../skills/paper-lookup/references/biorxiv.md | 163 + .../skills/paper-lookup/references/core.md | 150 + .../paper-lookup/references/crossref.md | 181 + .../paper-lookup/references/europepmc.md | 226 + .../skills/paper-lookup/references/medrxiv.md | 126 + .../paper-lookup/references/openalex.md | 174 + .agents/skills/paper-lookup/references/pmc.md | 228 + .../skills/paper-lookup/references/pubmed.md | 124 + .../references/semantic-scholar.md | 203 + .../paper-lookup/references/unpaywall.md | 127 + .../skills/paper-lookup/scripts/_common.py | 227 + .../skills/paper-lookup/scripts/arxiv_atom.py | 200 + .../paper-lookup/scripts/jats_to_text.py | 324 + .../paper-lookup/scripts/openalex_abstract.py | 163 + .../skills/paper-lookup/scripts/paginate.py | 490 + .agents/skills/paperclip/SKILL.md | 413 + .../paperclip/references/cli-reference.md | 389 + .../paperclip/references/installation.md | 341 + .../skills/paperclip/references/map-reduce.md | 252 + .../skills/paperclip/references/python-sdk.md | 323 + .../references/repos-and-workspace.md | 271 + .../references/search-and-retrieval.md | 281 + .agents/skills/paperzilla/SKILL.md | 159 + .agents/skills/parallel-web/SKILL.md | 128 + .../references/data-enrichment.md | 104 + .../parallel-web/references/deep-research.md | 91 + .../skills/parallel-web/references/findall.md | 81 + .../skills/parallel-web/references/monitor.md | 83 + .../parallel-web/references/web-extract.md | 59 + .../parallel-web/references/web-search.md | 100 + .agents/skills/pathml/SKILL.md | 222 + .../pathml/references/data_management.md | 357 + .agents/skills/pathml/references/graphs.md | 335 + .../skills/pathml/references/image_loading.md | 301 + .../pathml/references/machine_learning.md | 408 + .../pathml/references/multiparametric.md | 352 + .../skills/pathml/references/preprocessing.md | 371 + .agents/skills/pathml/scripts/_common.py | 385 + .agents/skills/pathml/scripts/image_qc.py | 325 + .../skills/pathml/scripts/plan_inference.py | 282 + .../skills/pathml/scripts/plan_pipeline.py | 239 + .../skills/pathml/scripts/slide_manifest.py | 405 + .../pathml/scripts/validate_spatial_schema.py | 420 + .../pathogen-variant-surveillance/SKILL.md | 208 + .../references/lapis-api.md | 209 + .../references/lineage-nomenclature.md | 126 + .../references/surveillance-caveats.md | 149 + .../scripts/lapis_client.py | 776 + .../scripts/lineage_prevalence.py | 310 + .../scripts/mutation_profile.py | 215 + .../scripts/reporting_lag.py | 217 + .../scripts/resolve_lineage.py | 198 + .agents/skills/pathway-enrichment/SKILL.md | 194 + .../references/databases-and-gene-sets.md | 140 + .../pathway-enrichment/references/gseapy.md | 189 + .../references/interpretation.md | 118 + .../scripts/run_enrichment.py | 231 + .agents/skills/pdf/LICENSE.txt | 30 + .agents/skills/pdf/SKILL.md | 322 + .agents/skills/pdf/forms.md | 294 + .agents/skills/pdf/reference.md | 612 + .../pdf/scripts/check_bounding_boxes.py | 65 + .../pdf/scripts/check_fillable_fields.py | 11 + .../pdf/scripts/convert_pdf_to_images.py | 33 + .../pdf/scripts/create_validation_image.py | 37 + .../pdf/scripts/extract_form_field_info.py | 122 + .../pdf/scripts/extract_form_structure.py | 115 + .../pdf/scripts/fill_fillable_fields.py | 98 + .../scripts/fill_pdf_form_with_annotations.py | 107 + .agents/skills/peer-review/SKILL.md | 288 + .../assets/citation_references_template.csv | 2 + .../assets/claim_evidence_matrix_template.csv | 4 + .../assets/reporting_checklist_template.csv | 31 + .../assets/reporting_guidelines.json | 466 + .../assets/review_intake_template.json | 52 + .../assets/review_scaffold_template.md | 68 + .../peer-review/assets/source_ledger.csv | 32 + .../statistical_reproducibility_template.json | 210 + .../assets/study_profile_template.json | 12 + .../peer-review/references/common_issues.md | 257 + .../references/ethical_review_practice.md | 233 + .../references/reporting_standards.md | 249 + .../references/security_validation.md | 75 + .../references/statistical_reproducibility.md | 329 + .../peer-review/references/tool_reference.md | 253 + .agents/skills/peer-review/scripts/_common.py | 398 + .../peer-review/scripts/audit_citations.py | 207 + .../audit_statistics_reproducibility.py | 305 + .../scripts/generate_review_scaffold.py | 81 + .../skills/peer-review/scripts/lint_review.py | 254 + .../scripts/select_reporting_guidelines.py | 383 + .../scripts/validate_claim_evidence.py | 221 + .../scripts/validate_review_intake.py | 452 + .agents/skills/pennylane/SKILL.md | 240 + .../pennylane/references/advanced_features.md | 667 + .../pennylane/references/devices_backends.md | 562 + .../pennylane/references/getting_started.md | 232 + .../pennylane/references/optimization.md | 670 + .../pennylane/references/quantum_chemistry.md | 576 + .../pennylane/references/quantum_circuits.md | 443 + .../skills/pennylane/references/quantum_ml.md | 555 + .agents/skills/phylogenetics/SKILL.md | 409 + .../references/iqtree_inference.md | 181 + .../scripts/phylogenetic_analysis.py | 272 + .agents/skills/pi-agent/SKILL.md | 83 + .../skills/pi-agent/references/compaction.md | 76 + .../pi-agent/references/containerization.md | 80 + .../pi-agent/references/custom-provider.md | 131 + .../skills/pi-agent/references/development.md | 61 + .../references/environment-variables.md | 57 + .../skills/pi-agent/references/extensions.md | 185 + .agents/skills/pi-agent/references/json.md | 69 + .../skills/pi-agent/references/keybindings.md | 58 + .../skills/pi-agent/references/llama-cpp.md | 69 + .agents/skills/pi-agent/references/models.md | 114 + .../skills/pi-agent/references/overview.md | 37 + .../skills/pi-agent/references/packages.md | 103 + .../pi-agent/references/pi-interview.md | 123 + .../pi-agent/references/pi-mcp-adapter.md | 191 + .../pi-agent/references/pi-subagents.md | 371 + .../pi-agent/references/pi-web-access.md | 243 + .../pi-agent/references/prompt-templates.md | 48 + .../skills/pi-agent/references/providers.md | 122 + .../skills/pi-agent/references/quickstart.md | 73 + .agents/skills/pi-agent/references/rpc.md | 95 + .agents/skills/pi-agent/references/sdk.md | 151 + .../skills/pi-agent/references/security.md | 52 + .../pi-agent/references/session-format.md | 90 + .../skills/pi-agent/references/sessions.md | 56 + .../skills/pi-agent/references/settings.md | 102 + .../pi-agent/references/shell-aliases.md | 15 + .agents/skills/pi-agent/references/skills.md | 83 + .../pi-agent/references/terminal-setup.md | 87 + .agents/skills/pi-agent/references/termux.md | 31 + .agents/skills/pi-agent/references/themes.md | 69 + .agents/skills/pi-agent/references/tmux.md | 44 + .agents/skills/pi-agent/references/tui.md | 97 + .agents/skills/pi-agent/references/usage.md | 129 + .agents/skills/pi-agent/references/windows.md | 23 + .agents/skills/pkpd-modeling/SKILL.md | 381 + .../assets/nca-reporting-checklist.md | 72 + .../assets/popk-analysis-plan.md | 136 + .../references/antimicrobial-and-tdm.md | 110 + .../references/bioequivalence.md | 132 + .../references/dataset-standards.md | 103 + .../pkpd-modeling/references/ddi-and-qt.md | 132 + .../references/nca-conventions.md | 128 + .../skills/pkpd-modeling/references/pbpk.md | 103 + .../references/pd-and-exposure-response.md | 149 + .../pkpd-modeling/references/population-pk.md | 133 + .../references/regulatory-guidance.md | 82 + .../references/software-ecosystem.md | 123 + .../pkpd-modeling/references/source-ledger.md | 89 + .../references/special-populations.md | 126 + .../references/structural-models.md | 140 + .../references/tmdd-and-biologics.md | 115 + .../skills/pkpd-modeling/scripts/_common.py | 327 + .../skills/pkpd-modeling/scripts/_models.py | 673 + .../scripts/allometry_and_fih.py | 346 + .../pkpd-modeling/scripts/bioequivalence.py | 480 + .../scripts/check_popk_dataset.py | 400 + .../pkpd-modeling/scripts/ddi_static.py | 346 + .../scripts/exposure_response.py | 328 + .../scripts/fit_compartmental.py | 558 + .agents/skills/pkpd-modeling/scripts/nca.py | 587 + .../pkpd-modeling/scripts/simulate_regimen.py | 323 + .../skills/pkpd-modeling/scripts/tdm_bayes.py | 312 + .agents/skills/polars-bio/SKILL.md | 379 + .../references/bioframe_migration.md | 250 + .../polars-bio/references/configuration.md | 187 + .../skills/polars-bio/references/file_io.md | 469 + .../references/interval_operations.md | 370 + .../references/pileup_operations.md | 176 + .../polars-bio/references/sql_processing.md | 224 + .agents/skills/polars/SKILL.md | 393 + .../polars/references/best_practices.md | 651 + .../skills/polars/references/core_concepts.md | 380 + .agents/skills/polars/references/io_guide.md | 564 + .../skills/polars/references/operations.md | 602 + .../polars/references/pandas_migration.md | 417 + .../polars/references/transformations.md | 549 + .agents/skills/pptx-posters/SKILL.md | 258 + .../assets/generation_dependencies.json | 5 + .../assets/poster_manifest_template.json | 154 + .../assets/poster_quality_checklist.md | 192 + .../pptx-posters/references/manifest_spec.md | 297 + .../references/poster_content_guide.md | 176 + .../references/poster_design_principles.md | 226 + .../references/poster_layout_design.md | 189 + .../pptx-posters/references/pptx_security.md | 202 + .../references/security_validation.md | 90 + .../pptx-posters/references/source_ledger.md | 256 + .../skills/pptx-posters/scripts/_common.py | 396 + .../skills/pptx-posters/scripts/_manifest.py | 1318 + .agents/skills/pptx-posters/scripts/_pptx.py | 1484 + .../pptx-posters/scripts/check_layout.py | 179 + .../pptx-posters/scripts/check_palette.py | 142 + .../pptx-posters/scripts/generate_poster.py | 427 + .../pptx-posters/scripts/inspect_pptx.py | 40 + .../pptx-posters/scripts/inventory_images.py | 252 + .../pptx-posters/scripts/plan_export.py | 201 + .../pptx-posters/scripts/validate_manifest.py | 55 + .agents/skills/pptx/LICENSE.txt | 30 + .agents/skills/pptx/SKILL.md | 246 + .agents/skills/pptx/scripts/__init__.py | 0 .agents/skills/pptx/scripts/add_slide.py | 367 + .agents/skills/pptx/scripts/clean.py | 309 + .../pptx/scripts/office/helpers/__init__.py | 111 + .../pptx/scripts/office/helpers/pptx_chart.py | 170 + .../pptx/scripts/office/helpers/pptx_slide.py | 60 + .../pptx/scripts/office/helpers/pptx_theme.py | 114 + .../schemas/ISO-IEC29500-4_2016/dml-chart.xsd | 1499 + .../ISO-IEC29500-4_2016/dml-chartDrawing.xsd | 146 + .../ISO-IEC29500-4_2016/dml-diagram.xsd | 1085 + .../ISO-IEC29500-4_2016/dml-lockedCanvas.xsd | 11 + .../schemas/ISO-IEC29500-4_2016/dml-main.xsd | 3081 + .../ISO-IEC29500-4_2016/dml-picture.xsd | 23 + .../dml-spreadsheetDrawing.xsd | 185 + .../dml-wordprocessingDrawing.xsd | 287 + .../schemas/ISO-IEC29500-4_2016/pml.xsd | 1676 + .../shared-additionalCharacteristics.xsd | 28 + .../shared-bibliography.xsd | 144 + .../shared-commonSimpleTypes.xsd | 174 + .../shared-customXmlDataProperties.xsd | 25 + .../shared-customXmlSchemaProperties.xsd | 18 + .../shared-documentPropertiesCustom.xsd | 59 + .../shared-documentPropertiesExtended.xsd | 56 + .../shared-documentPropertiesVariantTypes.xsd | 195 + .../ISO-IEC29500-4_2016/shared-math.xsd | 582 + .../shared-relationshipReference.xsd | 25 + .../schemas/ISO-IEC29500-4_2016/sml.xsd | 4439 ++ .../schemas/ISO-IEC29500-4_2016/vml-main.xsd | 570 + .../ISO-IEC29500-4_2016/vml-officeDrawing.xsd | 509 + .../vml-presentationDrawing.xsd | 12 + .../vml-spreadsheetDrawing.xsd | 108 + .../vml-wordprocessingDrawing.xsd | 96 + .../schemas/ISO-IEC29500-4_2016/wml.xsd | 3646 ++ .../schemas/ISO-IEC29500-4_2016/xml.xsd | 116 + .../ecma/fouth-edition/opc-contentTypes.xsd | 42 + .../ecma/fouth-edition/opc-coreProperties.xsd | 50 + .../schemas/ecma/fouth-edition/opc-digSig.xsd | 49 + .../ecma/fouth-edition/opc-relationships.xsd | 33 + .../pptx/scripts/office/schemas/mce/mc.xsd | 75 + .../office/schemas/microsoft/wml-2010.xsd | 560 + .../office/schemas/microsoft/wml-2012.xsd | 67 + .../office/schemas/microsoft/wml-2018.xsd | 14 + .../office/schemas/microsoft/wml-cex-2018.xsd | 20 + .../office/schemas/microsoft/wml-cid-2016.xsd | 13 + .../microsoft/wml-sdtdatahash-2020.xsd | 4 + .../schemas/microsoft/wml-symex-2015.xsd | 8 + .agents/skills/pptx/scripts/office/soffice.py | 232 + .../skills/pptx/scripts/office/validate.py | 173 + .../scripts/office/validators/__init__.py | 15 + .../pptx/scripts/office/validators/base.py | 875 + .../pptx/scripts/office/validators/docx.py | 466 + .../pptx/scripts/office/validators/pptx.py | 441 + .../scripts/office/validators/redlining.py | 299 + .agents/skills/pptx/scripts/thumbnail.py | 311 + .agents/skills/primekg/SKILL.md | 99 + .../skills/primekg/scripts/query_primekg.py | 128 + .../skills/protocolsio-integration/SKILL.md | 236 + .../assets/protocol-snapshot.schema.json | 179 + .../references/additional_features.md | 202 + .../references/authentication.md | 135 + .../references/discussions.md | 192 + .../references/file_manager.md | 256 + .../references/protocols_api.md | 256 + .../references/workspaces.md | 192 + .../scripts/__init__.py | 1 + .../scripts/_common.py | 613 + .../scripts/pagination_helper.py | 248 + .../scripts/plan_write_request.py | 657 + .../scripts/protocols_read.py | 493 + .../scripts/validate_auth_config.py | 128 + .../scripts/validate_protocol_json.py | 358 + .agents/skills/pufferlib/SKILL.md | 328 + .../pufferlib/references/environments.md | 260 + .../pufferlib/references/integration.md | 192 + .../skills/pufferlib/references/policies.md | 181 + .../skills/pufferlib/references/training.md | 287 + .../pufferlib/references/vectorization.md | 210 + .agents/skills/pufferlib/scripts/__init__.py | 1 + .agents/skills/pufferlib/scripts/_common.py | 199 + .../scripts/benchmark_vectorization.py | 244 + .../scripts/env_contract_validator.py | 198 + .../skills/pufferlib/scripts/env_template.py | 210 + .../pufferlib/scripts/inspect_checkpoint.py | 225 + .../skills/pufferlib/scripts/repro_plan.py | 177 + .../pufferlib/scripts/train_template.py | 282 + .../skills/pufferlib/scripts/validate_plan.py | 570 + .agents/skills/pydeseq2/SKILL.md | 369 + .../pydeseq2/references/analysis_patterns.md | 70 + .../pydeseq2/references/api_reference.md | 256 + .../references/core_workflow_steps.md | 162 + .../pydeseq2/references/workflow_guide.md | 603 + .../pydeseq2/scripts/run_deseq2_analysis.py | 388 + .agents/skills/pydicom/SKILL.md | 381 + .../skills/pydicom/references/common_tags.md | 277 + .../pydicom/references/transfer_syntaxes.md | 348 + .agents/skills/pydicom/scripts/__init__.py | 1 + .agents/skills/pydicom/scripts/_common.py | 916 + .../skills/pydicom/scripts/anonymize_dicom.py | 713 + .../pydicom/scripts/deidentification_audit.py | 380 + .../skills/pydicom/scripts/dicom_inventory.py | 402 + .../skills/pydicom/scripts/dicom_to_image.py | 459 + .../pydicom/scripts/extract_metadata.py | 330 + .../pydicom/scripts/pixel_frame_planner.py | 303 + .../scripts/transfer_syntax_inspector.py | 245 + .../pydicom/scripts/uid_mapping_validator.py | 240 + .agents/skills/pyhealth/SKILL.md | 124 + .../pyhealth/assets/starter_pipeline.py | 58 + .../skills/pyhealth/references/datasets.md | 126 + .../skills/pyhealth/references/examples.md | 237 + .../pyhealth/references/installation.md | 112 + .agents/skills/pyhealth/references/medcode.md | 94 + .agents/skills/pyhealth/references/models.md | 114 + .agents/skills/pyhealth/references/tasks.md | 143 + .agents/skills/pylabrobot/SKILL.md | 216 + .../assets/protocol-manifest.schema.json | 367 + .../references/analytical-equipment.md | 200 + .../references/hardware-backends.md | 215 + .../pylabrobot/references/liquid-handling.md | 223 + .../references/material-handling.md | 229 + .../skills/pylabrobot/references/resources.md | 238 + .../pylabrobot/references/visualization.md | 187 + .agents/skills/pylabrobot/scripts/__init__.py | 1 + .agents/skills/pylabrobot/scripts/_common.py | 718 + .../pylabrobot/scripts/check_deck_geometry.py | 52 + .../scripts/generate_simulation_plan.py | 140 + .../pylabrobot/scripts/inspect_backends.py | 216 + .../pylabrobot/scripts/plan_transfers.py | 59 + .../pylabrobot/scripts/validate_manifest.py | 47 + .agents/skills/pymatgen/SKILL.md | 404 + .../pymatgen/references/analysis_modules.md | 352 + .../pymatgen/references/core_classes.md | 290 + .../skills/pymatgen/references/io_formats.md | 323 + .../references/materials_project_api.md | 406 + .../references/transformations_workflows.md | 385 + .agents/skills/pymatgen/scripts/_common.py | 316 + .../pymatgen/scripts/artifact_manifest.py | 172 + .../composition_structure_validator.py | 300 + .../pymatgen/scripts/io_conversion_plan.py | 204 + .agents/skills/pymatgen/scripts/mp_query.py | 416 + .../scripts/phase_diagram_generator.py | 415 + .../pymatgen/scripts/structure_analyzer.py | 293 + .../pymatgen/scripts/structure_converter.py | 219 + .../scripts/symmetry_sensitivity_report.py | 212 + .agents/skills/pymc/SKILL.md | 310 + .../assets/hierarchical_model_template.py | 332 + .../pymc/assets/linear_regression_template.py | 244 + .../skills/pymc/references/distributions.md | 345 + .../skills/pymc/references/model_patterns.md | 130 + .../pymc/references/sampling_inference.md | 432 + .../pymc/references/standard_workflow.md | 176 + .agents/skills/pymc/references/workflows.md | 530 + .../skills/pymc/scripts/model_comparison.py | 409 + .../skills/pymc/scripts/model_diagnostics.py | 328 + .agents/skills/pymoo/SKILL.md | 276 + .agents/skills/pymoo/references/algorithms.md | 232 + .../pymoo/references/constraints_mcdm.md | 417 + .agents/skills/pymoo/references/operators.md | 345 + .../pymoo/references/parallelization.md | 80 + .agents/skills/pymoo/references/problems.md | 265 + .../pymoo/references/quick_start_workflows.md | 404 + .../skills/pymoo/references/visualization.md | 353 + .../pymoo/scripts/custom_problem_example.py | 181 + .../pymoo/scripts/decision_making_example.py | 161 + .../pymoo/scripts/many_objective_example.py | 74 + .../pymoo/scripts/multi_objective_example.py | 63 + .../pymoo/scripts/single_objective_example.py | 59 + .agents/skills/pyopenms/SKILL.md | 179 + .../pyopenms/references/data_structures.md | 498 + .../pyopenms/references/feature_detection.md | 495 + .agents/skills/pyopenms/references/file_io.md | 359 + .../pyopenms/references/identification.md | 431 + .../pyopenms/references/metabolomics.md | 548 + .../pyopenms/references/signal_processing.md | 444 + .../pyopenms/scripts/accurate_mass_search.py | 111 + .../pyopenms/scripts/align_link_quantify.py | 140 + .../pyopenms/scripts/consensus_to_matrix.py | 70 + .../skills/pyopenms/scripts/convert_format.py | 95 + .../skills/pyopenms/scripts/detect_adducts.py | 90 + .../scripts/detect_features_centroided.py | 80 + .../scripts/detect_features_metabo.py | 110 + .../skills/pyopenms/scripts/digest_protein.py | 102 + .../pyopenms/scripts/export_gnps_sirius.py | 90 + .../pyopenms/scripts/extract_chromatograms.py | 105 + .../pyopenms/scripts/inspect_ms_data.py | 167 + .../pyopenms/scripts/mass_calculator.py | 92 + .../skills/pyopenms/scripts/plot_ms_data.py | 129 + .../scripts/process_identifications.py | 111 + .../pyopenms/scripts/process_spectra.py | 124 + .../pyopenms/scripts/theoretical_spectrum.py | 75 + .agents/skills/pysam/SKILL.md | 330 + .../pysam/references/alignment_files.md | 374 + .../skills/pysam/references/api_reference.md | 421 + .../pysam/references/common_workflows.md | 442 + .../references/coordinates_and_indexing.md | 318 + .../pysam/references/cram_and_performance.md | 314 + .../pysam/references/migration_to_0_24.md | 177 + .../skills/pysam/references/sequence_files.md | 316 + .agents/skills/pysam/references/sources.md | 132 + .../skills/pysam/references/variant_files.md | 396 + .agents/skills/pysam/scripts/alignment_qc.py | 326 + .../skills/pysam/scripts/filter_alignments.py | 359 + .agents/skills/pysam/scripts/inspect_hts.py | 485 + .../skills/pysam/scripts/variant_summary.py | 362 + .agents/skills/pytdc/SKILL.md | 297 + .agents/skills/pytdc/references/datasets.md | 242 + .agents/skills/pytdc/references/oracles.md | 273 + .agents/skills/pytdc/references/sources.md | 165 + .agents/skills/pytdc/references/utilities.md | 364 + .agents/skills/pytdc/scripts/_common.py | 205 + .../pytdc/scripts/benchmark_evaluation.py | 367 + .agents/skills/pytdc/scripts/cache_audit.py | 146 + .../skills/pytdc/scripts/discover_metadata.py | 174 + .../pytdc/scripts/load_and_split_data.py | 374 + .../pytdc/scripts/molecular_generation.py | 417 + .agents/skills/pytorch-lightning/SKILL.md | 191 + .../references/best_practices.md | 724 + .../pytorch-lightning/references/callbacks.md | 564 + .../references/data_module.md | 565 + .../references/distributed_training.md | 644 + .../references/lightning_module.md | 487 + .../pytorch-lightning/references/logging.md | 636 + .../pytorch-lightning/references/trainer.md | 641 + .../scripts/quick_trainer_setup.py | 473 + .../scripts/template_datamodule.py | 328 + .../scripts/template_lightning_module.py | 220 + .agents/skills/pyzotero/SKILL.md | 137 + .../pyzotero/references/authentication.md | 105 + .agents/skills/pyzotero/references/cli.md | 102 + .../skills/pyzotero/references/collections.md | 113 + .../pyzotero/references/error-handling.md | 108 + .agents/skills/pyzotero/references/exports.md | 102 + .../pyzotero/references/files-attachments.md | 97 + .../skills/pyzotero/references/full-text.md | 68 + .agents/skills/pyzotero/references/mcp.md | 90 + .../skills/pyzotero/references/pagination.md | 79 + .../skills/pyzotero/references/read-api.md | 137 + .../pyzotero/references/saved-searches.md | 77 + .../pyzotero/references/search-params.md | 90 + .agents/skills/pyzotero/references/tags.md | 87 + .../skills/pyzotero/references/write-api.md | 123 + .agents/skills/qiskit/SKILL.md | 259 + .../skills/qiskit/references/algorithms.md | 311 + .agents/skills/qiskit/references/backends.md | 382 + .agents/skills/qiskit/references/circuits.md | 319 + .agents/skills/qiskit/references/migration.md | 338 + .agents/skills/qiskit/references/patterns.md | 386 + .../skills/qiskit/references/primitives.md | 400 + .agents/skills/qiskit/references/setup.md | 253 + .agents/skills/qiskit/references/sources.md | 156 + .agents/skills/qiskit/references/testing.md | 428 + .../skills/qiskit/references/transpilation.md | 333 + .../skills/qiskit/references/visualization.md | 361 + .../qiskit/scripts/check_environment.py | 260 + .../skills/qiskit/scripts/inspect_runtime.py | 224 + .../qiskit/scripts/run_local_primitives.py | 200 + .agents/skills/qutip/SKILL.md | 317 + .agents/skills/qutip/references/advanced.md | 413 + .agents/skills/qutip/references/analysis.md | 319 + .../skills/qutip/references/core_concepts.md | 300 + .../skills/qutip/references/time_evolution.md | 373 + .../skills/qutip/references/visualization.md | 334 + .agents/skills/qutip/scripts/_common.py | 370 + .../skills/qutip/scripts/convergence_sweep.py | 358 + .../qutip/scripts/qobj_model_validator.py | 327 + .agents/skills/qutip/scripts/result_audit.py | 395 + .../qutip/scripts/solver_config_planner.py | 297 + .../scripts/steady_state_spectrum_planner.py | 245 + .../qutip/scripts/two_level_simulation.py | 394 + .agents/skills/rdkit/SKILL.md | 94 + .../skills/rdkit/references/api_reference.md | 443 + .../rdkit/references/core_capabilities.md | 604 + .../rdkit/references/descriptors_reference.md | 595 + .../rdkit/references/smarts_patterns.md | 668 + .../workflows_and_best_practices.md | 169 + .../rdkit/scripts/molecular_properties.py | 243 + .../skills/rdkit/scripts/similarity_search.py | 297 + .../rdkit/scripts/substructure_filter.py | 386 + .../skills/relsa-severity-assessment/SKILL.md | 354 + .../assets/example_cohort.csv | 55 + .../references/forecasting.md | 155 + .../references/relsa-method.md | 175 + .../references/thresholds-and-zones.md | 154 + .../scripts/_common.py | 287 + .../scripts/forecast_relsa.py | 757 + .../scripts/kde_thresholds.py | 369 + .../scripts/relsa_score.py | 488 + .agents/skills/research-grants/SKILL.md | 296 + .../assets/budget_justification_template.md | 453 + .../assets/nih_specific_aims_template.md | 166 + .../assets/nsf_project_summary_template.md | 92 + .../research-grants/references/README.md | 287 + .../references/broader_impacts.md | 392 + .../references/core_components.md | 397 + .../references/darpa_guidelines.md | 636 + .../references/doe_guidelines.md | 586 + .../references/nih_guidelines.md | 853 + .../references/nsf_guidelines.md | 570 + .../references/nstc_guidelines.md | 733 + .../proposal_types_and_resubmission.md | 81 + .../references/review_criteria.md | 93 + .../references/specific_aims_guide.md | 458 + .../references/writing_principles.md | 94 + .agents/skills/research-lookup/README.md | 106 + .agents/skills/research-lookup/SKILL.md | 338 + .../scripts/manuscript_packet.py | 754 + .../scripts/research_lookup.py | 1204 + .agents/skills/rowan/SKILL.md | 398 + .../rowan/references/access_and_pricing.md | 37 + .../rowan/references/batch_and_webhooks.md | 255 + .../rowan/references/end_to_end_example.md | 119 + .../rowan/references/troubleshooting.md | 106 + .../rowan/references/workflow_catalog.md | 308 + .agents/skills/scanpy/SKILL.md | 303 + .../skills/scanpy/assets/analysis_template.py | 301 + .../scanpy/assets/celltype_mapping.json | 10 + .../skills/scanpy/assets/gene_signatures.json | 9 + .../skills/scanpy/assets/pipeline_config.json | 19 + .../scanpy/references/analysis_workflow.md | 236 + .../skills/scanpy/references/api_reference.md | 267 + .../scanpy/references/plotting_guide.md | 365 + .agents/skills/scanpy/references/r_interop.md | 292 + .../scanpy/references/standard_workflow.md | 223 + .agents/skills/scanpy/scripts/_common.py | 127 + .agents/skills/scanpy/scripts/annotate.py | 84 + .../skills/scanpy/scripts/batch_correct.py | 65 + .agents/skills/scanpy/scripts/cluster.py | 63 + .agents/skills/scanpy/scripts/convert.py | 43 + .agents/skills/scanpy/scripts/find_markers.py | 75 + .agents/skills/scanpy/scripts/inspect_data.py | 81 + .agents/skills/scanpy/scripts/plot.py | 78 + .agents/skills/scanpy/scripts/preprocess.py | 88 + .agents/skills/scanpy/scripts/pseudobulk.py | 74 + .agents/skills/scanpy/scripts/qc_analysis.py | 104 + .../scanpy/scripts/reduce_dimensions.py | 64 + .agents/skills/scanpy/scripts/run_pipeline.py | 182 + .agents/skills/scanpy/scripts/score_genes.py | 82 + .agents/skills/scanpy/scripts/subset.py | 64 + .agents/skills/scholar-evaluation/SKILL.md | 296 + .../assets/evaluation_template.json | 50 + .../assets/evidence_manifest_template.json | 63 + .../assets/process_checklist_template.json | 70 + .../assets/ratings_template.csv | 21 + .../assets/rubric_template.json | 301 + .../references/evaluation_framework.md | 264 + .../references/local_tooling.md | 232 + .../references/responsible_assessment.md | 196 + .../references/security_validation.md | 95 + .../references/source_ledger.md | 222 + .../scholar-evaluation/scripts/_common.py | 986 + .../scripts/calculate_scores.py | 57 + .../scripts/check_process.py | 231 + .../scripts/check_traceability.py | 233 + .../scripts/generate_report_scaffold.py | 231 + .../scripts/summarize_agreement.py | 235 + .../scripts/validate_rubric.py | 54 + .../scripts/weight_sensitivity.py | 251 + .../skills/scientific-brainstorming/SKILL.md | 282 + .../references/brainstorming_methods.md | 292 + .../references/facilitation_workflows.md | 284 + .../references/idea_evaluation.md | 268 + .../references/responsible_ai.md | 220 + .../references/sources.md | 364 + .../scripts/_common.py | 307 + .../scripts/evaluate_matrix.py | 518 + .../scripts/session_scaffold.py | 248 + .../scripts/validate_register.py | 654 + .../scientific-critical-thinking/SKILL.md | 180 + .../references/common_biases.md | 364 + .../references/core_capabilities.md | 407 + .../references/evidence_hierarchy.md | 485 + .../references/experimental_design.md | 496 + .../references/logical_fallacies.md | 478 + .../references/scientific_method.md | 169 + .../references/statistical_pitfalls.md | 506 + .agents/skills/scientific-schematics/SKILL.md | 370 + .../references/best_practices.md | 574 + .../references/iterative_refinement.md | 315 + .../scripts/example_usage.sh | 92 + .../scripts/generate_schematic.py | 198 + .../scripts/generate_schematic_ai.py | 950 + .agents/skills/scientific-slides/SKILL.md | 379 + .../assets/beamer_template_conference.tex | 407 + .../assets/beamer_template_defense.tex | 906 + .../assets/beamer_template_seminar.tex | 870 + .../assets/powerpoint_design_guide.md | 662 + .../assets/timing_guidelines.md | 597 + .../references/beamer_guide.md | 1019 + .../references/common_pitfalls.md | 85 + .../references/data_visualization_slides.md | 708 + .../references/presentation_structure.md | 642 + .../references/presentation_workflow.md | 196 + .../references/prompt_writing.md | 42 + .../references/script_reference.md | 143 + .../references/slide_capabilities.md | 360 + .../references/slide_design_principles.md | 849 + .../references/talk_types_guide.md | 687 + .../references/visual_review_workflow.md | 775 + .../scripts/generate_schematic.py | 198 + .../scripts/generate_schematic_ai.py | 950 + .../scripts/generate_slide_image.py | 197 + .../scripts/generate_slide_image_ai.py | 877 + .../scripts/pdf_to_images.py | 221 + .../scripts/slides_to_pdf.py | 235 + .../scripts/validate_presentation.py | 408 + .../skills/scientific-visualization/SKILL.md | 285 + .../assets/color_palettes.py | 263 + .../assets/nature.mplstyle | 68 + .../assets/presentation.mplstyle | 68 + .../assets/publication.mplstyle | 77 + .../assets/publisher_profiles.json | 269 + .../references/color_palettes.md | 227 + .../references/journal_requirements.md | 169 + .../references/matplotlib_examples.md | 336 + .../references/publication_guidelines.md | 196 + .../references/sources.md | 76 + .../scripts/_common.py | 136 + .../scripts/export_plan.py | 493 + .../scripts/figure_export.py | 642 + .../scripts/image_metadata.py | 731 + .../scripts/palette_audit.py | 327 + .../scripts/style_presets.py | 501 + .../scripts/style_preview.py | 232 + .agents/skills/scientific-writing/SKILL.md | 356 + .../assets/REPORT_FORMATTING_GUIDE.md | 60 + .../assets/authorship_template.json | 56 + .../assets/claim_evidence_template.csv | 2 + .../assets/consistency_manifest_template.json | 43 + .../assets/manuscript_manifest_template.json | 37 + .../assets/manuscript_scaffold.md | 65 + .../assets/reporting_coverage_template.json | 6 + .../assets/reporting_guidelines.json | 529 + .../assets/source_manifest_template.json | 27 + .../authorship_ai_confidentiality.md | 111 + .../references/citation_styles.md | 92 + .../references/cli_reference.md | 113 + .../references/evidence_workflow.md | 94 + .../references/figures_tables.md | 94 + .../references/imrad_structure.md | 114 + .../references/journal_policies.md | 56 + .../professional_report_formatting.md | 82 + .../references/reporting_guidelines.md | 107 + .../research_integrity_open_science.md | 97 + .../references/source_ledger.md | 268 + .../references/writing_principles.md | 97 + .../scientific-writing/scripts/_common.py | 240 + .../scripts/audit_claims.py | 241 + .../scripts/check_consistency.py | 408 + .../scripts/check_references.py | 219 + .../scripts/lint_manuscript.py | 171 + .../scripts/scaffold_manuscript.py | 143 + .../scripts/select_reporting_guidelines.py | 214 + .../scripts/validate_authorship.py | 322 + .../scripts/validate_manifest.py | 460 + .agents/skills/scikit-bio/SKILL.md | 470 + .../scikit-bio/references/api_reference.md | 766 + .agents/skills/scikit-learn/SKILL.md | 324 + .../references/common_workflows.md | 107 + .../references/core_capabilities.md | 133 + .../references/model_evaluation.md | 592 + .../references/pipelines_and_composition.md | 612 + .../scikit-learn/references/preprocessing.md | 606 + .../references/quick_reference.md | 436 + .../references/supervised_learning.md | 379 + .../references/unsupervised_learning.md | 517 + .../scripts/classification_pipeline.py | 257 + .../scripts/clustering_analysis.py | 386 + .agents/skills/scikit-survival/SKILL.md | 313 + .../references/competing-risks.md | 302 + .../scikit-survival/references/cox-models.md | 252 + .../references/data-handling.md | 278 + .../references/ensemble-models.md | 287 + .../references/evaluation-metrics.md | 391 + .../scikit-survival/references/svm-models.md | 277 + .../skills/scikit-survival/scripts/_common.py | 456 + .../scripts/competing_risk_cif.py | 286 + .../scripts/evaluate_survival_metrics.py | 296 + .../scikit-survival/scripts/model_report.py | 297 + .../scripts/train_survival_model.py | 583 + .../scripts/validate_survival_csv.py | 172 + .agents/skills/scvelo/SKILL.md | 328 + .../scvelo/references/velocity_models.md | 168 + .../scvelo/scripts/rna_velocity_workflow.py | 240 + .agents/skills/scvi-tools/SKILL.md | 201 + .../references/differential-expression.md | 597 + .../scvi-tools/references/models-atac-seq.md | 329 + .../references/models-multimodal.md | 400 + .../scvi-tools/references/models-scrna-seq.md | 333 + .../scvi-tools/references/models-spatial.md | 432 + .../references/models-specialized.md | 376 + .../references/theoretical-foundations.md | 438 + .../skills/scvi-tools/references/workflows.md | 559 + .agents/skills/seaborn/SKILL.md | 254 + .agents/skills/seaborn/references/examples.md | 824 + .../seaborn/references/function_reference.md | 772 + .../seaborn/references/grids_and_levels.md | 85 + .../seaborn/references/objects_interface.md | 963 + .../references/palettes_and_theming.md | 110 + .../patterns_and_troubleshooting.md | 114 + .../seaborn/references/plotting_functions.md | 178 + .agents/skills/shap/SKILL.md | 282 + .../skills/shap/references/data-maskers.md | 287 + .agents/skills/shap/references/explainers.md | 376 + .agents/skills/shap/references/migration.md | 415 + .agents/skills/shap/references/modalities.md | 353 + .agents/skills/shap/references/plots.md | 406 + .agents/skills/shap/references/theory.md | 352 + .../skills/shap/references/troubleshooting.md | 442 + .agents/skills/shap/references/workflows.md | 565 + .agents/skills/shap/scripts/tabular_report.py | 326 + .agents/skills/simpy/SKILL.md | 283 + .agents/skills/simpy/references/cli-guide.md | 266 + .agents/skills/simpy/references/events.md | 225 + .agents/skills/simpy/references/monitoring.md | 260 + .../simpy/references/process-interaction.md | 269 + .agents/skills/simpy/references/real-time.md | 174 + .agents/skills/simpy/references/resources.md | 274 + .../references/simulation-methodology.md | 293 + .agents/skills/simpy/references/sources.md | 167 + .agents/skills/simpy/scripts/_common.py | 473 + .../scripts/basic_simulation_template.py | 415 + .../simpy/scripts/bounded_queue_scenario.py | 126 + .../simpy/scripts/event_trace_summary.py | 296 + .../simpy/scripts/replication_runner.py | 194 + .../skills/simpy/scripts/resource_monitor.py | 474 + .../scripts/validate_simulation_config.py | 111 + .agents/skills/stable-baselines3/SKILL.md | 325 + .../references/algorithms.md | 348 + .../stable-baselines3/references/callbacks.md | 571 + .../references/custom_environments.md | 528 + .../references/vectorized_envs.md | 580 + .../scripts/custom_env_template.py | 314 + .../scripts/evaluate_agent.py | 245 + .../scripts/train_rl_agent.py | 165 + .agents/skills/statistical-analysis/SKILL.md | 446 + .../references/assumptions_and_diagnostics.md | 379 + .../references/bayesian_statistics.md | 686 + .../references/effect_sizes_and_power.md | 649 + .../references/reporting_standards.md | 482 + .../references/test_selection_guide.md | 129 + .../scripts/assumption_checks.py | 652 + .agents/skills/statistical-power/SKILL.md | 200 + .../references/closed_form_recipes.md | 174 + .../references/effect_sizes.md | 121 + .../references/simulation_based_power.md | 101 + .../skills/statistical-power/scripts/power.py | 320 + .../scripts/simulate_power.py | 217 + .agents/skills/statsmodels/SKILL.md | 238 + .../statsmodels/references/discrete_choice.md | 669 + .agents/skills/statsmodels/references/glm.md | 619 + .../statsmodels/references/linear_models.md | 447 + .../statsmodels/references/model_selection.md | 99 + .../references/modeling_capabilities.md | 168 + .../references/quick_start_guide.md | 154 + .../references/stats_diagnostics.md | 859 + .../statsmodels/references/time_series.md | 723 + .agents/skills/sympy/SKILL.md | 354 + .../sympy/references/advanced-topics.md | 635 + .../references/code-generation-printing.md | 628 + .../sympy/references/core-capabilities.md | 348 + .../sympy/references/core_capabilities.md | 190 + .../references/matrices-linear-algebra.md | 526 + .../sympy/references/physics-mechanics.md | 592 + .agents/skills/tamarind/SKILL.md | 285 + .../tamarind/references/api_reference.md | 165 + .../skills/tamarind/references/examples.md | 132 + .../tamarind/references/tool_catalog.md | 66 + .../skills/tamarind/references/workflows.md | 263 + .agents/skills/tiledbvcf/SKILL.md | 456 + .agents/skills/timesfm-forecasting/SKILL.md | 408 + .../anomaly-detection/detect_anomalies.py | 524 + .../output/anomaly_detection.json | 448 + .../output/anomaly_detection.png | Bin 0 -> 217028 bytes .../covariates-forecasting/demo_covariates.py | 568 + .../output/covariates_data.png | Bin 0 -> 458505 bytes .../output/covariates_metadata.json | 59 + .../output/sales_with_covariates.csv | 109 + .../examples/global-temperature/README.md | 178 + .../generate_animation_data.py | 147 + .../global-temperature/generate_gif.py | 248 + .../global-temperature/generate_html.py | 544 + .../output/animation_data.json | 5441 ++ .../output/forecast_animation.gif | Bin 0 -> 794231 bytes .../output/forecast_output.csv | 13 + .../output/forecast_output.json | 188 + .../output/forecast_visualization.png | Bin 0 -> 156559 bytes .../output/interactive_forecast.html | 5939 ++ .../global-temperature/run_example.sh | 53 + .../global-temperature/run_forecast.py | 167 + .../temperature_anomaly.csv | 37 + .../global-temperature/visualize_forecast.py | 123 + .../references/api_reference.md | 231 + .../references/data_preparation.md | 272 + .../references/examples_and_validation.md | 103 + .../references/output_and_config.md | 93 + .../references/performance_tuning.md | 80 + .../references/system_requirements.md | 201 + .../references/workflows.md | 126 + .../scripts/check_system.py | 521 + .../scripts/forecast_csv.py | 269 + .agents/skills/torch-geometric/SKILL.md | 458 + .../references/custom_datasets.md | 239 + .../references/explainability.md | 208 + .../references/heterogeneous.md | 241 + .../references/link_prediction.md | 226 + .../references/message_passing.md | 121 + .../torch-geometric/references/scaling.md | 269 + .agents/skills/torchdrug/SKILL.md | 241 + .../torchdrug/references/core_concepts.md | 241 + .../skills/torchdrug/references/datasets.md | 179 + .../torchdrug/references/knowledge_graphs.md | 226 + .../references/models_architectures.md | 223 + .../references/molecular_generation.md | 246 + .../molecular_property_prediction.md | 239 + .../torchdrug/references/protein_modeling.md | 221 + .../torchdrug/references/retrosynthesis.md | 247 + .agents/skills/transformers/SKILL.md | 195 + .../transformers/references/generation.md | 473 + .../skills/transformers/references/models.md | 371 + .../transformers/references/pipelines.md | 334 + .../transformers/references/tokenizers.md | 449 + .../transformers/references/training.md | 504 + .agents/skills/treatment-plans/SKILL.md | 174 + ...nician_authored_intervention_template.json | 11 + .../goals_monitoring_checkpoint_template.json | 13 + ...d_preference_shared_decision_template.json | 11 + .../assets/intended_use_handoff_template.json | 83 + .../assets/source_fact_manifest_template.json | 11 + .../transition_reconciliation_template.json | 27 + .../treatment-plans/references/README.md | 19 + .../references/documentation_workflow.md | 165 + .../references/privacy_governance.md | 119 + .../references/safety_scope.md | 101 + .../references/security_validation.md | 68 + .../references/shared_decision_handoff.md | 138 + .../references/source_boundaries.md | 127 + .../references/source_ledger.md | 131 + .../skills/treatment-plans/scripts/_common.py | 1160 + .../scripts/check_completeness.py | 572 + .../scripts/check_consistency.py | 386 + .../scripts/generate_template.py | 133 + .../scripts/privacy_process_check.py | 213 + .../scripts/timeline_generator.py | 260 + .../scripts/validate_traceability.py | 147 + .../scripts/validate_treatment_plan.py | 95 + .agents/skills/umap-learn/SKILL.md | 488 + .../umap-learn/references/api_reference.md | 574 + .agents/skills/uncertainty-and-units/SKILL.md | 384 + .../references/domain-conversions.md | 188 + .../references/gum-methodology.md | 219 + .../references/pint-recipes.md | 228 + .../references/plausibility-scales.md | 168 + .../references/reporting-rules.md | 133 + .../references/uncertainties-recipes.md | 167 + .../uncertainty-and-units/scripts/_common.py | 666 + .../scripts/audit_units.py | 575 + .../scripts/check_plausibility.py | 894 + .../scripts/convert_units.py | 280 + .../scripts/format_result.py | 326 + .../scripts/propagate_uncertainty.py | 662 + .../scripts/uncertainty_budget.py | 363 + .agents/skills/usfiscaldata/SKILL.md | 171 + .../usfiscaldata/references/api-basics.md | 105 + .../usfiscaldata/references/datasets-debt.md | 166 + .../references/datasets-fiscal.md | 212 + .../references/datasets-interest-rates.md | 188 + .../references/datasets-securities.md | 238 + .../usfiscaldata/references/examples.md | 258 + .../usfiscaldata/references/parameters.md | 182 + .../references/response-format.md | 178 + .agents/skills/vaex/SKILL.md | 204 + .../skills/vaex/references/core_dataframes.md | 373 + .../skills/vaex/references/data_processing.md | 555 + .../skills/vaex/references/io_operations.md | 718 + .../vaex/references/machine_learning.md | 728 + .agents/skills/vaex/references/performance.md | 571 + .../skills/vaex/references/visualization.md | 644 + .agents/skills/venue-templates/SKILL.md | 269 + .../assets/examples/cell_summary_example.md | 247 + .../examples/medical_structured_abstract.md | 313 + .../examples/nature_abstract_examples.md | 213 + .../examples/neurips_introduction_example.md | 245 + .../assets/grants/nih_specific_aims.tex | 237 + .../assets/grants/nsf_proposal_template.tex | 384 + .../assets/journals/elsarticle-harv.bst | 1598 + .../assets/journals/elsarticle-num-names.bst | 1535 + .../assets/journals/elsarticle-num.bst | 1509 + .../journals/elsarticle-template-harv.tex | 286 + .../elsarticle-template-num-names.tex | 284 + .../journals/elsarticle-template-num.tex | 286 + .../assets/journals/nature_article.tex | 174 + .../assets/journals/neurips_article.tex | 292 + .../assets/journals/plos_one.tex | 320 + .../assets/posters/beamerposter_academic.tex | 312 + .../references/cell_press_style.md | 486 + .../references/conferences_formatting.md | 175 + .../references/cs_conference_style.md | 465 + .../references/grants_requirements.md | 267 + .../references/journals_formatting.md | 200 + .../references/medical_journal_styles.md | 536 + .../references/ml_conference_style.md | 562 + .../references/nature_science_style.md | 407 + .../references/posters_guidelines.md | 630 + .../references/reviewer_expectations.md | 422 + .../references/venue_writing_styles.md | 323 + .../scripts/customize_template.py | 206 + .../venue-templates/scripts/query_template.py | 202 + .../scripts/validate_format.py | 321 + .agents/skills/waypoint-bio/SKILL.md | 273 + .../waypoint-bio/references/cli-reference.md | 210 + .../references/compass-benchmark.md | 124 + .../references/data-preparation.md | 200 + .../waypoint-bio/references/python-api.md | 219 + .../scripts/profiler_to_waypoint.py | 481 + .../waypoint-bio/scripts/vocab_coverage.py | 235 + .agents/skills/what-if-oracle/SKILL.md | 184 + .../references/scenario-templates.md | 137 + .agents/skills/xlsx/LICENSE.txt | 30 + .agents/skills/xlsx/SKILL.md | 110 + .../xlsx/scripts/office/helpers/__init__.py | 111 + .../xlsx/scripts/office/helpers/pptx_chart.py | 170 + .../xlsx/scripts/office/helpers/pptx_slide.py | 60 + .../xlsx/scripts/office/helpers/pptx_theme.py | 114 + .../schemas/ISO-IEC29500-4_2016/dml-chart.xsd | 1499 + .../ISO-IEC29500-4_2016/dml-chartDrawing.xsd | 146 + .../ISO-IEC29500-4_2016/dml-diagram.xsd | 1085 + .../ISO-IEC29500-4_2016/dml-lockedCanvas.xsd | 11 + .../schemas/ISO-IEC29500-4_2016/dml-main.xsd | 3081 + .../ISO-IEC29500-4_2016/dml-picture.xsd | 23 + .../dml-spreadsheetDrawing.xsd | 185 + .../dml-wordprocessingDrawing.xsd | 287 + .../schemas/ISO-IEC29500-4_2016/pml.xsd | 1676 + .../shared-additionalCharacteristics.xsd | 28 + .../shared-bibliography.xsd | 144 + .../shared-commonSimpleTypes.xsd | 174 + .../shared-customXmlDataProperties.xsd | 25 + .../shared-customXmlSchemaProperties.xsd | 18 + .../shared-documentPropertiesCustom.xsd | 59 + .../shared-documentPropertiesExtended.xsd | 56 + .../shared-documentPropertiesVariantTypes.xsd | 195 + .../ISO-IEC29500-4_2016/shared-math.xsd | 582 + .../shared-relationshipReference.xsd | 25 + .../schemas/ISO-IEC29500-4_2016/sml.xsd | 4439 ++ .../schemas/ISO-IEC29500-4_2016/vml-main.xsd | 570 + .../ISO-IEC29500-4_2016/vml-officeDrawing.xsd | 509 + .../vml-presentationDrawing.xsd | 12 + .../vml-spreadsheetDrawing.xsd | 108 + .../vml-wordprocessingDrawing.xsd | 96 + .../schemas/ISO-IEC29500-4_2016/wml.xsd | 3646 ++ .../schemas/ISO-IEC29500-4_2016/xml.xsd | 116 + .../ecma/fouth-edition/opc-contentTypes.xsd | 42 + .../ecma/fouth-edition/opc-coreProperties.xsd | 50 + .../schemas/ecma/fouth-edition/opc-digSig.xsd | 49 + .../ecma/fouth-edition/opc-relationships.xsd | 33 + .../xlsx/scripts/office/schemas/mce/mc.xsd | 75 + .../office/schemas/microsoft/wml-2010.xsd | 560 + .../office/schemas/microsoft/wml-2012.xsd | 67 + .../office/schemas/microsoft/wml-2018.xsd | 14 + .../office/schemas/microsoft/wml-cex-2018.xsd | 20 + .../office/schemas/microsoft/wml-cid-2016.xsd | 13 + .../microsoft/wml-sdtdatahash-2020.xsd | 4 + .../schemas/microsoft/wml-symex-2015.xsd | 8 + .agents/skills/xlsx/scripts/office/soffice.py | 232 + .../skills/xlsx/scripts/office/validate.py | 173 + .../scripts/office/validators/__init__.py | 15 + .../xlsx/scripts/office/validators/base.py | 875 + .../xlsx/scripts/office/validators/docx.py | 466 + .../xlsx/scripts/office/validators/pptx.py | 441 + .../scripts/office/validators/redlining.py | 299 + .agents/skills/xlsx/scripts/recalc.py | 308 + .agents/skills/zarr-python/SKILL.md | 241 + .../zarr-python/references/api_reference.md | 162 + .../references/chunking_and_compression.md | 138 + .../zarr-python/references/integration.md | 147 + .../references/performance_and_patterns.md | 198 + .../references/storage_backends.md | 91 + .../zarr-python/references/v3_migration.md | 127 + .claude/skills/adaptyv | 1 + .claude/skills/aeon | 1 + .claude/skills/analytical-method-validation | 1 + .claude/skills/anndata | 1 + .claude/skills/arbor | 1 + .claude/skills/arboreto | 1 + .claude/skills/astropy | 1 + .claude/skills/autoskill | 1 + .claude/skills/benchling-integration | 1 + .claude/skills/bgpt-paper-search | 1 + .claude/skills/bids | 1 + .claude/skills/biopython | 1 + .claude/skills/bioservices | 1 + .claude/skills/bulk-rnaseq | 1 + .claude/skills/cellxgene-census | 1 + .claude/skills/cirq | 1 + .claude/skills/citation-management | 1 + .claude/skills/clinical-decision-support | 1 + .claude/skills/clinical-reports | 1 + .claude/skills/cobrapy | 1 + .claude/skills/consciousness-council | 1 + .claude/skills/dask | 1 + .claude/skills/database-lookup | 1 + .claude/skills/datamol | 1 + .claude/skills/deepchem | 1 + .claude/skills/deepspot-m | 1 + .claude/skills/deeptools | 1 + .claude/skills/depmap | 1 + .claude/skills/dhdna-profiler | 1 + .claude/skills/diffdock | 1 + .claude/skills/dnanexus-integration | 1 + .claude/skills/docx | 1 + .claude/skills/esm | 1 + .claude/skills/etetoolkit | 1 + .claude/skills/exa-search | 1 + .claude/skills/experimental-design | 1 + .claude/skills/exploratory-data-analysis | 1 + .claude/skills/flowio | 1 + .claude/skills/fluidsim | 1 + .claude/skills/generate-image | 1 + .claude/skills/geniml | 1 + .claude/skills/genomic-coordinates | 1 + .claude/skills/genomic-intelligence | 1 + .claude/skills/geomaster | 1 + .claude/skills/geopandas | 1 + .claude/skills/get-available-resources | 1 + .claude/skills/gget | 1 + .claude/skills/ginkgo-cloud-lab | 1 + .claude/skills/glycoengineering | 1 + .claude/skills/gtars | 1 + .claude/skills/histolab | 1 + .claude/skills/hugging-science | 1 + .claude/skills/hypogenic | 1 + .claude/skills/hypothesis-generation | 1 + .claude/skills/imaging-data-commons | 1 + .claude/skills/infographics | 1 + .claude/skills/iso-standards-readiness | 1 + .claude/skills/lab-hardware-cad | 1 + .claude/skills/labarchive-integration | 1 + .claude/skills/lamindb | 1 + .claude/skills/latchbio-integration | 1 + .claude/skills/latex-posters | 1 + .claude/skills/liteparse | 1 + .claude/skills/literature-review | 1 + .claude/skills/markdown-mermaid-writing | 1 + .claude/skills/market-research-reports | 1 + .claude/skills/markitdown | 1 + .claude/skills/matchms | 1 + .claude/skills/matlab | 1 + .claude/skills/matplotlib | 1 + .claude/skills/medchem | 1 + .claude/skills/modal | 1 + .claude/skills/molecular-dynamics | 1 + .claude/skills/molfeat | 1 + .claude/skills/ncats-arax | 1 + .claude/skills/networkx | 1 + .claude/skills/neurokit2 | 1 + .claude/skills/neuropixels-analysis | 1 + .claude/skills/nextflow | 1 + .claude/skills/omero-integration | 1 + .claude/skills/onekgpd | 1 + .claude/skills/ontology-term-resolution | 1 + .claude/skills/open-notebook | 1 + .claude/skills/openpiv | 1 + .claude/skills/opentrons-integration | 1 + .claude/skills/optimize-for-gpu | 1 + .claude/skills/pacsomatic | 1 + .claude/skills/paper-lookup | 1 + .claude/skills/paperclip | 1 + .claude/skills/paperzilla | 1 + .claude/skills/parallel-web | 1 + .claude/skills/pathml | 1 + .claude/skills/pathogen-variant-surveillance | 1 + .claude/skills/pathway-enrichment | 1 + .claude/skills/pdf | 1 + .claude/skills/peer-review | 1 + .claude/skills/pennylane | 1 + .claude/skills/phylogenetics | 1 + .claude/skills/pi-agent | 1 + .claude/skills/pkpd-modeling | 1 + .claude/skills/polars | 1 + .claude/skills/polars-bio | 1 + .claude/skills/pptx | 1 + .claude/skills/pptx-posters | 1 + .claude/skills/primekg | 1 + .claude/skills/protocolsio-integration | 1 + .claude/skills/pufferlib | 1 + .claude/skills/pydeseq2 | 1 + .claude/skills/pydicom | 1 + .claude/skills/pyhealth | 1 + .claude/skills/pylabrobot | 1 + .claude/skills/pymatgen | 1 + .claude/skills/pymc | 1 + .claude/skills/pymoo | 1 + .claude/skills/pyopenms | 1 + .claude/skills/pysam | 1 + .claude/skills/pytdc | 1 + .claude/skills/pytorch-lightning | 1 + .claude/skills/pyzotero | 1 + .claude/skills/qiskit | 1 + .claude/skills/qutip | 1 + .claude/skills/rdkit | 1 + .claude/skills/relsa-severity-assessment | 1 + .claude/skills/research-grants | 1 + .claude/skills/research-lookup | 1 + .claude/skills/rowan | 1 + .claude/skills/scanpy | 1 + .claude/skills/scholar-evaluation | 1 + .claude/skills/scientific-brainstorming | 1 + .claude/skills/scientific-critical-thinking | 1 + .claude/skills/scientific-schematics | 1 + .claude/skills/scientific-slides | 1 + .claude/skills/scientific-visualization | 1 + .claude/skills/scientific-writing | 1 + .claude/skills/scikit-bio | 1 + .claude/skills/scikit-learn | 1 + .claude/skills/scikit-survival | 1 + .claude/skills/scvelo | 1 + .claude/skills/scvi-tools | 1 + .claude/skills/seaborn | 1 + .claude/skills/shap | 1 + .claude/skills/simpy | 1 + .claude/skills/stable-baselines3 | 1 + .claude/skills/statistical-analysis | 1 + .claude/skills/statistical-power | 1 + .claude/skills/statsmodels | 1 + .claude/skills/sympy | 1 + .claude/skills/tamarind | 1 + .claude/skills/tiledbvcf | 1 + .claude/skills/timesfm-forecasting | 1 + .claude/skills/torch-geometric | 1 + .claude/skills/torchdrug | 1 + .claude/skills/transformers | 1 + .claude/skills/treatment-plans | 1 + .claude/skills/umap-learn | 1 + .claude/skills/uncertainty-and-units | 1 + .claude/skills/usfiscaldata | 1 + .claude/skills/vaex | 1 + .claude/skills/venue-templates | 1 + .claude/skills/waypoint-bio | 1 + .claude/skills/what-if-oracle | 1 + .claude/skills/xlsx | 1 + .claude/skills/zarr-python | 1 + skills-lock.json | 983 + 2204 files changed, 669731 insertions(+) create mode 100644 .agents/skills/adaptyv/SKILL.md create mode 100644 .agents/skills/adaptyv/references/api-endpoints.md create mode 100644 .agents/skills/aeon/SKILL.md create mode 100644 .agents/skills/aeon/references/anomaly_detection.md create mode 100644 .agents/skills/aeon/references/classification.md create mode 100644 .agents/skills/aeon/references/clustering.md create mode 100644 .agents/skills/aeon/references/datasets_benchmarking.md create mode 100644 .agents/skills/aeon/references/distances.md create mode 100644 .agents/skills/aeon/references/forecasting.md create mode 100644 .agents/skills/aeon/references/networks.md create mode 100644 .agents/skills/aeon/references/regression.md create mode 100644 .agents/skills/aeon/references/segmentation.md create mode 100644 .agents/skills/aeon/references/similarity_search.md create mode 100644 .agents/skills/aeon/references/transformations.md create mode 100644 .agents/skills/analytical-method-validation/SKILL.md create mode 100644 .agents/skills/analytical-method-validation/assets/validation-protocol-template.md create mode 100644 .agents/skills/analytical-method-validation/assets/validation-report-template.md create mode 100644 .agents/skills/analytical-method-validation/references/compendial-and-clsi.md create mode 100644 .agents/skills/analytical-method-validation/references/framework-selection.md create mode 100644 .agents/skills/analytical-method-validation/references/ich-m10-bioanalytical.md create mode 100644 .agents/skills/analytical-method-validation/references/ich-q2r2.md create mode 100644 .agents/skills/analytical-method-validation/references/source-ledger.md create mode 100644 .agents/skills/analytical-method-validation/references/statistics.md create mode 100644 .agents/skills/analytical-method-validation/scripts/_catalog.py create mode 100644 .agents/skills/analytical-method-validation/scripts/_common.py create mode 100644 .agents/skills/analytical-method-validation/scripts/check_accuracy_precision.py create mode 100644 .agents/skills/analytical-method-validation/scripts/check_bioanalytical_run.py create mode 100644 .agents/skills/analytical-method-validation/scripts/check_detection_limits.py create mode 100644 .agents/skills/analytical-method-validation/scripts/check_response.py create mode 100644 .agents/skills/analytical-method-validation/scripts/compare_methods.py create mode 100644 .agents/skills/analytical-method-validation/scripts/plan_validation.py create mode 100644 .agents/skills/anndata/SKILL.md create mode 100644 .agents/skills/anndata/references/best_practices.md create mode 100644 .agents/skills/anndata/references/concatenation.md create mode 100644 .agents/skills/anndata/references/data_structure.md create mode 100644 .agents/skills/anndata/references/io_operations.md create mode 100644 .agents/skills/anndata/references/manipulation.md create mode 100644 .agents/skills/arbor/SKILL.md create mode 100644 .agents/skills/arbor/references/arbor-upstream.md create mode 100644 .agents/skills/arbor/references/executor-brief.md create mode 100644 .agents/skills/arbor/references/htr-methodology.md create mode 100644 .agents/skills/arbor/references/report-template.md create mode 100644 .agents/skills/arbor/scripts/tree.py create mode 100644 .agents/skills/arboreto/SKILL.md create mode 100644 .agents/skills/arboreto/references/algorithms.md create mode 100644 .agents/skills/arboreto/references/basic_inference.md create mode 100644 .agents/skills/arboreto/references/distributed_computing.md create mode 100644 .agents/skills/arboreto/scripts/basic_grn_inference.py create mode 100644 .agents/skills/astropy/SKILL.md create mode 100644 .agents/skills/astropy/references/coordinates.md create mode 100644 .agents/skills/astropy/references/cosmology.md create mode 100644 .agents/skills/astropy/references/fits.md create mode 100644 .agents/skills/astropy/references/tables.md create mode 100644 .agents/skills/astropy/references/time.md create mode 100644 .agents/skills/astropy/references/units.md create mode 100644 .agents/skills/astropy/references/wcs_and_other_modules.md create mode 100644 .agents/skills/autoskill/.gitignore create mode 100644 .agents/skills/autoskill/SKILL.md create mode 100644 .agents/skills/autoskill/config.yaml create mode 100644 .agents/skills/autoskill/references/https-proxy.md create mode 100644 .agents/skills/autoskill/references/screenpipe-config.yaml create mode 100644 .agents/skills/autoskill/scripts/autoskill.py create mode 100644 .agents/skills/autoskill/scripts/backends.py create mode 100644 .agents/skills/autoskill/scripts/cluster.py create mode 100644 .agents/skills/autoskill/scripts/doctor.py create mode 100644 .agents/skills/autoskill/scripts/fetch_window.py create mode 100644 .agents/skills/autoskill/scripts/match_skills.py create mode 100644 .agents/skills/autoskill/scripts/promote.py create mode 100644 .agents/skills/autoskill/scripts/redact.py create mode 100644 .agents/skills/autoskill/scripts/run.py create mode 100644 .agents/skills/autoskill/scripts/synthesize.py create mode 100644 .agents/skills/benchling-integration/SKILL.md create mode 100644 .agents/skills/benchling-integration/references/api_endpoints.md create mode 100644 .agents/skills/benchling-integration/references/authentication.md create mode 100644 .agents/skills/benchling-integration/references/core_capabilities.md create mode 100644 .agents/skills/benchling-integration/references/eventbridge.md create mode 100644 .agents/skills/benchling-integration/references/sdk_reference.md create mode 100644 .agents/skills/bgpt-paper-search/SKILL.md create mode 100644 .agents/skills/bids/SKILL.md create mode 100644 .agents/skills/bids/references/beps.yml create mode 100644 .agents/skills/bids/references/bids_schema.json create mode 100644 .agents/skills/bids/references/bids_specification.md create mode 100644 .agents/skills/bids/references/conversion_tools.md create mode 100644 .agents/skills/bids/references/core_workflows.md create mode 100644 .agents/skills/bids/references/metadata_fields.md create mode 100644 .agents/skills/bids/scripts/update_schema.py create mode 100644 .agents/skills/biopython/SKILL.md create mode 100644 .agents/skills/biopython/references/advanced.md create mode 100644 .agents/skills/biopython/references/alignment.md create mode 100644 .agents/skills/biopython/references/blast.md create mode 100644 .agents/skills/biopython/references/databases.md create mode 100644 .agents/skills/biopython/references/phylogenetics.md create mode 100644 .agents/skills/biopython/references/sequence_io.md create mode 100644 .agents/skills/biopython/references/structure.md create mode 100644 .agents/skills/bioservices/SKILL.md create mode 100644 .agents/skills/bioservices/references/identifier_mapping.md create mode 100644 .agents/skills/bioservices/references/services_reference.md create mode 100644 .agents/skills/bioservices/references/workflow_patterns.md create mode 100755 .agents/skills/bioservices/scripts/batch_id_converter.py create mode 100755 .agents/skills/bioservices/scripts/compound_cross_reference.py create mode 100755 .agents/skills/bioservices/scripts/pathway_analysis.py create mode 100755 .agents/skills/bioservices/scripts/protein_analysis_workflow.py create mode 100644 .agents/skills/bulk-rnaseq/SKILL.md create mode 100644 .agents/skills/bulk-rnaseq/references/counts-and-handoff.md create mode 100644 .agents/skills/bulk-rnaseq/references/design-and-qc.md create mode 100644 .agents/skills/bulk-rnaseq/references/upstream-manual.md create mode 100644 .agents/skills/bulk-rnaseq/references/upstream-nfcore.md create mode 100644 .agents/skills/bulk-rnaseq/scripts/build_counts_matrix.py create mode 100644 .agents/skills/bulk-rnaseq/scripts/validate_samplesheet.py create mode 100644 .agents/skills/cellxgene-census/SKILL.md create mode 100644 .agents/skills/cellxgene-census/references/census_schema.md create mode 100644 .agents/skills/cellxgene-census/references/common_patterns.md create mode 100644 .agents/skills/cellxgene-census/references/core_workflow_patterns.md create mode 100644 .agents/skills/cirq/SKILL.md create mode 100644 .agents/skills/cirq/references/building.md create mode 100644 .agents/skills/cirq/references/experiments.md create mode 100644 .agents/skills/cirq/references/hardware.md create mode 100644 .agents/skills/cirq/references/noise.md create mode 100644 .agents/skills/cirq/references/simulation.md create mode 100644 .agents/skills/cirq/references/transformation.md create mode 100644 .agents/skills/citation-management/SKILL.md create mode 100644 .agents/skills/citation-management/assets/bibtex_template.bib create mode 100644 .agents/skills/citation-management/assets/citation_checklist.md create mode 100644 .agents/skills/citation-management/references/best_practices.md create mode 100644 .agents/skills/citation-management/references/bibtex_formatting.md create mode 100644 .agents/skills/citation-management/references/citation_validation.md create mode 100644 .agents/skills/citation-management/references/core_workflow.md create mode 100644 .agents/skills/citation-management/references/example_workflows.md create mode 100644 .agents/skills/citation-management/references/google_scholar_search.md create mode 100644 .agents/skills/citation-management/references/metadata_extraction.md create mode 100644 .agents/skills/citation-management/references/pubmed_search.md create mode 100644 .agents/skills/citation-management/references/script_reference.md create mode 100644 .agents/skills/citation-management/references/search_strategies.md create mode 100644 .agents/skills/citation-management/scripts/_common.py create mode 100644 .agents/skills/citation-management/scripts/doi_to_bibtex.py create mode 100755 .agents/skills/citation-management/scripts/extract_metadata.py create mode 100755 .agents/skills/citation-management/scripts/format_bibtex.py create mode 100755 .agents/skills/citation-management/scripts/search_google_scholar.py create mode 100644 .agents/skills/citation-management/scripts/search_openalex.py create mode 100755 .agents/skills/citation-management/scripts/search_pubmed.py create mode 100755 .agents/skills/citation-management/scripts/validate_citations.py create mode 100644 .agents/skills/clinical-decision-support/SKILL.md create mode 100644 .agents/skills/clinical-decision-support/assets/aggregate_cohort_table_template.json create mode 100644 .agents/skills/clinical-decision-support/assets/aggregate_model_evaluation_template.json create mode 100644 .agents/skills/clinical-decision-support/assets/artifact_intended_use_template.json create mode 100644 .agents/skills/clinical-decision-support/assets/decision_logic_traceability_template.json create mode 100644 .agents/skills/clinical-decision-support/assets/deidentification_checklist_template.json create mode 100644 .agents/skills/clinical-decision-support/assets/evidence_profile_template.json create mode 100644 .agents/skills/clinical-decision-support/assets/survival_analysis_plan_template.json create mode 100644 .agents/skills/clinical-decision-support/references/README.md create mode 100644 .agents/skills/clinical-decision-support/references/cohort_evaluation.md create mode 100644 .agents/skills/clinical-decision-support/references/decision_logic_traceability.md create mode 100644 .agents/skills/clinical-decision-support/references/evidence_profiles.md create mode 100644 .agents/skills/clinical-decision-support/references/model_biomarker_evaluation.md create mode 100644 .agents/skills/clinical-decision-support/references/privacy_and_disclosure.md create mode 100644 .agents/skills/clinical-decision-support/references/regulatory_and_governance.md create mode 100644 .agents/skills/clinical-decision-support/references/safety_and_scope.md create mode 100644 .agents/skills/clinical-decision-support/references/security_validation.md create mode 100644 .agents/skills/clinical-decision-support/references/sources.md create mode 100644 .agents/skills/clinical-decision-support/references/study_reporting.md create mode 100644 .agents/skills/clinical-decision-support/references/survival_analysis.md create mode 100644 .agents/skills/clinical-decision-support/scripts/_common.py create mode 100644 .agents/skills/clinical-decision-support/scripts/cohort_table_generator.py create mode 100644 .agents/skills/clinical-decision-support/scripts/decision_logic_traceability.py create mode 100644 .agents/skills/clinical-decision-support/scripts/deidentification_checklist.py create mode 100644 .agents/skills/clinical-decision-support/scripts/evidence_profile_check.py create mode 100644 .agents/skills/clinical-decision-support/scripts/model_biomarker_evaluation.py create mode 100644 .agents/skills/clinical-decision-support/scripts/survival_plan_validator.py create mode 100644 .agents/skills/clinical-decision-support/scripts/validate_cds_artifact.py create mode 100644 .agents/skills/clinical-reports/SKILL.md create mode 100644 .agents/skills/clinical-reports/assets/adverse_event_aggregate_input_template.csv create mode 100644 .agents/skills/clinical-reports/assets/case_report_template.json create mode 100644 .agents/skills/clinical-reports/assets/clinical_trial_csr_template.json create mode 100644 .agents/skills/clinical-reports/assets/clinical_trial_results_template.json create mode 100644 .agents/skills/clinical-reports/assets/clinical_trial_safety_aggregate_template.json create mode 100644 .agents/skills/clinical-reports/assets/consistency_manifest_template.json create mode 100644 .agents/skills/clinical-reports/assets/deidentification_process_checklist.json create mode 100644 .agents/skills/clinical-reports/assets/lab_report_template.json create mode 100644 .agents/skills/clinical-reports/assets/pathology_report_template.json create mode 100644 .agents/skills/clinical-reports/assets/provenance_manifest_template.json create mode 100644 .agents/skills/clinical-reports/assets/quality_review_checklist.json create mode 100644 .agents/skills/clinical-reports/assets/radiology_report_template.json create mode 100644 .agents/skills/clinical-reports/assets/research_summary_template.json create mode 100644 .agents/skills/clinical-reports/assets/terminology_manifest_template.json create mode 100644 .agents/skills/clinical-reports/assets/trial_protocol_reporting_checklist.json create mode 100644 .agents/skills/clinical-reports/references/README.md create mode 100644 .agents/skills/clinical-reports/references/case_report_guidelines.md create mode 100644 .agents/skills/clinical-reports/references/clinical_trial_reporting.md create mode 100644 .agents/skills/clinical-reports/references/data_presentation.md create mode 100644 .agents/skills/clinical-reports/references/diagnostic_reports_standards.md create mode 100644 .agents/skills/clinical-reports/references/medical_terminology.md create mode 100644 .agents/skills/clinical-reports/references/privacy_and_deidentification.md create mode 100644 .agents/skills/clinical-reports/references/professional_review.md create mode 100644 .agents/skills/clinical-reports/references/report_type_routing.md create mode 100644 .agents/skills/clinical-reports/references/safety_reporting.md create mode 100644 .agents/skills/clinical-reports/references/sources.md create mode 100644 .agents/skills/clinical-reports/scripts/_common.py create mode 100644 .agents/skills/clinical-reports/scripts/check_deidentification.py create mode 100644 .agents/skills/clinical-reports/scripts/consistency_checker.py create mode 100644 .agents/skills/clinical-reports/scripts/format_adverse_events.py create mode 100644 .agents/skills/clinical-reports/scripts/generate_report_template.py create mode 100644 .agents/skills/clinical-reports/scripts/provenance_validator.py create mode 100644 .agents/skills/clinical-reports/scripts/terminology_validator.py create mode 100644 .agents/skills/clinical-reports/scripts/validate_case_report.py create mode 100644 .agents/skills/clinical-reports/scripts/validate_trial_report.py create mode 100644 .agents/skills/cobrapy/SKILL.md create mode 100644 .agents/skills/cobrapy/references/api_quick_reference.md create mode 100644 .agents/skills/cobrapy/references/workflows.md create mode 100644 .agents/skills/consciousness-council/SKILL.md create mode 100644 .agents/skills/consciousness-council/references/advanced-configurations.md create mode 100644 .agents/skills/dask/SKILL.md create mode 100644 .agents/skills/dask/references/arrays.md create mode 100644 .agents/skills/dask/references/bags.md create mode 100644 .agents/skills/dask/references/best-practices.md create mode 100644 .agents/skills/dask/references/dataframes.md create mode 100644 .agents/skills/dask/references/futures.md create mode 100644 .agents/skills/dask/references/schedulers.md create mode 100644 .agents/skills/database-lookup/SKILL.md create mode 100644 .agents/skills/database-lookup/references/addgene.md create mode 100644 .agents/skills/database-lookup/references/alphafold.md create mode 100644 .agents/skills/database-lookup/references/alphavantage.md create mode 100644 .agents/skills/database-lookup/references/bea.md create mode 100644 .agents/skills/database-lookup/references/bindingdb.md create mode 100644 .agents/skills/database-lookup/references/biogrid.md create mode 100644 .agents/skills/database-lookup/references/bls.md create mode 100644 .agents/skills/database-lookup/references/brenda.md create mode 100644 .agents/skills/database-lookup/references/cbioportal.md create mode 100644 .agents/skills/database-lookup/references/census.md create mode 100644 .agents/skills/database-lookup/references/chebi.md create mode 100644 .agents/skills/database-lookup/references/chembl.md create mode 100644 .agents/skills/database-lookup/references/clinicaltrials.md create mode 100644 .agents/skills/database-lookup/references/clinpgx.md create mode 100644 .agents/skills/database-lookup/references/clinvar.md create mode 100644 .agents/skills/database-lookup/references/cod.md create mode 100644 .agents/skills/database-lookup/references/cosmic.md create mode 100644 .agents/skills/database-lookup/references/dailymed.md create mode 100644 .agents/skills/database-lookup/references/database_selection_guide.md create mode 100644 .agents/skills/database-lookup/references/datacommons.md create mode 100644 .agents/skills/database-lookup/references/dbsnp.md create mode 100644 .agents/skills/database-lookup/references/disgenet.md create mode 100644 .agents/skills/database-lookup/references/drugbank.md create mode 100644 .agents/skills/database-lookup/references/ecb.md create mode 100644 .agents/skills/database-lookup/references/emdb.md create mode 100644 .agents/skills/database-lookup/references/ena.md create mode 100644 .agents/skills/database-lookup/references/encode.md create mode 100644 .agents/skills/database-lookup/references/ensembl.md create mode 100644 .agents/skills/database-lookup/references/epa.md create mode 100644 .agents/skills/database-lookup/references/eurostat.md create mode 100644 .agents/skills/database-lookup/references/fda.md create mode 100644 .agents/skills/database-lookup/references/federal-reserve.md create mode 100644 .agents/skills/database-lookup/references/fred.md create mode 100644 .agents/skills/database-lookup/references/gene-ontology.md create mode 100644 .agents/skills/database-lookup/references/geo.md create mode 100644 .agents/skills/database-lookup/references/gnomad.md create mode 100644 .agents/skills/database-lookup/references/gtex.md create mode 100644 .agents/skills/database-lookup/references/gwas-catalog.md create mode 100644 .agents/skills/database-lookup/references/hca.md create mode 100644 .agents/skills/database-lookup/references/hpo.md create mode 100644 .agents/skills/database-lookup/references/human-protein-atlas.md create mode 100644 .agents/skills/database-lookup/references/interpro.md create mode 100644 .agents/skills/database-lookup/references/jaspar.md create mode 100644 .agents/skills/database-lookup/references/kegg.md create mode 100644 .agents/skills/database-lookup/references/lincs-l1000.md create mode 100644 .agents/skills/database-lookup/references/materials-project.md create mode 100644 .agents/skills/database-lookup/references/metabolomics-workbench.md create mode 100644 .agents/skills/database-lookup/references/monarch.md create mode 100644 .agents/skills/database-lookup/references/mousemine.md create mode 100644 .agents/skills/database-lookup/references/nasa-exoplanet-archive.md create mode 100644 .agents/skills/database-lookup/references/nasa.md create mode 100644 .agents/skills/database-lookup/references/ncbi-gene.md create mode 100644 .agents/skills/database-lookup/references/ncbi-protein.md create mode 100644 .agents/skills/database-lookup/references/ncbi-taxonomy.md create mode 100644 .agents/skills/database-lookup/references/nist.md create mode 100644 .agents/skills/database-lookup/references/noaa.md create mode 100644 .agents/skills/database-lookup/references/omim.md create mode 100644 .agents/skills/database-lookup/references/opentargets.md create mode 100644 .agents/skills/database-lookup/references/openweathermap.md create mode 100644 .agents/skills/database-lookup/references/pdb.md create mode 100644 .agents/skills/database-lookup/references/pride.md create mode 100644 .agents/skills/database-lookup/references/pubchem.md create mode 100644 .agents/skills/database-lookup/references/quickgo.md create mode 100644 .agents/skills/database-lookup/references/reactome.md create mode 100644 .agents/skills/database-lookup/references/retrieval-contract.md create mode 100644 .agents/skills/database-lookup/references/rummageo.md create mode 100644 .agents/skills/database-lookup/references/sdss.md create mode 100644 .agents/skills/database-lookup/references/sec-edgar.md create mode 100644 .agents/skills/database-lookup/references/simbad.md create mode 100644 .agents/skills/database-lookup/references/sra.md create mode 100644 .agents/skills/database-lookup/references/string.md create mode 100644 .agents/skills/database-lookup/references/tcga-gdc.md create mode 100644 .agents/skills/database-lookup/references/treasury.md create mode 100644 .agents/skills/database-lookup/references/ucsc-genome.md create mode 100644 .agents/skills/database-lookup/references/uniprot.md create mode 100644 .agents/skills/database-lookup/references/usgs.md create mode 100644 .agents/skills/database-lookup/references/uspto.md create mode 100644 .agents/skills/database-lookup/references/who.md create mode 100644 .agents/skills/database-lookup/references/worldbank.md create mode 100644 .agents/skills/database-lookup/references/zinc.md create mode 100644 .agents/skills/datamol/SKILL.md create mode 100644 .agents/skills/datamol/references/conformers_module.md create mode 100644 .agents/skills/datamol/references/core_api.md create mode 100644 .agents/skills/datamol/references/core_workflows.md create mode 100644 .agents/skills/datamol/references/descriptors_viz.md create mode 100644 .agents/skills/datamol/references/fragments_scaffolds.md create mode 100644 .agents/skills/datamol/references/io_module.md create mode 100644 .agents/skills/datamol/references/reactions_data.md create mode 100644 .agents/skills/datamol/references/workflow_patterns.md create mode 100644 .agents/skills/deepchem/SKILL.md create mode 100644 .agents/skills/deepchem/references/api_reference.md create mode 100644 .agents/skills/deepchem/references/core_capabilities.md create mode 100644 .agents/skills/deepchem/references/typical_workflows.md create mode 100644 .agents/skills/deepchem/references/workflows.md create mode 100644 .agents/skills/deepchem/scripts/graph_neural_network.py create mode 100644 .agents/skills/deepchem/scripts/predict_solubility.py create mode 100644 .agents/skills/deepchem/scripts/transfer_learning.py create mode 100644 .agents/skills/deepspot-m/SKILL.md create mode 100644 .agents/skills/deepspot-m/references/api.md create mode 100644 .agents/skills/deepspot-m/references/whole_slide.md create mode 100644 .agents/skills/deeptools/SKILL.md create mode 100644 .agents/skills/deeptools/assets/quick_reference.md create mode 100644 .agents/skills/deeptools/references/core_workflows.md create mode 100644 .agents/skills/deeptools/references/effective_genome_sizes.md create mode 100644 .agents/skills/deeptools/references/normalization_methods.md create mode 100644 .agents/skills/deeptools/references/tools_reference.md create mode 100644 .agents/skills/deeptools/references/workflows.md create mode 100644 .agents/skills/deeptools/scripts/validate_files.py create mode 100644 .agents/skills/deeptools/scripts/workflow_generator.py create mode 100644 .agents/skills/depmap/SKILL.md create mode 100644 .agents/skills/depmap/references/dependency_analysis.md create mode 100644 .agents/skills/dhdna-profiler/SKILL.md create mode 100644 .agents/skills/dhdna-profiler/references/advanced-profiling.md create mode 100644 .agents/skills/diffdock/SKILL.md create mode 100644 .agents/skills/diffdock/assets/batch_template.csv create mode 100644 .agents/skills/diffdock/assets/custom_inference_config.yaml create mode 100644 .agents/skills/diffdock/references/confidence_and_limitations.md create mode 100644 .agents/skills/diffdock/references/parameters_reference.md create mode 100644 .agents/skills/diffdock/references/workflows_examples.md create mode 100755 .agents/skills/diffdock/scripts/analyze_results.py create mode 100755 .agents/skills/diffdock/scripts/prepare_batch_csv.py create mode 100755 .agents/skills/diffdock/scripts/setup_check.py create mode 100644 .agents/skills/dnanexus-integration/SKILL.md create mode 100644 .agents/skills/dnanexus-integration/references/app-development.md create mode 100644 .agents/skills/dnanexus-integration/references/authentication.md create mode 100644 .agents/skills/dnanexus-integration/references/configuration.md create mode 100644 .agents/skills/dnanexus-integration/references/data-operations.md create mode 100644 .agents/skills/dnanexus-integration/references/job-execution.md create mode 100644 .agents/skills/dnanexus-integration/references/operations-and-troubleshooting.md create mode 100644 .agents/skills/dnanexus-integration/references/python-sdk.md create mode 100644 .agents/skills/dnanexus-integration/references/sources.md create mode 100644 .agents/skills/dnanexus-integration/references/workflow-languages.md create mode 100644 .agents/skills/dnanexus-integration/scripts/inspect_dxpy.py create mode 100644 .agents/skills/dnanexus-integration/scripts/validate_dxapp.py create mode 100644 .agents/skills/docx/LICENSE.txt create mode 100644 .agents/skills/docx/SKILL.md create mode 100755 .agents/skills/docx/scripts/__init__.py create mode 100755 .agents/skills/docx/scripts/accept_changes.py create mode 100755 .agents/skills/docx/scripts/comment.py create mode 100755 .agents/skills/docx/scripts/merge_runs.py create mode 100644 .agents/skills/docx/scripts/office/helpers/__init__.py create mode 100644 .agents/skills/docx/scripts/office/helpers/pptx_chart.py create mode 100644 .agents/skills/docx/scripts/office/helpers/pptx_slide.py create mode 100644 .agents/skills/docx/scripts/office/helpers/pptx_theme.py create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-chart.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-main.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-picture.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/pml.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-math.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/sml.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-main.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/wml.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ISO-IEC29500-4_2016/xml.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ecma/fouth-edition/opc-contentTypes.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ecma/fouth-edition/opc-coreProperties.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ecma/fouth-edition/opc-digSig.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/ecma/fouth-edition/opc-relationships.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/mce/mc.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/microsoft/wml-2010.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/microsoft/wml-2012.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/microsoft/wml-2018.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/microsoft/wml-cex-2018.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/microsoft/wml-cid-2016.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/microsoft/wml-sdtdatahash-2020.xsd create mode 100644 .agents/skills/docx/scripts/office/schemas/microsoft/wml-symex-2015.xsd create mode 100644 .agents/skills/docx/scripts/office/soffice.py create mode 100755 .agents/skills/docx/scripts/office/validate.py create mode 100644 .agents/skills/docx/scripts/office/validators/__init__.py create mode 100644 .agents/skills/docx/scripts/office/validators/base.py create mode 100644 .agents/skills/docx/scripts/office/validators/docx.py create mode 100644 .agents/skills/docx/scripts/office/validators/pptx.py create mode 100644 .agents/skills/docx/scripts/office/validators/redlining.py create mode 100644 .agents/skills/docx/scripts/templates/comments.xml create mode 100644 .agents/skills/docx/scripts/templates/commentsExtended.xml create mode 100644 .agents/skills/docx/scripts/templates/commentsExtensible.xml create mode 100644 .agents/skills/docx/scripts/templates/commentsIds.xml create mode 100644 .agents/skills/docx/scripts/templates/people.xml create mode 100644 .agents/skills/esm/SKILL.md create mode 100644 .agents/skills/esm/references/biohub-platform.md create mode 100644 .agents/skills/esm/references/esm-c-api.md create mode 100644 .agents/skills/esm/references/esm3-api.md create mode 100644 .agents/skills/esm/references/forge-api.md create mode 100644 .agents/skills/esm/references/workflows.md create mode 100644 .agents/skills/etetoolkit/SKILL.md create mode 100644 .agents/skills/etetoolkit/references/api_reference.md create mode 100644 .agents/skills/etetoolkit/references/migration-ete3-to-ete4.md create mode 100644 .agents/skills/etetoolkit/references/taxonomy.md create mode 100644 .agents/skills/etetoolkit/references/visualization.md create mode 100644 .agents/skills/etetoolkit/references/workflows.md create mode 100755 .agents/skills/etetoolkit/scripts/quick_visualize.py create mode 100755 .agents/skills/etetoolkit/scripts/tree_operations.py create mode 100644 .agents/skills/exa-search/SKILL.md create mode 100644 .agents/skills/exa-search/references/web-extract.md create mode 100644 .agents/skills/exa-search/references/web-search.md create mode 100644 .agents/skills/exa-search/scripts/exa_extract.py create mode 100644 .agents/skills/exa-search/scripts/exa_search.py create mode 100644 .agents/skills/experimental-design/SKILL.md create mode 100644 .agents/skills/experimental-design/references/design_types.md create mode 100644 .agents/skills/experimental-design/references/factorial_and_doe.md create mode 100644 .agents/skills/experimental-design/references/randomization_and_blocking.md create mode 100644 .agents/skills/experimental-design/references/sequential_and_adaptive.md create mode 100644 .agents/skills/experimental-design/scripts/doe_designs.py create mode 100644 .agents/skills/experimental-design/scripts/randomization.py create mode 100644 .agents/skills/exploratory-data-analysis/SKILL.md create mode 100644 .agents/skills/exploratory-data-analysis/assets/report_template.md create mode 100644 .agents/skills/exploratory-data-analysis/references/bioinformatics_genomics_formats.md create mode 100644 .agents/skills/exploratory-data-analysis/references/chemistry_molecular_formats.md create mode 100644 .agents/skills/exploratory-data-analysis/references/general_scientific_formats.md create mode 100644 .agents/skills/exploratory-data-analysis/references/microscopy_imaging_formats.md create mode 100644 .agents/skills/exploratory-data-analysis/references/proteomics_metabolomics_formats.md create mode 100644 .agents/skills/exploratory-data-analysis/references/spectroscopy_analytical_formats.md create mode 100644 .agents/skills/exploratory-data-analysis/scripts/__init__.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/_capabilities.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/_common.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/_structured.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/_tabular.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/capability_manifest.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/distribution_sensitivity.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/eda_analyzer.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/image_inspector.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/missingness_leakage_audit.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/report_scaffold.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/sequence_inspector.py create mode 100644 .agents/skills/exploratory-data-analysis/scripts/tabular_profile.py create mode 100644 .agents/skills/flowio/SKILL.md create mode 100644 .agents/skills/flowio/references/api_reference.md create mode 100644 .agents/skills/flowio/references/fcs_semantics.md create mode 100644 .agents/skills/flowio/references/sources.md create mode 100644 .agents/skills/flowio/references/troubleshooting.md create mode 100644 .agents/skills/flowio/references/workflows.md create mode 100644 .agents/skills/flowio/scripts/inspect_fcs.py create mode 100644 .agents/skills/fluidsim/SKILL.md create mode 100644 .agents/skills/fluidsim/references/advanced_features.md create mode 100644 .agents/skills/fluidsim/references/installation.md create mode 100644 .agents/skills/fluidsim/references/output_analysis.md create mode 100644 .agents/skills/fluidsim/references/parameters.md create mode 100644 .agents/skills/fluidsim/references/simulation_workflow.md create mode 100644 .agents/skills/fluidsim/references/solvers.md create mode 100644 .agents/skills/fluidsim/scripts/__init__.py create mode 100644 .agents/skills/fluidsim/scripts/_common.py create mode 100644 .agents/skills/fluidsim/scripts/_schema.py create mode 100644 .agents/skills/fluidsim/scripts/budget_summary.py create mode 100644 .agents/skills/fluidsim/scripts/grid_resource_estimator.py create mode 100644 .agents/skills/fluidsim/scripts/output_inventory.py create mode 100644 .agents/skills/fluidsim/scripts/restart_compatibility.py create mode 100644 .agents/skills/fluidsim/scripts/simulation_dry_run.py create mode 100644 .agents/skills/fluidsim/scripts/solver_config_validator.py create mode 100644 .agents/skills/generate-image/SKILL.md create mode 100644 .agents/skills/generate-image/references/models.md create mode 100755 .agents/skills/generate-image/scripts/generate_image.py create mode 100644 .agents/skills/geniml/SKILL.md create mode 100644 .agents/skills/geniml/references/bedspace.md create mode 100644 .agents/skills/geniml/references/consensus_peaks.md create mode 100644 .agents/skills/geniml/references/region2vec.md create mode 100644 .agents/skills/geniml/references/scembed.md create mode 100644 .agents/skills/geniml/references/utilities.md create mode 100644 .agents/skills/geniml/scripts/__init__.py create mode 100644 .agents/skills/geniml/scripts/_common.py create mode 100644 .agents/skills/geniml/scripts/bed_validator.py create mode 100644 .agents/skills/geniml/scripts/consensus_plan.py create mode 100644 .agents/skills/geniml/scripts/corpus_auditor.py create mode 100644 .agents/skills/geniml/scripts/embedding_plan.py create mode 100644 .agents/skills/geniml/scripts/model_artifact_inspector.py create mode 100644 .agents/skills/geniml/scripts/tokenizer_compatibility.py create mode 100644 .agents/skills/genomic-coordinates/SKILL.md create mode 100644 .agents/skills/genomic-coordinates/references/format-conventions.md create mode 100644 .agents/skills/genomic-coordinates/references/reference-builds.md create mode 100644 .agents/skills/genomic-coordinates/references/transcript-coordinates.md create mode 100644 .agents/skills/genomic-coordinates/references/variant-representation.md create mode 100644 .agents/skills/genomic-coordinates/scripts/_common.py create mode 100644 .agents/skills/genomic-coordinates/scripts/audit_intervals.py create mode 100644 .agents/skills/genomic-coordinates/scripts/check_contigs.py create mode 100644 .agents/skills/genomic-coordinates/scripts/convert_coords.py create mode 100644 .agents/skills/genomic-coordinates/scripts/normalize_variant.py create mode 100644 .agents/skills/genomic-intelligence/SKILL.md create mode 100644 .agents/skills/genomic-intelligence/references/api-and-auth.md create mode 100644 .agents/skills/genomic-intelligence/references/mcp.md create mode 100644 .agents/skills/genomic-intelligence/references/sequence-acquisition.md create mode 100644 .agents/skills/genomic-intelligence/references/tasks.md create mode 100644 .agents/skills/geomaster/README.md create mode 100644 .agents/skills/geomaster/SKILL.md create mode 100644 .agents/skills/geomaster/references/advanced-gis.md create mode 100644 .agents/skills/geomaster/references/big-data.md create mode 100644 .agents/skills/geomaster/references/code-examples.md create mode 100644 .agents/skills/geomaster/references/coordinate-systems.md create mode 100644 .agents/skills/geomaster/references/core-libraries.md create mode 100644 .agents/skills/geomaster/references/data-sources.md create mode 100644 .agents/skills/geomaster/references/gis-software.md create mode 100644 .agents/skills/geomaster/references/industry-applications.md create mode 100644 .agents/skills/geomaster/references/machine-learning.md create mode 100644 .agents/skills/geomaster/references/programming-languages.md create mode 100644 .agents/skills/geomaster/references/remote-sensing.md create mode 100644 .agents/skills/geomaster/references/scientific-domains.md create mode 100644 .agents/skills/geomaster/references/specialized-topics.md create mode 100644 .agents/skills/geomaster/references/troubleshooting.md create mode 100644 .agents/skills/geopandas/SKILL.md create mode 100644 .agents/skills/geopandas/references/crs-management.md create mode 100644 .agents/skills/geopandas/references/data-io.md create mode 100644 .agents/skills/geopandas/references/data-structures.md create mode 100644 .agents/skills/geopandas/references/geometric-operations.md create mode 100644 .agents/skills/geopandas/references/spatial-analysis.md create mode 100644 .agents/skills/geopandas/references/visualization.md create mode 100644 .agents/skills/geopandas/scripts/_common.py create mode 100755 .agents/skills/geopandas/scripts/crs_reprojection_plan.py create mode 100755 .agents/skills/geopandas/scripts/export_plan.py create mode 100755 .agents/skills/geopandas/scripts/geometry_validity_report.py create mode 100755 .agents/skills/geopandas/scripts/sensitive_coordinates_checklist.py create mode 100755 .agents/skills/geopandas/scripts/spatial_join_audit.py create mode 100755 .agents/skills/geopandas/scripts/vector_inventory.py create mode 100644 .agents/skills/get-available-resources/SKILL.md create mode 100644 .agents/skills/get-available-resources/references/resource_semantics.md create mode 100644 .agents/skills/get-available-resources/references/snapshot_schema.md create mode 100644 .agents/skills/get-available-resources/references/sources.md create mode 100644 .agents/skills/get-available-resources/scripts/_common.py create mode 100644 .agents/skills/get-available-resources/scripts/accelerator_diagnostics.py create mode 100644 .agents/skills/get-available-resources/scripts/detect_resources.py create mode 100644 .agents/skills/get-available-resources/scripts/plan_workload.py create mode 100644 .agents/skills/get-available-resources/scripts/snapshot_tools.py create mode 100644 .agents/skills/gget/SKILL.md create mode 100644 .agents/skills/gget/references/common_workflows.md create mode 100644 .agents/skills/gget/references/database_info.md create mode 100644 .agents/skills/gget/references/module_catalog.md create mode 100644 .agents/skills/gget/references/module_reference.md create mode 100644 .agents/skills/gget/references/workflows.md create mode 100755 .agents/skills/gget/scripts/batch_sequence_analysis.py create mode 100755 .agents/skills/gget/scripts/enrichment_pipeline.py create mode 100755 .agents/skills/gget/scripts/gene_analysis.py create mode 100644 .agents/skills/ginkgo-cloud-lab/SKILL.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/cell-free-protein-expression-hibit.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/cell-free-protein-expression-optimization.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/cell-free-protein-expression-validation.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/cfps-expression-purification-quantification.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/cfps-strep-purification-thermal-shift.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/cfps-strep-tag-purification-a280.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/echo-ms-cfps-detection.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/echo-ms-method-onboarding.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/ecoli-expression-purification-quantification.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/ecoli-minibinder-expression-histag-a280.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/ecoli-protein-expression-hibit.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/ecoli-protein-expression-histag-a280.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/fluorescent-pixel-art-generation.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/ivt-rna-synthesis-qpcr.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/minibinder-strep-tag-a280.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/pichia-protein-expression-labchip.md create mode 100644 .agents/skills/ginkgo-cloud-lab/references/spr-target-onboarding.md create mode 100644 .agents/skills/glycoengineering/SKILL.md create mode 100644 .agents/skills/glycoengineering/references/glycan_databases.md create mode 100644 .agents/skills/gtars/SKILL.md create mode 100644 .agents/skills/gtars/references/cli.md create mode 100644 .agents/skills/gtars/references/coverage.md create mode 100644 .agents/skills/gtars/references/overlap.md create mode 100644 .agents/skills/gtars/references/python-api.md create mode 100644 .agents/skills/gtars/references/refget.md create mode 100644 .agents/skills/gtars/references/tokenizers.md create mode 100644 .agents/skills/gtars/scripts/__init__.py create mode 100644 .agents/skills/gtars/scripts/_common.py create mode 100644 .agents/skills/gtars/scripts/artifact_inspector.py create mode 100644 .agents/skills/gtars/scripts/bed_validator.py create mode 100644 .agents/skills/gtars/scripts/coverage_preflight.py create mode 100644 .agents/skills/gtars/scripts/execution_plan.py create mode 100644 .agents/skills/gtars/scripts/refget_digest_plan.py create mode 100644 .agents/skills/gtars/scripts/tokenizer_manifest.py create mode 100644 .agents/skills/histolab/SKILL.md create mode 100644 .agents/skills/histolab/references/core_capabilities.md create mode 100644 .agents/skills/histolab/references/filters_preprocessing.md create mode 100644 .agents/skills/histolab/references/slide_management.md create mode 100644 .agents/skills/histolab/references/tile_extraction.md create mode 100644 .agents/skills/histolab/references/tissue_masks.md create mode 100644 .agents/skills/histolab/references/typical_workflows.md create mode 100644 .agents/skills/histolab/references/visualization.md create mode 100644 .agents/skills/hugging-science/SKILL.md create mode 100644 .agents/skills/hugging-science/references/flagship-resources.md create mode 100644 .agents/skills/hugging-science/references/topics-and-slugs.md create mode 100644 .agents/skills/hugging-science/references/using-datasets.md create mode 100644 .agents/skills/hugging-science/references/using-models.md create mode 100644 .agents/skills/hugging-science/references/using-spaces.md create mode 100755 .agents/skills/hugging-science/scripts/fetch_catalog.py create mode 100644 .agents/skills/hypogenic/SKILL.md create mode 100644 .agents/skills/hypogenic/assets/dataset_manifest.example.json create mode 100644 .agents/skills/hypogenic/assets/result.example.json create mode 100644 .agents/skills/hypogenic/assets/run_config.example.json create mode 100644 .agents/skills/hypogenic/assets/task_config.example.yaml create mode 100644 .agents/skills/hypogenic/references/configuration.md create mode 100644 .agents/skills/hypogenic/references/datasets.md create mode 100644 .agents/skills/hypogenic/references/evaluation.md create mode 100644 .agents/skills/hypogenic/references/security.md create mode 100644 .agents/skills/hypogenic/references/sources.md create mode 100644 .agents/skills/hypogenic/references/upstream.md create mode 100644 .agents/skills/hypogenic/scripts/__init__.py create mode 100644 .agents/skills/hypogenic/scripts/_common.py create mode 100644 .agents/skills/hypogenic/scripts/audit_dataset.py create mode 100644 .agents/skills/hypogenic/scripts/evaluate_local.py create mode 100644 .agents/skills/hypogenic/scripts/inspect_outputs.py create mode 100644 .agents/skills/hypogenic/scripts/plan_run.py create mode 100644 .agents/skills/hypogenic/scripts/validate_config.py create mode 100644 .agents/skills/hypothesis-generation/SKILL.md create mode 100644 .agents/skills/hypothesis-generation/assets/evidence_ledger_template.csv create mode 100644 .agents/skills/hypothesis-generation/assets/falsification_controls_template.json create mode 100644 .agents/skills/hypothesis-generation/assets/hypothesis_record_template.json create mode 100644 .agents/skills/hypothesis-generation/assets/operationalization_template.json create mode 100644 .agents/skills/hypothesis-generation/assets/prediction_rival_matrix_template.csv create mode 100644 .agents/skills/hypothesis-generation/assets/preregistration_scaffold_template.md create mode 100644 .agents/skills/hypothesis-generation/assets/search_boundary_template.json create mode 100644 .agents/skills/hypothesis-generation/assets/source_ledger.csv create mode 100644 .agents/skills/hypothesis-generation/references/causal_inference_and_claims.md create mode 100644 .agents/skills/hypothesis-generation/references/concepts_and_workflow.md create mode 100644 .agents/skills/hypothesis-generation/references/ethics_safety_and_ai.md create mode 100644 .agents/skills/hypothesis-generation/references/experimental_design_patterns.md create mode 100644 .agents/skills/hypothesis-generation/references/hypothesis_quality_criteria.md create mode 100644 .agents/skills/hypothesis-generation/references/literature_search_strategies.md create mode 100644 .agents/skills/hypothesis-generation/references/preregistration_and_open_science.md create mode 100644 .agents/skills/hypothesis-generation/references/security_validation.md create mode 100644 .agents/skills/hypothesis-generation/references/source_ledger.md create mode 100644 .agents/skills/hypothesis-generation/references/tool_reference.md create mode 100644 .agents/skills/hypothesis-generation/scripts/_common.py create mode 100644 .agents/skills/hypothesis-generation/scripts/audit_evidence_ledger.py create mode 100644 .agents/skills/hypothesis-generation/scripts/check_falsification_controls.py create mode 100644 .agents/skills/hypothesis-generation/scripts/check_operationalization.py create mode 100644 .agents/skills/hypothesis-generation/scripts/generate_preregistration_scaffold.py create mode 100644 .agents/skills/hypothesis-generation/scripts/lint_causal_claims.py create mode 100644 .agents/skills/hypothesis-generation/scripts/validate_hypothesis_schema.py create mode 100644 .agents/skills/hypothesis-generation/scripts/validate_prediction_matrix.py create mode 100644 .agents/skills/imaging-data-commons/SKILL.md create mode 100644 .agents/skills/imaging-data-commons/references/bigquery_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/cli_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/clinical_data_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/cloud_storage_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/dicomweb_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/digital_pathology_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/index_tables_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/licensing_and_citation.md create mode 100644 .agents/skills/imaging-data-commons/references/mcp_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/parquet_access_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/rest_api_guide.md create mode 100644 .agents/skills/imaging-data-commons/references/sql_patterns.md create mode 100644 .agents/skills/imaging-data-commons/references/use_cases.md create mode 100644 .agents/skills/imaging-data-commons/scripts/check_version.py create mode 100644 .agents/skills/infographics/SKILL.md create mode 100644 .agents/skills/infographics/references/color_palettes.md create mode 100644 .agents/skills/infographics/references/design_principles.md create mode 100644 .agents/skills/infographics/references/infographic_type_catalog.md create mode 100644 .agents/skills/infographics/references/infographic_types.md create mode 100644 .agents/skills/infographics/references/iterative_refinement.md create mode 100755 .agents/skills/infographics/scripts/generate_infographic.py create mode 100644 .agents/skills/infographics/scripts/generate_infographic_ai.py create mode 100644 .agents/skills/iso-standards-readiness/SKILL.md create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/capa-record-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/document-register-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/evidence-manifest-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/laboratory-scope-intake-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/medical-laboratory-scope-intake-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/procedures/CAPA-procedure-template.md create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/procedures/document-control-procedure-template.md create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/qmsr-transition-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/quality-manual-template.md create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/scope-intake-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/supplier-controls-template.json create mode 100644 .agents/skills/iso-standards-readiness/assets/templates/traceability-matrix-template.json create mode 100644 .agents/skills/iso-standards-readiness/references/assurance-lanes.md create mode 100644 .agents/skills/iso-standards-readiness/references/evidence-architecture.md create mode 100644 .agents/skills/iso-standards-readiness/references/gap-analysis-checklist.md create mode 100644 .agents/skills/iso-standards-readiness/references/iso-13485.md create mode 100644 .agents/skills/iso-standards-readiness/references/iso-14971.md create mode 100644 .agents/skills/iso-standards-readiness/references/iso-15189.md create mode 100644 .agents/skills/iso-standards-readiness/references/iso-17025.md create mode 100644 .agents/skills/iso-standards-readiness/references/quality-manual-guide.md create mode 100644 .agents/skills/iso-standards-readiness/references/source-ledger.md create mode 100644 .agents/skills/iso-standards-readiness/scripts/_catalog.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/_common.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/audit_document_records.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/check_capa.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/check_qmsr_transition.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/check_supplier_controls.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/check_traceability.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/gap_analyzer.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/validate_evidence_manifest.py create mode 100644 .agents/skills/iso-standards-readiness/scripts/validate_scope_intake.py create mode 100644 .agents/skills/lab-hardware-cad/SKILL.md create mode 100644 .agents/skills/lab-hardware-cad/assets/standards.json create mode 100644 .agents/skills/lab-hardware-cad/references/behavior-rigs.md create mode 100644 .agents/skills/lab-hardware-cad/references/build123d-patterns.md create mode 100644 .agents/skills/lab-hardware-cad/references/fabrication-limits.md create mode 100644 .agents/skills/lab-hardware-cad/references/labware-adapters.md create mode 100644 .agents/skills/lab-hardware-cad/references/microfluidics.md create mode 100644 .agents/skills/lab-hardware-cad/references/optomechanics.md create mode 100644 .agents/skills/lab-hardware-cad/references/validation.md create mode 100644 .agents/skills/lab-hardware-cad/scripts/_common.py create mode 100644 .agents/skills/lab-hardware-cad/scripts/check.py create mode 100644 .agents/skills/lab-hardware-cad/scripts/gen.py create mode 100644 .agents/skills/lab-hardware-cad/scripts/snapshot.py create mode 100644 .agents/skills/labarchive-integration/SKILL.md create mode 100644 .agents/skills/labarchive-integration/references/api_reference.md create mode 100644 .agents/skills/labarchive-integration/references/authentication_guide.md create mode 100644 .agents/skills/labarchive-integration/references/integrations.md create mode 100644 .agents/skills/labarchive-integration/references/sources.md create mode 100644 .agents/skills/labarchive-integration/scripts/entry_operations.py create mode 100644 .agents/skills/labarchive-integration/scripts/notebook_operations.py create mode 100644 .agents/skills/labarchive-integration/scripts/setup_config.py create mode 100644 .agents/skills/lamindb/SKILL.md create mode 100644 .agents/skills/lamindb/references/annotation-validation.md create mode 100644 .agents/skills/lamindb/references/core-concepts.md create mode 100644 .agents/skills/lamindb/references/data-management.md create mode 100644 .agents/skills/lamindb/references/integrations.md create mode 100644 .agents/skills/lamindb/references/ontologies.md create mode 100644 .agents/skills/lamindb/references/setup-deployment.md create mode 100644 .agents/skills/latchbio-integration/SKILL.md create mode 100644 .agents/skills/latchbio-integration/references/data-management.md create mode 100644 .agents/skills/latchbio-integration/references/latch-mcp.md create mode 100644 .agents/skills/latchbio-integration/references/nextflow-snakemake.md create mode 100644 .agents/skills/latchbio-integration/references/operations-and-debugging.md create mode 100644 .agents/skills/latchbio-integration/references/registry.md create mode 100644 .agents/skills/latchbio-integration/references/resource-configuration.md create mode 100644 .agents/skills/latchbio-integration/references/ui-and-automation.md create mode 100644 .agents/skills/latchbio-integration/references/verified-workflows.md create mode 100644 .agents/skills/latchbio-integration/references/workflow-creation.md create mode 100644 .agents/skills/latchbio-integration/scripts/inspect_latch_sdk.py create mode 100644 .agents/skills/latex-posters/SKILL.md create mode 100644 .agents/skills/latex-posters/assets/baposter_template.tex create mode 100644 .agents/skills/latex-posters/assets/beamerposter_template.tex create mode 100644 .agents/skills/latex-posters/assets/poster_quality_checklist.md create mode 100644 .agents/skills/latex-posters/assets/tikzposter_template.tex create mode 100644 .agents/skills/latex-posters/references/README.md create mode 100644 .agents/skills/latex-posters/references/ai_graphics_for_posters.md create mode 100644 .agents/skills/latex-posters/references/compilation_and_quality_control.md create mode 100644 .agents/skills/latex-posters/references/latex_poster_packages.md create mode 100644 .agents/skills/latex-posters/references/latex_poster_reference.md create mode 100644 .agents/skills/latex-posters/references/poster_content_guide.md create mode 100644 .agents/skills/latex-posters/references/poster_design_principles.md create mode 100644 .agents/skills/latex-posters/references/poster_layout_design.md create mode 100644 .agents/skills/latex-posters/references/poster_patterns_and_presentation.md create mode 100644 .agents/skills/latex-posters/scripts/generate_schematic.py create mode 100644 .agents/skills/latex-posters/scripts/generate_schematic_ai.py create mode 100755 .agents/skills/latex-posters/scripts/review_poster.sh create mode 100644 .agents/skills/liteparse/SKILL.md create mode 100644 .agents/skills/liteparse/references/api_reference.md create mode 100644 .agents/skills/liteparse/references/choosing_a_parser.md create mode 100644 .agents/skills/liteparse/references/cli_reference.md create mode 100644 .agents/skills/liteparse/references/ocr_and_formats.md create mode 100644 .agents/skills/liteparse/references/output_formats.md create mode 100644 .agents/skills/liteparse/scripts/batch_parse_dir.py create mode 100644 .agents/skills/literature-review/SKILL.md create mode 100644 .agents/skills/literature-review/assets/review_template.md create mode 100644 .agents/skills/literature-review/references/citation_styles.md create mode 100644 .agents/skills/literature-review/references/core_workflow.md create mode 100644 .agents/skills/literature-review/references/database_strategies.md create mode 100644 .agents/skills/literature-review/references/example_workflow.md create mode 100644 .agents/skills/literature-review/references/search_and_citation.md create mode 100644 .agents/skills/literature-review/scripts/generate_pdf.py create mode 100644 .agents/skills/literature-review/scripts/generate_schematic.py create mode 100644 .agents/skills/literature-review/scripts/generate_schematic_ai.py create mode 100644 .agents/skills/literature-review/scripts/search_databases.py create mode 100644 .agents/skills/literature-review/scripts/verify_citations.py create mode 100644 .agents/skills/markdown-mermaid-writing/SKILL.md create mode 100644 .agents/skills/markdown-mermaid-writing/assets/examples/example-research-report.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/architecture.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/block.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/c4.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/class.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/complex_examples.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/er.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/flowchart.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/gantt.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/git_graph.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/kanban.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/mindmap.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/packet.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/pie.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/quadrant.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/radar.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/requirement.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/sankey.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/sequence.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/state.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/timeline.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/treemap.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/user_journey.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/xy_chart.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/diagrams/zenuml.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/markdown_style_guide.md create mode 100644 .agents/skills/markdown-mermaid-writing/references/mermaid_style_guide.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/decision_record.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/how_to_guide.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/issue.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/kanban.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/presentation.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/project_documentation.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/pull_request.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/research_paper.md create mode 100644 .agents/skills/markdown-mermaid-writing/templates/status_report.md create mode 100644 .agents/skills/market-research-reports/SKILL.md create mode 100644 .agents/skills/market-research-reports/assets/FORMATTING_GUIDE.md create mode 100644 .agents/skills/market-research-reports/assets/claims_ledger_template.csv create mode 100644 .agents/skills/market-research-reports/assets/competitor_feature_matrix_template.csv create mode 100644 .agents/skills/market-research-reports/assets/consistency_check_template.csv create mode 100644 .agents/skills/market-research-reports/assets/forecast_sensitivity_template.json create mode 100644 .agents/skills/market-research-reports/assets/market_report_template.tex create mode 100644 .agents/skills/market-research-reports/assets/market_research.sty create mode 100644 .agents/skills/market-research-reports/assets/market_sizing_scenarios_template.json create mode 100644 .agents/skills/market-research-reports/assets/report_manifest_template.json create mode 100644 .agents/skills/market-research-reports/assets/source_ledger_template.csv create mode 100644 .agents/skills/market-research-reports/references/data_analysis_patterns.md create mode 100644 .agents/skills/market-research-reports/references/evidence_model.md create mode 100644 .agents/skills/market-research-reports/references/methods_and_ethics.md create mode 100644 .agents/skills/market-research-reports/references/official_data_sources.md create mode 100644 .agents/skills/market-research-reports/references/report_structure_guide.md create mode 100644 .agents/skills/market-research-reports/references/sources.md create mode 100644 .agents/skills/market-research-reports/references/visual_generation_guide.md create mode 100644 .agents/skills/market-research-reports/scripts/_common.py create mode 100644 .agents/skills/market-research-reports/scripts/audit_claim_citations.py create mode 100644 .agents/skills/market-research-reports/scripts/calculate_market_sizing.py create mode 100644 .agents/skills/market-research-reports/scripts/check_unit_consistency.py create mode 100644 .agents/skills/market-research-reports/scripts/forecast_sensitivity.py create mode 100644 .agents/skills/market-research-reports/scripts/generate_report_scaffold.py create mode 100644 .agents/skills/market-research-reports/scripts/validate_competitor_matrix.py create mode 100644 .agents/skills/market-research-reports/scripts/validate_evidence_ledger.py create mode 100644 .agents/skills/markitdown/SKILL.md create mode 100644 .agents/skills/markitdown/references/api_reference.md create mode 100644 .agents/skills/markitdown/references/cloud_and_ocr.md create mode 100644 .agents/skills/markitdown/references/file_formats.md create mode 100644 .agents/skills/markitdown/references/mcp_and_plugins.md create mode 100644 .agents/skills/markitdown/references/migration.md create mode 100644 .agents/skills/markitdown/references/security.md create mode 100644 .agents/skills/markitdown/references/workflows.md create mode 100644 .agents/skills/markitdown/scripts/batch_convert.py create mode 100644 .agents/skills/markitdown/scripts/convert_literature.py create mode 100644 .agents/skills/markitdown/scripts/inspect_installation.py create mode 100644 .agents/skills/matchms/SKILL.md create mode 100644 .agents/skills/matchms/references/filtering.md create mode 100644 .agents/skills/matchms/references/importing_exporting.md create mode 100644 .agents/skills/matchms/references/migration.md create mode 100644 .agents/skills/matchms/references/similarity.md create mode 100644 .agents/skills/matchms/references/sources.md create mode 100644 .agents/skills/matchms/references/workflows.md create mode 100644 .agents/skills/matchms/scripts/library_search.py create mode 100644 .agents/skills/matlab/SKILL.md create mode 100644 .agents/skills/matlab/assets/project_manifest_template.json create mode 100644 .agents/skills/matlab/assets/python_compatibility_r2026a.json create mode 100644 .agents/skills/matlab/assets/reproducibility_manifest_template.json create mode 100644 .agents/skills/matlab/references/data-import-export.md create mode 100644 .agents/skills/matlab/references/executing-scripts.md create mode 100644 .agents/skills/matlab/references/graphics-visualization.md create mode 100644 .agents/skills/matlab/references/mathematics.md create mode 100644 .agents/skills/matlab/references/matrices-arrays.md create mode 100644 .agents/skills/matlab/references/octave-compatibility.md create mode 100644 .agents/skills/matlab/references/programming.md create mode 100644 .agents/skills/matlab/references/python-integration.md create mode 100644 .agents/skills/matlab/scripts/_common.py create mode 100644 .agents/skills/matlab/scripts/generate_function_scaffold.py create mode 100644 .agents/skills/matlab/scripts/inventory_mat_file.py create mode 100644 .agents/skills/matlab/scripts/plan_batch_command.py create mode 100644 .agents/skills/matlab/scripts/plan_python_compatibility.py create mode 100644 .agents/skills/matlab/scripts/reproducibility_report.py create mode 100644 .agents/skills/matlab/scripts/scan_m_code.py create mode 100644 .agents/skills/matlab/scripts/validate_project_manifest.py create mode 100644 .agents/skills/matplotlib/SKILL.md create mode 100644 .agents/skills/matplotlib/references/api_reference.md create mode 100644 .agents/skills/matplotlib/references/common_issues.md create mode 100644 .agents/skills/matplotlib/references/plot_types.md create mode 100644 .agents/skills/matplotlib/references/styling_guide.md create mode 100644 .agents/skills/matplotlib/scripts/plot_template.py create mode 100644 .agents/skills/matplotlib/scripts/style_configurator.py create mode 100644 .agents/skills/medchem/SKILL.md create mode 100644 .agents/skills/medchem/references/api_guide.md create mode 100644 .agents/skills/medchem/references/rules_catalog.md create mode 100644 .agents/skills/medchem/scripts/filter_molecules.py create mode 100644 .agents/skills/modal/SKILL.md create mode 100644 .agents/skills/modal/references/api_reference.md create mode 100644 .agents/skills/modal/references/examples.md create mode 100644 .agents/skills/modal/references/functions.md create mode 100644 .agents/skills/modal/references/getting-started.md create mode 100644 .agents/skills/modal/references/gpu.md create mode 100644 .agents/skills/modal/references/images.md create mode 100644 .agents/skills/modal/references/resources.md create mode 100644 .agents/skills/modal/references/scaling.md create mode 100644 .agents/skills/modal/references/scheduled-jobs.md create mode 100644 .agents/skills/modal/references/secrets.md create mode 100644 .agents/skills/modal/references/volumes.md create mode 100644 .agents/skills/modal/references/web-endpoints.md create mode 100644 .agents/skills/molecular-dynamics/SKILL.md create mode 100644 .agents/skills/molecular-dynamics/references/mdanalysis_analysis.md create mode 100644 .agents/skills/molfeat/SKILL.md create mode 100644 .agents/skills/molfeat/references/api_reference.md create mode 100644 .agents/skills/molfeat/references/available_featurizers.md create mode 100644 .agents/skills/molfeat/references/choosing_a_featurizer.md create mode 100644 .agents/skills/molfeat/references/examples.md create mode 100644 .agents/skills/ncats-arax/SKILL.md create mode 100644 .agents/skills/ncats-arax/references/output-schema.md create mode 100644 .agents/skills/ncats-arax/references/query-contract.md create mode 100644 .agents/skills/ncats-arax/scripts/arax_client.py create mode 100644 .agents/skills/networkx/SKILL.md create mode 100644 .agents/skills/networkx/references/algorithms.md create mode 100644 .agents/skills/networkx/references/generators.md create mode 100644 .agents/skills/networkx/references/graph-basics.md create mode 100644 .agents/skills/networkx/references/io.md create mode 100644 .agents/skills/networkx/references/visualization.md create mode 100644 .agents/skills/neurokit2/SKILL.md create mode 100644 .agents/skills/neurokit2/references/bio_module.md create mode 100644 .agents/skills/neurokit2/references/complexity.md create mode 100644 .agents/skills/neurokit2/references/ecg_cardiac.md create mode 100644 .agents/skills/neurokit2/references/eda.md create mode 100644 .agents/skills/neurokit2/references/eeg.md create mode 100644 .agents/skills/neurokit2/references/emg.md create mode 100644 .agents/skills/neurokit2/references/eog.md create mode 100644 .agents/skills/neurokit2/references/epochs_events.md create mode 100644 .agents/skills/neurokit2/references/hrv.md create mode 100644 .agents/skills/neurokit2/references/ppg.md create mode 100644 .agents/skills/neurokit2/references/rsp.md create mode 100644 .agents/skills/neurokit2/references/signal_processing.md create mode 100644 .agents/skills/neurokit2/scripts/_common.py create mode 100644 .agents/skills/neurokit2/scripts/ecg_hrv_pipeline.py create mode 100644 .agents/skills/neurokit2/scripts/eda_pipeline.py create mode 100644 .agents/skills/neurokit2/scripts/generate_synthetic.py create mode 100644 .agents/skills/neurokit2/scripts/inspect_signal.py create mode 100644 .agents/skills/neurokit2/scripts/plan_epochs.py create mode 100644 .agents/skills/neurokit2/scripts/validate_multimodal.py create mode 100644 .agents/skills/neuropixels-analysis/SKILL.md create mode 100644 .agents/skills/neuropixels-analysis/assets/analysis_template.py create mode 100644 .agents/skills/neuropixels-analysis/references/AI_CURATION.md create mode 100644 .agents/skills/neuropixels-analysis/references/ANALYSIS.md create mode 100644 .agents/skills/neuropixels-analysis/references/AUTOMATED_CURATION.md create mode 100644 .agents/skills/neuropixels-analysis/references/MOTION_CORRECTION.md create mode 100644 .agents/skills/neuropixels-analysis/references/PREPROCESSING.md create mode 100644 .agents/skills/neuropixels-analysis/references/QUALITY_METRICS.md create mode 100644 .agents/skills/neuropixels-analysis/references/SPIKE_SORTING.md create mode 100644 .agents/skills/neuropixels-analysis/references/api_reference.md create mode 100644 .agents/skills/neuropixels-analysis/references/plotting_guide.md create mode 100644 .agents/skills/neuropixels-analysis/references/standard_workflow.md create mode 100644 .agents/skills/neuropixels-analysis/scripts/compute_metrics.py create mode 100644 .agents/skills/neuropixels-analysis/scripts/explore_recording.py create mode 100644 .agents/skills/neuropixels-analysis/scripts/export_to_phy.py create mode 100644 .agents/skills/neuropixels-analysis/scripts/neuropixels_pipeline.py create mode 100644 .agents/skills/neuropixels-analysis/scripts/preprocess_recording.py create mode 100644 .agents/skills/neuropixels-analysis/scripts/run_sorting.py create mode 100644 .agents/skills/nextflow/SKILL.md create mode 100644 .agents/skills/nextflow/references/configuration.md create mode 100644 .agents/skills/nextflow/references/containers.md create mode 100644 .agents/skills/nextflow/references/developing.md create mode 100644 .agents/skills/nextflow/references/language.md create mode 100644 .agents/skills/nextflow/references/nf-core-tools.md create mode 100644 .agents/skills/nextflow/references/running-pipelines.md create mode 100644 .agents/skills/nextflow/references/testing.md create mode 100644 .agents/skills/omero-integration/SKILL.md create mode 100644 .agents/skills/omero-integration/references/advanced.md create mode 100644 .agents/skills/omero-integration/references/connection.md create mode 100644 .agents/skills/omero-integration/references/data_access.md create mode 100644 .agents/skills/omero-integration/references/image_processing.md create mode 100644 .agents/skills/omero-integration/references/metadata.md create mode 100644 .agents/skills/omero-integration/references/rois.md create mode 100644 .agents/skills/omero-integration/references/scripts.md create mode 100644 .agents/skills/omero-integration/references/sources.md create mode 100644 .agents/skills/omero-integration/references/tables.md create mode 100644 .agents/skills/omero-integration/scripts/export_image_metadata.py create mode 100644 .agents/skills/omero-integration/scripts/inventory.py create mode 100644 .agents/skills/omero-integration/scripts/omero_common.py create mode 100644 .agents/skills/omero-integration/scripts/plan_transfer.py create mode 100644 .agents/skills/omero-integration/scripts/validate_config.py create mode 100644 .agents/skills/onekgpd/SKILL.md create mode 100644 .agents/skills/onekgpd/assets/kgpe.json create mode 100644 .agents/skills/onekgpd/references/annotation_vocabularies.md create mode 100644 .agents/skills/onekgpd/references/onekgpd_commands.md create mode 100644 .agents/skills/onekgpd/scripts/onekgpd_api.py create mode 100644 .agents/skills/onekgpd/scripts/onekgpd_meta.py create mode 100644 .agents/skills/ontology-term-resolution/SKILL.md create mode 100644 .agents/skills/ontology-term-resolution/references/curation-rules.md create mode 100644 .agents/skills/ontology-term-resolution/references/ols4-api.md create mode 100644 .agents/skills/ontology-term-resolution/references/ontology-registry.md create mode 100644 .agents/skills/ontology-term-resolution/scripts/ols_client.py create mode 100644 .agents/skills/ontology-term-resolution/scripts/resolve_terms.py create mode 100644 .agents/skills/ontology-term-resolution/scripts/validate_terms.py create mode 100644 .agents/skills/open-notebook/SKILL.md create mode 100644 .agents/skills/open-notebook/references/api_reference.md create mode 100644 .agents/skills/open-notebook/references/architecture.md create mode 100644 .agents/skills/open-notebook/references/configuration.md create mode 100644 .agents/skills/open-notebook/references/examples.md create mode 100644 .agents/skills/open-notebook/scripts/chat_interaction.py create mode 100644 .agents/skills/open-notebook/scripts/notebook_management.py create mode 100644 .agents/skills/open-notebook/scripts/source_ingestion.py create mode 100644 .agents/skills/openpiv/SKILL.md create mode 100644 .agents/skills/openpiv/references/advanced_algorithms.md create mode 100644 .agents/skills/openpiv/scripts/__init__.py create mode 100644 .agents/skills/openpiv/scripts/analyze.py create mode 100644 .agents/skills/openpiv/scripts/run_example.py create mode 100644 .agents/skills/openpiv/scripts/runner.py create mode 100644 .agents/skills/opentrons-integration/SKILL.md create mode 100644 .agents/skills/opentrons-integration/references/api_reference.md create mode 100644 .agents/skills/opentrons-integration/references/liquid_handling.md create mode 100644 .agents/skills/opentrons-integration/references/migration-api-2-19-to-2-29.md create mode 100644 .agents/skills/opentrons-integration/references/modules_and_deck.md create mode 100644 .agents/skills/opentrons-integration/references/protocol_authoring.md create mode 100644 .agents/skills/opentrons-integration/references/sources.md create mode 100644 .agents/skills/opentrons-integration/references/validation_and_operations.md create mode 100644 .agents/skills/opentrons-integration/requirements-flex.txt create mode 100644 .agents/skills/opentrons-integration/requirements-ot2.txt create mode 100644 .agents/skills/opentrons-integration/scripts/absorbance_reader_template.py create mode 100644 .agents/skills/opentrons-integration/scripts/basic_protocol_template.py create mode 100644 .agents/skills/opentrons-integration/scripts/ot2_basic_protocol_template.py create mode 100644 .agents/skills/opentrons-integration/scripts/pcr_setup_template.py create mode 100644 .agents/skills/opentrons-integration/scripts/runtime_parameters_template.py create mode 100644 .agents/skills/opentrons-integration/scripts/serial_dilution_template.py create mode 100644 .agents/skills/optimize-for-gpu/SKILL.md create mode 100644 .agents/skills/optimize-for-gpu/references/code_transformation_patterns.md create mode 100644 .agents/skills/optimize-for-gpu/references/cucim.md create mode 100644 .agents/skills/optimize-for-gpu/references/cudf.md create mode 100644 .agents/skills/optimize-for-gpu/references/cugraph.md create mode 100644 .agents/skills/optimize-for-gpu/references/cuml.md create mode 100644 .agents/skills/optimize-for-gpu/references/cupy.md create mode 100644 .agents/skills/optimize-for-gpu/references/cuspatial.md create mode 100644 .agents/skills/optimize-for-gpu/references/cuvs.md create mode 100644 .agents/skills/optimize-for-gpu/references/cuxfilter.md create mode 100644 .agents/skills/optimize-for-gpu/references/decision_framework.md create mode 100644 .agents/skills/optimize-for-gpu/references/installation.md create mode 100644 .agents/skills/optimize-for-gpu/references/kvikio.md create mode 100644 .agents/skills/optimize-for-gpu/references/numba.md create mode 100644 .agents/skills/optimize-for-gpu/references/raft.md create mode 100644 .agents/skills/optimize-for-gpu/references/warp.md create mode 100644 .agents/skills/pacsomatic/LICENSE create mode 100644 .agents/skills/pacsomatic/SKILL.md create mode 100644 .agents/skills/pacsomatic/config.yaml create mode 100644 .agents/skills/pacsomatic/references/agent-playbook.md create mode 100644 .agents/skills/pacsomatic/references/config-and-output.md create mode 100644 .agents/skills/pacsomatic/references/pacsomatic_guide.md create mode 100644 .agents/skills/pacsomatic/scripts/run_pacsomatic.py create mode 100644 .agents/skills/paper-lookup/SKILL.md create mode 100644 .agents/skills/paper-lookup/references/arxiv.md create mode 100644 .agents/skills/paper-lookup/references/biorxiv.md create mode 100644 .agents/skills/paper-lookup/references/core.md create mode 100644 .agents/skills/paper-lookup/references/crossref.md create mode 100644 .agents/skills/paper-lookup/references/europepmc.md create mode 100644 .agents/skills/paper-lookup/references/medrxiv.md create mode 100644 .agents/skills/paper-lookup/references/openalex.md create mode 100644 .agents/skills/paper-lookup/references/pmc.md create mode 100644 .agents/skills/paper-lookup/references/pubmed.md create mode 100644 .agents/skills/paper-lookup/references/semantic-scholar.md create mode 100644 .agents/skills/paper-lookup/references/unpaywall.md create mode 100644 .agents/skills/paper-lookup/scripts/_common.py create mode 100644 .agents/skills/paper-lookup/scripts/arxiv_atom.py create mode 100644 .agents/skills/paper-lookup/scripts/jats_to_text.py create mode 100644 .agents/skills/paper-lookup/scripts/openalex_abstract.py create mode 100644 .agents/skills/paper-lookup/scripts/paginate.py create mode 100644 .agents/skills/paperclip/SKILL.md create mode 100644 .agents/skills/paperclip/references/cli-reference.md create mode 100644 .agents/skills/paperclip/references/installation.md create mode 100644 .agents/skills/paperclip/references/map-reduce.md create mode 100644 .agents/skills/paperclip/references/python-sdk.md create mode 100644 .agents/skills/paperclip/references/repos-and-workspace.md create mode 100644 .agents/skills/paperclip/references/search-and-retrieval.md create mode 100644 .agents/skills/paperzilla/SKILL.md create mode 100644 .agents/skills/parallel-web/SKILL.md create mode 100644 .agents/skills/parallel-web/references/data-enrichment.md create mode 100644 .agents/skills/parallel-web/references/deep-research.md create mode 100644 .agents/skills/parallel-web/references/findall.md create mode 100644 .agents/skills/parallel-web/references/monitor.md create mode 100644 .agents/skills/parallel-web/references/web-extract.md create mode 100644 .agents/skills/parallel-web/references/web-search.md create mode 100644 .agents/skills/pathml/SKILL.md create mode 100644 .agents/skills/pathml/references/data_management.md create mode 100644 .agents/skills/pathml/references/graphs.md create mode 100644 .agents/skills/pathml/references/image_loading.md create mode 100644 .agents/skills/pathml/references/machine_learning.md create mode 100644 .agents/skills/pathml/references/multiparametric.md create mode 100644 .agents/skills/pathml/references/preprocessing.md create mode 100644 .agents/skills/pathml/scripts/_common.py create mode 100644 .agents/skills/pathml/scripts/image_qc.py create mode 100644 .agents/skills/pathml/scripts/plan_inference.py create mode 100644 .agents/skills/pathml/scripts/plan_pipeline.py create mode 100644 .agents/skills/pathml/scripts/slide_manifest.py create mode 100644 .agents/skills/pathml/scripts/validate_spatial_schema.py create mode 100644 .agents/skills/pathogen-variant-surveillance/SKILL.md create mode 100644 .agents/skills/pathogen-variant-surveillance/references/lapis-api.md create mode 100644 .agents/skills/pathogen-variant-surveillance/references/lineage-nomenclature.md create mode 100644 .agents/skills/pathogen-variant-surveillance/references/surveillance-caveats.md create mode 100644 .agents/skills/pathogen-variant-surveillance/scripts/lapis_client.py create mode 100644 .agents/skills/pathogen-variant-surveillance/scripts/lineage_prevalence.py create mode 100644 .agents/skills/pathogen-variant-surveillance/scripts/mutation_profile.py create mode 100644 .agents/skills/pathogen-variant-surveillance/scripts/reporting_lag.py create mode 100644 .agents/skills/pathogen-variant-surveillance/scripts/resolve_lineage.py create mode 100644 .agents/skills/pathway-enrichment/SKILL.md create mode 100644 .agents/skills/pathway-enrichment/references/databases-and-gene-sets.md create mode 100644 .agents/skills/pathway-enrichment/references/gseapy.md create mode 100644 .agents/skills/pathway-enrichment/references/interpretation.md create mode 100644 .agents/skills/pathway-enrichment/scripts/run_enrichment.py create mode 100644 .agents/skills/pdf/LICENSE.txt create mode 100644 .agents/skills/pdf/SKILL.md create mode 100644 .agents/skills/pdf/forms.md create mode 100644 .agents/skills/pdf/reference.md create mode 100644 .agents/skills/pdf/scripts/check_bounding_boxes.py create mode 100644 .agents/skills/pdf/scripts/check_fillable_fields.py create mode 100644 .agents/skills/pdf/scripts/convert_pdf_to_images.py create mode 100644 .agents/skills/pdf/scripts/create_validation_image.py create mode 100644 .agents/skills/pdf/scripts/extract_form_field_info.py create mode 100755 .agents/skills/pdf/scripts/extract_form_structure.py create mode 100644 .agents/skills/pdf/scripts/fill_fillable_fields.py create mode 100644 .agents/skills/pdf/scripts/fill_pdf_form_with_annotations.py create mode 100644 .agents/skills/peer-review/SKILL.md create mode 100644 .agents/skills/peer-review/assets/citation_references_template.csv create mode 100644 .agents/skills/peer-review/assets/claim_evidence_matrix_template.csv create mode 100644 .agents/skills/peer-review/assets/reporting_checklist_template.csv create mode 100644 .agents/skills/peer-review/assets/reporting_guidelines.json create mode 100644 .agents/skills/peer-review/assets/review_intake_template.json create mode 100644 .agents/skills/peer-review/assets/review_scaffold_template.md create mode 100644 .agents/skills/peer-review/assets/source_ledger.csv create mode 100644 .agents/skills/peer-review/assets/statistical_reproducibility_template.json create mode 100644 .agents/skills/peer-review/assets/study_profile_template.json create mode 100644 .agents/skills/peer-review/references/common_issues.md create mode 100644 .agents/skills/peer-review/references/ethical_review_practice.md create mode 100644 .agents/skills/peer-review/references/reporting_standards.md create mode 100644 .agents/skills/peer-review/references/security_validation.md create mode 100644 .agents/skills/peer-review/references/statistical_reproducibility.md create mode 100644 .agents/skills/peer-review/references/tool_reference.md create mode 100644 .agents/skills/peer-review/scripts/_common.py create mode 100644 .agents/skills/peer-review/scripts/audit_citations.py create mode 100644 .agents/skills/peer-review/scripts/audit_statistics_reproducibility.py create mode 100644 .agents/skills/peer-review/scripts/generate_review_scaffold.py create mode 100644 .agents/skills/peer-review/scripts/lint_review.py create mode 100644 .agents/skills/peer-review/scripts/select_reporting_guidelines.py create mode 100644 .agents/skills/peer-review/scripts/validate_claim_evidence.py create mode 100644 .agents/skills/peer-review/scripts/validate_review_intake.py create mode 100644 .agents/skills/pennylane/SKILL.md create mode 100644 .agents/skills/pennylane/references/advanced_features.md create mode 100644 .agents/skills/pennylane/references/devices_backends.md create mode 100644 .agents/skills/pennylane/references/getting_started.md create mode 100644 .agents/skills/pennylane/references/optimization.md create mode 100644 .agents/skills/pennylane/references/quantum_chemistry.md create mode 100644 .agents/skills/pennylane/references/quantum_circuits.md create mode 100644 .agents/skills/pennylane/references/quantum_ml.md create mode 100644 .agents/skills/phylogenetics/SKILL.md create mode 100644 .agents/skills/phylogenetics/references/iqtree_inference.md create mode 100644 .agents/skills/phylogenetics/scripts/phylogenetic_analysis.py create mode 100644 .agents/skills/pi-agent/SKILL.md create mode 100644 .agents/skills/pi-agent/references/compaction.md create mode 100644 .agents/skills/pi-agent/references/containerization.md create mode 100644 .agents/skills/pi-agent/references/custom-provider.md create mode 100644 .agents/skills/pi-agent/references/development.md create mode 100644 .agents/skills/pi-agent/references/environment-variables.md create mode 100644 .agents/skills/pi-agent/references/extensions.md create mode 100644 .agents/skills/pi-agent/references/json.md create mode 100644 .agents/skills/pi-agent/references/keybindings.md create mode 100644 .agents/skills/pi-agent/references/llama-cpp.md create mode 100644 .agents/skills/pi-agent/references/models.md create mode 100644 .agents/skills/pi-agent/references/overview.md create mode 100644 .agents/skills/pi-agent/references/packages.md create mode 100644 .agents/skills/pi-agent/references/pi-interview.md create mode 100644 .agents/skills/pi-agent/references/pi-mcp-adapter.md create mode 100644 .agents/skills/pi-agent/references/pi-subagents.md create mode 100644 .agents/skills/pi-agent/references/pi-web-access.md create mode 100644 .agents/skills/pi-agent/references/prompt-templates.md create mode 100644 .agents/skills/pi-agent/references/providers.md create mode 100644 .agents/skills/pi-agent/references/quickstart.md create mode 100644 .agents/skills/pi-agent/references/rpc.md create mode 100644 .agents/skills/pi-agent/references/sdk.md create mode 100644 .agents/skills/pi-agent/references/security.md create mode 100644 .agents/skills/pi-agent/references/session-format.md create mode 100644 .agents/skills/pi-agent/references/sessions.md create mode 100644 .agents/skills/pi-agent/references/settings.md create mode 100644 .agents/skills/pi-agent/references/shell-aliases.md create mode 100644 .agents/skills/pi-agent/references/skills.md create mode 100644 .agents/skills/pi-agent/references/terminal-setup.md create mode 100644 .agents/skills/pi-agent/references/termux.md create mode 100644 .agents/skills/pi-agent/references/themes.md create mode 100644 .agents/skills/pi-agent/references/tmux.md create mode 100644 .agents/skills/pi-agent/references/tui.md create mode 100644 .agents/skills/pi-agent/references/usage.md create mode 100644 .agents/skills/pi-agent/references/windows.md create mode 100644 .agents/skills/pkpd-modeling/SKILL.md create mode 100644 .agents/skills/pkpd-modeling/assets/nca-reporting-checklist.md create mode 100644 .agents/skills/pkpd-modeling/assets/popk-analysis-plan.md create mode 100644 .agents/skills/pkpd-modeling/references/antimicrobial-and-tdm.md create mode 100644 .agents/skills/pkpd-modeling/references/bioequivalence.md create mode 100644 .agents/skills/pkpd-modeling/references/dataset-standards.md create mode 100644 .agents/skills/pkpd-modeling/references/ddi-and-qt.md create mode 100644 .agents/skills/pkpd-modeling/references/nca-conventions.md create mode 100644 .agents/skills/pkpd-modeling/references/pbpk.md create mode 100644 .agents/skills/pkpd-modeling/references/pd-and-exposure-response.md create mode 100644 .agents/skills/pkpd-modeling/references/population-pk.md create mode 100644 .agents/skills/pkpd-modeling/references/regulatory-guidance.md create mode 100644 .agents/skills/pkpd-modeling/references/software-ecosystem.md create mode 100644 .agents/skills/pkpd-modeling/references/source-ledger.md create mode 100644 .agents/skills/pkpd-modeling/references/special-populations.md create mode 100644 .agents/skills/pkpd-modeling/references/structural-models.md create mode 100644 .agents/skills/pkpd-modeling/references/tmdd-and-biologics.md create mode 100644 .agents/skills/pkpd-modeling/scripts/_common.py create mode 100644 .agents/skills/pkpd-modeling/scripts/_models.py create mode 100644 .agents/skills/pkpd-modeling/scripts/allometry_and_fih.py create mode 100644 .agents/skills/pkpd-modeling/scripts/bioequivalence.py create mode 100644 .agents/skills/pkpd-modeling/scripts/check_popk_dataset.py create mode 100644 .agents/skills/pkpd-modeling/scripts/ddi_static.py create mode 100644 .agents/skills/pkpd-modeling/scripts/exposure_response.py create mode 100644 .agents/skills/pkpd-modeling/scripts/fit_compartmental.py create mode 100644 .agents/skills/pkpd-modeling/scripts/nca.py create mode 100644 .agents/skills/pkpd-modeling/scripts/simulate_regimen.py create mode 100644 .agents/skills/pkpd-modeling/scripts/tdm_bayes.py create mode 100644 .agents/skills/polars-bio/SKILL.md create mode 100644 .agents/skills/polars-bio/references/bioframe_migration.md create mode 100644 .agents/skills/polars-bio/references/configuration.md create mode 100644 .agents/skills/polars-bio/references/file_io.md create mode 100644 .agents/skills/polars-bio/references/interval_operations.md create mode 100644 .agents/skills/polars-bio/references/pileup_operations.md create mode 100644 .agents/skills/polars-bio/references/sql_processing.md create mode 100644 .agents/skills/polars/SKILL.md create mode 100644 .agents/skills/polars/references/best_practices.md create mode 100644 .agents/skills/polars/references/core_concepts.md create mode 100644 .agents/skills/polars/references/io_guide.md create mode 100644 .agents/skills/polars/references/operations.md create mode 100644 .agents/skills/polars/references/pandas_migration.md create mode 100644 .agents/skills/polars/references/transformations.md create mode 100644 .agents/skills/pptx-posters/SKILL.md create mode 100644 .agents/skills/pptx-posters/assets/generation_dependencies.json create mode 100644 .agents/skills/pptx-posters/assets/poster_manifest_template.json create mode 100644 .agents/skills/pptx-posters/assets/poster_quality_checklist.md create mode 100644 .agents/skills/pptx-posters/references/manifest_spec.md create mode 100644 .agents/skills/pptx-posters/references/poster_content_guide.md create mode 100644 .agents/skills/pptx-posters/references/poster_design_principles.md create mode 100644 .agents/skills/pptx-posters/references/poster_layout_design.md create mode 100644 .agents/skills/pptx-posters/references/pptx_security.md create mode 100644 .agents/skills/pptx-posters/references/security_validation.md create mode 100644 .agents/skills/pptx-posters/references/source_ledger.md create mode 100644 .agents/skills/pptx-posters/scripts/_common.py create mode 100644 .agents/skills/pptx-posters/scripts/_manifest.py create mode 100644 .agents/skills/pptx-posters/scripts/_pptx.py create mode 100644 .agents/skills/pptx-posters/scripts/check_layout.py create mode 100644 .agents/skills/pptx-posters/scripts/check_palette.py create mode 100644 .agents/skills/pptx-posters/scripts/generate_poster.py create mode 100644 .agents/skills/pptx-posters/scripts/inspect_pptx.py create mode 100644 .agents/skills/pptx-posters/scripts/inventory_images.py create mode 100644 .agents/skills/pptx-posters/scripts/plan_export.py create mode 100644 .agents/skills/pptx-posters/scripts/validate_manifest.py create mode 100644 .agents/skills/pptx/LICENSE.txt create mode 100644 .agents/skills/pptx/SKILL.md create mode 100644 .agents/skills/pptx/scripts/__init__.py create mode 100755 .agents/skills/pptx/scripts/add_slide.py create mode 100755 .agents/skills/pptx/scripts/clean.py create mode 100644 .agents/skills/pptx/scripts/office/helpers/__init__.py create mode 100644 .agents/skills/pptx/scripts/office/helpers/pptx_chart.py create mode 100644 .agents/skills/pptx/scripts/office/helpers/pptx_slide.py create mode 100644 .agents/skills/pptx/scripts/office/helpers/pptx_theme.py create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-chart.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-main.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-picture.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/pml.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-math.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/sml.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-main.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/wml.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ISO-IEC29500-4_2016/xml.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ecma/fouth-edition/opc-contentTypes.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ecma/fouth-edition/opc-coreProperties.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ecma/fouth-edition/opc-digSig.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/ecma/fouth-edition/opc-relationships.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/mce/mc.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/microsoft/wml-2010.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/microsoft/wml-2012.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/microsoft/wml-2018.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/microsoft/wml-cex-2018.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/microsoft/wml-cid-2016.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/microsoft/wml-sdtdatahash-2020.xsd create mode 100644 .agents/skills/pptx/scripts/office/schemas/microsoft/wml-symex-2015.xsd create mode 100644 .agents/skills/pptx/scripts/office/soffice.py create mode 100755 .agents/skills/pptx/scripts/office/validate.py create mode 100644 .agents/skills/pptx/scripts/office/validators/__init__.py create mode 100644 .agents/skills/pptx/scripts/office/validators/base.py create mode 100644 .agents/skills/pptx/scripts/office/validators/docx.py create mode 100644 .agents/skills/pptx/scripts/office/validators/pptx.py create mode 100644 .agents/skills/pptx/scripts/office/validators/redlining.py create mode 100755 .agents/skills/pptx/scripts/thumbnail.py create mode 100644 .agents/skills/primekg/SKILL.md create mode 100644 .agents/skills/primekg/scripts/query_primekg.py create mode 100644 .agents/skills/protocolsio-integration/SKILL.md create mode 100644 .agents/skills/protocolsio-integration/assets/protocol-snapshot.schema.json create mode 100644 .agents/skills/protocolsio-integration/references/additional_features.md create mode 100644 .agents/skills/protocolsio-integration/references/authentication.md create mode 100644 .agents/skills/protocolsio-integration/references/discussions.md create mode 100644 .agents/skills/protocolsio-integration/references/file_manager.md create mode 100644 .agents/skills/protocolsio-integration/references/protocols_api.md create mode 100644 .agents/skills/protocolsio-integration/references/workspaces.md create mode 100644 .agents/skills/protocolsio-integration/scripts/__init__.py create mode 100644 .agents/skills/protocolsio-integration/scripts/_common.py create mode 100644 .agents/skills/protocolsio-integration/scripts/pagination_helper.py create mode 100644 .agents/skills/protocolsio-integration/scripts/plan_write_request.py create mode 100644 .agents/skills/protocolsio-integration/scripts/protocols_read.py create mode 100644 .agents/skills/protocolsio-integration/scripts/validate_auth_config.py create mode 100644 .agents/skills/protocolsio-integration/scripts/validate_protocol_json.py create mode 100644 .agents/skills/pufferlib/SKILL.md create mode 100644 .agents/skills/pufferlib/references/environments.md create mode 100644 .agents/skills/pufferlib/references/integration.md create mode 100644 .agents/skills/pufferlib/references/policies.md create mode 100644 .agents/skills/pufferlib/references/training.md create mode 100644 .agents/skills/pufferlib/references/vectorization.md create mode 100644 .agents/skills/pufferlib/scripts/__init__.py create mode 100644 .agents/skills/pufferlib/scripts/_common.py create mode 100644 .agents/skills/pufferlib/scripts/benchmark_vectorization.py create mode 100644 .agents/skills/pufferlib/scripts/env_contract_validator.py create mode 100644 .agents/skills/pufferlib/scripts/env_template.py create mode 100644 .agents/skills/pufferlib/scripts/inspect_checkpoint.py create mode 100644 .agents/skills/pufferlib/scripts/repro_plan.py create mode 100644 .agents/skills/pufferlib/scripts/train_template.py create mode 100644 .agents/skills/pufferlib/scripts/validate_plan.py create mode 100644 .agents/skills/pydeseq2/SKILL.md create mode 100644 .agents/skills/pydeseq2/references/analysis_patterns.md create mode 100644 .agents/skills/pydeseq2/references/api_reference.md create mode 100644 .agents/skills/pydeseq2/references/core_workflow_steps.md create mode 100644 .agents/skills/pydeseq2/references/workflow_guide.md create mode 100644 .agents/skills/pydeseq2/scripts/run_deseq2_analysis.py create mode 100644 .agents/skills/pydicom/SKILL.md create mode 100644 .agents/skills/pydicom/references/common_tags.md create mode 100644 .agents/skills/pydicom/references/transfer_syntaxes.md create mode 100644 .agents/skills/pydicom/scripts/__init__.py create mode 100644 .agents/skills/pydicom/scripts/_common.py create mode 100755 .agents/skills/pydicom/scripts/anonymize_dicom.py create mode 100755 .agents/skills/pydicom/scripts/deidentification_audit.py create mode 100755 .agents/skills/pydicom/scripts/dicom_inventory.py create mode 100755 .agents/skills/pydicom/scripts/dicom_to_image.py create mode 100755 .agents/skills/pydicom/scripts/extract_metadata.py create mode 100755 .agents/skills/pydicom/scripts/pixel_frame_planner.py create mode 100755 .agents/skills/pydicom/scripts/transfer_syntax_inspector.py create mode 100755 .agents/skills/pydicom/scripts/uid_mapping_validator.py create mode 100644 .agents/skills/pyhealth/SKILL.md create mode 100644 .agents/skills/pyhealth/assets/starter_pipeline.py create mode 100644 .agents/skills/pyhealth/references/datasets.md create mode 100644 .agents/skills/pyhealth/references/examples.md create mode 100644 .agents/skills/pyhealth/references/installation.md create mode 100644 .agents/skills/pyhealth/references/medcode.md create mode 100644 .agents/skills/pyhealth/references/models.md create mode 100644 .agents/skills/pyhealth/references/tasks.md create mode 100644 .agents/skills/pylabrobot/SKILL.md create mode 100644 .agents/skills/pylabrobot/assets/protocol-manifest.schema.json create mode 100644 .agents/skills/pylabrobot/references/analytical-equipment.md create mode 100644 .agents/skills/pylabrobot/references/hardware-backends.md create mode 100644 .agents/skills/pylabrobot/references/liquid-handling.md create mode 100644 .agents/skills/pylabrobot/references/material-handling.md create mode 100644 .agents/skills/pylabrobot/references/resources.md create mode 100644 .agents/skills/pylabrobot/references/visualization.md create mode 100644 .agents/skills/pylabrobot/scripts/__init__.py create mode 100644 .agents/skills/pylabrobot/scripts/_common.py create mode 100644 .agents/skills/pylabrobot/scripts/check_deck_geometry.py create mode 100644 .agents/skills/pylabrobot/scripts/generate_simulation_plan.py create mode 100644 .agents/skills/pylabrobot/scripts/inspect_backends.py create mode 100644 .agents/skills/pylabrobot/scripts/plan_transfers.py create mode 100644 .agents/skills/pylabrobot/scripts/validate_manifest.py create mode 100644 .agents/skills/pymatgen/SKILL.md create mode 100644 .agents/skills/pymatgen/references/analysis_modules.md create mode 100644 .agents/skills/pymatgen/references/core_classes.md create mode 100644 .agents/skills/pymatgen/references/io_formats.md create mode 100644 .agents/skills/pymatgen/references/materials_project_api.md create mode 100644 .agents/skills/pymatgen/references/transformations_workflows.md create mode 100644 .agents/skills/pymatgen/scripts/_common.py create mode 100644 .agents/skills/pymatgen/scripts/artifact_manifest.py create mode 100644 .agents/skills/pymatgen/scripts/composition_structure_validator.py create mode 100644 .agents/skills/pymatgen/scripts/io_conversion_plan.py create mode 100644 .agents/skills/pymatgen/scripts/mp_query.py create mode 100644 .agents/skills/pymatgen/scripts/phase_diagram_generator.py create mode 100644 .agents/skills/pymatgen/scripts/structure_analyzer.py create mode 100644 .agents/skills/pymatgen/scripts/structure_converter.py create mode 100644 .agents/skills/pymatgen/scripts/symmetry_sensitivity_report.py create mode 100644 .agents/skills/pymc/SKILL.md create mode 100644 .agents/skills/pymc/assets/hierarchical_model_template.py create mode 100644 .agents/skills/pymc/assets/linear_regression_template.py create mode 100644 .agents/skills/pymc/references/distributions.md create mode 100644 .agents/skills/pymc/references/model_patterns.md create mode 100644 .agents/skills/pymc/references/sampling_inference.md create mode 100644 .agents/skills/pymc/references/standard_workflow.md create mode 100644 .agents/skills/pymc/references/workflows.md create mode 100644 .agents/skills/pymc/scripts/model_comparison.py create mode 100644 .agents/skills/pymc/scripts/model_diagnostics.py create mode 100644 .agents/skills/pymoo/SKILL.md create mode 100644 .agents/skills/pymoo/references/algorithms.md create mode 100644 .agents/skills/pymoo/references/constraints_mcdm.md create mode 100644 .agents/skills/pymoo/references/operators.md create mode 100644 .agents/skills/pymoo/references/parallelization.md create mode 100644 .agents/skills/pymoo/references/problems.md create mode 100644 .agents/skills/pymoo/references/quick_start_workflows.md create mode 100644 .agents/skills/pymoo/references/visualization.md create mode 100644 .agents/skills/pymoo/scripts/custom_problem_example.py create mode 100644 .agents/skills/pymoo/scripts/decision_making_example.py create mode 100644 .agents/skills/pymoo/scripts/many_objective_example.py create mode 100644 .agents/skills/pymoo/scripts/multi_objective_example.py create mode 100644 .agents/skills/pymoo/scripts/single_objective_example.py create mode 100644 .agents/skills/pyopenms/SKILL.md create mode 100644 .agents/skills/pyopenms/references/data_structures.md create mode 100644 .agents/skills/pyopenms/references/feature_detection.md create mode 100644 .agents/skills/pyopenms/references/file_io.md create mode 100644 .agents/skills/pyopenms/references/identification.md create mode 100644 .agents/skills/pyopenms/references/metabolomics.md create mode 100644 .agents/skills/pyopenms/references/signal_processing.md create mode 100644 .agents/skills/pyopenms/scripts/accurate_mass_search.py create mode 100644 .agents/skills/pyopenms/scripts/align_link_quantify.py create mode 100644 .agents/skills/pyopenms/scripts/consensus_to_matrix.py create mode 100644 .agents/skills/pyopenms/scripts/convert_format.py create mode 100644 .agents/skills/pyopenms/scripts/detect_adducts.py create mode 100644 .agents/skills/pyopenms/scripts/detect_features_centroided.py create mode 100644 .agents/skills/pyopenms/scripts/detect_features_metabo.py create mode 100644 .agents/skills/pyopenms/scripts/digest_protein.py create mode 100644 .agents/skills/pyopenms/scripts/export_gnps_sirius.py create mode 100644 .agents/skills/pyopenms/scripts/extract_chromatograms.py create mode 100644 .agents/skills/pyopenms/scripts/inspect_ms_data.py create mode 100644 .agents/skills/pyopenms/scripts/mass_calculator.py create mode 100644 .agents/skills/pyopenms/scripts/plot_ms_data.py create mode 100644 .agents/skills/pyopenms/scripts/process_identifications.py create mode 100644 .agents/skills/pyopenms/scripts/process_spectra.py create mode 100644 .agents/skills/pyopenms/scripts/theoretical_spectrum.py create mode 100644 .agents/skills/pysam/SKILL.md create mode 100644 .agents/skills/pysam/references/alignment_files.md create mode 100644 .agents/skills/pysam/references/api_reference.md create mode 100644 .agents/skills/pysam/references/common_workflows.md create mode 100644 .agents/skills/pysam/references/coordinates_and_indexing.md create mode 100644 .agents/skills/pysam/references/cram_and_performance.md create mode 100644 .agents/skills/pysam/references/migration_to_0_24.md create mode 100644 .agents/skills/pysam/references/sequence_files.md create mode 100644 .agents/skills/pysam/references/sources.md create mode 100644 .agents/skills/pysam/references/variant_files.md create mode 100644 .agents/skills/pysam/scripts/alignment_qc.py create mode 100644 .agents/skills/pysam/scripts/filter_alignments.py create mode 100644 .agents/skills/pysam/scripts/inspect_hts.py create mode 100644 .agents/skills/pysam/scripts/variant_summary.py create mode 100644 .agents/skills/pytdc/SKILL.md create mode 100644 .agents/skills/pytdc/references/datasets.md create mode 100644 .agents/skills/pytdc/references/oracles.md create mode 100644 .agents/skills/pytdc/references/sources.md create mode 100644 .agents/skills/pytdc/references/utilities.md create mode 100644 .agents/skills/pytdc/scripts/_common.py create mode 100644 .agents/skills/pytdc/scripts/benchmark_evaluation.py create mode 100644 .agents/skills/pytdc/scripts/cache_audit.py create mode 100644 .agents/skills/pytdc/scripts/discover_metadata.py create mode 100644 .agents/skills/pytdc/scripts/load_and_split_data.py create mode 100644 .agents/skills/pytdc/scripts/molecular_generation.py create mode 100644 .agents/skills/pytorch-lightning/SKILL.md create mode 100644 .agents/skills/pytorch-lightning/references/best_practices.md create mode 100644 .agents/skills/pytorch-lightning/references/callbacks.md create mode 100644 .agents/skills/pytorch-lightning/references/data_module.md create mode 100644 .agents/skills/pytorch-lightning/references/distributed_training.md create mode 100644 .agents/skills/pytorch-lightning/references/lightning_module.md create mode 100644 .agents/skills/pytorch-lightning/references/logging.md create mode 100644 .agents/skills/pytorch-lightning/references/trainer.md create mode 100644 .agents/skills/pytorch-lightning/scripts/quick_trainer_setup.py create mode 100644 .agents/skills/pytorch-lightning/scripts/template_datamodule.py create mode 100644 .agents/skills/pytorch-lightning/scripts/template_lightning_module.py create mode 100644 .agents/skills/pyzotero/SKILL.md create mode 100644 .agents/skills/pyzotero/references/authentication.md create mode 100644 .agents/skills/pyzotero/references/cli.md create mode 100644 .agents/skills/pyzotero/references/collections.md create mode 100644 .agents/skills/pyzotero/references/error-handling.md create mode 100644 .agents/skills/pyzotero/references/exports.md create mode 100644 .agents/skills/pyzotero/references/files-attachments.md create mode 100644 .agents/skills/pyzotero/references/full-text.md create mode 100644 .agents/skills/pyzotero/references/mcp.md create mode 100644 .agents/skills/pyzotero/references/pagination.md create mode 100644 .agents/skills/pyzotero/references/read-api.md create mode 100644 .agents/skills/pyzotero/references/saved-searches.md create mode 100644 .agents/skills/pyzotero/references/search-params.md create mode 100644 .agents/skills/pyzotero/references/tags.md create mode 100644 .agents/skills/pyzotero/references/write-api.md create mode 100644 .agents/skills/qiskit/SKILL.md create mode 100644 .agents/skills/qiskit/references/algorithms.md create mode 100644 .agents/skills/qiskit/references/backends.md create mode 100644 .agents/skills/qiskit/references/circuits.md create mode 100644 .agents/skills/qiskit/references/migration.md create mode 100644 .agents/skills/qiskit/references/patterns.md create mode 100644 .agents/skills/qiskit/references/primitives.md create mode 100644 .agents/skills/qiskit/references/setup.md create mode 100644 .agents/skills/qiskit/references/sources.md create mode 100644 .agents/skills/qiskit/references/testing.md create mode 100644 .agents/skills/qiskit/references/transpilation.md create mode 100644 .agents/skills/qiskit/references/visualization.md create mode 100644 .agents/skills/qiskit/scripts/check_environment.py create mode 100644 .agents/skills/qiskit/scripts/inspect_runtime.py create mode 100644 .agents/skills/qiskit/scripts/run_local_primitives.py create mode 100644 .agents/skills/qutip/SKILL.md create mode 100644 .agents/skills/qutip/references/advanced.md create mode 100644 .agents/skills/qutip/references/analysis.md create mode 100644 .agents/skills/qutip/references/core_concepts.md create mode 100644 .agents/skills/qutip/references/time_evolution.md create mode 100644 .agents/skills/qutip/references/visualization.md create mode 100644 .agents/skills/qutip/scripts/_common.py create mode 100644 .agents/skills/qutip/scripts/convergence_sweep.py create mode 100644 .agents/skills/qutip/scripts/qobj_model_validator.py create mode 100644 .agents/skills/qutip/scripts/result_audit.py create mode 100644 .agents/skills/qutip/scripts/solver_config_planner.py create mode 100644 .agents/skills/qutip/scripts/steady_state_spectrum_planner.py create mode 100644 .agents/skills/qutip/scripts/two_level_simulation.py create mode 100644 .agents/skills/rdkit/SKILL.md create mode 100644 .agents/skills/rdkit/references/api_reference.md create mode 100644 .agents/skills/rdkit/references/core_capabilities.md create mode 100644 .agents/skills/rdkit/references/descriptors_reference.md create mode 100644 .agents/skills/rdkit/references/smarts_patterns.md create mode 100644 .agents/skills/rdkit/references/workflows_and_best_practices.md create mode 100644 .agents/skills/rdkit/scripts/molecular_properties.py create mode 100644 .agents/skills/rdkit/scripts/similarity_search.py create mode 100644 .agents/skills/rdkit/scripts/substructure_filter.py create mode 100644 .agents/skills/relsa-severity-assessment/SKILL.md create mode 100644 .agents/skills/relsa-severity-assessment/assets/example_cohort.csv create mode 100644 .agents/skills/relsa-severity-assessment/references/forecasting.md create mode 100644 .agents/skills/relsa-severity-assessment/references/relsa-method.md create mode 100644 .agents/skills/relsa-severity-assessment/references/thresholds-and-zones.md create mode 100644 .agents/skills/relsa-severity-assessment/scripts/_common.py create mode 100644 .agents/skills/relsa-severity-assessment/scripts/forecast_relsa.py create mode 100644 .agents/skills/relsa-severity-assessment/scripts/kde_thresholds.py create mode 100644 .agents/skills/relsa-severity-assessment/scripts/relsa_score.py create mode 100644 .agents/skills/research-grants/SKILL.md create mode 100644 .agents/skills/research-grants/assets/budget_justification_template.md create mode 100644 .agents/skills/research-grants/assets/nih_specific_aims_template.md create mode 100644 .agents/skills/research-grants/assets/nsf_project_summary_template.md create mode 100644 .agents/skills/research-grants/references/README.md create mode 100644 .agents/skills/research-grants/references/broader_impacts.md create mode 100644 .agents/skills/research-grants/references/core_components.md create mode 100644 .agents/skills/research-grants/references/darpa_guidelines.md create mode 100644 .agents/skills/research-grants/references/doe_guidelines.md create mode 100644 .agents/skills/research-grants/references/nih_guidelines.md create mode 100644 .agents/skills/research-grants/references/nsf_guidelines.md create mode 100644 .agents/skills/research-grants/references/nstc_guidelines.md create mode 100644 .agents/skills/research-grants/references/proposal_types_and_resubmission.md create mode 100644 .agents/skills/research-grants/references/review_criteria.md create mode 100644 .agents/skills/research-grants/references/specific_aims_guide.md create mode 100644 .agents/skills/research-grants/references/writing_principles.md create mode 100644 .agents/skills/research-lookup/README.md create mode 100644 .agents/skills/research-lookup/SKILL.md create mode 100644 .agents/skills/research-lookup/scripts/manuscript_packet.py create mode 100644 .agents/skills/research-lookup/scripts/research_lookup.py create mode 100644 .agents/skills/rowan/SKILL.md create mode 100644 .agents/skills/rowan/references/access_and_pricing.md create mode 100644 .agents/skills/rowan/references/batch_and_webhooks.md create mode 100644 .agents/skills/rowan/references/end_to_end_example.md create mode 100644 .agents/skills/rowan/references/troubleshooting.md create mode 100644 .agents/skills/rowan/references/workflow_catalog.md create mode 100644 .agents/skills/scanpy/SKILL.md create mode 100644 .agents/skills/scanpy/assets/analysis_template.py create mode 100644 .agents/skills/scanpy/assets/celltype_mapping.json create mode 100644 .agents/skills/scanpy/assets/gene_signatures.json create mode 100644 .agents/skills/scanpy/assets/pipeline_config.json create mode 100644 .agents/skills/scanpy/references/analysis_workflow.md create mode 100644 .agents/skills/scanpy/references/api_reference.md create mode 100644 .agents/skills/scanpy/references/plotting_guide.md create mode 100644 .agents/skills/scanpy/references/r_interop.md create mode 100644 .agents/skills/scanpy/references/standard_workflow.md create mode 100644 .agents/skills/scanpy/scripts/_common.py create mode 100644 .agents/skills/scanpy/scripts/annotate.py create mode 100644 .agents/skills/scanpy/scripts/batch_correct.py create mode 100644 .agents/skills/scanpy/scripts/cluster.py create mode 100644 .agents/skills/scanpy/scripts/convert.py create mode 100644 .agents/skills/scanpy/scripts/find_markers.py create mode 100644 .agents/skills/scanpy/scripts/inspect_data.py create mode 100644 .agents/skills/scanpy/scripts/plot.py create mode 100644 .agents/skills/scanpy/scripts/preprocess.py create mode 100644 .agents/skills/scanpy/scripts/pseudobulk.py create mode 100755 .agents/skills/scanpy/scripts/qc_analysis.py create mode 100644 .agents/skills/scanpy/scripts/reduce_dimensions.py create mode 100644 .agents/skills/scanpy/scripts/run_pipeline.py create mode 100644 .agents/skills/scanpy/scripts/score_genes.py create mode 100644 .agents/skills/scanpy/scripts/subset.py create mode 100644 .agents/skills/scholar-evaluation/SKILL.md create mode 100644 .agents/skills/scholar-evaluation/assets/evaluation_template.json create mode 100644 .agents/skills/scholar-evaluation/assets/evidence_manifest_template.json create mode 100644 .agents/skills/scholar-evaluation/assets/process_checklist_template.json create mode 100644 .agents/skills/scholar-evaluation/assets/ratings_template.csv create mode 100644 .agents/skills/scholar-evaluation/assets/rubric_template.json create mode 100644 .agents/skills/scholar-evaluation/references/evaluation_framework.md create mode 100644 .agents/skills/scholar-evaluation/references/local_tooling.md create mode 100644 .agents/skills/scholar-evaluation/references/responsible_assessment.md create mode 100644 .agents/skills/scholar-evaluation/references/security_validation.md create mode 100644 .agents/skills/scholar-evaluation/references/source_ledger.md create mode 100644 .agents/skills/scholar-evaluation/scripts/_common.py create mode 100644 .agents/skills/scholar-evaluation/scripts/calculate_scores.py create mode 100644 .agents/skills/scholar-evaluation/scripts/check_process.py create mode 100644 .agents/skills/scholar-evaluation/scripts/check_traceability.py create mode 100644 .agents/skills/scholar-evaluation/scripts/generate_report_scaffold.py create mode 100644 .agents/skills/scholar-evaluation/scripts/summarize_agreement.py create mode 100644 .agents/skills/scholar-evaluation/scripts/validate_rubric.py create mode 100644 .agents/skills/scholar-evaluation/scripts/weight_sensitivity.py create mode 100644 .agents/skills/scientific-brainstorming/SKILL.md create mode 100644 .agents/skills/scientific-brainstorming/references/brainstorming_methods.md create mode 100644 .agents/skills/scientific-brainstorming/references/facilitation_workflows.md create mode 100644 .agents/skills/scientific-brainstorming/references/idea_evaluation.md create mode 100644 .agents/skills/scientific-brainstorming/references/responsible_ai.md create mode 100644 .agents/skills/scientific-brainstorming/references/sources.md create mode 100644 .agents/skills/scientific-brainstorming/scripts/_common.py create mode 100644 .agents/skills/scientific-brainstorming/scripts/evaluate_matrix.py create mode 100644 .agents/skills/scientific-brainstorming/scripts/session_scaffold.py create mode 100644 .agents/skills/scientific-brainstorming/scripts/validate_register.py create mode 100644 .agents/skills/scientific-critical-thinking/SKILL.md create mode 100644 .agents/skills/scientific-critical-thinking/references/common_biases.md create mode 100644 .agents/skills/scientific-critical-thinking/references/core_capabilities.md create mode 100644 .agents/skills/scientific-critical-thinking/references/evidence_hierarchy.md create mode 100644 .agents/skills/scientific-critical-thinking/references/experimental_design.md create mode 100644 .agents/skills/scientific-critical-thinking/references/logical_fallacies.md create mode 100644 .agents/skills/scientific-critical-thinking/references/scientific_method.md create mode 100644 .agents/skills/scientific-critical-thinking/references/statistical_pitfalls.md create mode 100644 .agents/skills/scientific-schematics/SKILL.md create mode 100644 .agents/skills/scientific-schematics/references/best_practices.md create mode 100644 .agents/skills/scientific-schematics/references/iterative_refinement.md create mode 100755 .agents/skills/scientific-schematics/scripts/example_usage.sh create mode 100644 .agents/skills/scientific-schematics/scripts/generate_schematic.py create mode 100644 .agents/skills/scientific-schematics/scripts/generate_schematic_ai.py create mode 100644 .agents/skills/scientific-slides/SKILL.md create mode 100644 .agents/skills/scientific-slides/assets/beamer_template_conference.tex create mode 100644 .agents/skills/scientific-slides/assets/beamer_template_defense.tex create mode 100644 .agents/skills/scientific-slides/assets/beamer_template_seminar.tex create mode 100644 .agents/skills/scientific-slides/assets/powerpoint_design_guide.md create mode 100644 .agents/skills/scientific-slides/assets/timing_guidelines.md create mode 100644 .agents/skills/scientific-slides/references/beamer_guide.md create mode 100644 .agents/skills/scientific-slides/references/common_pitfalls.md create mode 100644 .agents/skills/scientific-slides/references/data_visualization_slides.md create mode 100644 .agents/skills/scientific-slides/references/presentation_structure.md create mode 100644 .agents/skills/scientific-slides/references/presentation_workflow.md create mode 100644 .agents/skills/scientific-slides/references/prompt_writing.md create mode 100644 .agents/skills/scientific-slides/references/script_reference.md create mode 100644 .agents/skills/scientific-slides/references/slide_capabilities.md create mode 100644 .agents/skills/scientific-slides/references/slide_design_principles.md create mode 100644 .agents/skills/scientific-slides/references/talk_types_guide.md create mode 100644 .agents/skills/scientific-slides/references/visual_review_workflow.md create mode 100644 .agents/skills/scientific-slides/scripts/generate_schematic.py create mode 100644 .agents/skills/scientific-slides/scripts/generate_schematic_ai.py create mode 100644 .agents/skills/scientific-slides/scripts/generate_slide_image.py create mode 100644 .agents/skills/scientific-slides/scripts/generate_slide_image_ai.py create mode 100644 .agents/skills/scientific-slides/scripts/pdf_to_images.py create mode 100644 .agents/skills/scientific-slides/scripts/slides_to_pdf.py create mode 100644 .agents/skills/scientific-slides/scripts/validate_presentation.py create mode 100644 .agents/skills/scientific-visualization/SKILL.md create mode 100644 .agents/skills/scientific-visualization/assets/color_palettes.py create mode 100644 .agents/skills/scientific-visualization/assets/nature.mplstyle create mode 100644 .agents/skills/scientific-visualization/assets/presentation.mplstyle create mode 100644 .agents/skills/scientific-visualization/assets/publication.mplstyle create mode 100644 .agents/skills/scientific-visualization/assets/publisher_profiles.json create mode 100644 .agents/skills/scientific-visualization/references/color_palettes.md create mode 100644 .agents/skills/scientific-visualization/references/journal_requirements.md create mode 100644 .agents/skills/scientific-visualization/references/matplotlib_examples.md create mode 100644 .agents/skills/scientific-visualization/references/publication_guidelines.md create mode 100644 .agents/skills/scientific-visualization/references/sources.md create mode 100644 .agents/skills/scientific-visualization/scripts/_common.py create mode 100644 .agents/skills/scientific-visualization/scripts/export_plan.py create mode 100644 .agents/skills/scientific-visualization/scripts/figure_export.py create mode 100644 .agents/skills/scientific-visualization/scripts/image_metadata.py create mode 100644 .agents/skills/scientific-visualization/scripts/palette_audit.py create mode 100644 .agents/skills/scientific-visualization/scripts/style_presets.py create mode 100644 .agents/skills/scientific-visualization/scripts/style_preview.py create mode 100644 .agents/skills/scientific-writing/SKILL.md create mode 100644 .agents/skills/scientific-writing/assets/REPORT_FORMATTING_GUIDE.md create mode 100644 .agents/skills/scientific-writing/assets/authorship_template.json create mode 100644 .agents/skills/scientific-writing/assets/claim_evidence_template.csv create mode 100644 .agents/skills/scientific-writing/assets/consistency_manifest_template.json create mode 100644 .agents/skills/scientific-writing/assets/manuscript_manifest_template.json create mode 100644 .agents/skills/scientific-writing/assets/manuscript_scaffold.md create mode 100644 .agents/skills/scientific-writing/assets/reporting_coverage_template.json create mode 100644 .agents/skills/scientific-writing/assets/reporting_guidelines.json create mode 100644 .agents/skills/scientific-writing/assets/source_manifest_template.json create mode 100644 .agents/skills/scientific-writing/references/authorship_ai_confidentiality.md create mode 100644 .agents/skills/scientific-writing/references/citation_styles.md create mode 100644 .agents/skills/scientific-writing/references/cli_reference.md create mode 100644 .agents/skills/scientific-writing/references/evidence_workflow.md create mode 100644 .agents/skills/scientific-writing/references/figures_tables.md create mode 100644 .agents/skills/scientific-writing/references/imrad_structure.md create mode 100644 .agents/skills/scientific-writing/references/journal_policies.md create mode 100644 .agents/skills/scientific-writing/references/professional_report_formatting.md create mode 100644 .agents/skills/scientific-writing/references/reporting_guidelines.md create mode 100644 .agents/skills/scientific-writing/references/research_integrity_open_science.md create mode 100644 .agents/skills/scientific-writing/references/source_ledger.md create mode 100644 .agents/skills/scientific-writing/references/writing_principles.md create mode 100644 .agents/skills/scientific-writing/scripts/_common.py create mode 100644 .agents/skills/scientific-writing/scripts/audit_claims.py create mode 100644 .agents/skills/scientific-writing/scripts/check_consistency.py create mode 100644 .agents/skills/scientific-writing/scripts/check_references.py create mode 100644 .agents/skills/scientific-writing/scripts/lint_manuscript.py create mode 100644 .agents/skills/scientific-writing/scripts/scaffold_manuscript.py create mode 100644 .agents/skills/scientific-writing/scripts/select_reporting_guidelines.py create mode 100644 .agents/skills/scientific-writing/scripts/validate_authorship.py create mode 100644 .agents/skills/scientific-writing/scripts/validate_manifest.py create mode 100644 .agents/skills/scikit-bio/SKILL.md create mode 100644 .agents/skills/scikit-bio/references/api_reference.md create mode 100644 .agents/skills/scikit-learn/SKILL.md create mode 100644 .agents/skills/scikit-learn/references/common_workflows.md create mode 100644 .agents/skills/scikit-learn/references/core_capabilities.md create mode 100644 .agents/skills/scikit-learn/references/model_evaluation.md create mode 100644 .agents/skills/scikit-learn/references/pipelines_and_composition.md create mode 100644 .agents/skills/scikit-learn/references/preprocessing.md create mode 100644 .agents/skills/scikit-learn/references/quick_reference.md create mode 100644 .agents/skills/scikit-learn/references/supervised_learning.md create mode 100644 .agents/skills/scikit-learn/references/unsupervised_learning.md create mode 100644 .agents/skills/scikit-learn/scripts/classification_pipeline.py create mode 100644 .agents/skills/scikit-learn/scripts/clustering_analysis.py create mode 100644 .agents/skills/scikit-survival/SKILL.md create mode 100644 .agents/skills/scikit-survival/references/competing-risks.md create mode 100644 .agents/skills/scikit-survival/references/cox-models.md create mode 100644 .agents/skills/scikit-survival/references/data-handling.md create mode 100644 .agents/skills/scikit-survival/references/ensemble-models.md create mode 100644 .agents/skills/scikit-survival/references/evaluation-metrics.md create mode 100644 .agents/skills/scikit-survival/references/svm-models.md create mode 100644 .agents/skills/scikit-survival/scripts/_common.py create mode 100644 .agents/skills/scikit-survival/scripts/competing_risk_cif.py create mode 100644 .agents/skills/scikit-survival/scripts/evaluate_survival_metrics.py create mode 100644 .agents/skills/scikit-survival/scripts/model_report.py create mode 100644 .agents/skills/scikit-survival/scripts/train_survival_model.py create mode 100644 .agents/skills/scikit-survival/scripts/validate_survival_csv.py create mode 100644 .agents/skills/scvelo/SKILL.md create mode 100644 .agents/skills/scvelo/references/velocity_models.md create mode 100644 .agents/skills/scvelo/scripts/rna_velocity_workflow.py create mode 100644 .agents/skills/scvi-tools/SKILL.md create mode 100644 .agents/skills/scvi-tools/references/differential-expression.md create mode 100644 .agents/skills/scvi-tools/references/models-atac-seq.md create mode 100644 .agents/skills/scvi-tools/references/models-multimodal.md create mode 100644 .agents/skills/scvi-tools/references/models-scrna-seq.md create mode 100644 .agents/skills/scvi-tools/references/models-spatial.md create mode 100644 .agents/skills/scvi-tools/references/models-specialized.md create mode 100644 .agents/skills/scvi-tools/references/theoretical-foundations.md create mode 100644 .agents/skills/scvi-tools/references/workflows.md create mode 100644 .agents/skills/seaborn/SKILL.md create mode 100644 .agents/skills/seaborn/references/examples.md create mode 100644 .agents/skills/seaborn/references/function_reference.md create mode 100644 .agents/skills/seaborn/references/grids_and_levels.md create mode 100644 .agents/skills/seaborn/references/objects_interface.md create mode 100644 .agents/skills/seaborn/references/palettes_and_theming.md create mode 100644 .agents/skills/seaborn/references/patterns_and_troubleshooting.md create mode 100644 .agents/skills/seaborn/references/plotting_functions.md create mode 100644 .agents/skills/shap/SKILL.md create mode 100644 .agents/skills/shap/references/data-maskers.md create mode 100644 .agents/skills/shap/references/explainers.md create mode 100644 .agents/skills/shap/references/migration.md create mode 100644 .agents/skills/shap/references/modalities.md create mode 100644 .agents/skills/shap/references/plots.md create mode 100644 .agents/skills/shap/references/theory.md create mode 100644 .agents/skills/shap/references/troubleshooting.md create mode 100644 .agents/skills/shap/references/workflows.md create mode 100644 .agents/skills/shap/scripts/tabular_report.py create mode 100644 .agents/skills/simpy/SKILL.md create mode 100644 .agents/skills/simpy/references/cli-guide.md create mode 100644 .agents/skills/simpy/references/events.md create mode 100644 .agents/skills/simpy/references/monitoring.md create mode 100644 .agents/skills/simpy/references/process-interaction.md create mode 100644 .agents/skills/simpy/references/real-time.md create mode 100644 .agents/skills/simpy/references/resources.md create mode 100644 .agents/skills/simpy/references/simulation-methodology.md create mode 100644 .agents/skills/simpy/references/sources.md create mode 100644 .agents/skills/simpy/scripts/_common.py create mode 100644 .agents/skills/simpy/scripts/basic_simulation_template.py create mode 100644 .agents/skills/simpy/scripts/bounded_queue_scenario.py create mode 100644 .agents/skills/simpy/scripts/event_trace_summary.py create mode 100644 .agents/skills/simpy/scripts/replication_runner.py create mode 100644 .agents/skills/simpy/scripts/resource_monitor.py create mode 100644 .agents/skills/simpy/scripts/validate_simulation_config.py create mode 100644 .agents/skills/stable-baselines3/SKILL.md create mode 100644 .agents/skills/stable-baselines3/references/algorithms.md create mode 100644 .agents/skills/stable-baselines3/references/callbacks.md create mode 100644 .agents/skills/stable-baselines3/references/custom_environments.md create mode 100644 .agents/skills/stable-baselines3/references/vectorized_envs.md create mode 100644 .agents/skills/stable-baselines3/scripts/custom_env_template.py create mode 100644 .agents/skills/stable-baselines3/scripts/evaluate_agent.py create mode 100644 .agents/skills/stable-baselines3/scripts/train_rl_agent.py create mode 100644 .agents/skills/statistical-analysis/SKILL.md create mode 100644 .agents/skills/statistical-analysis/references/assumptions_and_diagnostics.md create mode 100644 .agents/skills/statistical-analysis/references/bayesian_statistics.md create mode 100644 .agents/skills/statistical-analysis/references/effect_sizes_and_power.md create mode 100644 .agents/skills/statistical-analysis/references/reporting_standards.md create mode 100644 .agents/skills/statistical-analysis/references/test_selection_guide.md create mode 100644 .agents/skills/statistical-analysis/scripts/assumption_checks.py create mode 100644 .agents/skills/statistical-power/SKILL.md create mode 100644 .agents/skills/statistical-power/references/closed_form_recipes.md create mode 100644 .agents/skills/statistical-power/references/effect_sizes.md create mode 100644 .agents/skills/statistical-power/references/simulation_based_power.md create mode 100644 .agents/skills/statistical-power/scripts/power.py create mode 100644 .agents/skills/statistical-power/scripts/simulate_power.py create mode 100644 .agents/skills/statsmodels/SKILL.md create mode 100644 .agents/skills/statsmodels/references/discrete_choice.md create mode 100644 .agents/skills/statsmodels/references/glm.md create mode 100644 .agents/skills/statsmodels/references/linear_models.md create mode 100644 .agents/skills/statsmodels/references/model_selection.md create mode 100644 .agents/skills/statsmodels/references/modeling_capabilities.md create mode 100644 .agents/skills/statsmodels/references/quick_start_guide.md create mode 100644 .agents/skills/statsmodels/references/stats_diagnostics.md create mode 100644 .agents/skills/statsmodels/references/time_series.md create mode 100644 .agents/skills/sympy/SKILL.md create mode 100644 .agents/skills/sympy/references/advanced-topics.md create mode 100644 .agents/skills/sympy/references/code-generation-printing.md create mode 100644 .agents/skills/sympy/references/core-capabilities.md create mode 100644 .agents/skills/sympy/references/core_capabilities.md create mode 100644 .agents/skills/sympy/references/matrices-linear-algebra.md create mode 100644 .agents/skills/sympy/references/physics-mechanics.md create mode 100644 .agents/skills/tamarind/SKILL.md create mode 100644 .agents/skills/tamarind/references/api_reference.md create mode 100644 .agents/skills/tamarind/references/examples.md create mode 100644 .agents/skills/tamarind/references/tool_catalog.md create mode 100644 .agents/skills/tamarind/references/workflows.md create mode 100644 .agents/skills/tiledbvcf/SKILL.md create mode 100644 .agents/skills/timesfm-forecasting/SKILL.md create mode 100644 .agents/skills/timesfm-forecasting/examples/anomaly-detection/detect_anomalies.py create mode 100644 .agents/skills/timesfm-forecasting/examples/anomaly-detection/output/anomaly_detection.json create mode 100644 .agents/skills/timesfm-forecasting/examples/anomaly-detection/output/anomaly_detection.png create mode 100644 .agents/skills/timesfm-forecasting/examples/covariates-forecasting/demo_covariates.py create mode 100644 .agents/skills/timesfm-forecasting/examples/covariates-forecasting/output/covariates_data.png create mode 100644 .agents/skills/timesfm-forecasting/examples/covariates-forecasting/output/covariates_metadata.json create mode 100644 .agents/skills/timesfm-forecasting/examples/covariates-forecasting/output/sales_with_covariates.csv create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/README.md create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/generate_animation_data.py create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/generate_gif.py create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/generate_html.py create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/output/animation_data.json create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/output/forecast_animation.gif create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/output/forecast_output.csv create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/output/forecast_output.json create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/output/forecast_visualization.png create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/output/interactive_forecast.html create mode 100755 .agents/skills/timesfm-forecasting/examples/global-temperature/run_example.sh create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/run_forecast.py create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/temperature_anomaly.csv create mode 100644 .agents/skills/timesfm-forecasting/examples/global-temperature/visualize_forecast.py create mode 100644 .agents/skills/timesfm-forecasting/references/api_reference.md create mode 100644 .agents/skills/timesfm-forecasting/references/data_preparation.md create mode 100644 .agents/skills/timesfm-forecasting/references/examples_and_validation.md create mode 100644 .agents/skills/timesfm-forecasting/references/output_and_config.md create mode 100644 .agents/skills/timesfm-forecasting/references/performance_tuning.md create mode 100644 .agents/skills/timesfm-forecasting/references/system_requirements.md create mode 100644 .agents/skills/timesfm-forecasting/references/workflows.md create mode 100644 .agents/skills/timesfm-forecasting/scripts/check_system.py create mode 100644 .agents/skills/timesfm-forecasting/scripts/forecast_csv.py create mode 100644 .agents/skills/torch-geometric/SKILL.md create mode 100644 .agents/skills/torch-geometric/references/custom_datasets.md create mode 100644 .agents/skills/torch-geometric/references/explainability.md create mode 100644 .agents/skills/torch-geometric/references/heterogeneous.md create mode 100644 .agents/skills/torch-geometric/references/link_prediction.md create mode 100644 .agents/skills/torch-geometric/references/message_passing.md create mode 100644 .agents/skills/torch-geometric/references/scaling.md create mode 100644 .agents/skills/torchdrug/SKILL.md create mode 100644 .agents/skills/torchdrug/references/core_concepts.md create mode 100644 .agents/skills/torchdrug/references/datasets.md create mode 100644 .agents/skills/torchdrug/references/knowledge_graphs.md create mode 100644 .agents/skills/torchdrug/references/models_architectures.md create mode 100644 .agents/skills/torchdrug/references/molecular_generation.md create mode 100644 .agents/skills/torchdrug/references/molecular_property_prediction.md create mode 100644 .agents/skills/torchdrug/references/protein_modeling.md create mode 100644 .agents/skills/torchdrug/references/retrosynthesis.md create mode 100644 .agents/skills/transformers/SKILL.md create mode 100644 .agents/skills/transformers/references/generation.md create mode 100644 .agents/skills/transformers/references/models.md create mode 100644 .agents/skills/transformers/references/pipelines.md create mode 100644 .agents/skills/transformers/references/tokenizers.md create mode 100644 .agents/skills/transformers/references/training.md create mode 100644 .agents/skills/treatment-plans/SKILL.md create mode 100644 .agents/skills/treatment-plans/assets/clinician_authored_intervention_template.json create mode 100644 .agents/skills/treatment-plans/assets/goals_monitoring_checkpoint_template.json create mode 100644 .agents/skills/treatment-plans/assets/informed_preference_shared_decision_template.json create mode 100644 .agents/skills/treatment-plans/assets/intended_use_handoff_template.json create mode 100644 .agents/skills/treatment-plans/assets/source_fact_manifest_template.json create mode 100644 .agents/skills/treatment-plans/assets/transition_reconciliation_template.json create mode 100644 .agents/skills/treatment-plans/references/README.md create mode 100644 .agents/skills/treatment-plans/references/documentation_workflow.md create mode 100644 .agents/skills/treatment-plans/references/privacy_governance.md create mode 100644 .agents/skills/treatment-plans/references/safety_scope.md create mode 100644 .agents/skills/treatment-plans/references/security_validation.md create mode 100644 .agents/skills/treatment-plans/references/shared_decision_handoff.md create mode 100644 .agents/skills/treatment-plans/references/source_boundaries.md create mode 100644 .agents/skills/treatment-plans/references/source_ledger.md create mode 100644 .agents/skills/treatment-plans/scripts/_common.py create mode 100644 .agents/skills/treatment-plans/scripts/check_completeness.py create mode 100644 .agents/skills/treatment-plans/scripts/check_consistency.py create mode 100644 .agents/skills/treatment-plans/scripts/generate_template.py create mode 100644 .agents/skills/treatment-plans/scripts/privacy_process_check.py create mode 100644 .agents/skills/treatment-plans/scripts/timeline_generator.py create mode 100644 .agents/skills/treatment-plans/scripts/validate_traceability.py create mode 100644 .agents/skills/treatment-plans/scripts/validate_treatment_plan.py create mode 100644 .agents/skills/umap-learn/SKILL.md create mode 100644 .agents/skills/umap-learn/references/api_reference.md create mode 100644 .agents/skills/uncertainty-and-units/SKILL.md create mode 100644 .agents/skills/uncertainty-and-units/references/domain-conversions.md create mode 100644 .agents/skills/uncertainty-and-units/references/gum-methodology.md create mode 100644 .agents/skills/uncertainty-and-units/references/pint-recipes.md create mode 100644 .agents/skills/uncertainty-and-units/references/plausibility-scales.md create mode 100644 .agents/skills/uncertainty-and-units/references/reporting-rules.md create mode 100644 .agents/skills/uncertainty-and-units/references/uncertainties-recipes.md create mode 100644 .agents/skills/uncertainty-and-units/scripts/_common.py create mode 100644 .agents/skills/uncertainty-and-units/scripts/audit_units.py create mode 100644 .agents/skills/uncertainty-and-units/scripts/check_plausibility.py create mode 100644 .agents/skills/uncertainty-and-units/scripts/convert_units.py create mode 100644 .agents/skills/uncertainty-and-units/scripts/format_result.py create mode 100644 .agents/skills/uncertainty-and-units/scripts/propagate_uncertainty.py create mode 100644 .agents/skills/uncertainty-and-units/scripts/uncertainty_budget.py create mode 100644 .agents/skills/usfiscaldata/SKILL.md create mode 100644 .agents/skills/usfiscaldata/references/api-basics.md create mode 100644 .agents/skills/usfiscaldata/references/datasets-debt.md create mode 100644 .agents/skills/usfiscaldata/references/datasets-fiscal.md create mode 100644 .agents/skills/usfiscaldata/references/datasets-interest-rates.md create mode 100644 .agents/skills/usfiscaldata/references/datasets-securities.md create mode 100644 .agents/skills/usfiscaldata/references/examples.md create mode 100644 .agents/skills/usfiscaldata/references/parameters.md create mode 100644 .agents/skills/usfiscaldata/references/response-format.md create mode 100644 .agents/skills/vaex/SKILL.md create mode 100644 .agents/skills/vaex/references/core_dataframes.md create mode 100644 .agents/skills/vaex/references/data_processing.md create mode 100644 .agents/skills/vaex/references/io_operations.md create mode 100644 .agents/skills/vaex/references/machine_learning.md create mode 100644 .agents/skills/vaex/references/performance.md create mode 100644 .agents/skills/vaex/references/visualization.md create mode 100644 .agents/skills/venue-templates/SKILL.md create mode 100644 .agents/skills/venue-templates/assets/examples/cell_summary_example.md create mode 100644 .agents/skills/venue-templates/assets/examples/medical_structured_abstract.md create mode 100644 .agents/skills/venue-templates/assets/examples/nature_abstract_examples.md create mode 100644 .agents/skills/venue-templates/assets/examples/neurips_introduction_example.md create mode 100644 .agents/skills/venue-templates/assets/grants/nih_specific_aims.tex create mode 100644 .agents/skills/venue-templates/assets/grants/nsf_proposal_template.tex create mode 100644 .agents/skills/venue-templates/assets/journals/elsarticle-harv.bst create mode 100644 .agents/skills/venue-templates/assets/journals/elsarticle-num-names.bst create mode 100644 .agents/skills/venue-templates/assets/journals/elsarticle-num.bst create mode 100644 .agents/skills/venue-templates/assets/journals/elsarticle-template-harv.tex create mode 100644 .agents/skills/venue-templates/assets/journals/elsarticle-template-num-names.tex create mode 100644 .agents/skills/venue-templates/assets/journals/elsarticle-template-num.tex create mode 100644 .agents/skills/venue-templates/assets/journals/nature_article.tex create mode 100644 .agents/skills/venue-templates/assets/journals/neurips_article.tex create mode 100644 .agents/skills/venue-templates/assets/journals/plos_one.tex create mode 100644 .agents/skills/venue-templates/assets/posters/beamerposter_academic.tex create mode 100644 .agents/skills/venue-templates/references/cell_press_style.md create mode 100644 .agents/skills/venue-templates/references/conferences_formatting.md create mode 100644 .agents/skills/venue-templates/references/cs_conference_style.md create mode 100644 .agents/skills/venue-templates/references/grants_requirements.md create mode 100644 .agents/skills/venue-templates/references/journals_formatting.md create mode 100644 .agents/skills/venue-templates/references/medical_journal_styles.md create mode 100644 .agents/skills/venue-templates/references/ml_conference_style.md create mode 100644 .agents/skills/venue-templates/references/nature_science_style.md create mode 100644 .agents/skills/venue-templates/references/posters_guidelines.md create mode 100644 .agents/skills/venue-templates/references/reviewer_expectations.md create mode 100644 .agents/skills/venue-templates/references/venue_writing_styles.md create mode 100755 .agents/skills/venue-templates/scripts/customize_template.py create mode 100755 .agents/skills/venue-templates/scripts/query_template.py create mode 100755 .agents/skills/venue-templates/scripts/validate_format.py create mode 100644 .agents/skills/waypoint-bio/SKILL.md create mode 100644 .agents/skills/waypoint-bio/references/cli-reference.md create mode 100644 .agents/skills/waypoint-bio/references/compass-benchmark.md create mode 100644 .agents/skills/waypoint-bio/references/data-preparation.md create mode 100644 .agents/skills/waypoint-bio/references/python-api.md create mode 100755 .agents/skills/waypoint-bio/scripts/profiler_to_waypoint.py create mode 100755 .agents/skills/waypoint-bio/scripts/vocab_coverage.py create mode 100644 .agents/skills/what-if-oracle/SKILL.md create mode 100644 .agents/skills/what-if-oracle/references/scenario-templates.md create mode 100644 .agents/skills/xlsx/LICENSE.txt create mode 100644 .agents/skills/xlsx/SKILL.md create mode 100644 .agents/skills/xlsx/scripts/office/helpers/__init__.py create mode 100644 .agents/skills/xlsx/scripts/office/helpers/pptx_chart.py create mode 100644 .agents/skills/xlsx/scripts/office/helpers/pptx_slide.py create mode 100644 .agents/skills/xlsx/scripts/office/helpers/pptx_theme.py create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-chart.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-main.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-picture.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/pml.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-math.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/sml.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-main.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/wml.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ISO-IEC29500-4_2016/xml.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ecma/fouth-edition/opc-contentTypes.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ecma/fouth-edition/opc-coreProperties.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ecma/fouth-edition/opc-digSig.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/ecma/fouth-edition/opc-relationships.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/mce/mc.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/microsoft/wml-2010.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/microsoft/wml-2012.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/microsoft/wml-2018.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/microsoft/wml-cex-2018.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/microsoft/wml-cid-2016.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/microsoft/wml-sdtdatahash-2020.xsd create mode 100644 .agents/skills/xlsx/scripts/office/schemas/microsoft/wml-symex-2015.xsd create mode 100644 .agents/skills/xlsx/scripts/office/soffice.py create mode 100755 .agents/skills/xlsx/scripts/office/validate.py create mode 100644 .agents/skills/xlsx/scripts/office/validators/__init__.py create mode 100644 .agents/skills/xlsx/scripts/office/validators/base.py create mode 100644 .agents/skills/xlsx/scripts/office/validators/docx.py create mode 100644 .agents/skills/xlsx/scripts/office/validators/pptx.py create mode 100644 .agents/skills/xlsx/scripts/office/validators/redlining.py create mode 100755 .agents/skills/xlsx/scripts/recalc.py create mode 100644 .agents/skills/zarr-python/SKILL.md create mode 100644 .agents/skills/zarr-python/references/api_reference.md create mode 100644 .agents/skills/zarr-python/references/chunking_and_compression.md create mode 100644 .agents/skills/zarr-python/references/integration.md create mode 100644 .agents/skills/zarr-python/references/performance_and_patterns.md create mode 100644 .agents/skills/zarr-python/references/storage_backends.md create mode 100644 .agents/skills/zarr-python/references/v3_migration.md create mode 120000 .claude/skills/adaptyv create mode 120000 .claude/skills/aeon create mode 120000 .claude/skills/analytical-method-validation create mode 120000 .claude/skills/anndata create mode 120000 .claude/skills/arbor create mode 120000 .claude/skills/arboreto create mode 120000 .claude/skills/astropy create mode 120000 .claude/skills/autoskill create mode 120000 .claude/skills/benchling-integration create mode 120000 .claude/skills/bgpt-paper-search create mode 120000 .claude/skills/bids create mode 120000 .claude/skills/biopython create mode 120000 .claude/skills/bioservices create mode 120000 .claude/skills/bulk-rnaseq create mode 120000 .claude/skills/cellxgene-census create mode 120000 .claude/skills/cirq create mode 120000 .claude/skills/citation-management create mode 120000 .claude/skills/clinical-decision-support create mode 120000 .claude/skills/clinical-reports create mode 120000 .claude/skills/cobrapy create mode 120000 .claude/skills/consciousness-council create mode 120000 .claude/skills/dask create mode 120000 .claude/skills/database-lookup create mode 120000 .claude/skills/datamol create mode 120000 .claude/skills/deepchem create mode 120000 .claude/skills/deepspot-m create mode 120000 .claude/skills/deeptools create mode 120000 .claude/skills/depmap create mode 120000 .claude/skills/dhdna-profiler create mode 120000 .claude/skills/diffdock create mode 120000 .claude/skills/dnanexus-integration create mode 120000 .claude/skills/docx create mode 120000 .claude/skills/esm create mode 120000 .claude/skills/etetoolkit create mode 120000 .claude/skills/exa-search create mode 120000 .claude/skills/experimental-design create mode 120000 .claude/skills/exploratory-data-analysis create mode 120000 .claude/skills/flowio create mode 120000 .claude/skills/fluidsim create mode 120000 .claude/skills/generate-image create mode 120000 .claude/skills/geniml create mode 120000 .claude/skills/genomic-coordinates create mode 120000 .claude/skills/genomic-intelligence create mode 120000 .claude/skills/geomaster create mode 120000 .claude/skills/geopandas create mode 120000 .claude/skills/get-available-resources create mode 120000 .claude/skills/gget create mode 120000 .claude/skills/ginkgo-cloud-lab create mode 120000 .claude/skills/glycoengineering create mode 120000 .claude/skills/gtars create mode 120000 .claude/skills/histolab create mode 120000 .claude/skills/hugging-science create mode 120000 .claude/skills/hypogenic create mode 120000 .claude/skills/hypothesis-generation create mode 120000 .claude/skills/imaging-data-commons create mode 120000 .claude/skills/infographics create mode 120000 .claude/skills/iso-standards-readiness create mode 120000 .claude/skills/lab-hardware-cad create mode 120000 .claude/skills/labarchive-integration create mode 120000 .claude/skills/lamindb create mode 120000 .claude/skills/latchbio-integration create mode 120000 .claude/skills/latex-posters create mode 120000 .claude/skills/liteparse create mode 120000 .claude/skills/literature-review create mode 120000 .claude/skills/markdown-mermaid-writing create mode 120000 .claude/skills/market-research-reports create mode 120000 .claude/skills/markitdown create mode 120000 .claude/skills/matchms create mode 120000 .claude/skills/matlab create mode 120000 .claude/skills/matplotlib create mode 120000 .claude/skills/medchem create mode 120000 .claude/skills/modal create mode 120000 .claude/skills/molecular-dynamics create mode 120000 .claude/skills/molfeat create mode 120000 .claude/skills/ncats-arax create mode 120000 .claude/skills/networkx create mode 120000 .claude/skills/neurokit2 create mode 120000 .claude/skills/neuropixels-analysis create mode 120000 .claude/skills/nextflow create mode 120000 .claude/skills/omero-integration create mode 120000 .claude/skills/onekgpd create mode 120000 .claude/skills/ontology-term-resolution create mode 120000 .claude/skills/open-notebook create mode 120000 .claude/skills/openpiv create mode 120000 .claude/skills/opentrons-integration create mode 120000 .claude/skills/optimize-for-gpu create mode 120000 .claude/skills/pacsomatic create mode 120000 .claude/skills/paper-lookup create mode 120000 .claude/skills/paperclip create mode 120000 .claude/skills/paperzilla create mode 120000 .claude/skills/parallel-web create mode 120000 .claude/skills/pathml create mode 120000 .claude/skills/pathogen-variant-surveillance create mode 120000 .claude/skills/pathway-enrichment create mode 120000 .claude/skills/pdf create mode 120000 .claude/skills/peer-review create mode 120000 .claude/skills/pennylane create mode 120000 .claude/skills/phylogenetics create mode 120000 .claude/skills/pi-agent create mode 120000 .claude/skills/pkpd-modeling create mode 120000 .claude/skills/polars create mode 120000 .claude/skills/polars-bio create mode 120000 .claude/skills/pptx create mode 120000 .claude/skills/pptx-posters create mode 120000 .claude/skills/primekg create mode 120000 .claude/skills/protocolsio-integration create mode 120000 .claude/skills/pufferlib create mode 120000 .claude/skills/pydeseq2 create mode 120000 .claude/skills/pydicom create mode 120000 .claude/skills/pyhealth create mode 120000 .claude/skills/pylabrobot create mode 120000 .claude/skills/pymatgen create mode 120000 .claude/skills/pymc create mode 120000 .claude/skills/pymoo create mode 120000 .claude/skills/pyopenms create mode 120000 .claude/skills/pysam create mode 120000 .claude/skills/pytdc create mode 120000 .claude/skills/pytorch-lightning create mode 120000 .claude/skills/pyzotero create mode 120000 .claude/skills/qiskit create mode 120000 .claude/skills/qutip create mode 120000 .claude/skills/rdkit create mode 120000 .claude/skills/relsa-severity-assessment create mode 120000 .claude/skills/research-grants create mode 120000 .claude/skills/research-lookup create mode 120000 .claude/skills/rowan create mode 120000 .claude/skills/scanpy create mode 120000 .claude/skills/scholar-evaluation create mode 120000 .claude/skills/scientific-brainstorming create mode 120000 .claude/skills/scientific-critical-thinking create mode 120000 .claude/skills/scientific-schematics create mode 120000 .claude/skills/scientific-slides create mode 120000 .claude/skills/scientific-visualization create mode 120000 .claude/skills/scientific-writing create mode 120000 .claude/skills/scikit-bio create mode 120000 .claude/skills/scikit-learn create mode 120000 .claude/skills/scikit-survival create mode 120000 .claude/skills/scvelo create mode 120000 .claude/skills/scvi-tools create mode 120000 .claude/skills/seaborn create mode 120000 .claude/skills/shap create mode 120000 .claude/skills/simpy create mode 120000 .claude/skills/stable-baselines3 create mode 120000 .claude/skills/statistical-analysis create mode 120000 .claude/skills/statistical-power create mode 120000 .claude/skills/statsmodels create mode 120000 .claude/skills/sympy create mode 120000 .claude/skills/tamarind create mode 120000 .claude/skills/tiledbvcf create mode 120000 .claude/skills/timesfm-forecasting create mode 120000 .claude/skills/torch-geometric create mode 120000 .claude/skills/torchdrug create mode 120000 .claude/skills/transformers create mode 120000 .claude/skills/treatment-plans create mode 120000 .claude/skills/umap-learn create mode 120000 .claude/skills/uncertainty-and-units create mode 120000 .claude/skills/usfiscaldata create mode 120000 .claude/skills/vaex create mode 120000 .claude/skills/venue-templates create mode 120000 .claude/skills/waypoint-bio create mode 120000 .claude/skills/what-if-oracle create mode 120000 .claude/skills/xlsx create mode 120000 .claude/skills/zarr-python create mode 100644 skills-lock.json diff --git a/.agents/skills/adaptyv/SKILL.md b/.agents/skills/adaptyv/SKILL.md new file mode 100644 index 0000000..7e99d0a --- /dev/null +++ b/.agents/skills/adaptyv/SKILL.md @@ -0,0 +1,240 @@ +--- +name: adaptyv +description: "How to use the Adaptyv Bio Foundry API and Python SDK for protein experiment design, submission, and results retrieval. Use this skill whenever the user mentions Adaptyv, Foundry API, protein binding assays, protein screening experiments, BLI/SPR assays, thermostability assays, or wants to submit protein sequences for experimental characterization. Also trigger when code imports `adaptyv`, `adaptyv_sdk`, or `FoundryClient`, or references `foundry-api-public.adaptyvbio.com`." +license: MIT +compatibility: Requires Python 3.10+, an Adaptyv Foundry account, and an API key from foundry.adaptyvbio.com. Install adaptyv-sdk from GitHub with uv pip install. +metadata: + version: "1.2" + skill-author: K-Dense Inc. +--- + +# Adaptyv Bio Foundry API + +Adaptyv Bio is a cloud lab that turns protein sequences into experimental data. Users submit amino acid sequences via API or UI; Adaptyv's automated lab runs assays (binding, thermostability, expression, fluorescence) and delivers results in ~21 days. + +**Official docs:** [docs.adaptyvbio.com/api-reference](https://docs.adaptyvbio.com/api-reference) · [llms.txt index](https://docs.adaptyvbio.com/llms.txt) · [OpenAPI spec](https://foundry-api-public.adaptyvbio.com/api/v1/openapi.json) + +## Quick Start + +**Base URL:** `https://foundry-api-public.adaptyvbio.com/api/v1` + +**Authentication:** Bearer token in the `Authorization` header. Tokens are obtained from [foundry.adaptyvbio.com](https://foundry.adaptyvbio.com/) sidebar. + +When writing code, always read the API key from the environment variable `ADAPTYV_API_KEY` or from a `.env` file — never hardcode tokens. Check for a `.env` file in the project root first; if one exists, use a library like `python-dotenv` to load it. + +The [official API docs](https://docs.adaptyvbio.com/api-reference/api-introduction) use `FOUNDRY_API_TOKEN` in curl examples; that is the same bearer token — prefer `ADAPTYV_API_KEY` in Python and new shell scripts for consistency with the SDK. + +```bash +export ADAPTYV_API_KEY="abs0_..." +curl https://foundry-api-public.adaptyvbio.com/api/v1/targets?limit=3 \ + -H "Authorization: Bearer $ADAPTYV_API_KEY" +``` + +Every request except `GET /openapi.json` requires authentication. Store tokens in environment variables or `.env` files — never commit them to source control. + +## Python SDK + +**Version note:** `adaptyv-sdk` **0.1.0** (beta) is not yet on PyPI — install from GitHub: + +```bash +uv pip install "git+https://github.com/adaptyvbio/adaptyv-sdk.git" +``` + +In a project with `pyproject.toml`: + +```bash +uv add "adaptyv-sdk @ git+https://github.com/adaptyvbio/adaptyv-sdk.git" +``` + +**Environment variables** (set in shell or `.env` file): + +```bash +ADAPTYV_API_KEY=your_api_key +ADAPTYV_API_URL=https://foundry-api-public.adaptyvbio.com/api/v1 +ADAPTYV_ORGANIZATION_ID=your_org_id # optional +``` + +The `@lab.experiment` decorator and `FoundryClient` both read `ADAPTYV_API_KEY` and `ADAPTYV_API_URL` from the environment when not passed explicitly. + +### Decorator Pattern + +```python +from adaptyv import lab + +@lab.experiment(target="PD-L1", experiment_type="screening", method="bli") +def design_binders(): + return {"design_a": "MVKVGVNG...", "design_b": "MKVLVAG..."} + +result = design_binders() +print(f"Experiment: {result.experiment_url}") +``` + +### Client Pattern + +```python +import os +from adaptyv import FoundryClient + +client = FoundryClient( + api_key=os.environ["ADAPTYV_API_KEY"], + base_url=os.environ.get( + "ADAPTYV_API_URL", + "https://foundry-api-public.adaptyvbio.com/api/v1", + ), +) + +# Browse targets +targets = client.targets.list(search="EGFR", selfservice_only=True) + +# Estimate cost +estimate = client.experiments.cost_estimate({ + "experiment_spec": { + "experiment_type": "screening", + "method": "bli", + "target_id": "target-uuid", + "sequences": {"seq1": "EVQLVESGGGLVQ..."}, + "n_replicates": 3 + } +}) + +# Create and submit +exp = client.experiments.create({...}) +client.experiments.submit(exp.experiment_id) + +# Later: retrieve results +results = client.experiments.get_results(exp.experiment_id) +``` + +## Experiment Types + +| Type | Method | Measures | Requires Target | +|---|---|---|---| +| `affinity` | `bli` or `spr` | KD, kon, koff kinetics | Yes | +| `screening` | `bli` or `spr` | Yes/no binding | Yes | +| `thermostability` | — | Melting temperature (Tm) | No | +| `expression` | — | Expression yield | No | +| `fluorescence` | — | Fluorescence intensity | No | + +## Experiment Lifecycle + +``` +Draft → WaitingForConfirmation → QuoteSent → WaitingForMaterials → InQueue → InProduction → DataAnalysis → InReview → Done +``` + +| Status | Who Acts | Description | +|---|---|---| +| `Draft` | You | Editable, no cost commitment | +| `WaitingForConfirmation` | Adaptyv | Under review, quote being prepared | +| `QuoteSent` | You | Review and confirm the quote | +| `WaitingForMaterials` | Adaptyv | Gene fragments and target ordered | +| `InQueue` | Adaptyv | Materials arrived, queued for lab | +| `InProduction` | Adaptyv | Assay running | +| `DataAnalysis` | Adaptyv | Raw data processing and QC | +| `InReview` | Adaptyv | Final validation | +| `Done` | You | Results available | +| `Canceled` | Either | Experiment canceled | + +The `results_status` field on an experiment tracks: `none`, `partial`, or `all`. + +## Common Workflows + +### 1. Submit a Binding Screen (Step by Step) + +```python +# 1. Find a target +targets = client.targets.list(search="EGFR", selfservice_only=True) +target_id = targets.items[0].id + +# 2. Preview cost +estimate = client.experiments.cost_estimate({ + "experiment_spec": { + "experiment_type": "screening", + "method": "bli", + "target_id": target_id, + "sequences": {"seq1": "EVQLVESGGGLVQ...", "seq2": "MKVLVAG..."}, + "n_replicates": 3 + } +}) + +# 3. Create experiment (starts as Draft) +exp = client.experiments.create({ + "name": "EGFR binder screen batch 1", + "experiment_spec": { + "experiment_type": "screening", + "method": "bli", + "target_id": target_id, + "sequences": {"seq1": "EVQLVESGGGLVQ...", "seq2": "MKVLVAG..."}, + "n_replicates": 3 + } +}) + +# 4. Submit for review +client.experiments.submit(exp.experiment_id) + +# 5. Poll or use webhooks until Done +# 6. Retrieve results +results = client.experiments.get_results(exp.experiment_id) +``` + +### 2. Automated Pipeline (Skip Draft + Auto-Accept Quote) + +```python +exp = client.experiments.create({ + "name": "Auto pipeline run", + "experiment_spec": {...}, + "skip_draft": True, + "auto_accept_quote": True, + "webhook_url": "https://my-server.com/webhook" +}) +# Webhook fires on each status transition; poll or wait for Done +``` + +### 3. Using Webhooks + +Pass `webhook_url` when creating an experiment. Adaptyv POSTs to that URL on every status transition with the experiment ID, previous status, and new status. + +## Sequences + +- Simple format: `{"seq1": "EVQLVESGGGLVQPGGSLRLSCAAS"}` +- Rich format: `{"seq1": {"aa_string": "EVQLVESGGGLVQ...", "control": false, "metadata": {"type": "scfv"}}}` +- Multi-chain: use colon separator — `"MVLS:EVQL"` +- Valid amino acids: A, C, D, E, F, G, H, I, K, L, M, N, P, Q, R, S, T, V, W, Y (case-insensitive, stored uppercase) +- Sequences can only be added to experiments in `Draft` status + +## Filtering, Sorting, and Pagination + +All list endpoints support pagination (`limit` 1-100, default 50; `offset`), search (free-text on name fields), and sorting. + +**Filtering** uses s-expression syntax via the `filter` query parameter: +- Comparison: `eq(field,value)`, `neq`, `gt`, `gte`, `lt`, `lte`, `contains(field,substring)` +- Range/set: `between(field,lo,hi)`, `in(field,v1,v2,...)` +- Logic: `and(expr1,expr2,...)`, `or(...)`, `not(expr)` +- Null: `is_null(field)`, `is_not_null(field)` +- JSONB: `at(field,key)` — e.g., `eq(at(metadata,score),42)` +- Cast: `float()`, `int()`, `text()`, `timestamp()`, `date()` + +**Sorting** uses `asc(field)` or `desc(field)`, comma-separated (max 8): +``` +sort=desc(created_at),asc(name) +``` + +**Example:** `filter=and(gte(created_at,2026-01-01),eq(status,done))` + +## Error Handling + +All errors return: +```json +{ + "error": "Human-readable description", + "request_id": "req_019462a4-b1c2-7def-8901-23456789abcd" +} +``` +The `request_id` is also in the `x-request-id` response header — include it when contacting support. + +## Token Management + +Tokens use Biscuit-based cryptographic attenuation. You can create restricted tokens scoped by organization, resource type, actions (read/create/update), and expiry via `POST /tokens/attenuate`. Revoking a token (`POST /tokens/revoke`) revokes it and all its descendants. + +## Detailed API Reference + +For the full list of all 32 endpoints with request/response schemas, read `references/api-endpoints.md`. diff --git a/.agents/skills/adaptyv/references/api-endpoints.md b/.agents/skills/adaptyv/references/api-endpoints.md new file mode 100644 index 0000000..2a6d5f0 --- /dev/null +++ b/.agents/skills/adaptyv/references/api-endpoints.md @@ -0,0 +1,690 @@ +# Adaptyv Bio Foundry API — Complete Endpoint Reference + +Base URL: `https://foundry-api-public.adaptyvbio.com/api/v1` +OpenAPI spec: `GET /openapi.json` + +## Table of Contents + +- [Experiments](#experiments) +- [Sequences](#sequences) +- [Results](#results) +- [Targets](#targets) +- [Quotes](#quotes) +- [Tokens](#tokens) +- [Updates](#updates) +- [Feedback](#feedback) + +--- + +## Experiments + +### POST /experiments — Create experiment + +Creates a new experiment. Starts in `Draft` status by default. + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | string | Yes | Human-readable name | +| `experiment_spec` | ExperimentSpec | Yes | Experiment definition (see below) | +| `skip_draft` | boolean | No (default false) | Bypass Draft, go straight to WaitingForConfirmation | +| `auto_accept_quote` | boolean | No (default false) | Auto-accept quote and create invoice | +| `webhook_url` | string/null | No | URL for status-change POST notifications | + +**ExperimentSpec:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `experiment_type` | string | Yes | `affinity`, `screening`, `thermostability`, `fluorescence`, or `expression` | +| `method` | string | Required for binding types | `bli` or `spr` | +| `target_id` | uuid | Required for binding types | Target UUID from catalog | +| `sequences` | object | Yes | Map of name → amino acid string or rich object | +| `n_replicates` | integer | Recommended (default 3) | Technical replicates (min 1) | +| `antigen_concentrations` | number[] | No (affinity only) | Defaults to `[1000.0, 316.2, 100.0, 31.6, 0.0]` nM | +| `parameters` | object | No | Experiment-specific settings | + +**Field requirements by experiment type:** + +| Field | Affinity | Screening | Thermostability | Fluorescence | Expression | +|---|---|---|---|---|---| +| `experiment_type` | required | required | required | required | required | +| `method` | required | required | — | — | — | +| `target_id` | required | required | — | — | — | +| `sequences` | required | required | required | required | required | +| `n_replicates` | recommended | recommended | optional | optional | optional | +| `antigen_concentrations` | optional | — | — | — | — | + +**Response (201):** + +| Field | Type | Description | +|---|---|---| +| `experiment_id` | string | UUID of new experiment | +| `error` | string/null | Error message if validation fails | +| `stripe_hosted_invoice_url` | string/null | Present when `auto_accept_quote` created an invoice | +| `stripe_invoice_id` | string/null | Stripe invoice ID | + +**Status codes:** 201, 400, 401, 403, 404 + +--- + +### GET /experiments — List experiments + +Lists experiments accessible to caller, sorted by creation date (newest first). + +**Query params:** `limit`, `offset`, `filter`, `search`, `sort` + +**Response item:** + +| Field | Type | Description | +|---|---|---| +| `id` | uuid | Unique identifier | +| `code` | string | e.g., "EXP-2024-001" | +| `name` | string/null | Human-readable name | +| `status` | ExperimentStatus | Current lifecycle status | +| `experiment_type` | ExperimentType | affinity/screening/thermostability/fluorescence/expression | +| `results_status` | ResultsStatus | none/partial/all | +| `created_at` | datetime | ISO 8601 | +| `experiment_url` | string | URL to Foundry portal | +| `stripe_invoice_url` | string/null | Invoice URL | +| `stripe_quote_url` | string/null | Quote URL | + +**Status codes:** 200, 401 + +--- + +### GET /experiments/{experiment_id} — Get experiment + +Returns full metadata for a single experiment. + +**Path param:** `experiment_id` (uuid) + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `id` | uuid | Unique identifier | +| `code` | string | Experiment code | +| `status` | ExperimentStatus | Current status | +| `experiment_spec` | ExperimentSpec | Full experiment definition | +| `results_status` | ResultsStatus | none/partial/all | +| `created_at` | datetime | ISO 8601 | +| `experiment_url` | string | Portal URL | +| `costs` | object | Cost breakdown | + +**Status codes:** 200, 401, 404, 500 + +--- + +### PATCH /experiments/{experiment_id} — Update experiment + +Modify an existing experiment. Draft experiments allow full edits; after quote generation, only `name`, `description`, and `webhook_url` are editable. + +**Path param:** `experiment_id` (uuid) + +**Request body:** All fields optional — only provided fields are updated. + +**Status codes:** 200, 400, 401, 404, 409 + +--- + +### POST /experiments/{experiment_id}/submit — Submit experiment + +Submits a draft experiment for review. Advances from `Draft` to `WaitingForConfirmation`. + +**Path param:** `experiment_id` (uuid) + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `experiment_id` | string | Experiment UUID | + +**Status codes:** 200, 401, 403, 404, 409, 500 + +--- + +### POST /experiments/cost-estimate — Estimate cost + +Calculates cost without creating an experiment. + +**Request body:** +```json +{ + "experiment_spec": { + "experiment_type": "screening", + "method": "bli", + "target_id": "...", + "sequences": {"seq1": "MKTL..."}, + "n_replicates": 3 + } +} +``` + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `pricing_version` | string | e.g., "v1_2026-01-20" | +| `assay` | object | Per-type costs with base and replicate pricing | +| `materials` | object | Target material costs (binding experiments) | +| `total_cents` | integer | Sum in USD cents | + +All prices exclude VAT; taxes calculated at invoicing. Targets without self-service pricing return incomplete estimates. + +**Status codes:** 200, 400, 401 + +--- + +### GET /experiments/{experiment_id}/quote — Get quote + +Returns quote metadata (totals, currency, status, expiration). + +**Path param:** `experiment_id` (uuid) + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `experiment_id` | string | Experiment UUID | +| `stripe_quote_url` | string | Stripe quote URL | +| `amount_total` | int64 | Total in smallest currency unit | +| `amount_subtotal` | int64 | Subtotal | +| `currency` | string | ISO currency code (e.g., "usd") | +| `status` | string | Quote status | +| `expires_at` | datetime/null | Expiration time | + +**Status codes:** 200, 401, 403, 404, 500 + +--- + +### GET /experiments/{experiment_id}/quote/pdf — Get quote PDF + +Returns the quote as a PDF file (`application/pdf`). + +**Path param:** `experiment_id` (uuid) + +**Status codes:** 200, 401, 403, 404, 500 + +--- + +### POST /experiments/{experiment_id}/quote/confirm — Accept quote (by experiment) + +Accepts Stripe quote, creates draft invoice, transitions to `WaitingForMaterials`. + +**Path param:** `experiment_id` (uuid) + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `purchase_order_number` | string/null | No | PO number for your records | +| `notes` | string/null | No | Reserved | + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `id` | string | Quote ID | +| `status` | StripeQuoteStatus | New status | +| `hosted_invoice_url` | string/null | Stripe payment URL | +| `invoice_id` | string/null | Generated invoice ID | + +**Status codes:** 200, 401, 403, 404, 409 + +--- + +### GET /experiments/{experiment_id}/invoice — Get invoice + +Returns invoice metadata including hosted payment URL. + +**Path param:** `experiment_id` (uuid) + +**Status codes:** 200, 401, 403, 404, 500 + +--- + +### GET /experiments/{experiment_id}/results — List results for experiment + +Returns all analysis results for a specific experiment. + +**Path param:** `experiment_id` (uuid) +**Query params:** `limit`, `offset`, `filter`, `sort` + +**Status codes:** 200, 400, 401, 403, 404 + +--- + +### GET /experiments/{experiment_id}/sequences — List sequences for experiment + +Returns all sequences for a specific experiment, sorted newest first. + +**Path param:** `experiment_id` (uuid) +**Query params:** `limit`, `offset`, `search`, `sort` + +**Status codes:** 200, 400, 401, 403, 404 + +--- + +### GET /experiments/{experiment_id}/updates — List experiment updates + +Returns updates for one experiment, oldest first. Types: `status_change`, `progress`, `error`. + +**Path param:** `experiment_id` (uuid) +**Query params:** `limit`, `offset`, `filter`, `sort` + +Filter example: `filter=eq(type,status_change)` + +--- + +## Sequences + +### GET /sequences — List sequences + +Returns sequences from all experiments, sorted newest first. + +**Query params:** `limit`, `offset`, `search`, `sort`, `experiment_id` (filter by experiment UUID) + +**Response item:** + +| Field | Type | Description | +|---|---|---| +| `id` | uuid | Unique identifier | +| `name` | string/null | Optional name | +| `aa_preview` | string/null | Truncated preview (first 50 chars) | +| `length` | int32 | Sequence length in amino acids | +| `experiment_id` | uuid | Parent experiment | +| `experiment_code` | string | Human-readable experiment code | +| `is_control` | boolean | Whether this is a control | +| `created_at` | datetime | Creation timestamp | + +**Status codes:** 200, 401 + +--- + +### GET /sequences/{sequence_id} — Get sequence + +Returns full details including complete amino acid string. + +**Path param:** `sequence_id` (uuid) + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `id` | uuid | Unique identifier | +| `aa_string` | string/null | Complete amino acid sequence | +| `length` | int32 | Length in amino acids | +| `is_control` | boolean | Control flag | +| `metadata` | object | Sequence-level annotations | +| `experiment` | object | Parent experiment reference | +| `created_at` | datetime | Creation timestamp | + +**Status codes:** 200, 401, 403, 404, 500 + +--- + +### POST /sequences — Add sequences to experiment + +Appends sequences to a **Draft** experiment identified by its human-readable code. + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `experiment_code` | string | Yes | e.g., "PROJ-001" | +| `sequences` | array | Yes | Array of sequence entries | + +**Each sequence entry:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `aa_string` | string | Yes | Amino acid sequence | +| `name` | string | No | Human-readable name | +| `control` | boolean | No | Whether this is a control | +| `metadata` | object | No | Annotations | + +**Response (201):** + +| Field | Type | Description | +|---|---|---| +| `added_count` | int32 | Number of sequences added | +| `experiment_id` | string | Experiment UUID | +| `experiment_code` | string | Experiment code | +| `sequence_ids` | array | IDs of added sequences | + +**Status codes:** 201, 400, 404, 409 (experiment not in Draft), 500 + +--- + +## Results + +### GET /results — List results + +Lists completed analysis results, sorted newest first. Results appear when `results_status` reaches `partial` or `all`. + +**Query params:** `limit`, `offset`, `filter`, `search`, `sort` + +**Response item:** + +| Field | Type | Description | +|---|---|---| +| `id` | uuid | Result identifier | +| `title` | string | Human-readable title | +| `experiment_id` | uuid | Associated experiment | +| `result_type` | string | e.g., "affinity", "thermostability" | +| `summary` | array | Key results (type-specific, see below) | +| `metadata` | object | Extended metadata (e.g., instrument info) | +| `data_package_url` | string/null | Download URL for raw data package | +| `created_at` | datetime | When result was generated | + +**AffinityResult summary fields:** `kd_mean`, `kd_std`, `kon_mean`, `kon_log_std`, `koff_mean`, `koff_std`, `replicates` (array with per-replicate `kd`, `kon`, `koff`, `binding_strength`, `kon_method`, `koff_method`, `replicate` index), `sequence`, `target_id` + +**ThermostabilityResult summary fields:** Tm values and melting curves + +**Status codes:** 200, 401 + +--- + +### GET /results/{result_id} — Get result + +Returns detailed result data including full summary array. + +**Path param:** `result_id` (uuid) + +**Status codes:** 200, 401, 403, 404, 500 + +--- + +## Targets + +### GET /targets — List targets + +Lists validated antigens available for experiments. + +**Query params:** + +| Parameter | Type | Description | +|---|---|---| +| `limit` | int | Max items (1-100, default 50) | +| `offset` | int | Skip count | +| `search` | string | Free-text search on product name | +| `sort` | string | Sort expression | +| `selfservice_only` | boolean | Only targets with self-service pricing | +| `show_conjugated` | boolean | Include conjugated targets (default: unconjugated only) | +| `detailed` | boolean | Populate `details` block with enrichment data | + +**Response item:** + +| Field | Type | Description | +|---|---|---| +| `id` | uuid | Target UUID (use as `experiment_spec.target_id`) | +| `name` | string | Target name | +| `vendor_name` | string | Vendor name | +| `catalog_number` | string | Vendor catalog/SKU number | +| `url` | string | Target URL | +| `pricing` | object/null | Self-service pricing (null = custom quote required) | +| `details` | object/null | Enrichment data (gene names, structures, sequence, bioactivity) | + +**Status codes:** 200, 401 + +--- + +### GET /targets/{target_id} — Get target + +Returns catalog record for a single target. + +**Path param:** `target_id` (uuid) + +**Status codes:** 200, 400, 401, 403, 404, 500 + +--- + +### POST /targets/request-custom — Submit custom target request + +Submit a new custom target for staff review. At least one of `sequence` or `pdb_id` must be provided. + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | string | Yes | Display name | +| `product_id` | string | Yes | Must be unique within organization | +| `sequence` | string/null | At least one | Amino acid sequence | +| `pdb_id` | string/null | At least one | PDB identifier | +| `pdb_file` | string/null | No | PDB file content | +| `molecular_weight` | number/null | No | Weight in kDa | +| `note` | string/null | No | Additional notes | + +**Status codes:** 201, 400, 401, 403, 500 + +--- + +### GET /targets/request-custom — List custom target requests + +Returns custom target requests for your organization, sorted newest first. + +**Query params:** `limit`, `offset`, `filter`, `sort` + +Filter example: `filter=eq(status,pending_review)` + +--- + +### GET /targets/request-custom/{request_id} — Get custom target request + +**Path param:** `request_id` (uuid) + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `id` | uuid | Request identifier | +| `name` | string | Target name | +| `product_id` | string | Your product ID | +| `status` | string | e.g., "pending_review" | +| `material_id` | string/null | Linked catalog ID if approved | +| `molecular_weight` | number/null | Weight in kDa | +| `note` | string/null | User notes | +| `created_at` | datetime | Created | +| `updated_at` | datetime | Last updated | + +**Status codes:** 200, 401, 403, 404, 500 + +--- + +## Quotes + +### GET /quotes — List quotes + +Returns all quotes for caller's organization. + +**Query params:** `limit`, `offset`, `filter`, `sort` + +**Response item:** + +| Field | Type | Description | +|---|---|---| +| `id` | string | Quote identifier | +| `quote_number` | string | Human-readable quote number | +| `organization_id` | uuid | Organization | +| `amount_cents` | int | Amount in cents | +| `currency` | string | ISO 4217 code | +| `status` | StripeQuoteStatus | Quote status | +| `valid_until` | datetime | Expiration | +| `created_at` | datetime | Creation timestamp | + +--- + +### GET /quotes/{quote_id} — Get quote + +Returns full quote document with itemized pricing. + +**Path param:** `quote_id` (string, e.g., "qt_1Abc2DefGhi") + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `id` | string | Quote identifier | +| `quote_number` | string | Reference number | +| `organization_id` | uuid | Organization | +| `organization_name` | string | Organization name | +| `line_items` | array | Itemized pricing | +| `subtotal_cents` | int | Subtotal in cents | +| `tax_cents` | int | Tax in cents | +| `total_cents` | int | Total in cents | +| `currency` | string | ISO 4217 | +| `status` | StripeQuoteStatus | Current status | +| `valid_until` | datetime | Expiration | +| `notes` | string | Special pricing info | +| `terms_and_conditions` | string | Terms | +| `stripe_quote_url` | string | Stripe URL | +| `created_at` | datetime | Created | + +**Status codes:** 200, 401, 403, 404, 500 + +--- + +### POST /quotes/{quote_id}/confirm — Accept quote + +Finalizes quote, creates draft invoice, advances experiment to `WaitingForMaterials`. + +**Path param:** `quote_id` (string) + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `purchase_order_number` | string/null | No | PO number | +| `notes` | string/null | No | Reserved | + +**Response:** `id`, `status`, `hosted_invoice_url`, `invoice_id` + +**Status codes:** 200, 403, 404, 409, 500 + +--- + +### POST /quotes/{quote_id}/reject — Reject quote + +Cancels quote; linked experiment reverts to `Draft`. + +**Path param:** `quote_id` (string) + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `reason` | QuoteRejectionReason | Yes | Primary reason | +| `feedback` | string/null | No | Additional feedback | + +**Response:** `id`, `status` (canceled) + +**Status codes:** 200, 403, 404, 409, 500 + +--- + +## Tokens + +### GET /tokens — List tokens + +Returns all tokens (root and attenuated) the caller owns. + +**Query params:** `limit`, `offset` + +**Response item:** + +| Field | Type | Description | +|---|---|---| +| `id` | string | Token identifier | +| `name` | string | Human-readable label | +| `kind` | string | "root" or "attenuated" | +| `created_at` | datetime | Created | +| `expires_at` | datetime/null | Expiration (null = no expiry) | +| `revoked_at` | datetime/null | Revocation timestamp | +| `parent_token_id` | string/null | Parent (null for root) | +| `root_token_id` | string/null | Root of derivation tree | +| `attenuation_spec` | object/null | Restrictions (null for root) | + +--- + +### POST /tokens/attenuate — Attenuate token + +Creates a restricted version of an existing token using Biscuit cryptographic attenuation. + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `token` | string | Yes | Existing token (`abs0_{slug}{biscuit_base64}`) | +| `attenuation` | AttenuationSpec | Yes | Restrictions to apply | +| `name` | string | Yes | Human-readable label | +| `attenuated_parent_token_id` | uuid/null | No | Parent ID for chained attenuation | + +**Restriction types:** Organization, Resource (experiments/results), Action (read/create/update), Expiry + +**Response (201):** `id` (database ID), `token` (new attenuated token string) + +**Status codes:** 201, 400, 401, 403 + +--- + +### POST /tokens/revoke — Revoke token and lineage + +Revokes the calling token's root and all attenuated descendants. Idempotent. + +**Response:** + +| Field | Type | Description | +|---|---|---| +| `token_id` | string | Root token ID revoked | +| `revoked_at` | datetime | Revocation timestamp | +| `children_revoked` | int64 | Child tokens newly revoked | + +**Status codes:** 200, 403, 404 + +--- + +## Updates + +### GET /updates — List updates + +Returns the experiment update feed (newest first): status changes, progress, errors. + +**Query params:** `limit`, `offset`, `filter`, `sort` + +**Filter examples:** +- `filter=eq(experiment_id,)` +- `filter=in(experiment_id,uuid1,uuid2)` +- `filter=eq(type,status_change)` + +**Response item:** + +| Field | Type | Description | +|---|---|---| +| `id` | string | Update identifier | +| `experiment_id` | uuid | Associated experiment | +| `experiment_code` | string | Human-readable code | +| `name` | string | Update description | +| `timestamp` | datetime | When the update occurred | + +--- + +## Feedback + +### POST /feedback/submit — Submit feedback + +For bug reports, feature requests, or general feedback. + +**Request body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `request_uuid` | uuid | Yes | UUID from the problematic API request | +| `feedback_type` | FeedbackType | Yes | `feature_request`, `feedback`, or `bug_report` | +| `title` | string/null | No | Short title | +| `json_body` | object/null | At least one | Structured error details | +| `human_note` | string/null | At least one | Free-form description | + +**Response (201):** `reference` (feedback reference), `message` (confirmation) + +**Status codes:** 201, 400, 401, 500 diff --git a/.agents/skills/aeon/SKILL.md b/.agents/skills/aeon/SKILL.md new file mode 100644 index 0000000..d39f473 --- /dev/null +++ b/.agents/skills/aeon/SKILL.md @@ -0,0 +1,402 @@ +--- +name: aeon +description: This skill should be used for time series machine learning tasks including classification, regression, clustering, forecasting, anomaly detection, segmentation, and similarity search. Use when working with temporal data, sequential patterns, or time-indexed observations requiring specialized algorithms beyond standard ML approaches. Particularly suited for univariate and multivariate time series analysis with scikit-learn compatible APIs. +license: BSD-3-Clause license +allowed-tools: Read Write Edit Bash +compatibility: Requires Python 3.10+ and the aeon package (uv pip install). Optional aeon[all_extras] for deep learning and extended dependencies. +metadata: + version: "1.0" + skill-author: K-Dense Inc. +--- + +# Aeon Time Series Machine Learning + +## Overview + +Aeon is a scikit-learn compatible Python toolkit for time series machine learning ([aeon-toolkit.org](https://www.aeon-toolkit.org/)). It provides algorithms across classification, regression, clustering, forecasting, anomaly detection, segmentation, similarity search, distances, transformations, benchmarking, and visualization — with a consistent estimator API. + +**Version note:** Examples target **aeon 1.x** (stable docs: v1.4.0, March 2026). The v1.0 release reworked forecasting and transformations; import paths differ from aeon 0.x/sktime-era code. + +## When to Use This Skill + +Apply this skill when: +- Classifying or predicting from time series data +- Detecting anomalies or change points in temporal sequences +- Clustering similar time series patterns +- Forecasting future values +- Finding repeated patterns (motifs) or unusual subsequences (discords) +- Comparing time series with specialized distance metrics +- Extracting features from temporal data + +## Installation + +Requires **Python 3.10+** (3.11+ recommended). Pin a 1.x release for reproducibility: + +```bash +uv pip install "aeon>=1.4,<2" +``` + +For deep learning forecasters/classifiers and other optional estimators: + +```bash +uv pip install "aeon[all_extras]>=1.4,<2" +``` + +On zsh, quote the extras: `uv pip install "aeon[all_extras]>=1.4,<2"`. + +### Experimental modules + +Upstream treats **forecasting**, **anomaly_detection**, **segmentation**, **similarity_search**, and **visualisation** as experimental — interfaces may change between minor releases. Prefer stable modules (classification, regression, clustering, distances, transformations) for production pipelines unless you need these tasks. + +## Core Capabilities + +### 1. Time Series Classification + +Categorize time series into predefined classes. See `references/classification.md` for complete algorithm catalog. + +**Quick Start:** +```python +from aeon.classification.convolution_based import RocketClassifier +from aeon.datasets import load_classification + +# Load data +X_train, y_train = load_classification("GunPoint", split="train") +X_test, y_test = load_classification("GunPoint", split="test") + +# Train classifier +clf = RocketClassifier(n_kernels=10000) +clf.fit(X_train, y_train) +accuracy = clf.score(X_test, y_test) +``` + +**Algorithm Selection:** +- **Speed + Performance**: `MiniRocketClassifier`, `Arsenal` +- **Maximum Accuracy**: `HIVECOTEV2`, `InceptionTimeClassifier` +- **Interpretability**: `ShapeletTransformClassifier`, `Catch22Classifier` +- **Small Datasets**: `KNeighborsTimeSeriesClassifier` with DTW distance + +### 2. Time Series Regression + +Predict continuous values from time series. See `references/regression.md` for algorithms. + +**Quick Start:** +```python +from aeon.regression.convolution_based import RocketRegressor +from aeon.datasets import load_regression + +X_train, y_train = load_regression("Covid3Month", split="train") +X_test, y_test = load_regression("Covid3Month", split="test") + +reg = RocketRegressor() +reg.fit(X_train, y_train) +predictions = reg.predict(X_test) +``` + +### 3. Time Series Clustering + +Group similar time series without labels. See `references/clustering.md` for methods. + +**Quick Start:** +```python +from aeon.clustering import TimeSeriesKMeans + +clusterer = TimeSeriesKMeans( + n_clusters=3, + distance="dtw", + averaging_method="ba" +) +labels = clusterer.fit_predict(X_train) +centers = clusterer.cluster_centers_ +``` + +### 4. Forecasting + +Predict future time series values (experimental module in aeon 1.x). See `references/forecasting.md` for forecasters. + +**Quick Start:** +```python +import numpy as np +from aeon.forecasting import NaiveForecaster +from aeon.forecasting.stats import ARIMA + +y_train = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) + +# Set horizon in the constructor; predict passes the series to forecast from +naive = NaiveForecaster(strategy="last", horizon=5) +naive.fit(y_train) +y_pred = naive.predict(y_train) + +# ARIMA uses p/d/q (not order=); multi-step via iterative_forecast +arima = ARIMA(p=1, d=1, q=1) +arima.fit(y_train) +y_pred = arima.iterative_forecast(y_train, prediction_horizon=5) +``` + +### 5. Anomaly Detection + +Identify unusual patterns or outliers. See `references/anomaly_detection.md` for detectors. + +**Quick Start:** +```python +from aeon.anomaly_detection import STOMP + +detector = STOMP(window_size=50) +anomaly_scores = detector.fit_predict(y) + +# Higher scores indicate anomalies +threshold = np.percentile(anomaly_scores, 95) +anomalies = anomaly_scores > threshold +``` + +### 6. Segmentation + +Partition time series into regions with change points. See `references/segmentation.md`. + +**Quick Start:** +```python +from aeon.segmentation import ClaSPSegmenter + +segmenter = ClaSPSegmenter() +change_points = segmenter.fit_predict(y) +``` + +### 7. Similarity Search + +Find similar patterns within or across time series. See `references/similarity_search.md`. + +**Quick Start:** +```python +from aeon.similarity_search import StompMotif + +# Find recurring patterns +motif_finder = StompMotif(window_size=50, k=3) +motifs = motif_finder.fit_predict(y) +``` + +## Feature Extraction and Transformations + +Transform time series for feature engineering. See `references/transformations.md`. + +**ROCKET Features:** +```python +from aeon.transformations.collection.convolution_based import RocketTransformer + +rocket = RocketTransformer() +X_features = rocket.fit_transform(X_train) + +# Use features with any sklearn classifier +from sklearn.ensemble import RandomForestClassifier +clf = RandomForestClassifier() +clf.fit(X_features, y_train) +``` + +**Statistical Features:** +```python +from aeon.transformations.collection.feature_based import Catch22 + +catch22 = Catch22() +X_features = catch22.fit_transform(X_train) +``` + +**Preprocessing:** +```python +from aeon.transformations.collection import MinMaxScaler, Normalizer + +scaler = Normalizer() # Z-normalization +X_normalized = scaler.fit_transform(X_train) +``` + +## Distance Metrics + +Specialized temporal distance measures. See `references/distances.md` for complete catalog. + +**Usage:** +```python +from aeon.distances import dtw_distance, dtw_pairwise_distance + +# Single distance +distance = dtw_distance(x, y, window=0.1) + +# Pairwise distances +distance_matrix = dtw_pairwise_distance(X_train) + +# Use with classifiers +from aeon.classification.distance_based import KNeighborsTimeSeriesClassifier + +clf = KNeighborsTimeSeriesClassifier( + n_neighbors=5, + distance="dtw", + distance_params={"window": 0.2} +) +``` + +**Available Distances:** +- **Elastic**: DTW, DDTW, WDTW, ERP, EDR, LCSS, TWE, MSM +- **Lock-step**: Euclidean, Manhattan, Minkowski +- **Shape-based**: Shape DTW, SBD + +## Deep Learning Networks + +Neural architectures for time series. See `references/networks.md`. + +**Architectures:** +- Convolutional: `FCNClassifier`, `ResNetClassifier`, `InceptionTimeClassifier` +- Recurrent: `RecurrentNetwork`, `TCNNetwork` +- Autoencoders: `AEFCNClusterer`, `AEResNetClusterer` + +**Usage:** +```python +from aeon.classification.deep_learning import InceptionTimeClassifier + +clf = InceptionTimeClassifier(n_epochs=100, batch_size=32) +clf.fit(X_train, y_train) +predictions = clf.predict(X_test) +``` + +## Datasets and Benchmarking + +Load standard benchmarks and evaluate performance. See `references/datasets_benchmarking.md`. + +**Load Datasets:** +```python +from aeon.datasets import load_classification, load_gunpoint, load_regression + +# Classification (generic loader or dataset-specific helper) +X_train, y_train = load_classification("GunPoint", split="train") +X_train, y_train = load_gunpoint(split="train") # same UCR dataset + +# Regression +X_train, y_train = load_regression("Covid3Month", split="train") +``` + +**Benchmarking:** +```python +from aeon.benchmarking import get_estimator_results + +# Compare with published results +published = get_estimator_results("ROCKET", "GunPoint") +``` + +## Common Workflows + +### Classification Pipeline + +```python +from aeon.transformations.collection import Normalizer +from aeon.classification.convolution_based import RocketClassifier +from sklearn.pipeline import Pipeline + +pipeline = Pipeline([ + ('normalize', Normalizer()), + ('classify', RocketClassifier()) +]) + +pipeline.fit(X_train, y_train) +accuracy = pipeline.score(X_test, y_test) +``` + +### Feature Extraction + Traditional ML + +```python +from aeon.transformations.collection import RocketTransformer +from sklearn.ensemble import GradientBoostingClassifier + +# Extract features +rocket = RocketTransformer() +X_train_features = rocket.fit_transform(X_train) +X_test_features = rocket.transform(X_test) + +# Train traditional ML +clf = GradientBoostingClassifier() +clf.fit(X_train_features, y_train) +predictions = clf.predict(X_test_features) +``` + +### Anomaly Detection with Visualization + +```python +from aeon.anomaly_detection import STOMP +import matplotlib.pyplot as plt + +detector = STOMP(window_size=50) +scores = detector.fit_predict(y) + +plt.figure(figsize=(15, 5)) +plt.subplot(2, 1, 1) +plt.plot(y, label='Time Series') +plt.subplot(2, 1, 2) +plt.plot(scores, label='Anomaly Scores', color='red') +plt.axhline(np.percentile(scores, 95), color='k', linestyle='--') +plt.show() +``` + +## Best Practices + +### Data Preparation + +1. **Normalize**: Most algorithms benefit from z-normalization + ```python + from aeon.transformations.collection import Normalizer + normalizer = Normalizer() + X_train = normalizer.fit_transform(X_train) + X_test = normalizer.transform(X_test) + ``` + +2. **Handle Missing Values**: Impute before analysis + ```python + from aeon.transformations.collection import SimpleImputer + imputer = SimpleImputer(strategy='mean') + X_train = imputer.fit_transform(X_train) + ``` + +3. **Check Data Format**: Collections use `(n_cases, n_channels, n_timepoints)`; single series use `(n_channels, n_timepoints)` (see [data format](https://www.aeon-toolkit.org/en/stable/api_reference/data_format.html)) + +### Model Selection + +1. **Start Simple**: Begin with ROCKET variants before deep learning +2. **Use Validation**: Split training data for hyperparameter tuning +3. **Compare Baselines**: Test against simple methods (1-NN Euclidean, Naive) +4. **Consider Resources**: ROCKET for speed, deep learning if GPU available + +### Algorithm Selection Guide + +**For Fast Prototyping:** +- Classification: `MiniRocketClassifier` +- Regression: `MiniRocketRegressor` +- Clustering: `TimeSeriesKMeans` with Euclidean + +**For Maximum Accuracy:** +- Classification: `HIVECOTEV2`, `InceptionTimeClassifier` +- Regression: `InceptionTimeRegressor` +- Forecasting: `AutoARIMA`, `AutoETS`, `TCNForecaster` (requires `[all_extras]` for deep learning) + +**For Interpretability:** +- Classification: `ShapeletTransformClassifier`, `Catch22Classifier` +- Features: `Catch22`, `TSFresh` + +**For Small Datasets:** +- Distance-based: `KNeighborsTimeSeriesClassifier` with DTW +- Avoid: Deep learning (requires large data) + +## Reference Documentation + +Detailed information available in `references/`: +- `classification.md` - All classification algorithms +- `regression.md` - Regression methods +- `clustering.md` - Clustering algorithms +- `forecasting.md` - Forecasting approaches +- `anomaly_detection.md` - Anomaly detection methods +- `segmentation.md` - Segmentation algorithms +- `similarity_search.md` - Pattern matching and motif discovery +- `transformations.md` - Feature extraction and preprocessing +- `distances.md` - Time series distance metrics +- `networks.md` - Deep learning architectures +- `datasets_benchmarking.md` - Data loading and evaluation tools + +## Additional Resources + +- Documentation: https://www.aeon-toolkit.org/ +- GitHub: https://github.com/aeon-toolkit/aeon +- Examples: https://www.aeon-toolkit.org/en/stable/examples.html +- API Reference: https://www.aeon-toolkit.org/en/stable/api_reference.html + diff --git a/.agents/skills/aeon/references/anomaly_detection.md b/.agents/skills/aeon/references/anomaly_detection.md new file mode 100644 index 0000000..a205c7c --- /dev/null +++ b/.agents/skills/aeon/references/anomaly_detection.md @@ -0,0 +1,154 @@ +# Anomaly Detection + +Aeon provides anomaly detection methods for identifying unusual patterns in time series at both series and collection levels. + +## Collection Anomaly Detectors + +Detect anomalous time series within a collection: + +- `ClassificationAdapter` - Adapts classifiers for anomaly detection + - Train on normal data, flag outliers during prediction + - **Use when**: Have labeled normal data, want classification-based approach + +- `OutlierDetectionAdapter` - Wraps sklearn outlier detectors + - Works with IsolationForest, LOF, OneClassSVM + - **Use when**: Want to use sklearn anomaly detectors on collections + +## Series Anomaly Detectors + +Detect anomalous points or subsequences within a single time series. + +### Distance-Based Methods + +Use similarity metrics to identify anomalies: + +- `CBLOF` - Cluster-Based Local Outlier Factor + - Clusters data, identifies outliers based on cluster properties + - **Use when**: Anomalies form sparse clusters + +- `KMeansAD` - K-means based anomaly detection + - Distance to nearest cluster center indicates anomaly + - **Use when**: Normal patterns cluster well + +- `LeftSTAMPi` - Left STAMP incremental + - Matrix profile for online anomaly detection + - **Use when**: Streaming data, need online detection + +- `STOMP` - Scalable Time series Ordered-search Matrix Profile + - Computes matrix profile for subsequence anomalies + - **Use when**: Discord discovery, motif detection + +- `MERLIN` - Matrix profile-based method + - Efficient matrix profile computation + - **Use when**: Large time series, need scalability + +- `LOF` - Local Outlier Factor adapted for time series + - Density-based outlier detection + - **Use when**: Anomalies in low-density regions + +- `ROCKAD` - ROCKET-based semi-supervised detection + - Uses ROCKET features for anomaly identification + - **Use when**: Have some labeled data, want feature-based approach + +### Distribution-Based Methods + +Analyze statistical distributions: + +- `COPOD` - Copula-Based Outlier Detection + - Models marginal and joint distributions + - **Use when**: Multi-dimensional time series, complex dependencies + +- `DWT_MLEAD` - Discrete Wavelet Transform Multi-Level Anomaly Detection + - Decomposes series into frequency bands + - **Use when**: Anomalies at specific frequencies + +### Isolation-Based Methods + +Use isolation principles: + +- `IsolationForest` - Random forest-based isolation + - Anomalies easier to isolate than normal points + - **Use when**: High-dimensional data, no assumptions about distribution + +- `OneClassSVM` - Support vector machine for novelty detection + - Learns boundary around normal data + - **Use when**: Well-defined normal region, need robust boundary + +- `STRAY` - Streaming Robust Anomaly Detection + - Robust to data distribution changes + - **Use when**: Streaming data, distribution shifts + +### External Library Integration + +- `PyODAdapter` - Bridges PyOD library to aeon + - Access 40+ PyOD anomaly detectors + - **Use when**: Need specific PyOD algorithm + +## Quick Start + +```python +from aeon.anomaly_detection import STOMP +import numpy as np + +# Create time series with anomaly +y = np.concatenate([ + np.sin(np.linspace(0, 10, 100)), + [5.0], # Anomaly spike + np.sin(np.linspace(10, 20, 100)) +]) + +# Detect anomalies +detector = STOMP(window_size=10) +anomaly_scores = detector.fit_predict(y) + +# Higher scores indicate more anomalous points +threshold = np.percentile(anomaly_scores, 95) +anomalies = anomaly_scores > threshold +``` + +## Point vs Subsequence Anomalies + +- **Point anomalies**: Single unusual values + - Use: COPOD, DWT_MLEAD, IsolationForest + +- **Subsequence anomalies** (discords): Unusual patterns + - Use: STOMP, LeftSTAMPi, MERLIN + +- **Collective anomalies**: Groups of points forming unusual pattern + - Use: Matrix profile methods, clustering-based + +## Evaluation Metrics + +Specialized metrics for anomaly detection: + +```python +from aeon.benchmarking.metrics.anomaly_detection import ( + range_precision, + range_recall, + range_f_score, + roc_auc_score +) + +# Range-based metrics account for window detection +precision = range_precision(y_true, y_pred, alpha=0.5) +recall = range_recall(y_true, y_pred, alpha=0.5) +f1 = range_f_score(y_true, y_pred, alpha=0.5) +``` + +## Algorithm Selection + +- **Speed priority**: KMeansAD, IsolationForest +- **Accuracy priority**: STOMP, COPOD +- **Streaming data**: LeftSTAMPi, STRAY +- **Discord discovery**: STOMP, MERLIN +- **Multi-dimensional**: COPOD, PyODAdapter +- **Semi-supervised**: ROCKAD, OneClassSVM +- **No training data**: IsolationForest, STOMP + +## Best Practices + +1. **Normalize data**: Many methods sensitive to scale +2. **Choose window size**: For matrix profile methods, window size critical +3. **Set threshold**: Use percentile-based or domain-specific thresholds +4. **Validate results**: Visualize detections to verify meaningfulness +5. **Handle seasonality**: Detrend/deseasonalize before detection diff --git a/.agents/skills/aeon/references/classification.md b/.agents/skills/aeon/references/classification.md new file mode 100644 index 0000000..af5b8be --- /dev/null +++ b/.agents/skills/aeon/references/classification.md @@ -0,0 +1,144 @@ +# Time Series Classification + +Aeon provides 13 categories of time series classifiers with scikit-learn compatible APIs. + +## Convolution-Based Classifiers + +Apply random convolutional transformations for efficient feature extraction: + +- `Arsenal` - Ensemble of ROCKET classifiers with varied kernels +- `HydraClassifier` - Multi-resolution convolution with dilation +- `RocketClassifier` - Random convolution kernels with ridge regression +- `MiniRocketClassifier` - Simplified ROCKET variant for speed +- `MultiRocketClassifier` - Combines multiple ROCKET variants + +**Use when**: Need fast, scalable classification with strong performance across diverse datasets. + +## Deep Learning Classifiers + +Neural network architectures optimized for temporal sequences: + +- `FCNClassifier` - Fully convolutional network +- `ResNetClassifier` - Residual networks with skip connections +- `InceptionTimeClassifier` - Multi-scale inception modules +- `TimeCNNClassifier` - Standard CNN for time series +- `MLPClassifier` - Multi-layer perceptron baseline +- `EncoderClassifier` - Generic encoder wrapper +- `DisjointCNNClassifier` - Shapelet-focused architecture + +**Use when**: Large datasets available, need end-to-end learning, or complex temporal patterns. + +## Dictionary-Based Classifiers + +Transform time series into symbolic representations: + +- `BOSSEnsemble` - Bag-of-SFA-Symbols with ensemble voting +- `TemporalDictionaryEnsemble` - Multiple dictionary methods combined +- `WEASEL` - Word ExtrAction for time SEries cLassification +- `MrSEQLClassifier` - Multiple symbolic sequence learning + +**Use when**: Need interpretable models, sparse patterns, or symbolic reasoning. + +## Distance-Based Classifiers + +Leverage specialized time series distance metrics: + +- `KNeighborsTimeSeriesClassifier` - k-NN with temporal distances (DTW, LCSS, ERP, etc.) +- `ElasticEnsemble` - Combines multiple elastic distance measures +- `ProximityForest` - Tree ensemble using distance-based splits + +**Use when**: Small datasets, need similarity-based classification, or interpretable decisions. + +## Feature-Based Classifiers + +Extract statistical and signature features before classification: + +- `Catch22Classifier` - 22 canonical time-series characteristics +- `TSFreshClassifier` - Automated feature extraction via tsfresh +- `SignatureClassifier` - Path signature transformations +- `SummaryClassifier` - Summary statistics extraction +- `FreshPRINCEClassifier` - Combines multiple feature extractors + +**Use when**: Need interpretable features, domain expertise available, or feature engineering approach. + +## Interval-Based Classifiers + +Extract features from random or supervised intervals: + +- `CanonicalIntervalForestClassifier` - Random interval features with decision trees +- `DrCIFClassifier` - Diverse Representation CIF with catch22 features +- `TimeSeriesForestClassifier` - Random intervals with summary statistics +- `RandomIntervalClassifier` - Simple interval-based approach +- `RandomIntervalSpectralEnsembleClassifier` - Spectral features from intervals +- `SupervisedTimeSeriesForest` - Supervised interval selection + +**Use when**: Discriminative patterns occur in specific time windows. + +## Shapelet-Based Classifiers + +Identify discriminative subsequences (shapelets): + +- `ShapeletTransformClassifier` - Discovers and uses discriminative shapelets +- `LearningShapeletClassifier` - Learns shapelets via gradient descent +- `SASTClassifier` - Scalable approximate shapelet transform +- `RDSTClassifier` - Random dilated shapelet transform + +**Use when**: Need interpretable discriminative patterns or phase-invariant features. + +## Hybrid Classifiers + +Combine multiple classification paradigms: + +- `HIVECOTEV1` - Hierarchical Vote Collective of Transformation-based Ensembles (version 1) +- `HIVECOTEV2` - Enhanced version with updated components + +**Use when**: Maximum accuracy required, computational resources available. + +## Early Classification + +Make predictions before observing entire time series: + +- `TEASER` - Two-tier Early and Accurate Series Classifier +- `ProbabilityThresholdEarlyClassifier` - Prediction when confidence exceeds threshold + +**Use when**: Real-time decisions needed, or observations have cost. + +## Ordinal Classification + +Handle ordered class labels: + +- `OrdinalTDE` - Temporal dictionary ensemble for ordinal outputs + +**Use when**: Classes have natural ordering (e.g., severity levels). + +## Composition Tools + +Build custom pipelines and ensembles: + +- `ClassifierPipeline` - Chain transformers with classifiers +- `WeightedEnsembleClassifier` - Weighted combination of classifiers +- `SklearnClassifierWrapper` - Adapt sklearn classifiers for time series + +## Quick Start + +```python +from aeon.classification.convolution_based import RocketClassifier +from aeon.datasets import load_classification + +# Load data +X_train, y_train = load_classification("GunPoint", split="train") +X_test, y_test = load_classification("GunPoint", split="test") + +# Train and predict +clf = RocketClassifier() +clf.fit(X_train, y_train) +accuracy = clf.score(X_test, y_test) +``` + +## Algorithm Selection + +- **Speed priority**: MiniRocketClassifier, Arsenal +- **Accuracy priority**: HIVECOTEV2, InceptionTimeClassifier +- **Interpretability**: ShapeletTransformClassifier, Catch22Classifier +- **Small data**: KNeighborsTimeSeriesClassifier, Distance-based methods +- **Large data**: Deep learning classifiers, ROCKET variants diff --git a/.agents/skills/aeon/references/clustering.md b/.agents/skills/aeon/references/clustering.md new file mode 100644 index 0000000..87dd469 --- /dev/null +++ b/.agents/skills/aeon/references/clustering.md @@ -0,0 +1,123 @@ +# Time Series Clustering + +Aeon provides clustering algorithms adapted for temporal data with specialized distance metrics and averaging methods. + +## Partitioning Algorithms + +Standard k-means/k-medoids adapted for time series: + +- `TimeSeriesKMeans` - K-means with temporal distance metrics (DTW, Euclidean, etc.) +- `TimeSeriesKMedoids` - Uses actual time series as cluster centers +- `TimeSeriesKShape` - Shape-based clustering algorithm +- `TimeSeriesKernelKMeans` - Kernel-based variant for nonlinear patterns + +**Use when**: Known number of clusters, spherical cluster shapes expected. + +## Large Dataset Methods + +Efficient clustering for large collections: + +- `TimeSeriesCLARA` - Clustering Large Applications with sampling +- `TimeSeriesCLARANS` - Randomized search variant of CLARA + +**Use when**: Dataset too large for standard k-medoids, need scalability. + +## Elastic Distance Clustering + +Specialized for alignment-based similarity: + +- `KASBA` - K-means with shift-invariant elastic averaging +- `ElasticSOM` - Self-organizing map using elastic distances + +**Use when**: Time series have temporal shifts or warping. + +## Spectral Methods + +Graph-based clustering: + +- `KSpectralCentroid` - Spectral clustering with centroid computation + +**Use when**: Non-convex cluster shapes, need graph-based approach. + +## Deep Learning Clustering + +Neural network-based clustering with auto-encoders: + +- `AEFCNClusterer` - Fully convolutional auto-encoder +- `AEResNetClusterer` - Residual network auto-encoder +- `AEDCNNClusterer` - Dilated CNN auto-encoder +- `AEDRNNClusterer` - Dilated RNN auto-encoder +- `AEBiGRUClusterer` - Bidirectional GRU auto-encoder +- `AEAttentionBiGRUClusterer` - Attention-enhanced BiGRU auto-encoder + +**Use when**: Large datasets, need learned representations, or complex patterns. + +## Feature-Based Clustering + +Transform to feature space before clustering: + +- `Catch22Clusterer` - Clusters on 22 canonical features +- `SummaryClusterer` - Uses summary statistics +- `TSFreshClusterer` - Automated tsfresh features + +**Use when**: Raw time series not informative, need interpretable features. + +## Composition + +Build custom clustering pipelines: + +- `ClustererPipeline` - Chain transformers with clusterers + +## Averaging Methods + +Compute cluster centers for time series: + +- `mean_average` - Arithmetic mean +- `ba_average` - Barycentric averaging with DTW +- `kasba_average` - Shift-invariant averaging +- `shift_invariant_average` - General shift-invariant method + +**Use when**: Need representative cluster centers for visualization or initialization. + +## Quick Start + +```python +from aeon.clustering import TimeSeriesKMeans +from aeon.datasets import load_classification + +# Load data (using classification data for clustering) +X_train, _ = load_classification("GunPoint", split="train") + +# Cluster time series +clusterer = TimeSeriesKMeans( + n_clusters=3, + distance="dtw", # Use DTW distance + averaging_method="ba" # Barycentric averaging +) +labels = clusterer.fit_predict(X_train) +centers = clusterer.cluster_centers_ +``` + +## Algorithm Selection + +- **Speed priority**: TimeSeriesKMeans with Euclidean distance +- **Temporal alignment**: KASBA, TimeSeriesKMeans with DTW +- **Large datasets**: TimeSeriesCLARA, TimeSeriesCLARANS +- **Complex patterns**: Deep learning clusterers +- **Interpretability**: Catch22Clusterer, SummaryClusterer +- **Non-convex clusters**: KSpectralCentroid + +## Distance Metrics + +Compatible distance metrics include: +- Euclidean, Manhattan, Minkowski (lock-step) +- DTW, DDTW, WDTW (elastic with alignment) +- ERP, EDR, LCSS (edit-based) +- MSM, TWE (specialized elastic) + +## Evaluation + +Use clustering metrics from sklearn or aeon benchmarking: +- Silhouette score +- Davies-Bouldin index +- Calinski-Harabasz index diff --git a/.agents/skills/aeon/references/datasets_benchmarking.md b/.agents/skills/aeon/references/datasets_benchmarking.md new file mode 100644 index 0000000..01fbc08 --- /dev/null +++ b/.agents/skills/aeon/references/datasets_benchmarking.md @@ -0,0 +1,392 @@ +# Datasets and Benchmarking + +Aeon provides comprehensive tools for loading datasets and benchmarking time series algorithms. + +From **aeon 1.4** onward, most classification and regression archives are hosted on **Zenodo** (including the relaunched Multiverse multivariate classification archive). Loaders download on first use; cache location follows aeon defaults. + +## Dataset Loading + +### Task-Specific Loaders + +**Classification Datasets**: +```python +from aeon.datasets import load_classification + +# Load train/test split (or use load_gunpoint for this benchmark) +from aeon.datasets import load_gunpoint + +X_train, y_train = load_classification("GunPoint", split="train") +X_test, y_test = load_classification("GunPoint", split="test") +# X_train, y_train = load_gunpoint(split="train") + +# Load entire dataset +X, y = load_classification("GunPoint") +``` + +**Regression Datasets**: +```python +from aeon.datasets import load_regression + +X_train, y_train = load_regression("Covid3Month", split="train") +X_test, y_test = load_regression("Covid3Month", split="test") + +# Bulk download +from aeon.datasets import download_all_regression +download_all_regression() # Downloads Monash TSER archive +``` + +**Forecasting Datasets**: +```python +from aeon.datasets import load_forecasting + +# Load from forecastingdata.org +y, X = load_forecasting("airline", return_X_y=True) +``` + +**Anomaly Detection Datasets**: +```python +from aeon.datasets import load_anomaly_detection + +X, y = load_anomaly_detection("NAB_realKnownCause") +``` + +### File Format Loaders + +**Load from .ts files**: +```python +from aeon.datasets import load_from_ts_file + +X, y = load_from_ts_file("path/to/data.ts") +``` + +**Load from .tsf files**: +```python +from aeon.datasets import load_from_tsf_file + +df, metadata = load_from_tsf_file("path/to/data.tsf") +``` + +**Load from ARFF files**: +```python +from aeon.datasets import load_from_arff_file + +X, y = load_from_arff_file("path/to/data.arff") +``` + +**Load from TSV files**: +```python +from aeon.datasets import load_from_tsv_file + +data = load_from_tsv_file("path/to/data.tsv") +``` + +**Load TimeEval CSV**: +```python +from aeon.datasets import load_from_timeeval_csv_file + +X, y = load_from_timeeval_csv_file("path/to/timeeval.csv") +``` + +### Writing Datasets + +**Write to .ts format**: +```python +from aeon.datasets import write_to_ts_file + +write_to_ts_file(X, "output.ts", y=y, problem_name="MyDataset") +``` + +**Write to ARFF format**: +```python +from aeon.datasets import write_to_arff_file + +write_to_arff_file(X, "output.arff", y=y) +``` + +## Built-in Datasets + +Aeon includes several benchmark datasets for quick testing: + +### Classification +- `ArrowHead` - Shape classification +- `GunPoint` - Gesture recognition +- `ItalyPowerDemand` - Energy demand +- `BasicMotions` - Motion classification +- And 100+ more from UCR/UEA archives + +### Regression +- `Covid3Month` - COVID forecasting +- Various datasets from Monash TSER archive + +### Segmentation +- Time series segmentation datasets +- Human activity data +- Sensor data collections + +### Special Collections +- `RehabPile` - Rehabilitation data (classification & regression) + +## Dataset Metadata + +Get information about datasets: + +```python +from aeon.datasets import get_dataset_meta_data + +metadata = get_dataset_meta_data("GunPoint") +print(metadata) +# {'n_train': 50, 'n_test': 150, 'length': 150, 'n_classes': 2, ...} +``` + +## Benchmarking Tools + +### Loading Published Results + +Access pre-computed benchmark results: + +```python +from aeon.benchmarking import get_estimator_results + +# Get results for specific algorithm on dataset +results = get_estimator_results( + estimator_name="ROCKET", + dataset_name="GunPoint" +) + +# Get all available estimators for a dataset +estimators = get_available_estimators("GunPoint") +``` + +### Resampling Strategies + +Create reproducible train/test splits: + +```python +from aeon.benchmarking import stratified_resample + +# Stratified resampling maintaining class distribution +X_train, X_test, y_train, y_test = stratified_resample( + X, y, + random_state=42, + test_size=0.3 +) +``` + +### Performance Metrics + +Specialized metrics for time series tasks: + +**Anomaly Detection Metrics**: +```python +from aeon.benchmarking.metrics.anomaly_detection import ( + range_precision, + range_recall, + range_f_score, + range_roc_auc_score +) + +# Range-based metrics for window detection +precision = range_precision(y_true, y_pred, alpha=0.5) +recall = range_recall(y_true, y_pred, alpha=0.5) +f1 = range_f_score(y_true, y_pred, alpha=0.5) +auc = range_roc_auc_score(y_true, y_scores) +``` + +**Clustering Metrics**: +```python +from aeon.benchmarking.metrics.clustering import clustering_accuracy + +# Clustering accuracy with label matching +accuracy = clustering_accuracy(y_true, y_pred) +``` + +**Segmentation Metrics**: +```python +from aeon.benchmarking.metrics.segmentation import ( + count_error, + hausdorff_error +) + +# Number of change points difference +count_err = count_error(y_true, y_pred) + +# Maximum distance between predicted and true change points +hausdorff_err = hausdorff_error(y_true, y_pred) +``` + +### Statistical Testing + +Post-hoc analysis for algorithm comparison: + +```python +from aeon.benchmarking import ( + nemenyi_test, + wilcoxon_test +) + +# Nemenyi test for multiple algorithms +results = nemenyi_test(scores_matrix, alpha=0.05) + +# Pairwise Wilcoxon signed-rank test +stat, p_value = wilcoxon_test(scores_alg1, scores_alg2) +``` + +## Benchmark Collections + +### UCR/UEA Time Series Archives + +Access to comprehensive benchmark repositories: + +```python +# Classification: 112 univariate + 30 multivariate datasets +X_train, y_train = load_classification("Chinatown", split="train") + +# Automatically downloads from timeseriesclassification.com +``` + +### Monash Forecasting Archive + +```python +# Load forecasting datasets +y = load_forecasting("nn5_daily", return_X_y=False) +``` + +### Published Benchmark Results + +Pre-computed results from major competitions: + +- 2017 Univariate Bake-off +- 2021 Multivariate Classification +- 2023 Univariate Bake-off + +## Workflow Example + +Complete benchmarking workflow: + +```python +from aeon.datasets import load_classification +from aeon.classification.convolution_based import RocketClassifier +from aeon.benchmarking import get_estimator_results +from sklearn.metrics import accuracy_score +import numpy as np + +# Load dataset +dataset_name = "GunPoint" +X_train, y_train = load_classification(dataset_name, split="train") +X_test, y_test = load_classification(dataset_name, split="test") + +# Train model +clf = RocketClassifier(n_kernels=10000, random_state=42) +clf.fit(X_train, y_train) +y_pred = clf.predict(X_test) + +# Evaluate +accuracy = accuracy_score(y_test, y_pred) +print(f"Accuracy: {accuracy:.4f}") + +# Compare with published results +published = get_estimator_results("ROCKET", dataset_name) +print(f"Published ROCKET accuracy: {published['accuracy']:.4f}") +``` + +## Best Practices + +### 1. Use Standard Splits + +For reproducibility, use provided train/test splits: + +```python +# Good: Use standard splits +X_train, y_train = load_classification("GunPoint", split="train") +X_test, y_test = load_classification("GunPoint", split="test") + +# Avoid: Creating custom splits +X, y = load_classification("GunPoint") +X_train, X_test, y_train, y_test = train_test_split(X, y) +``` + +### 2. Set Random Seeds + +Ensure reproducibility: + +```python +clf = RocketClassifier(random_state=42) +results = stratified_resample(X, y, random_state=42) +``` + +### 3. Report Multiple Metrics + +Don't rely on single metric: + +```python +from sklearn.metrics import accuracy_score, f1_score, precision_score + +accuracy = accuracy_score(y_test, y_pred) +f1 = f1_score(y_test, y_pred, average='weighted') +precision = precision_score(y_test, y_pred, average='weighted') +``` + +### 4. Cross-Validation + +For robust evaluation on small datasets: + +```python +from sklearn.model_selection import cross_val_score + +scores = cross_val_score( + clf, X_train, y_train, + cv=5, + scoring='accuracy' +) +print(f"CV Accuracy: {scores.mean():.4f} (+/- {scores.std():.4f})") +``` + +### 5. Compare Against Baselines + +Always compare with simple baselines: + +```python +from aeon.classification.distance_based import KNeighborsTimeSeriesClassifier + +# Simple baseline: 1-NN with Euclidean distance +baseline = KNeighborsTimeSeriesClassifier(n_neighbors=1, distance="euclidean") +baseline.fit(X_train, y_train) +baseline_acc = baseline.score(X_test, y_test) + +print(f"Baseline: {baseline_acc:.4f}") +print(f"Your model: {accuracy:.4f}") +``` + +### 6. Statistical Significance + +Test if improvements are statistically significant: + +```python +from aeon.benchmarking import wilcoxon_test + +# Run on multiple datasets +accuracies_alg1 = [0.85, 0.92, 0.78, 0.88] +accuracies_alg2 = [0.83, 0.90, 0.76, 0.86] + +stat, p_value = wilcoxon_test(accuracies_alg1, accuracies_alg2) +if p_value < 0.05: + print("Difference is statistically significant") +``` + +## Dataset Discovery + +Find datasets matching criteria: + +```python +# List all available classification datasets +from aeon.datasets import get_available_datasets + +datasets = get_available_datasets("classification") +print(f"Found {len(datasets)} classification datasets") + +# Filter by properties +univariate_datasets = [ + d for d in datasets + if get_dataset_meta_data(d)['n_channels'] == 1 +] +``` diff --git a/.agents/skills/aeon/references/distances.md b/.agents/skills/aeon/references/distances.md new file mode 100644 index 0000000..4710203 --- /dev/null +++ b/.agents/skills/aeon/references/distances.md @@ -0,0 +1,256 @@ +# Distance Metrics + +Aeon provides specialized distance functions for measuring similarity between time series, compatible with both aeon and scikit-learn estimators. + +## Distance Categories + +### Elastic Distances + +Allow flexible temporal alignment between series: + +**Dynamic Time Warping Family:** +- `dtw` - Classic Dynamic Time Warping +- `ddtw` - Derivative DTW (compares derivatives) +- `wdtw` - Weighted DTW (penalizes warping by location) +- `wddtw` - Weighted Derivative DTW +- `shape_dtw` - Shape-based DTW + +**Edit-Based:** +- `erp` - Edit distance with Real Penalty +- `edr` - Edit Distance on Real sequences +- `lcss` - Longest Common SubSequence +- `twe` - Time Warp Edit distance + +**Specialized:** +- `msm` - Move-Split-Merge distance +- `adtw` - Amerced DTW +- `sbd` - Shape-Based Distance + +**Use when**: Time series may have temporal shifts, speed variations, or phase differences. + +### Lock-Step Distances + +Compare time series point-by-point without alignment: + +- `euclidean` - Euclidean distance (L2 norm) +- `manhattan` - Manhattan distance (L1 norm) +- `minkowski` - Generalized Minkowski distance (Lp norm) +- `squared` - Squared Euclidean distance + +**Use when**: Series already aligned, need computational speed, or no temporal warping expected. + +## Usage Patterns + +### Computing Single Distance + +```python +from aeon.distances import dtw_distance + +# Distance between two time series +distance = dtw_distance(x, y) + +# With window constraint (Sakoe-Chiba band) +distance = dtw_distance(x, y, window=0.1) +``` + +### Pairwise Distance Matrix + +```python +from aeon.distances import dtw_pairwise_distance + +# All pairwise distances in collection +X = [series1, series2, series3, series4] +distance_matrix = dtw_pairwise_distance(X) + +# Cross-collection distances +distance_matrix = dtw_pairwise_distance(X_train, X_test) +``` + +### Cost Matrix and Alignment Path + +```python +from aeon.distances import dtw_cost_matrix, dtw_alignment_path + +# Get full cost matrix +cost_matrix = dtw_cost_matrix(x, y) + +# Get optimal alignment path +path = dtw_alignment_path(x, y) +# Returns indices: [(0,0), (1,1), (2,1), (2,2), ...] +``` + +### Using with Estimators + +```python +from aeon.classification.distance_based import KNeighborsTimeSeriesClassifier + +# Use DTW distance in classifier +clf = KNeighborsTimeSeriesClassifier( + n_neighbors=5, + distance="dtw", + distance_params={"window": 0.2} +) +clf.fit(X_train, y_train) +``` + +## Distance Parameters + +### Window Constraints + +Limit warping path deviation (improves speed and prevents pathological warping): + +```python +# Sakoe-Chiba band: window as fraction of series length +dtw_distance(x, y, window=0.1) # Allow 10% deviation + +# Itakura parallelogram: slopes constrain path +dtw_distance(x, y, itakura_max_slope=2.0) +``` + +### Normalization + +Control whether to z-normalize series before distance computation: + +```python +# Most elastic distances support normalization +distance = dtw_distance(x, y, normalize=True) +``` + +### Distance-Specific Parameters + +```python +# ERP: penalty for gaps +distance = erp_distance(x, y, g=0.5) + +# TWE: stiffness and penalty parameters +distance = twe_distance(x, y, nu=0.001, lmbda=1.0) + +# LCSS: epsilon threshold for matching +distance = lcss_distance(x, y, epsilon=0.5) +``` + +## Algorithm Selection + +### By Use Case: + +**Temporal misalignment**: DTW, DDTW, WDTW +**Speed variations**: DTW with window constraint +**Shape similarity**: Shape DTW, SBD +**Edit operations**: ERP, EDR, LCSS +**Derivative matching**: DDTW +**Computational speed**: Euclidean, Manhattan +**Outlier robustness**: Manhattan, LCSS + +### By Computational Cost: + +**Fastest**: Euclidean (O(n)) +**Fast**: Constrained DTW (O(nw) where w is window) +**Medium**: Full DTW (O(n²)) +**Slower**: Complex elastic distances (ERP, TWE, MSM) + +## Quick Reference Table + +| Distance | Alignment | Speed | Robustness | Interpretability | +|----------|-----------|-------|------------|------------------| +| Euclidean | Lock-step | Very Fast | Low | High | +| DTW | Elastic | Medium | Medium | Medium | +| DDTW | Elastic | Medium | High | Medium | +| WDTW | Elastic | Medium | Medium | Medium | +| ERP | Edit-based | Slow | High | Low | +| LCSS | Edit-based | Slow | Very High | Low | +| Shape DTW | Elastic | Medium | Medium | High | + +## Best Practices + +### 1. Normalization + +Most distances sensitive to scale; normalize when appropriate: + +```python +from aeon.transformations.collection import Normalizer + +normalizer = Normalizer() +X_normalized = normalizer.fit_transform(X) +``` + +### 2. Window Constraints + +For DTW variants, use window constraints for speed and better generalization: + +```python +# Start with 10-20% window +distance = dtw_distance(x, y, window=0.1) +``` + +### 3. Series Length + +- Equal-length required: Most lock-step distances +- Unequal-length supported: Elastic distances (DTW, ERP, etc.) + +### 4. Multivariate Series + +Most distances support multivariate time series: + +```python +# x.shape = (n_channels, n_timepoints) +distance = dtw_distance(x_multivariate, y_multivariate) +``` + +### 5. Performance Optimization + +- Use numba-compiled implementations (default in aeon) +- Consider lock-step distances if alignment not needed +- Use windowed DTW instead of full DTW +- Precompute distance matrices for repeated use + +### 6. Choosing the Right Distance + +```python +# Quick decision tree: +if series_aligned: + use_distance = "euclidean" +elif need_speed: + use_distance = "dtw" # with window constraint +elif temporal_shifts_expected: + use_distance = "dtw" or "shape_dtw" +elif outliers_present: + use_distance = "lcss" or "manhattan" +elif derivatives_matter: + use_distance = "ddtw" or "wddtw" +``` + +## Integration with scikit-learn + +Aeon distances work with sklearn estimators: + +```python +from sklearn.neighbors import KNeighborsClassifier +from aeon.distances import dtw_pairwise_distance + +# Precompute distance matrix +X_train_distances = dtw_pairwise_distance(X_train) + +# Use with sklearn +clf = KNeighborsClassifier(metric='precomputed') +clf.fit(X_train_distances, y_train) +``` + +## Available Distance Functions + +Get list of all available distances: + +```python +from aeon.distances import get_distance_function_names + +print(get_distance_function_names()) +# ['dtw', 'ddtw', 'wdtw', 'euclidean', 'erp', 'edr', ...] +``` + +Retrieve specific distance function: + +```python +from aeon.distances import get_distance_function + +distance_func = get_distance_function("dtw") +result = distance_func(x, y, window=0.1) +``` diff --git a/.agents/skills/aeon/references/forecasting.md b/.agents/skills/aeon/references/forecasting.md new file mode 100644 index 0000000..769c9d1 --- /dev/null +++ b/.agents/skills/aeon/references/forecasting.md @@ -0,0 +1,109 @@ +# Time Series Forecasting + +The `aeon.forecasting` module provides forecasters for univariate and multivariate series. In aeon **1.x**, forecasting was rebuilt on array-native `BaseForecaster` estimators (replacing the old sktime-style `fh` API). The module is marked **experimental** — expect API evolution between releases. + +Import paths (aeon 1.4+): + +- `from aeon.forecasting import NaiveForecaster, RegressionForecaster` +- `from aeon.forecasting.stats import ARIMA, AutoARIMA, ETS, AutoETS, Theta, TAR, AutoTAR, TVP` +- `from aeon.forecasting.deep_learning import TCNForecaster, DeepARForecaster` + +List all forecasters: `aeon.utils.discovery.all_estimators(type_filter="forecaster")`. + +## Naive and Baseline Methods + +- `NaiveForecaster` — `strategy` in `"last"`, `"mean"`, `"seasonal_last"`; set `horizon` and `seasonal_period` in the constructor + - **Use when**: Establishing baselines or simple patterns + +## Statistical Models + +- `ARIMA` / `AutoARIMA` — `p`, `d`, `q` orders (not `order=(p,d,q)`); supports exogenous variables via `exog` +- `ETS` / `AutoETS` — exponential smoothing (native implementations in aeon 1.4+) +- `Theta` — classical Theta method +- `TAR` / `AutoTAR` — threshold autoregressive models for regime switching +- `TVP` — time-varying parameter (Kalman-style) models + +## Deep Learning Forecasters + +Requires `aeon[all_extras]` (PyTorch stack): + +- `TCNForecaster` — temporal convolutional network +- `DeepARForecaster` — probabilistic RNN forecaster (replaces legacy `DeepARNetwork` naming) + +## Regression-Based Forecasting + +- `RegressionForecaster` — sliding `window` over history, `horizon` steps ahead, any sklearn/aeon regressor + +## Quick Start + +```python +import numpy as np +from aeon.forecasting import NaiveForecaster +from aeon.forecasting.stats import ARIMA, AutoETS + +y = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) + +# Naive — horizon is a constructor argument; predict(y) forecasts from series y +naive = NaiveForecaster(strategy="last", horizon=3) +naive.fit(y) +pred_naive = naive.predict(y) + +# ARIMA — one-step by default; multi-step via iterative_forecast +arima = ARIMA(p=1, d=1, q=1) +arima.fit(y) +pred_arima = arima.iterative_forecast(y, prediction_horizon=3) + +# Auto model selection +auto_ets = AutoETS(horizon=3) +auto_ets.fit(y) +pred_ets = auto_ets.predict(y) +``` + +## Forecasting Horizon + +In aeon 1.x, set `horizon` on the estimator (number of steps ahead). `predict(y)` returns the forecast `horizon` steps beyond the end of `y`. + +Multi-step strategies: + +- **`iterative_forecast(y, prediction_horizon)`** — reuse one fitted model, feed predictions back (ARIMA, many stats models) +- **`direct_forecast(y, prediction_horizon)`** — refit per horizon (requires `capability:horizon` tag; e.g. `RegressionForecaster`) +- **`NaiveForecaster`** — set `horizon>1` directly when `strategy` supports it + +There is no `ForecastingHorizon` / `fh=[1,2,3]` API in aeon 1.x. + +## Model Selection + +- **Baseline**: `NaiveForecaster(strategy="seasonal_last", seasonal_period=12, horizon=h)` +- **Linear / stationary**: `ARIMA`, `AutoARIMA` +- **Trend + seasonality**: `ETS`, `AutoETS` +- **Regime changes**: `TAR`, `AutoTAR` +- **Complex patterns**: `TCNForecaster`, `RegressionForecaster` with aeon regressors +- **Probabilistic**: `DeepARForecaster` + +## Evaluation Metrics + +Use scikit-learn or standard numpy metrics on hold-out forecasts: + +```python +from sklearn.metrics import mean_absolute_error, mean_squared_error + +mae = mean_absolute_error(y_true, y_pred) +mse = mean_squared_error(y_true, y_pred) +``` + +## Exogenous Variables + +Pass aligned exogenous arrays as `exog` (not `X`): + +```python +forecaster.fit(y_train, exog=exog_train) +y_pred = forecaster.predict(y_test, exog=exog_test) +``` + +## Base Classes + +- `BaseForecaster` — `horizon`, `axis`, `fit`, `predict`, `forecast` +- `DirectForecastingMixin` / `IterativeForecastingMixin` — multi-step helpers +- `BaseDeepForecaster` — deep learning forecasters + +Extend `BaseForecaster` for custom forecasters. diff --git a/.agents/skills/aeon/references/networks.md b/.agents/skills/aeon/references/networks.md new file mode 100644 index 0000000..f908180 --- /dev/null +++ b/.agents/skills/aeon/references/networks.md @@ -0,0 +1,289 @@ +# Deep Learning Networks + +Aeon provides neural network architectures specifically designed for time series tasks. These networks serve as building blocks for classification, regression, clustering, and forecasting. + +## Core Network Architectures + +### Convolutional Networks + +**FCNNetwork** - Fully Convolutional Network +- Three convolutional blocks with batch normalization +- Global average pooling for dimensionality reduction +- **Use when**: Need simple yet effective CNN baseline + +**ResNetNetwork** - Residual Network +- Residual blocks with skip connections +- Prevents vanishing gradients in deep networks +- **Use when**: Deep networks needed, training stability important + +**InceptionNetwork** - Inception Modules +- Multi-scale feature extraction with parallel convolutions +- Different kernel sizes capture patterns at various scales +- **Use when**: Patterns exist at multiple temporal scales + +**TimeCNNNetwork** - Standard CNN +- Basic convolutional architecture +- **Use when**: Simple CNN sufficient, interpretability valued + +**DisjointCNNNetwork** - Separate Pathways +- Disjoint convolutional pathways +- **Use when**: Different feature extraction strategies needed + +**DCNNNetwork** - Dilated CNN +- Dilated convolutions for large receptive fields +- **Use when**: Long-range dependencies without many layers + +### Recurrent Networks + +**RecurrentNetwork** - RNN/LSTM/GRU +- Configurable cell type (RNN, LSTM, GRU) +- Sequential modeling of temporal dependencies +- **Use when**: Sequential dependencies critical, variable-length series + +### Temporal Convolutional Network + +**TCNNetwork** - Temporal Convolutional Network +- Dilated causal convolutions +- Large receptive field without recurrence +- **Use when**: Long sequences, need parallelizable architecture + +### Multi-Layer Perceptron + +**MLPNetwork** - Basic Feedforward +- Simple fully-connected layers +- Flattens time series before processing +- **Use when**: Baseline needed, computational limits, or simple patterns + +## Encoder-Based Architectures + +Networks designed for representation learning and clustering. + +### Autoencoder Variants + +**EncoderNetwork** - Generic Encoder +- Flexible encoder structure +- **Use when**: Custom encoding needed + +**AEFCNNetwork** - FCN-based Autoencoder +- Fully convolutional encoder-decoder +- **Use when**: Need convolutional representation learning + +**AEResNetNetwork** - ResNet Autoencoder +- Residual blocks in encoder-decoder +- **Use when**: Deep autoencoding with skip connections + +**AEDCNNNetwork** - Dilated CNN Autoencoder +- Dilated convolutions for compression +- **Use when**: Need large receptive field in autoencoder + +**AEDRNNNetwork** - Dilated RNN Autoencoder +- Dilated recurrent connections +- **Use when**: Sequential patterns with long-range dependencies + +**AEBiGRUNetwork** - Bidirectional GRU +- Bidirectional recurrent encoding +- **Use when**: Context from both directions helpful + +**AEAttentionBiGRUNetwork** - Attention + BiGRU +- Attention mechanism on BiGRU outputs +- **Use when**: Need to focus on important time steps + +## Specialized Architectures + +**LITENetwork** - Lightweight Inception Time Ensemble +- Efficient inception-based architecture +- LITEMV variant for multivariate series +- **Use when**: Need efficiency with strong performance + +**DeepARForecaster** - Probabilistic forecasting (use via `aeon.forecasting.deep_learning`) +- Autoregressive RNN for forecasting +- Produces probabilistic predictions +- **Use when**: Need forecast uncertainty quantification + +## Usage with Estimators + +Networks are typically used within estimators, not directly: + +```python +from aeon.classification.deep_learning import FCNClassifier +from aeon.regression.deep_learning import ResNetRegressor +from aeon.clustering.deep_learning import AEFCNClusterer + +# Classification with FCN +clf = FCNClassifier(n_epochs=100, batch_size=16) +clf.fit(X_train, y_train) + +# Regression with ResNet +reg = ResNetRegressor(n_epochs=100) +reg.fit(X_train, y_train) + +# Clustering with autoencoder +clusterer = AEFCNClusterer(n_clusters=3, n_epochs=100) +labels = clusterer.fit_predict(X_train) +``` + +## Custom Network Configuration + +Many networks accept configuration parameters: + +```python +# Configure FCN layers +clf = FCNClassifier( + n_epochs=200, + batch_size=32, + kernel_size=[7, 5, 3], # Kernel sizes for each layer + n_filters=[128, 256, 128], # Filters per layer + learning_rate=0.001 +) +``` + +## Base Classes + +- `BaseDeepLearningNetwork` - Abstract base for all networks +- `BaseDeepRegressor` - Base for deep regression +- `BaseDeepClassifier` - Base for deep classification +- `BaseDeepForecaster` - Base for deep forecasting + +Extend these to implement custom architectures. + +## Training Considerations + +### Hyperparameters + +Key hyperparameters to tune: + +- `n_epochs` - Training iterations (50-200 typical) +- `batch_size` - Samples per batch (16-64 typical) +- `learning_rate` - Step size (0.0001-0.01) +- Network-specific: layers, filters, kernel sizes + +### Callbacks + +Many networks support callbacks for training monitoring: + +```python +from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau + +clf = FCNClassifier( + n_epochs=200, + callbacks=[ + EarlyStopping(patience=20, restore_best_weights=True), + ReduceLROnPlateau(patience=10, factor=0.5) + ] +) +``` + +### GPU Acceleration + +Deep learning networks benefit from GPU: + +```python +import os +os.environ['CUDA_VISIBLE_DEVICES'] = '0' # Use first GPU + +# Networks automatically use GPU if available +clf = InceptionTimeClassifier(n_epochs=100) +clf.fit(X_train, y_train) +``` + +## Architecture Selection + +### By Task: + +**Classification**: InceptionNetwork, ResNetNetwork, FCNNetwork +**Regression**: InceptionNetwork, ResNetNetwork, TCNNetwork +**Forecasting**: TCNForecaster, DeepARForecaster, RecurrentNetwork +**Clustering**: AEFCNNetwork, AEResNetNetwork, AEAttentionBiGRUNetwork + +### By Data Characteristics: + +**Long sequences**: TCNNetwork, DCNNNetwork (dilated convolutions) +**Short sequences**: MLPNetwork, FCNNetwork +**Multivariate**: InceptionNetwork, FCNNetwork, LITENetwork +**Variable length**: RecurrentNetwork with masking +**Multi-scale patterns**: InceptionNetwork + +### By Computational Resources: + +**Limited compute**: MLPNetwork, LITENetwork +**Moderate compute**: FCNNetwork, TimeCNNNetwork +**High compute available**: InceptionNetwork, ResNetNetwork +**GPU available**: Any deep network (major speedup) + +## Best Practices + +### 1. Data Preparation + +Normalize input data: + +```python +from aeon.transformations.collection import Normalizer + +normalizer = Normalizer() +X_train_norm = normalizer.fit_transform(X_train) +X_test_norm = normalizer.transform(X_test) +``` + +### 2. Training/Validation Split + +Use validation set for early stopping: + +```python +from sklearn.model_selection import train_test_split + +X_train_fit, X_val, y_train_fit, y_val = train_test_split( + X_train, y_train, test_size=0.2, stratify=y_train +) + +clf = FCNClassifier(n_epochs=200) +clf.fit(X_train_fit, y_train_fit, validation_data=(X_val, y_val)) +``` + +### 3. Start Simple + +Begin with simpler architectures before complex ones: + +1. Try MLPNetwork or FCNNetwork first +2. If insufficient, try ResNetNetwork or InceptionNetwork +3. Consider ensembles if single models insufficient + +### 4. Hyperparameter Tuning + +Use grid search or random search: + +```python +from sklearn.model_selection import GridSearchCV + +param_grid = { + 'n_epochs': [100, 200], + 'batch_size': [16, 32], + 'learning_rate': [0.001, 0.0001] +} + +clf = FCNClassifier() +grid = GridSearchCV(clf, param_grid, cv=3) +grid.fit(X_train, y_train) +``` + +### 5. Regularization + +Prevent overfitting: +- Use dropout (if network supports) +- Early stopping +- Data augmentation (if available) +- Reduce model complexity + +### 6. Reproducibility + +Set random seeds: + +```python +import numpy as np +import random +import tensorflow as tf + +seed = 42 +np.random.seed(seed) +random.seed(seed) +tf.random.set_seed(seed) +``` diff --git a/.agents/skills/aeon/references/regression.md b/.agents/skills/aeon/references/regression.md new file mode 100644 index 0000000..f3580e7 --- /dev/null +++ b/.agents/skills/aeon/references/regression.md @@ -0,0 +1,118 @@ +# Time Series Regression + +Aeon provides time series regressors across 9 categories for predicting continuous values from temporal sequences. + +## Convolution-Based Regressors + +Apply convolutional kernels for feature extraction: + +- `HydraRegressor` - Multi-resolution dilated convolutions +- `RocketRegressor` - Random convolutional kernels +- `MiniRocketRegressor` - Simplified ROCKET for speed +- `MultiRocketRegressor` - Combined ROCKET variants +- `MultiRocketHydraRegressor` - Merges ROCKET and Hydra approaches + +**Use when**: Need fast regression with strong baseline performance. + +## Deep Learning Regressors + +Neural architectures for end-to-end temporal regression: + +- `FCNRegressor` - Fully convolutional network +- `ResNetRegressor` - Residual blocks with skip connections +- `InceptionTimeRegressor` - Multi-scale inception modules +- `TimeCNNRegressor` - Standard CNN architecture +- `RecurrentRegressor` - RNN/LSTM/GRU variants +- `MLPRegressor` - Multi-layer perceptron +- `EncoderRegressor` - Generic encoder wrapper +- `LITERegressor` - Lightweight inception time ensemble +- `DisjointCNNRegressor` - Specialized CNN architecture + +**Use when**: Large datasets, complex patterns, or need feature learning. + +## Distance-Based Regressors + +k-nearest neighbors with temporal distance metrics: + +- `KNeighborsTimeSeriesRegressor` - k-NN with DTW, LCSS, ERP, or other distances + +**Use when**: Small datasets, local similarity patterns, or interpretable predictions. + +## Feature-Based Regressors + +Extract statistical features before regression: + +- `Catch22Regressor` - 22 canonical time-series characteristics +- `FreshPRINCERegressor` - Pipeline combining multiple feature extractors +- `SummaryRegressor` - Summary statistics features +- `TSFreshRegressor` - Automated tsfresh feature extraction + +**Use when**: Need interpretable features or domain-specific feature engineering. + +## Hybrid Regressors + +Combine multiple approaches: + +- `RISTRegressor` - Randomized Interval-Shapelet Transformation + +**Use when**: Benefit from combining interval and shapelet methods. + +## Interval-Based Regressors + +Extract features from time intervals: + +- `CanonicalIntervalForestRegressor` - Random intervals with decision trees +- `DrCIFRegressor` - Diverse Representation CIF +- `TimeSeriesForestRegressor` - Random interval ensemble +- `RandomIntervalRegressor` - Simple interval-based approach +- `RandomIntervalSpectralEnsembleRegressor` - Spectral interval features +- `QUANTRegressor` - Quantile-based interval features + +**Use when**: Predictive patterns occur in specific time windows. + +## Shapelet-Based Regressors + +Use discriminative subsequences for prediction: + +- `RDSTRegressor` - Random Dilated Shapelet Transform + +**Use when**: Need phase-invariant discriminative patterns. + +## Composition Tools + +Build custom regression pipelines: + +- `RegressorPipeline` - Chain transformers with regressors +- `RegressorEnsemble` - Weighted ensemble with learnable weights +- `SklearnRegressorWrapper` - Adapt sklearn regressors for time series + +## Utilities + +- `DummyRegressor` - Baseline strategies (mean, median) +- `BaseRegressor` - Abstract base for custom regressors +- `BaseDeepRegressor` - Base for deep learning regressors + +## Quick Start + +```python +from aeon.regression.convolution_based import RocketRegressor +from aeon.datasets import load_regression + +# Load data +X_train, y_train = load_regression("Covid3Month", split="train") +X_test, y_test = load_regression("Covid3Month", split="test") + +# Train and predict +reg = RocketRegressor() +reg.fit(X_train, y_train) +predictions = reg.predict(X_test) +``` + +## Algorithm Selection + +- **Speed priority**: MiniRocketRegressor +- **Accuracy priority**: InceptionTimeRegressor, MultiRocketHydraRegressor +- **Interpretability**: Catch22Regressor, SummaryRegressor +- **Small data**: KNeighborsTimeSeriesRegressor +- **Large data**: Deep learning regressors, ROCKET variants +- **Interval patterns**: DrCIFRegressor, CanonicalIntervalForestRegressor diff --git a/.agents/skills/aeon/references/segmentation.md b/.agents/skills/aeon/references/segmentation.md new file mode 100644 index 0000000..9807d93 --- /dev/null +++ b/.agents/skills/aeon/references/segmentation.md @@ -0,0 +1,163 @@ +# Time Series Segmentation + +Aeon provides algorithms to partition time series into regions with distinct characteristics, identifying change points and boundaries. + +## Segmentation Algorithms + +### Binary Segmentation +- `BinSegmenter` - Recursive binary segmentation + - Iteratively splits series at most significant change points + - Parameters: `n_segments`, `cost_function` + - **Use when**: Known number of segments, hierarchical structure + +### Classification-Based +- `ClaSPSegmenter` - Classification Score Profile + - Uses classification performance to identify boundaries + - Discovers segments where classification distinguishes neighbors + - **Use when**: Segments have different temporal patterns + +### Fast Pattern-Based +- `FLUSSSegmenter` - Fast Low-cost Unipotent Semantic Segmentation + - Efficient semantic segmentation using arc crossings + - Based on matrix profile + - **Use when**: Large time series, need speed and pattern discovery + +### Information Theory +- `InformationGainSegmenter` - Information gain maximization + - Finds boundaries maximizing information gain + - **Use when**: Statistical differences between segments + +### Gaussian Modeling +- `GreedyGaussianSegmenter` - Greedy Gaussian approximation + - Models segments as Gaussian distributions + - Incrementally adds change points + - **Use when**: Segments follow Gaussian distributions + +### Hierarchical Agglomerative +- `EAggloSegmenter` - Bottom-up merging approach + - Estimates change points via agglomeration + - **Use when**: Want hierarchical segmentation structure + +### Hidden Markov Models +- `HMMSegmenter` - HMM with Viterbi decoding + - Probabilistic state-based segmentation + - **Use when**: Segments represent hidden states + +### Dimensionality-Based +- `HidalgoSegmenter` - Heterogeneous Intrinsic Dimensionality Algorithm + - Detects changes in local dimensionality + - **Use when**: Dimensionality shifts between segments + +### Baseline +- `RandomSegmenter` - Random change point generation + - **Use when**: Need null hypothesis baseline + +## Quick Start + +```python +from aeon.segmentation import ClaSPSegmenter +import numpy as np + +# Create time series with regime changes +y = np.concatenate([ + np.sin(np.linspace(0, 10, 100)), # Segment 1 + np.cos(np.linspace(0, 10, 100)), # Segment 2 + np.sin(2 * np.linspace(0, 10, 100)) # Segment 3 +]) + +# Segment the series +segmenter = ClaSPSegmenter() +change_points = segmenter.fit_predict(y) + +print(f"Detected change points: {change_points}") +``` + +## Output Format + +Segmenters return change point indices: + +```python +# change_points = [100, 200] # Boundaries between segments +# This divides series into: [0:100], [100:200], [200:end] +``` + +## Algorithm Selection + +- **Speed priority**: FLUSSSegmenter, BinSegmenter +- **Accuracy priority**: ClaSPSegmenter, HMMSegmenter +- **Known segment count**: BinSegmenter with n_segments parameter +- **Unknown segment count**: ClaSPSegmenter, InformationGainSegmenter +- **Pattern changes**: FLUSSSegmenter, ClaSPSegmenter +- **Statistical changes**: InformationGainSegmenter, GreedyGaussianSegmenter +- **State transitions**: HMMSegmenter + +## Common Use Cases + +### Regime Change Detection +Identify when time series behavior fundamentally changes: + +```python +from aeon.segmentation import InformationGainSegmenter + +segmenter = InformationGainSegmenter(k=3) # Up to 3 change points +change_points = segmenter.fit_predict(stock_prices) +``` + +### Activity Segmentation +Segment sensor data into activities: + +```python +from aeon.segmentation import ClaSPSegmenter + +segmenter = ClaSPSegmenter() +boundaries = segmenter.fit_predict(accelerometer_data) +``` + +### Seasonal Boundary Detection +Find season transitions in time series: + +```python +from aeon.segmentation import HMMSegmenter + +segmenter = HMMSegmenter(n_states=4) # 4 seasons +segments = segmenter.fit_predict(temperature_data) +``` + +## Evaluation Metrics + +Use segmentation quality metrics: + +```python +from aeon.benchmarking.metrics.segmentation import ( + count_error, + hausdorff_error +) + +# Count error: difference in number of change points +count_err = count_error(y_true, y_pred) + +# Hausdorff: maximum distance between predicted and true points +hausdorff_err = hausdorff_error(y_true, y_pred) +``` + +## Best Practices + +1. **Normalize data**: Ensures change detection not dominated by scale +2. **Choose appropriate metric**: Different algorithms optimize different criteria +3. **Validate segments**: Visualize to verify meaningful boundaries +4. **Handle noise**: Consider smoothing before segmentation +5. **Domain knowledge**: Use expected segment count if known +6. **Parameter tuning**: Adjust sensitivity parameters (thresholds, penalties) + +## Visualization + +```python +import matplotlib.pyplot as plt + +plt.figure(figsize=(12, 4)) +plt.plot(y, label='Time Series') +for cp in change_points: + plt.axvline(cp, color='r', linestyle='--', label='Change Point') +plt.legend() +plt.show() +``` diff --git a/.agents/skills/aeon/references/similarity_search.md b/.agents/skills/aeon/references/similarity_search.md new file mode 100644 index 0000000..d5a1e5c --- /dev/null +++ b/.agents/skills/aeon/references/similarity_search.md @@ -0,0 +1,187 @@ +# Similarity Search + +Aeon provides tools for finding similar patterns within and across time series, including subsequence search, motif discovery, and approximate nearest neighbors. + +## Subsequence Nearest Neighbors (SNN) + +Find most similar subsequences within a time series. + +### MASS Algorithm +- `MassSNN` - Mueen's Algorithm for Similarity Search + - Fast normalized cross-correlation for similarity + - Computes distance profile efficiently + - **Use when**: Need exact nearest neighbor distances, large series + +### STOMP-Based Motif Discovery +- `StompMotif` - Discovers recurring patterns (motifs) + - Finds top-k most similar subsequence pairs + - Based on matrix profile computation + - **Use when**: Want to discover repeated patterns + +### Brute Force Baseline +- `DummySNN` - Exhaustive distance computation + - Computes all pairwise distances + - **Use when**: Small series, need exact baseline + +## Collection-Level Search + +Find similar time series across collections. + +### Approximate Nearest Neighbors (ANN) +- `RandomProjectionIndexANN` - Locality-sensitive hashing + - Uses random projections with cosine similarity + - Builds index for fast approximate search + - **Use when**: Large collection, speed more important than exactness + +## Quick Start: Motif Discovery + +```python +from aeon.similarity_search import StompMotif +import numpy as np + +# Create time series with repeated patterns +pattern = np.sin(np.linspace(0, 2*np.pi, 50)) +y = np.concatenate([ + pattern + np.random.normal(0, 0.1, 50), + np.random.normal(0, 1, 100), + pattern + np.random.normal(0, 0.1, 50), + np.random.normal(0, 1, 100) +]) + +# Find top-3 motifs +motif_finder = StompMotif(window_size=50, k=3) +motifs = motif_finder.fit_predict(y) + +# motifs contains indices of motif occurrences +for i, (idx1, idx2) in enumerate(motifs): + print(f"Motif {i+1} at positions {idx1} and {idx2}") +``` + +## Quick Start: Subsequence Search + +```python +from aeon.similarity_search import MassSNN +import numpy as np + +# Time series to search within +y = np.sin(np.linspace(0, 20, 500)) + +# Query subsequence +query = np.sin(np.linspace(0, 2, 50)) + +# Find nearest subsequences +searcher = MassSNN() +distances = searcher.fit_transform(y, query) + +# Find best match +best_match_idx = np.argmin(distances) +print(f"Best match at index {best_match_idx}") +``` + +## Quick Start: Approximate NN on Collections + +```python +from aeon.similarity_search import RandomProjectionIndexANN +from aeon.datasets import load_classification + +# Load time series collection +X_train, _ = load_classification("GunPoint", split="train") + +# Build index +ann = RandomProjectionIndexANN(n_projections=8, n_bits=4) +ann.fit(X_train) + +# Find approximate nearest neighbors +query = X_train[0] +neighbors, distances = ann.kneighbors(query, k=5) +``` + +## Matrix Profile + +The matrix profile is a fundamental data structure for many similarity search tasks: + +- **Distance Profile**: Distances from a query to all subsequences +- **Matrix Profile**: Minimum distance for each subsequence to any other +- **Motif**: Pair of subsequences with minimum distance +- **Discord**: Subsequence with maximum minimum distance (anomaly) + +```python +from aeon.similarity_search import StompMotif + +# Compute matrix profile and find motifs/discords +mp = StompMotif(window_size=50) +mp.fit(y) + +# Access matrix profile +profile = mp.matrix_profile_ +profile_indices = mp.matrix_profile_index_ + +# Find discords (anomalies) +discord_idx = np.argmax(profile) +``` + +## Algorithm Selection + +- **Exact subsequence search**: MassSNN +- **Motif discovery**: StompMotif +- **Anomaly detection**: Matrix profile (see anomaly_detection.md) +- **Fast approximate search**: RandomProjectionIndexANN +- **Small data**: DummySNN for exact results + +## Use Cases + +### Pattern Matching +Find where a pattern occurs in a long series: + +```python +# Find heartbeat pattern in ECG data +searcher = MassSNN() +distances = searcher.fit_transform(ecg_data, heartbeat_pattern) +occurrences = np.where(distances < threshold)[0] +``` + +### Motif Discovery +Identify recurring patterns: + +```python +# Find repeated behavioral patterns +motif_finder = StompMotif(window_size=100, k=5) +motifs = motif_finder.fit_predict(activity_data) +``` + +### Time Series Retrieval +Find similar time series in database: + +```python +# Build searchable index +ann = RandomProjectionIndexANN() +ann.fit(time_series_database) + +# Query for similar series +neighbors = ann.kneighbors(query_series, k=10) +``` + +## Best Practices + +1. **Window size**: Critical parameter for subsequence methods + - Too small: Captures noise + - Too large: Misses fine-grained patterns + - Rule of thumb: 10-20% of series length + +2. **Normalization**: Most methods assume z-normalized subsequences + - Handles amplitude variations + - Focus on shape similarity + +3. **Distance metrics**: Different metrics for different needs + - Euclidean: Fast, shape-based + - DTW: Handles temporal warping + - Cosine: Scale-invariant + +4. **Exclusion zone**: For motif discovery, exclude trivial matches + - Typically set to 0.5-1.0 × window_size + - Prevents finding overlapping occurrences + +5. **Performance**: + - MASS is O(n log n) vs O(n²) brute force + - ANN trades accuracy for speed + - GPU acceleration available for some methods diff --git a/.agents/skills/aeon/references/transformations.md b/.agents/skills/aeon/references/transformations.md new file mode 100644 index 0000000..17ec86e --- /dev/null +++ b/.agents/skills/aeon/references/transformations.md @@ -0,0 +1,246 @@ +# Transformations + +Aeon provides extensive transformation capabilities for preprocessing, feature extraction, and representation learning from time series data. + +## Transformation Types + +Aeon distinguishes between: +- **CollectionTransformers**: Transform multiple time series (collections) +- **SeriesTransformers**: Transform individual time series + +## Collection Transformers + +### Convolution-Based Feature Extraction + +Fast, scalable feature generation using random kernels: + +- `RocketTransformer` - Random convolutional kernels +- `MiniRocketTransformer` - Simplified ROCKET for speed +- `MultiRocketTransformer` - Enhanced ROCKET variant +- `HydraTransformer` - Multi-resolution dilated convolutions +- `MultiRocketHydraTransformer` - Combines ROCKET and Hydra +- `ROCKETGPU` - GPU-accelerated variant + +**Use when**: Need fast, scalable features for any ML algorithm, strong baseline performance. + +### Statistical Feature Extraction + +Domain-agnostic features based on time series characteristics: + +- `Catch22` - 22 canonical time-series characteristics +- `TSFresh` - Comprehensive automated feature extraction (100+ features) +- `TSFreshRelevant` - Feature extraction with relevance filtering +- `SevenNumberSummary` - Descriptive statistics (mean, std, quantiles) + +**Use when**: Need interpretable features, domain-agnostic approach, or feeding traditional ML. + +### Dictionary-Based Representations + +Symbolic approximations for discrete representations: + +- `SAX` - Symbolic Aggregate approXimation +- `PAA` - Piecewise Aggregate Approximation +- `SFA` - Symbolic Fourier Approximation +- `SFAFast` - Optimized SFA +- `SFAWhole` - SFA on entire series (no windowing) +- `BORF` - Bag-of-Receptive-Fields + +**Use when**: Need discrete/symbolic representation, dimensionality reduction, interpretability. + +### Shapelet-Based Features + +Discriminative subsequence extraction: + +- `RandomShapeletTransform` - Random discriminative shapelets +- `RandomDilatedShapeletTransform` - Dilated shapelets for multi-scale +- `SAST` - Scalable And Accurate Subsequence Transform +- `RSAST` - Randomized SAST + +**Use when**: Need interpretable discriminative patterns, phase-invariant features. + +### Interval-Based Features + +Statistical summaries from time intervals: + +- `RandomIntervals` - Features from random intervals +- `SupervisedIntervals` - Supervised interval selection +- `QUANTTransformer` - Quantile-based interval features + +**Use when**: Predictive patterns localized to specific windows. + +### Preprocessing Transformations + +Data preparation and normalization: + +- `MinMaxScaler` - Scale to [0, 1] range +- `Normalizer` - Z-normalization (zero mean, unit variance) +- `Centerer` - Center to zero mean +- `SimpleImputer` - Fill missing values +- `DownsampleTransformer` - Reduce temporal resolution +- `Tabularizer` - Convert time series to tabular format + +**Use when**: Need standardization, missing value handling, format conversion. + +### Specialized Transformations + +Advanced analysis methods: + +- `MatrixProfile` - Computes distance profiles for pattern discovery +- `DWTTransformer` - Discrete Wavelet Transform +- `AutocorrelationFunctionTransformer` - ACF computation +- `Dobin` - Distance-based Outlier BasIs using Neighbors +- `SignatureTransformer` - Path signature methods +- `PLATransformer` - Piecewise Linear Approximation + +### Class Imbalance Handling + +- `ADASYN` - Adaptive Synthetic Sampling +- `SMOTE` - Synthetic Minority Over-sampling +- `OHIT` - Over-sampling with Highly Imbalanced Time series + +**Use when**: Classification with imbalanced classes. + +### Pipeline Composition + +- `CollectionTransformerPipeline` - Chain multiple transformers + +## Series Transformers + +Transform individual time series (e.g., for preprocessing in forecasting). + +### Statistical Analysis + +- `AutoCorrelationSeriesTransformer` - Autocorrelation +- `StatsModelsACF` - ACF using statsmodels +- `StatsModelsPACF` - Partial autocorrelation + +### Smoothing and Filtering + +- `ExponentialSmoothing` - Exponentially weighted moving average +- `MovingAverage` - Simple or weighted moving average +- `SavitzkyGolayFilter` - Polynomial smoothing +- `GaussianFilter` - Gaussian kernel smoothing +- `BKFilter` - Baxter-King bandpass filter +- `DiscreteFourierApproximation` - Fourier-based filtering + +**Use when**: Need noise reduction, trend extraction, or frequency filtering. + +### Dimensionality Reduction + +- `PCASeriesTransformer` - Principal component analysis +- `PlASeriesTransformer` - Piecewise Linear Approximation + +### Transformations + +- `BoxCoxTransformer` - Variance stabilization +- `LogTransformer` - Logarithmic scaling +- `ClaSPTransformer` - Classification Score Profile + +### Pipeline Composition + +- `SeriesTransformerPipeline` - Chain series transformers + +## Quick Start: Feature Extraction + +```python +from aeon.transformations.collection.convolution_based import RocketTransformer +from aeon.classification.sklearn import RotationForest +from aeon.datasets import load_classification + +# Load data +X_train, y_train = load_classification("GunPoint", split="train") +X_test, y_test = load_classification("GunPoint", split="test") + +# Extract ROCKET features +rocket = RocketTransformer() +X_train_features = rocket.fit_transform(X_train) +X_test_features = rocket.transform(X_test) + +# Use with any sklearn classifier +clf = RotationForest() +clf.fit(X_train_features, y_train) +accuracy = clf.score(X_test_features, y_test) +``` + +## Quick Start: Preprocessing Pipeline + +```python +from aeon.transformations.collection import ( + MinMaxScaler, + SimpleImputer, + CollectionTransformerPipeline +) + +# Build preprocessing pipeline +pipeline = CollectionTransformerPipeline([ + ('imputer', SimpleImputer(strategy='mean')), + ('scaler', MinMaxScaler()) +]) + +X_transformed = pipeline.fit_transform(X_train) +``` + +## Quick Start: Series Smoothing + +```python +from aeon.transformations.series import MovingAverage + +# Smooth individual time series +smoother = MovingAverage(window_size=5) +y_smoothed = smoother.fit_transform(y) +``` + +## Algorithm Selection + +### For Feature Extraction: +- **Speed + Performance**: MiniRocketTransformer +- **Interpretability**: Catch22, TSFresh +- **Dimensionality reduction**: PAA, SAX, PCA +- **Discriminative patterns**: Shapelet transforms +- **Comprehensive features**: TSFresh (with longer runtime) + +### For Preprocessing: +- **Normalization**: Normalizer, MinMaxScaler +- **Smoothing**: MovingAverage, SavitzkyGolayFilter +- **Missing values**: SimpleImputer +- **Frequency analysis**: DWTTransformer, Fourier methods + +### For Symbolic Representation: +- **Fast approximation**: PAA +- **Alphabet-based**: SAX +- **Frequency-based**: SFA, SFAFast + +## Best Practices + +1. **Fit on training data only**: Avoid data leakage + ```python + transformer.fit(X_train) + X_train_tf = transformer.transform(X_train) + X_test_tf = transformer.transform(X_test) + ``` + +2. **Pipeline composition**: Chain transformers for complex workflows + ```python + pipeline = CollectionTransformerPipeline([ + ('imputer', SimpleImputer()), + ('scaler', Normalizer()), + ('features', RocketTransformer()) + ]) + ``` + +3. **Feature selection**: TSFresh can generate many features; consider selection + ```python + from sklearn.feature_selection import SelectKBest + selector = SelectKBest(k=100) + X_selected = selector.fit_transform(X_features, y) + ``` + +4. **Memory considerations**: Some transformers memory-intensive on large datasets + - Use MiniRocket instead of ROCKET for speed + - Consider downsampling for very long series + - Use ROCKETGPU for GPU acceleration + +5. **Domain knowledge**: Choose transformations matching domain: + - Periodic data: Fourier-based methods + - Noisy data: Smoothing filters + - Spike detection: Wavelet transforms diff --git a/.agents/skills/analytical-method-validation/SKILL.md b/.agents/skills/analytical-method-validation/SKILL.md new file mode 100644 index 0000000..de386c1 --- /dev/null +++ b/.agents/skills/analytical-method-validation/SKILL.md @@ -0,0 +1,299 @@ +--- +name: analytical-method-validation +description: Plan, execute, and document validation, verification, and transfer of analytical procedures under the governing framework - ICH Q2(R2) and Q14, USP <1220>/<1225>/<1226>, ICH M10 bioanalytical, CLSI EP, or ISO/IEC 17025. Use for HPLC, LC-MS/MS, GC, CE, ICP-MS, dissolution, qNMR, qPCR, NIR, and ligand binding or cell-based assays whenever the question is whether a procedure is fit for its intended purpose. Triggers include "method validation", "analytical method validation", "AMV", "validation protocol", "acceptance criteria", "linearity", "reportable range", "accuracy and precision", "repeatability", "intermediate precision", "recovery", "LOD", "LOQ", "detection limit", "quantitation limit", "specificity", "robustness", "method transfer", "method comparison", "Deming", "Passing-Bablok", "Bland-Altman", "equivalence testing", "OOS investigation", "ICH Q2", "Q2(R2)", "Q14", "USP 1225", "ICH M10", "incurred sample reanalysis", "ISR", "CLSI EP", and any request to show that an assay works. +license: MIT +compatibility: Requires Python 3.11+. Scripts use only the standard library - no numpy, scipy, or network access. Statistical distributions are computed from first principles so results are reproducible in any conforming interpreter. +allowed-tools: Read Write Edit Bash +metadata: + version: "1.0" + skill-author: K-Dense Inc. + last-reviewed: "2026-07-27" +--- + +# Analytical Method Validation + +## When to use + +Any time the question is whether an analytical procedure is fit for its intended purpose: +designing a validation study, evaluating validation data, verifying a compendial procedure, +transferring a procedure to another laboratory or instrument, or defending any of these in a +report. + +## The two rules + +**1. Establish which framework governs before designing anything.** The same assay validates +differently under ICH Q2(R2), USP <1225>, ICH M10, CLSI EP, and ISO/IEC 17025. They differ in +which characteristics are required, how the studies are laid out, and whether numeric acceptance +criteria are supplied at all. Blending them produces a protocol that satisfies none of them. + +**2. State acceptance criteria before collecting data.** Criteria chosen after seeing results are +not acceptance criteria, and deciding them post hoc is a standing audit finding. ICH Q2(R2) +deliberately supplies almost no numeric criteria — they have to come from the specification, the +analytical target profile (ICH Q14 section 3), or development data. ICH M10 is the exception: it +supplies explicit numbers, and they differ between chromatographic assays and ligand binding +assays. + +## Scope + +This skill plans studies, computes the statistics correctly, and structures the documentation. It +does **not** decide that a procedure is validated, release a batch, accept or reject a run, close +an investigation, or substitute for the analyst, the technical reviewer, the quality unit, or the +regulator. Every script reports; none of them concludes. + +## Copyright boundary + +ICH guidelines are published openly and licensed for reuse with acknowledgement, so their +requirements are encoded directly in this skill. **USP general chapters, CLSI EP documents, and +ISO standards are copyrighted and paywalled.** For those, this skill supplies the designation, +scope, and where to obtain an authorised copy — never the text, never invented thresholds. Do not +ask an agent to retrieve, transcribe, or reconstruct their content. If a number matters and it +lives in a paywalled document, read it from the authorised copy. + +## Frameworks + +```bash +cd skills/analytical-method-validation/scripts +python3 plan_validation.py --list-frameworks +``` + +| Key | Governs | Numeric criteria supplied | +| --- | --- | --- | +| `ich-q2r2` | Release and stability testing of drug substances and products | Almost none — you derive them | +| `ich-m10` | Bioanalytical concentration measurement (PK, TK, BE) | Yes, and they differ by modality | +| `usp-1220` | Compendial procedure lifecycle, three stages | Paywalled | +| `usp-1225` / `usp-1226` | Validation / verification of compendial procedures | Paywalled | +| `clsi` | Clinical laboratory measurement procedures (EP series) | Paywalled | +| `iso-17025` | Lab-developed and modified methods under accreditation | No — "to the extent necessary" | + +**Q2(R2) replaced Q2(R1) in November 2023 and restructured the characteristics.** Range is now +the parent characteristic (section 3.2), containing *response* (linearity) and *validation of +lower range limits* (DL/QL). Accuracy and precision are section 3.3 and may be evaluated in +combination against a single criterion. Robustness is treated as a development activity and +cross-refers to ICH Q14. Multivariate procedures are addressed explicitly (2.5 and 3.2.2.3), and +Annex 2 adds worked examples for techniques Q2(R1) never covered — quantitative ¹H-NMR, NIR, +quantitative LC/MS, qPCR, biological assays, and particle size. A Q2(R1)-shaped protocol — a flat +list of linearity, range, accuracy, precision, specificity, LOD, LOQ, robustness — is out of date. +Note also the error correction dated 30 November 2023 to Table 5 and Tables 6–11. + +## Scripts + +```bash +cd skills/analytical-method-validation/scripts +``` + +| Script | Question answered | +| --- | --- | +| `plan_validation.py` | Which framework, which characteristics, what study layout, what protocol? | +| `check_response.py` | Does the calibration model actually hold across the range? | +| `check_accuracy_precision.py` | What is the recovery, and how much of the variability is between days? | +| `check_detection_limits.py` | What are DL and QL by each allowed approach, and do they serve the reporting threshold? | +| `check_bioanalytical_run.py` | Does this run meet ICH M10 for its modality? | +| `compare_methods.py` | Are two procedures equivalent, at a pre-stated margin? | + +All take `--format table|tsv|json`. Provenance, guideline citations, and caveats go to stderr; +data goes to stdout, so `> out.tsv` keeps them separate. Exit code is `0` for no findings, `1` +when findings were raised, `2` for bad input — so any of them can gate a workflow. + +## Workflow + +### 1. Fix the framework and the required characteristics + +```bash +python3 plan_validation.py --framework ich-q2r2 --attribute assay --technique hplc --range-use assay +``` + +Q2(R2) Table 1 decides what is required from the *measured attribute*, not from the technique. For +an assay: specificity, response, accuracy, repeatability, intermediate precision. For a limit +test: specificity and DL only. For an identity test: specificity alone. Attributes accepted include +`assay`, `impurity` (quantitative), `impurity-limit`, and `identity`. + +Reportable range comes from the specification. Q2(R2) Table 2 gives worked examples — 80–120% of +declared content for an assay, 70–130% for content uniformity, reporting threshold to 120% of the +specification for an impurity. + +### 2. Generate the protocol and fill in the criteria + +```bash +python3 plan_validation.py --framework ich-q2r2 --attribute impurity --protocol > protocol.md +``` + +Every bracketed field is a decision to make and record *before* data collection. The protocol +skeleton deliberately refuses to pre-fill acceptance criteria for Q2(R2) work, because there is no +defensible default. + +### 3. Evaluate the response + +```bash +python3 check_response.py -i calibration.csv --max-back-calc-error 2 +``` + +Input is `level,response`, one row per injection; repeated rows at the same level are replicates, +and supplying them is what makes the linearity test possible. + +Real output from a curve that a coefficient of determination would wave through: + +``` +statistic value +distinct levels 5 +slope 166.6000 +intercept 2495.0000 +intercept CI includes 0 no +coefficient of determination (r2) 0.9830 +lack-of-fit F 469.5294 +lack-of-fit p 1.5139e-06 +runs test p 0.0492 + +level n mean_response mean_back_calculated relative_error_pct +50.0000 2 10075.0000 45.4982 -9.0036 +75.0000 2 15150.0000 75.9604 1.2805 +100.0000 2 20050.0000 105.3721 5.3721 +125.0000 2 24050.0000 129.3818 3.5054 +150.0000 2 26450.0000 143.7875 -4.1417 +``` + +r² = 0.983 and the model is unusable: −9.0% back-calculated error at the bottom of the range, +lack-of-fit p = 1.5 × 10⁻⁶, non-random residual signs. **r² is not evidence of linearity** — it +rises with range and is nearly insensitive to curvature. The lack-of-fit F test against pure error +and the residual pattern are the evidence, which is why Q2(R2) 3.2.2.1 asks for an analysis of the +deviation of points from the line rather than a correlation coefficient alone. + +Add `--weight 1/x2` for a wide-range curve. The script flags heteroscedasticity when the residual +variance in the top third of the range exceeds the bottom third by more than 10×, because an +unweighted fit then biases exactly the low end where a reporting threshold lives. + +### 4. Evaluate accuracy and precision + +```bash +python3 check_accuracy_precision.py -i ap.csv --accuracy-limit 2 --rsd-limit 1.0 --design-check assay +``` + +Input is `level,measured,group`, where `group` is the intermediate-precision factor — day, analyst, +or instrument. + +``` +level component sd rsd_pct df ci90_low_sd ci90_high_sd +100 repeatability (within group) 0.0707 0.0707 3 0.0438 0.2065 +100 between-group 1.6515 1.6515 2 n/a n/a +100 intermediate precision (total) 1.6530 1.6530 2.0037 0.9554 7.2821 +``` + +Repeatability of 0.07% RSD looks superb; intermediate precision is 1.65%, twenty-three times +larger, because the variability lives entirely between days. Reporting the within-day figure as +the procedure's precision would understate routine performance by more than an order of magnitude. +This is why the script fits a one-way random-effects model rather than pooling. + +Two traps the script handles for you: + +- **Precision is estimated within each level, never pooled across levels.** Pooling 80/100/120% + results into one standard deviation turns the range itself into apparent imprecision. The script + reports per level, plus a level-independent view as percent of nominal. +- **`--require-ci-within-limit`** enforces that the whole confidence interval sits inside the + limit, not just the mean. Q2(R2) 3.3.1.4 asks for the interval to be *compatible with* the + criterion; a mean that scrapes inside on six replicates has not demonstrated much. + +### 5. Establish DL and QL, and confirm them + +```bash +python3 check_detection_limits.py --calibration lowcal.csv --blanks blanks.csv \ + --confirm-ql 0.05 --confirm-data ql_check.csv --reporting-threshold 0.05 +``` + +``` +approach sigma slope DL QL +sd-and-slope (sigma = residual SD of regression) 7.2816 5033.3490 0.0048 0.0145 +sd-and-slope (sigma = SD of y-intercept) 4.3303 5033.3490 0.0028 0.0086 +sd-and-slope (sigma = SD of 8 blanks) 3.7702 5033.3490 0.0025 0.0075 +``` + +The same data give QL estimates spanning 1.9×, purely from the choice of σ. Q2(R2) 3.2.3.5 +therefore requires the limit **and the approach used to determine it** to be reported, and an +estimated limit to be confirmed with samples at or near it. For an impurity procedure the QL must +be at or below the reporting threshold. Reaching for `3.3σ/slope` reflexively, reporting one number +with no named approach, and never confirming it are three separate findings. + +### 6. Bioanalytical runs under ICH M10 + +```bash +python3 check_bioanalytical_run.py --modality chromatographic --run run1.csv +python3 check_bioanalytical_run.py --modality lba --isr isr.csv +python3 check_bioanalytical_run.py --modality lba --criteria +``` + +`--modality` is mandatory and has no default, because the criteria genuinely differ: + +| | Chromatographic | Ligand binding assay | +| --- | --- | --- | +| Calibration tolerance | ±15%, ±20% at LLOQ | ±20%, ±25% at LLOQ and ULOQ | +| Accuracy / precision | ±15% / ≤15% CV (±20% / ≤20% at LLOQ) | ±20% / ≤20% CV (±25% / ≤25% at LLOQ and ULOQ) | +| A&P design | 4 QC levels, 5 replicates/run, ≥3 runs over ≥2 days | 5 QC levels, 3 replicates/run, ≥6 runs over ≥2 days | +| Total error | no such criterion | ≤30%, ≤40% at LLOQ and ULOQ | +| ISR agreement | ±20% for ≥2/3 of repeats | ±30% for ≥2/3 of repeats | + +Applying the ±15% chromatographic numbers to a ligand binding assay, or importing the LBA total-error +criterion into a chromatographic method, are both common and both wrong. + +The run check enforces the per-level rule that gets missed: at least 2/3 of *all* QCs **and** at +least 50% at *each* level. A run can pass the overall fraction while a single level fails +completely. + +``` +finding: QC level high: 0/2 within tolerance (0%); M10 requires at least 50% at each level +``` + +### 7. Transfer and method comparison + +```bash +python3 compare_methods.py -i paired.csv --margin 2 --relative --slope-tolerance 0.05 +``` + +``` +mean difference (%) 1.4646 +TOST margin 2.0000 +TOST p-value 1.0528e-13 +90% CI (TOST) 1.44127 to 1.48797 +equivalent at stated margin yes +--- for contrast only --- +paired t-test p (NOT equivalence) 0.0000 +OLS slope (biased here) 1.0396 +Deming slope 1.0398 +Passing-Bablok slope 1.0351 +``` + +Two errors this replaces: + +- **"p > 0.05, no significant difference, therefore the methods are equivalent."** Failing to + detect a difference is not evidence of equivalence, and on a small transfer dataset that outcome + is close to guaranteed. TOST tests the hypothesis that matters — that the true difference lies + inside a pre-stated margin. Here the t test says the difference is highly significant *and* TOST + says the methods are equivalent at ±2%; both are true, and only one answers the question. +- **Ordinary least squares for method comparison.** OLS assumes the reference values carry no + error, which is false when comparing two procedures, and biases the slope toward zero. Deming + (with a stated error-variance ratio) and Passing–Bablok (non-parametric, outlier-resistant) are + the appropriate regressions and are reported side by side with OLS for contrast. + +The script also flags proportional bias — when the difference trends with concentration, a single +mean bias and its limits of agreement are misleading regardless of how tight they look. + +## What this skill exists to prevent + +1. Validating against ICH Q2(R1)'s structure three years after Q2(R2) replaced it. +2. Acceptance criteria written after the data were seen. +3. r² presented as evidence of linearity. +4. Repeatability reported as the procedure's precision, with the between-day component invisible. +5. One DL/QL number with no named approach and no confirmation. +6. Chromatographic M10 criteria applied to a ligand binding assay, or the reverse. +7. A t test's non-significance presented as equivalence at a method transfer. + +## References + +- `references/framework-selection.md` — which framework governs, and the questions that decide it +- `references/ich-q2r2.md` — structure, Table 1 and Table 2, per-characteristic recommended data +- `references/ich-m10-bioanalytical.md` — the full chromatographic and LBA criteria side by side +- `references/compendial-and-clsi.md` — USP, CLSI and ISO designations, scope, and how to cite them +- `references/statistics.md` — the statistical methods, why each one, and the common errors +- `references/source-ledger.md` — provenance and research dates for every claim in this skill + +## Assets + +- `assets/validation-protocol-template.md` — protocol structure with criteria stated up front +- `assets/validation-report-template.md` — report structure with raw-data traceability diff --git a/.agents/skills/analytical-method-validation/assets/validation-protocol-template.md b/.agents/skills/analytical-method-validation/assets/validation-protocol-template.md new file mode 100644 index 0000000..99f64fd --- /dev/null +++ b/.agents/skills/analytical-method-validation/assets/validation-protocol-template.md @@ -0,0 +1,106 @@ +# Analytical Procedure Validation Protocol + +> Every bracketed field is a decision to make and record **before** data collection. +> `plan_validation.py --protocol` generates a framework-specific version of this document with the +> required characteristics already listed. + +| Field | Value | +| --- | --- | +| Protocol number / version | [ ] | +| Analytical procedure identifier and version | [ ] | +| Product / analyte / matrix | [ ] | +| Measured quality attribute | [ ] assay / impurity (quantitative) / impurity (limit) / identity / other | +| Governing framework and section | [ ] | +| Regional expectation confirmed with | [ ] | +| Related development report (ICH Q14) | [ ] | +| Author / date | [ ] | +| Technical reviewer / date | [ ] | +| Quality unit approval / date | [ ] | + +## 1. Intended purpose and analytical target profile + +- Measurand and reporting unit: [ ] +- Decision the result supports: [ ] release / stability / in-process / clinical / other +- Specification or reporting limits served: [ ] +- Required reportable range, derived from the specification: [ ] +- Performance characteristics and criteria (the ATP): [ ] + +## 2. Pre-stated acceptance criteria + +State a numeric criterion and its justification for every characteristic to be validated. A +criterion with no justification traceable to the specification, the ATP, or development data is not +defensible. + +| Characteristic | Criterion | Justification | Framework reference | +| --- | --- | --- | --- | +| Specificity / selectivity | [ ] | [ ] | [ ] | +| Response (calibration model) | [ ] | [ ] | [ ] | +| Lower range limit (DL / QL) | [ ] | [ ] | [ ] | +| Accuracy | [ ] | [ ] | [ ] | +| Repeatability | [ ] | [ ] | [ ] | +| Intermediate precision | [ ] | [ ] | [ ] | +| Combined accuracy and precision, if used | [ ] | [ ] | [ ] | + +- Interval to be reported alongside accuracy and precision: [ ] confidence level [ ] +- Does the criterion apply to the point estimate or to the whole interval? [ ] + +## 3. Study design + +| Characteristic | Levels | Replicates | Runs / days / analysts / instruments | +| --- | --- | --- | --- | +| Response | [ ] (minimum 5 for ICH Q2(R2)) | [ ] | [ ] | +| Accuracy | [ ] | [ ] | [ ] | +| Repeatability | [ ] | [ ] | [ ] | +| Intermediate precision | [ ] | [ ] | [ ] | +| Lower range limit | [ ] | [ ] | [ ] | + +- Replicate count matches the routine reportable result: [ ] yes / [ ] justified deviation: [ ] +- Calibration model and weighting, fixed in advance: [ ] unweighted / 1/x / 1/x² / non-linear / multivariate +- Randomisation and run order: [ ] +- Prior knowledge or development data used in place of a test, with justification: [ ] + +## 4. Materials + +| Item | Identity / grade | Lot | Assigned value and uncertainty | Expiry | +| --- | --- | --- | --- | --- | +| Reference material | [ ] | [ ] | [ ] | [ ] | +| Impurity standards | [ ] | [ ] | [ ] | [ ] | +| Blank / placebo matrix | [ ] | [ ] | — | [ ] | + +## 5. Sample and solution handling + +- Preparation procedure and dilution scheme: [ ] +- Solution stability window to be demonstrated: [ ] +- Storage conditions: [ ] + +## 6. Specificity and stability-indicating properties + +- Interferences to be challenged: [ ] +- Forced degradation conditions, if a stability-indicating claim is made: [ ] +- Orthogonal procedure, if used, and its accuracy: [ ] + +## 7. Robustness (normally development, ICH Q14) + +| Parameter | Nominal | Range varied | Effect assessed on | +| --- | --- | --- | --- | +| [ ] | [ ] | [ ] | [ ] | + +## 8. Statistical treatment + +- Software, version, and how calculations are verified: [ ] +- Handling of outliers, stated in advance: [ ] +- Scripts to be used and their output retained as records: [ ] + +## 9. Deviations and data integrity + +- Deviation identification, assessment and approval route: [ ] +- All results will be reported, including out-of-criteria values: [ ] confirmed +- Raw data location, audit trail, and review: [ ] + +## 10. Approvals + +| Role | Name | Signature | Date | +| --- | --- | --- | --- | +| Author | | | | +| Technical reviewer | | | | +| Quality unit | | | | diff --git a/.agents/skills/analytical-method-validation/assets/validation-report-template.md b/.agents/skills/analytical-method-validation/assets/validation-report-template.md new file mode 100644 index 0000000..ea91d19 --- /dev/null +++ b/.agents/skills/analytical-method-validation/assets/validation-report-template.md @@ -0,0 +1,116 @@ +# Analytical Procedure Validation Report + +> Reports the outcome against criteria stated in the approved protocol. If a criterion here differs +> from the protocol, that is a deviation to be documented, not an edit to be made. + +| Field | Value | +| --- | --- | +| Report number / version | [ ] | +| Protocol number / version executed | [ ] | +| Analytical procedure identifier and version | [ ] | +| Governing framework | [ ] | +| Execution dates | [ ] | +| Analysts and instruments | [ ] | +| Author / date | [ ] | +| Technical reviewer / date | [ ] | +| Quality unit approval / date | [ ] | + +## 1. Summary of outcome + +| Characteristic | Criterion (from protocol) | Result | Interval reported | Met | +| --- | --- | --- | --- | --- | +| Specificity / selectivity | [ ] | [ ] | — | [ ] | +| Response | [ ] | [ ] | [ ] | [ ] | +| Lower range limit (DL / QL) | [ ] | [ ] | — | [ ] | +| Accuracy | [ ] | [ ] | [ ] | [ ] | +| Repeatability | [ ] | [ ] | [ ] | [ ] | +| Intermediate precision | [ ] | [ ] | [ ] | [ ] | + +- Validated reportable range: [ ] +- Statement of fitness for the intended purpose, and who is making it: [ ] + +## 2. Response + +- Levels and replicates actually run: [ ] +- Calibration model and weighting: [ ] +- Slope, intercept, and their confidence intervals: [ ] +- Coefficient of determination: [ ] +- **Analysis of deviation from the regression line** (residual plot, lack-of-fit test, back-calculated + relative error per level): [ ] + +## 3. Accuracy + +| Level | n | Mean recovery (%) | Bias (%) | Confidence interval | Met | +| --- | --- | --- | --- | --- | --- | +| [ ] | [ ] | [ ] | [ ] | [ ] | [ ] | + +- Approach used: [ ] reference material / spiking / orthogonal comparison +- For impurities, basis of determination: [ ] w/w / area % + +## 4. Precision + +| Level | Component | SD | %RSD | df | Interval | Met | +| --- | --- | --- | --- | --- | --- | --- | +| [ ] | repeatability | [ ] | [ ] | [ ] | [ ] | [ ] | +| [ ] | between-group | [ ] | [ ] | [ ] | — | — | +| [ ] | intermediate precision | [ ] | [ ] | [ ] | [ ] | [ ] | + +- Intermediate precision factors varied: [ ] days / analysts / instruments / environment +- Reproducibility, if performed: [ ] + +## 5. Lower range limits + +- DL, and **the approach used to determine it**: [ ] +- QL, and **the approach used to determine it**: [ ] +- Confirmation of the estimated limit with samples at or near it: [ ] +- For impurity procedures, QL relative to the reporting threshold: [ ] + +## 6. Specificity and stability-indicating properties + +- Interference results: [ ] +- Forced degradation results and peak purity / mass balance: [ ] +- Relative response factors, and any correction factor applied: [ ] + +## 7. Robustness + +| Parameter | Range varied | Effect on the reportable result | Conclusion | +| --- | --- | --- | --- | +| [ ] | [ ] | [ ] | [ ] | + +- Solution stability demonstrated over: [ ] + +## 8. Deviations + +| # | Description | Assessment of impact | Disposition | Approved by | +| --- | --- | --- | --- | --- | +| [ ] | [ ] | [ ] | [ ] | [ ] | + +- Out-of-criteria individual results, and whether they were included in the reported statistics: [ ] + +## 9. Raw data traceability + +Every reported number must be traceable to a retained record. A report whose numbers cannot be +reproduced from the raw data is the finding that costs the most to remediate. + +| Reported item | Raw data location | Instrument / system | Acquisition date | Reviewed by | +| --- | --- | --- | --- | --- | +| [ ] | [ ] | [ ] | [ ] | [ ] | + +- Software and version used for calculations: [ ] +- Calculation verification method: [ ] +- Script outputs retained as records: [ ] + +## 10. Conclusion and lifecycle + +- Conclusion against the ATP / intended purpose: [ ] +- Conditions or limitations on use: [ ] +- Ongoing performance monitoring planned: [ ] +- Revalidation triggers identified: [ ] + +## 11. Approvals + +| Role | Name | Signature | Date | +| --- | --- | --- | --- | +| Author | | | | +| Technical reviewer | | | | +| Quality unit | | | | diff --git a/.agents/skills/analytical-method-validation/references/compendial-and-clsi.md b/.agents/skills/analytical-method-validation/references/compendial-and-clsi.md new file mode 100644 index 0000000..c1f76b9 --- /dev/null +++ b/.agents/skills/analytical-method-validation/references/compendial-and-clsi.md @@ -0,0 +1,96 @@ +# Compendial, CLSI, and ISO Sources (No Standard Text) + +Research basis: **2026-07-27**. This reference identifies documents, their scope, and where to obtain +them. **It does not reproduce their requirements, thresholds, or study designs**, because they are +copyrighted and paywalled. + +## Copyright boundary + +USP–NF general chapters, CLSI documents, and ISO/IEC standards are copyrighted works sold by their +publishers. Do not ask an agent to retrieve, transcribe, summarise clause-by-clause, reconstruct, or +store their text. Vendor application notes and training decks that quote them are equally +constrained, and a paraphrase that carries the same numbers is still a reproduction of the +substantive content. + +The practical consequence: **when a numeric criterion or a study design lives in one of these +documents, read it from the authorised copy.** An agent asked for "the USP <621> tailing factor +limit" or "the CLSI EP15 number of days" will produce a plausible number. Plausible is not the same +as correct, and the difference is discovered at audit. + +Record publisher, title, designation, edition, amendments, authorised location, access date, and +review date in the laboratory's controlled source register. + +## USP–NF general chapters + +| Chapter | Title | Scope | +| --- | --- | --- | +| `<1220>` | Analytical Procedure Life Cycle | Three-stage lifecycle: procedure design (Stage 1), performance qualification (Stage 2), ongoing performance verification (Stage 3), organised around an analytical target profile. Official 1 May 2022 (incorporated into USP–NF 2022 Issue 1 on 1 Nov 2021). Integrates the concepts previously spread across `<1224>`, `<1225>`, and `<1226>`. | +| `<1225>` | Validation of Compendial Procedures | Validation of non-compendial procedures, and of compendial procedures used outside their stated scope. Stage 2 activity under `<1220>`. | +| `<1226>` | Verification of Compendial Procedures | Assessment of selected performance characteristics showing a compendial procedure works under actual conditions of use. **Verification is not revalidation** and does not repeat the full validation. | +| `<1224>` | Transfer of Analytical Procedures | Transfer between laboratories. | +| `<1010>` | Analytical Data — Interpretation and Treatment | Statistical treatment of analytical data. | +| `<621>` | Chromatography | System suitability and chromatographic operating parameters, including the extent to which a compendial procedure may be adjusted without triggering revalidation. | +| `<711>` / `<1092>` | Dissolution / The Dissolution Procedure | Dissolution testing and development/validation of the procedure. | + +Obtain from the USP–NF (). Regional pharmacopoeias — Ph. Eur., JP, ChP — +carry their own general chapters; check which pharmacopoeia the specification cites, because +adjustment allowances and system suitability requirements differ between them. + +**The `<1226>` decision.** Verification applies when using a compendial procedure as written and +within its scope. Two situations push you back to `<1225>` validation: using the procedure outside +its stated scope (a different matrix, a different dosage form, a concentration range it does not +cover), or modifying it beyond the adjustments the relevant chapter permits. Getting this wrong in +either direction is expensive — unnecessary full validation, or an unsupported claim of verification. + +## CLSI EP series + +Designations and titles below were taken from clsi.org listings and secondary sources on the +research date. **Editions change; confirm the current edition on before designing +a study.** Marked `[confirm]` where the edition was not read from the publisher directly. + +| Designation | Subject | Note | +| --- | --- | --- | +| EP05 | Evaluation of precision of quantitative measurement procedures | Establishment of precision; the multi-day/multi-run designs. `[confirm edition]` | +| EP06 | Evaluation of linearity of quantitative measurement procedures | 2nd edition reported. `[confirm edition]` | +| EP07 | Interference testing in clinical chemistry | Screening, quantifying and confirming interferents; verifying manufacturer interference claims. 3rd edition reported. `[confirm edition]` | +| EP09 | Measurement procedure comparison and bias estimation using patient samples | The method-comparison document. 3rd edition reported. `[confirm edition]` | +| EP15 | User verification of precision and estimation of bias | The short study a laboratory runs to verify a manufacturer's claims. 3rd edition reported. `[confirm edition]` | +| EP17 | Evaluation of detection capability | Limit of blank, limit of detection, limit of quantitation; verification of manufacturer claims. `[confirm edition]` | +| EP25 | Evaluation of stability of in vitro diagnostic reagents | `[confirm edition]` | +| EP28 | Defining, establishing, and verifying reference intervals | Formerly designated C28. An implementation guide (EP28IG) also exists. `[confirm edition]` | + +**Vocabulary.** CLSI distinguishes *limit of blank*, *limit of detection*, and *limit of quantitation* +as three separate quantities with separate protocols. This is not the same taxonomy as ICH Q2(R2)'s +detection limit and quantitation limit, and the two should not be translated into each other +casually — the underlying definitions and the experiments differ. + +**Verification versus establishment.** For an FDA-cleared or CE-marked assay used as intended, a +laboratory *verifies* the manufacturer's performance claims — a bounded study. For a +laboratory-developed test, or an assay used off-label, the laboratory *establishes* performance, +which is a much larger exercise. Under CLIA the distinction has direct regulatory consequences and +also depends on test complexity. Determine which applies before designing anything. + +## ISO standards + +| Standard | Relevance | +| --- | --- | +| ISO/IEC 17025:2017 | Clause 7.2 selection, verification and validation of methods; clause 7.6 measurement uncertainty. Validation "to the extent necessary" for the intended application — no characteristic list, no numeric criteria. | +| ISO 15189 | Medical laboratories: quality and competence. The clinical-laboratory counterpart to 17025. | +| ISO 21748 / ISO 5725 series | Using repeatability, reproducibility and trueness estimates in measurement uncertainty; accuracy of measurement methods. | + +Obtain from ISO () or a national member body. A laboratory is **accredited** to +ISO/IEC 17025 by an accreditation body — it is not "17025 certified", and writing "certified" is a +substantive error assessors notice. + +For accreditation readiness, the quality manual, and the surrounding management system, use this +repository's `iso-standards-readiness` skill. This skill stays at the level of the individual +procedure. + +## Environmental, food, and forensic method systems + +Where a prescribed method system governs — a published EPA method, an AOAC Official Method, a +standard method for water or food analysis — the validation and quality-control requirements are +written into the method or the programme, and they take precedence. Do not substitute a +pharmaceutical framework. Common differences: matrix spike and duplicate requirements per batch, +prescribed calibration-verification frequencies, method detection limit procedures that differ from +both ICH and CLSI, and mandatory participation in proficiency testing schemes. diff --git a/.agents/skills/analytical-method-validation/references/framework-selection.md b/.agents/skills/analytical-method-validation/references/framework-selection.md new file mode 100644 index 0000000..c6cb6e9 --- /dev/null +++ b/.agents/skills/analytical-method-validation/references/framework-selection.md @@ -0,0 +1,87 @@ +# Which Framework Governs + +Research basis: **2026-07-27**. Confirm every date and edition against the official source before +relying on it; see `source-ledger.md`. + +Framework selection is the first decision and the one most often skipped. Getting it wrong +invalidates the protocol regardless of how well the studies are executed, because each framework +requires a different set of characteristics, a different study layout, and a different treatment of +acceptance criteria. + +## The deciding questions, in order + +**1. Is the measurand a drug concentration in a biological matrix, supporting a nonclinical or +clinical study?** +→ **ICH M10.** This covers pharmacokinetics, toxicokinetics, and bioequivalence. M10 supplies +explicit numeric criteria, and they differ between chromatographic assays and ligand binding +assays. Q2(R2) does not govern here. + +**2. Is it a quality attribute of a drug substance or drug product — assay, potency, impurity, +identity, dissolution, content uniformity?** +→ **ICH Q2(R2)** for validation, with **ICH Q14** for development, robustness, the analytical +target profile, and lifecycle change management. If the procedure is compendial and being used as +written, see question 3 first. + +**3. Is the procedure a compendial (pharmacopoeial) procedure?** +→ **USP <1226> verification** if it is used as written and within its stated scope. Verification +assesses selected characteristics to show the procedure works under actual conditions of use; it is +not revalidation and does not repeat the full study. → **USP <1225> validation** if the procedure +is non-compendial, or compendial but used outside its scope. Both sit inside the **USP <1220>** +three-stage lifecycle. Regional pharmacopoeias (Ph. Eur., JP) have their own general chapters — +check which pharmacopoeia the specification cites. + +**4. Is it a clinical laboratory measurement procedure reporting patient results?** +→ **CLSI EP series**, inside a CLIA/CAP or ISO 15189 quality system. The vocabulary differs from +pharmaceutical work: *verification* of a manufacturer's claims for an FDA-cleared assay is a much +smaller exercise than *establishment* of performance for a laboratory-developed test, and the +distinction is regulatory, not stylistic. + +**5. Is the laboratory accredited to ISO/IEC 17025 and the method non-standard, laboratory-developed, +or a modified standard method?** +→ **ISO/IEC 17025 clause 7.2.2** requires validation as extensive as necessary to meet the needs of +the intended application, plus measurement uncertainty under clause 7.6. It sets no characteristic +list and no numeric criteria; the laboratory justifies both. + +**6. Is it an environmental, food, or forensic method under a prescribed method system?** +→ The method system governs (for example a published EPA method, an AOAC Official Method, or a +regulator's prescribed procedure), usually with its own validation and QC requirements written into +the method itself. Do not substitute a pharmaceutical framework. + +## More than one can apply + +Common and legitimate. A contract laboratory accredited to ISO/IEC 17025 running a compendial assay +for a pharmaceutical client satisfies <1226> for the procedure and 17025 clause 7.2 for the +accreditation scope, with the client's specification supplying the criteria. Record which framework +each requirement traces to, so a later change can be assessed against the right one. + +## Do not blend them + +The failure mode is a protocol that mixes Q2(R1)-era characteristic names, an M10 numeric tolerance +imported because it was memorable, and a CLSI study layout. It satisfies none of the three and is +hard to defend because no single source can be cited for any of it. If a requirement is in the +protocol, name the framework and section it comes from. + +## Where the numbers come from + +| Framework | Numeric acceptance criteria | +| --- | --- | +| ICH Q2(R2) | Almost none. Derive from the specification, the ATP, or development data, and justify. | +| ICH Q14 | None. It supplies the ATP concept and the development/robustness framework. | +| ICH M10 | Explicit, and modality-dependent. Use them as written. | +| USP <1225>/<1226>/<1220> | Consult the authorised text. | +| CLSI EP | Consult the authorised text; many EP documents supply study designs rather than limits. | +| ISO/IEC 17025 | None. The laboratory sets and justifies them. | + +Q2(R2)'s reticence is deliberate: a criterion that is not tied to what the result is used for is +arbitrary. An assay releasing product against a 95.0–105.0% specification needs different precision +than one supporting a 70–130% content-uniformity limit. Deriving the criterion from the decision the +result supports is the substance of the exercise, not paperwork around it. + +## Related skills in this repository + +- `iso-standards-readiness` — the surrounding quality system (ISO/IEC 17025, ISO 15189 + accreditation readiness, quality manual, CAPA). That skill operates at the laboratory level; this + one operates at the level of a single procedure. +- `statistical-analysis`, `statistical-power` — general inference and study sizing. +- `uncertainty-and-units` — unit handling and measurement uncertainty propagation, which ISO/IEC + 17025 clause 7.6 requires alongside validation. diff --git a/.agents/skills/analytical-method-validation/references/ich-m10-bioanalytical.md b/.agents/skills/analytical-method-validation/references/ich-m10-bioanalytical.md new file mode 100644 index 0000000..de95df1 --- /dev/null +++ b/.agents/skills/analytical-method-validation/references/ich-m10-bioanalytical.md @@ -0,0 +1,123 @@ +# ICH M10 — Bioanalytical Criteria, by Modality + +Research basis: **2026-07-27**, read from the ICH Harmonised Guideline *Bioanalytical Method +Validation and Study Sample Analysis M10*, Step 4 dated 24 May 2022. ICH licenses its documents for +reuse with acknowledgement. Confirm the current text and your region's implementation at +. + +M10 harmonised what had been separate FDA and EMA bioanalytical guidance for studies in its scope: +methods quantifying drug and metabolite concentrations in biological matrices supporting nonclinical +and clinical studies, plus the analysis of study samples. + +## The distinction that matters most + +**Chromatographic assays (section 3) and ligand binding assays (section 4) have different numeric +criteria throughout.** They are not stylistic variants of one set. Applying chromatographic +tolerances to an LBA is the most common error in this area, and importing the LBA total-error +criterion into a chromatographic method is its mirror image. + +| | Chromatographic | Ligand binding assay | +| --- | --- | --- | +| Calibration levels (minimum) | 6, including LLOQ | 6, including LLOQ | +| Calibration standard tolerance | ±15% | ±20% | +| … at LLOQ | ±20% | ±25% | +| … at ULOQ | ±15% | ±25% | +| Calibration standards that must pass | ≥75% | ≥75%, excluding anchor points | +| Accuracy | ±15% | ±20% | +| … at limits | ±20% at LLOQ | ±25% at LLOQ **and** ULOQ | +| Precision (%CV) | ≤15% | ≤20% | +| … at limits | ≤20% at LLOQ | ≤25% at LLOQ **and** ULOQ | +| A&P QC levels | minimum 4 | 5 (LLOQ, low, medium, high, ULOQ) | +| A&P replicates per level per run | ≥5 (within-run) | ≥3 | +| A&P runs | ≥3 runs over ≥2 days | ≥6 runs over ≥2 days | +| **Total error** | **no such criterion** | **≤30%; ≤40% at LLOQ and ULOQ** | +| Routine run QC tolerance | ±15% | ±20% | +| Routine run QC pass rule | ≥2/3 of all QCs **and** ≥50% at each level | same rule, ±20% | +| Dilution integrity | mean within ±15% | mean within ±20% | +| Stability | mean at each QC level within ±15% | mean within ±20% | +| ISR agreement | within ±20% for ≥2/3 of repeats | within ±30% for ≥2/3 of repeats | +| Selectivity sources/lots | ≥6 individual sources | ≥6 individual sources | +| Carry-over in blank | ≤20% of LLOQ analyte response and ≤5% of IS response | per guideline | + +Verify any figure against the guideline before using it in a protocol; regional implementation and +subsequent revisions can change the picture. + +## Chromatographic QC placement (section 3) + +Accuracy and precision validation QCs at a minimum of **four** concentration levels: + +- the **LLOQ** +- **low QC** — within three times the LLOQ +- **medium QC** — around 30–50% of the calibration curve range +- **high QC** — at least 75% of the ULOQ + +For runs that are not accuracy-and-precision runs, low, medium and high QCs may be analysed in +duplicate; these plus the calibration standards form the basis for accepting or rejecting the run. + +Calibration standards and QCs should be prepared from **separate stock solutions**, to avoid a bias +that is not a property of the analytical performance. If a single stock must serve both, verify the +accuracy and stability of that stock. A single source of blank matrix may be used if it is free of +interference and matrix effects. + +Calibration curves for accuracy and precision assessment should use freshly spiked standards in at +least one run; if other runs use frozen standards, demonstrate their stability. + +## Reporting obligations that catch people out + +**Report everything.** Validation data and the determination of accuracy and precision must include +*all* results obtained, including individual QCs outside the acceptance criteria — except cases where +errors are obvious and documented. Silently dropping an out-of-criteria QC is a data integrity +problem, not a rounding decision. + +**Within-run accuracy and precision are reported per run.** If the within-run criteria are not met in +every run, calculate an overall estimate of within-run accuracy and precision for each QC level. +Between-run (intermediate) accuracy and precision combine data from all runs. + +**Trend within a run.** It is recommended to demonstrate accuracy and precision over at least one run +sized like a prospective study-sample run, so time-dependent drift is visible. + +## Incurred sample reanalysis (section 5) + +ISR repeats the analysis of a subset of study samples in separate runs, to verify that measured +concentrations in real samples are reproducible. It is not a substitute for QCs — QCs are spiked, +incurred samples are not, and only incurred samples can reveal metabolite back-conversion, protein +binding effects, or matrix instability. + +- The extent depends on the analyte and the samples and should be justified. +- Objective criteria for choosing the subset should be **predefined**; selecting samples around + Cmax and the elimination phase is recommended. +- **Do not pool samples** — pooling masks anomalous findings. +- ISR samples and QCs are processed and analysed in the same manner as the original analysis. +- Percent difference is `(repeat value - initial value) / mean value x 100` -- assessed + against the **mean of the two**, not against the initial value. +- Repeats are performed within the analyte's stability window, but **not on the same day** + as the original analysis. +- Acceptance: within ±20% for at least 2/3 of repeats (chromatographic), or within ±30% for at least + 2/3 (LBA). + +For nonclinical studies in scope, ISR should in general be performed; the guideline notes incurred +samples need only be included if available, so inclusion was not felt to be mandatory in every case. +Confirm the situations requiring ISR against the guideline text for your study type. + +## Study sample reanalysis is a separate thing + +ISR is a method-reliability check. *Reanalysis of study samples* for a reportable-value decision is +different, and the reasons for reanalysis, the number of replicates, and the criteria for selecting +the value to report must be **predefined in the protocol, study plan, or SOP before study sample +analysis begins.** Deciding after the fact which of two values to report is the classic finding. + +## Partial and cross validation + +M10 addresses partial validation (a change to a validated method — matrix, anticoagulant, species, +instrument, or a range change) and cross validation (comparing data from two methods or two +laboratories contributing to the same study). Both are scoped by the change and the risk; consult +the guideline for what each requires. For a cross validation between sites or methods, the +statistics in `compare_methods.py` — equivalence testing against a pre-stated margin, and a +regression that allows error in both measurements — are the appropriate treatment. + +## Biomarkers and other contexts + +M10's scope centres on drug and metabolite concentration measurement. Biomarker assays, immunogenicity +assays, and diagnostic measurements are addressed differently or fall outside scope; do not assume the +concentration-assay criteria transfer. Where a biomarker assay supports a regulatory decision, the +fit-for-purpose framework and the applicable regional guidance govern the extent of validation. diff --git a/.agents/skills/analytical-method-validation/references/ich-q2r2.md b/.agents/skills/analytical-method-validation/references/ich-q2r2.md new file mode 100644 index 0000000..6081902 --- /dev/null +++ b/.agents/skills/analytical-method-validation/references/ich-q2r2.md @@ -0,0 +1,229 @@ +# ICH Q2(R2) — Structure and Recommended Data + +Research basis: **2026-07-27**, read from the ICH Harmonised Guideline *Validation of Analytical +Procedures Q2(R2)*, Final Version adopted 1 November 2023, with the error correction dated +30 November 2023. ICH licenses its documents for reuse with acknowledgement, so requirements are +summarised here directly. Confirm the current text and your region's implementation date at +. + +## Document history that matters + +| Version | Date | Note | +| --- | --- | --- | +| Q2A | Oct 1994 | Text | +| Q2B | Nov 1996 | Methodology | +| Q2(R1) | Nov 2005 | Q2B merged into the parent guideline | +| Q2(R2) | 1 Nov 2023 | Complete revision, aligned with the new Q14 | +| Q2(R2) correction | 30 Nov 2023 | Table 5 reportable-range linearity formulae; Tables 6–11 | + +If a protocol cites "ICH Q2(R1)" or lists characteristics in the R1 order, it is working from the +superseded structure. The error correction is easy to miss and applies to the dissolution example +and to Annex 2 Tables 6–11. + +## The restructure + +Q2(R1) presented a flat list. Q2(R2) groups methodology under section 3 by performance +characteristic: + +``` +3.1 Specificity/Selectivity + 3.1.1 General considerations (absence of interference, orthogonal comparison, + technology-inherent justification) +3.2 Range <-- parent characteristic + 3.2.2 Response + 3.2.2.1 Linear response <-- what R1 called "linearity" + 3.2.2.2 Non-linear response + 3.2.2.3 Multivariate calibration + 3.2.3 Validation of lower range limits <-- what R1 called LOD and LOQ +3.3 Accuracy and Precision + 3.3.1 Accuracy + 3.3.2 Precision (repeatability, intermediate precision, reproducibility) + 3.3.3 Combined approaches for accuracy and precision <-- new +3.4 Robustness --> largely a development activity, see ICH Q14 +``` + +Section 2 carries the general considerations, including two concepts absent from R1: **reportable +range** (2.3) and **considerations for multivariate analytical procedures** (2.5). + +## Table 1 — which tests for which measured attribute + +Required tests follow the *measured quality attribute*, not the instrument. + +| Characteristic | Identity | Impurity: quantitative | Impurity: limit test | Assay (content/potency) | +| --- | --- | --- | --- | --- | +| Specificity test | yes | yes | yes | yes | +| Response (calibration model) | no | yes | no | yes | +| Lower range limit | no | QL† | DL | no | +| Accuracy test | no | yes | no | yes | +| Repeatability test | no | yes | no | yes | +| Intermediate precision test | no | yes‡ | no | yes‡ | + +† In some complex cases DL may also be evaluated. +‡ Not required independently where reproducibility has been performed and intermediate precision +can be derived from that dataset. + +Further notes from Table 1: other quantitative measurements follow the impurity scheme when the +range limit is close to DL/QL, and the assay scheme when it is not. Some characteristics may be +substituted by technology-inherent justification for physicochemical properties. Lack of specificity +in one procedure should be compensated by one or more supporting procedures unless justified. + +## Table 2 — reportable range examples + +The reportable range derives from the specification and must include the upper and lower +specification or reporting limits. Other ranges are acceptable if justified; at low amounts a wider +upper range may be more practical. + +| Use | Low end | High end | +| --- | --- | --- | +| Assay of a product | 80% of declared content, or 80% of the lower specification limit | 120% of declared content, or 120% of the upper specification limit | +| Potency | lowest specification limit −20% | highest specification limit +20% | +| Content uniformity | 70% of declared content | 130% of declared content | +| Dissolution, IR, one point | Q − 45% of the lowest strength specification | per specification | +| Dissolution, IR, multi-point | lower limit as justified, or QL | 130% of declared content of the highest strength | +| Dissolution, modified release | lower limit as justified, or QL | per specification | +| Impurity | reporting threshold | 120% of the specification limit | +| Purity (area %) | 80% of the lower specification limit | upper specification limit, or 100% | + +Where assay and impurity run as a single test with one standard, linearity must be shown both at the +impurity reporting level and up to 120% of the assay specification limit. + +**Reportable range vs working range.** The reportable range is the interval of *reported results*. +A working range is what is presented to the instrument, and may differ because of dilution or other +sample preparation. They can be identical. Mathematical calculation normally links the two. + +## Recommended data, by characteristic + +**Specificity (3.1).** Demonstrate absence of relevant interference, or compare against an +orthogonal procedure, or justify from the technology. For a stability-indicating claim (2.4), +include samples containing relevant degradation products: spiked with target analytes and known +interferences, stressed physically and chemically, and aged or stress-stored product samples. + +**Response — linear (3.2.2.1).** Evaluate across the range. **A minimum of five concentrations, +appropriately distributed, is recommended.** Report the plot, the correlation coefficient or +coefficient of determination, the y-intercept, the slope, and *an analysis of the deviation of the +actual data points from the regression line* — for a linear response, assess the impact of any +non-random pattern in the residual plot. Data may be transformed (for example logarithmically) if +necessary. Other approaches require justification. + +**Response — non-linear (3.2.2.2).** Some procedures are legitimately non-linear; immunoassays and +cell-based assays commonly give an S-shaped curve, typically modelled with four- or five-parameter +logistic functions. For these, **linearity of the concentration–response relationship is not +required.** Assess the model by non-linear regression, and evaluate whether results are proportional +to the true values across the range. + +**Response — multivariate (3.2.2.3).** Algorithms may be linear or non-linear. Accuracy depends on +the distribution of calibration samples across the range and on the reference procedure's error. +Assess how the residuals change across the calibration range, graphically. + +**Lower range limits (3.2.3).** Four approaches: + +| Approach | DL | QL | +| --- | --- | --- | +| Visual evaluation (3.2.3.1) | lowest reliably detected | lowest reliably quantitated | +| Signal-to-noise (3.2.3.2) | S/N 3:1 generally acceptable | S/N at least 10:1 | +| SD of response and slope (3.2.3.3) | 3.3σ / S | 10σ / S | +| Accuracy and precision at the limit (3.2.3.4) | — | validated directly, not estimated | + +σ may come from the SD of blank responses, the residual SD of the regression line, or the SD of +y-intercepts of regression lines. S is the calibration slope. Signal-to-noise applies only where +there is baseline noise, and the noise region should sit around where the peak would appear. + +Reporting (3.2.3.5): give the limit **and the approach used**. An estimated limit should then be +validated by analysing a suitable number of samples at or near it. **For impurity tests the QL must +be at or below the reporting threshold.** Where the QL is well below the reporting limit — roughly +ten times lower — the confirmatory validation may be omitted with justification. + +**Accuracy (3.3.1).** Establish across the reportable range under regular test conditions, including +the sample matrix and the described preparation steps. Three routes: comparison against a reference +material of known purity, a spiking study into matrix, or comparison against an orthogonal +procedure. Accuracy can be inferred once precision, response within the range, and specificity are +established. + +Recommended data (3.3.1.4): an appropriate number of determinations and levels across the reportable +range — **for example 3 concentrations × 3 replicates of the full procedure.** Report as mean percent +recovery of a known added amount, or as the difference between the mean and the accepted true value, +**together with an appropriate 100(1−α)% confidence interval** or justified alternative interval. The +observed interval should be compatible with the accuracy criterion. For impurities, state whether +the determination is weight/weight or area percent. For quantitative multivariate procedures use +RMSEP, compared against an acceptable RMSEC. + +**Precision (3.3.2).** Use authentic homogeneous samples, or artificially prepared ones if +unavailable. + +- *Repeatability (3.3.2.1)*: **a minimum of 9 determinations covering the reportable range** (for + example 3 concentrations × 3 replicates), **or a minimum of 6 determinations at 100% of the test + concentration.** +- *Intermediate precision (3.3.2.2)*: establish the effects of random events — typically different + days, environmental conditions, analysts, and equipment. **Studying these effects individually is + not necessary**, and design of experiments is encouraged. The extent should be justified from + development understanding and risk assessment (ICH Q14). +- *Reproducibility (3.3.2.3)*: an inter-laboratory trial. **Usually not required for a regulatory + submission**, but consider it for pharmacopoeial standardisation or multi-site procedures. + +Recommended data (3.3.2.4): report the standard deviation, the relative standard deviation, and an +appropriate 100(1−α)% confidence interval. + +**Combined accuracy and precision (3.3.3).** Instead of separate criteria, assess total impact +against a single combined criterion, using a prediction interval, a tolerance interval, or a +confidence interval. Report the combined value, describe the approach, and supply the individual +results as supplemental information where they help justify suitability. + +**Robustness (3.4).** Deliberate variation of procedure parameters, plus stability of sample +preparations and reagents over the duration of the procedure. Considered during development; may be +submitted as development data case-by-case or made available on request. See ICH Q14 section 5. + +## Lifecycle, transfer, and prior knowledge + +Section 2.1 permits suitable development data (ICH Q14) to form part of the validation data, and +allows abbreviated validation testing for an established platform procedure used for a new purpose, +with scientific justification. A validation protocol must exist before the study, stating the +intended purpose, the characteristics to be validated, and the associated criteria; where prior +knowledge is used, justify it. Results are summarised in a validation report. + +The experimental design should reflect the number of replicates used in routine analysis to generate +a reportable result, unless a different number is justified. + +Section 2.2 covers change: partial or full revalidation may be needed, decided on science and risk, +and scoped to the characteristics the change affects. **Transfer** to another laboratory calls for +partial or full revalidation and/or comparative analysis of representative samples; not performing +transfer experiments requires justification. **Co-validation** across multiple sites can demonstrate +the criteria are met and can simultaneously satisfy transfer at the participating sites. + +## Annex 2 — illustrative technique examples + +Non-mandatory worked examples, useful as a starting point for the robustness parameter list: + +| Table | Technique | +| --- | --- | +| 3 | Quantitative separation techniques (HPLC, GC, CE) for impurities or assay, and relative-area quantitation | +| 4 | Elemental impurities by ICP-OES or ICP-MS | +| 5 | Dissolution with HPLC as product performance test (corrected 30 Nov 2023) | +| 6 | Quantitative ¹H-NMR for assay of a drug substance | +| 7 | Biological assays | +| 8 | Quantitative PCR | +| 9 | Particle size measurement | +| 10 | NIR analytical procedure | +| 11 | Quantitative LC/MS | + +From Table 3, a detail worth carrying forward: **relative response factors.** Where the analyte +responds differently from the reference material, calculate the RRF from the appropriate ratio of +responses under final procedure conditions and document it. **If the RRF falls outside 0.8–1.2, +apply a correction factor.** Where an impurity is overestimated, omitting the correction may be +acceptable. + +## Multivariate procedures (2.5) + +Results come from a model relating many input variables to the property of interest. Validate in two +phases: + +1. **Model development** — calibration plus internal testing. Test data may be a separate set or + part of the calibration set used rotationally, and are used to estimate performance and tune + parameters such as the number of PLS latent variables. See ICH Q14. +2. **Model validation** — an independent validation set. For identification libraries, analyse + challenge samples *not* represented in the library to demonstrate discriminative ability. + +Samples need reference values or categories, normally from a validated or pharmacopoeial reference +procedure whose performance **equals or exceeds** the expected performance of the multivariate +procedure. Reference measurement and multivariate data collection should be on the same samples +within a period short enough to assure sample and measurement stability. Describe any correlation or +unit conversion, and any assumptions. diff --git a/.agents/skills/analytical-method-validation/references/source-ledger.md b/.agents/skills/analytical-method-validation/references/source-ledger.md new file mode 100644 index 0000000..be0d3cf --- /dev/null +++ b/.agents/skills/analytical-method-validation/references/source-ledger.md @@ -0,0 +1,125 @@ +# Official Source Ledger + +**Research date: 2026-07-27.** Every framework claim in this skill traces to an entry below. +Re-check each source before operational use — guidelines are revised, editions change, and regional +implementation dates differ from adoption dates. + +This ledger is a version baseline. It is not legal advice, an applicability determination, or a +substitute for a controlled copy held under the laboratory's document control. + +## Documents read directly + +These were downloaded and read as full text on the research date, so the requirements encoded in +`scripts/_catalog.py` and summarised in `references/ich-q2r2.md` and +`references/ich-m10-bioanalytical.md` come from the primary source rather than from secondary +summaries. + +### ICH Q2(R2) Validation of Analytical Procedures + +- Source read: +- Verified metadata: Final Version, adopted by the ICH Assembly Regulatory Members under Step 4 on + **1 November 2023**. Step 2 endorsement 24 March 2022. Supersedes Q2(R1) (November 2005). +- Verified detail: an **error correction dated 30 November 2023** covers Table 5 (dissolution with + HPLC, reportable range linearity formulae, page 25) and Tables 6–11 (pages 26–32). +- Content taken: section structure; Table 1 (tests by measured attribute); Table 2 (reportable range + examples); recommended data for specificity, response, lower range limits, accuracy, precision, + and robustness; sections 2.1–2.5; Annex 1 and Annex 2 table inventory; the relative response factor + 0.8–1.2 rule from Annex 2 Table 3. +- Licence: ICH permits use, reproduction, adaptation and distribution under a public licence provided + ICH's copyright is acknowledged. Acknowledged here and in `scripts/_catalog.py`. +- Limitation: **adoption is not implementation.** Confirm the date from which your regional regulator + expects Q2(R2) with that regulator. + +### ICH M10 Bioanalytical Method Validation and Study Sample Analysis + +- Source read: +- Verified metadata: Step 4, dated **24 May 2022**. +- Content taken: chromatographic criteria (section 3) — calibration levels and tolerances, QC + placement at four levels with the low/medium/high definitions, within-run and between-run accuracy + and precision design and criteria, routine-run QC pass rules, carry-over, selectivity source count, + dilution integrity, stability; ligand binding assay criteria (section 4) — calibration tolerances + including anchor point exclusion, five QC levels, run and replicate structure, accuracy and + precision criteria at LLOQ and ULOQ, and the total error criterion; incurred sample reanalysis + (section 5) including the percent-difference basis and the pass fractions. +- Verified distinction: the **total error criterion (≤30%, ≤40% at LLOQ and ULOQ) appears for ligand + binding assays**. No equivalent criterion was found for chromatographic assays. +- Licence: as for Q2(R2). +- Limitation: regional implementation dates differ. Confirm with the regional regulator. + +### ICH Q14 Analytical Procedure Development + +- Source read: +- Content taken: section structure; the minimal versus enhanced approaches (section 2.1); the + analytical target profile (section 3) and that its formal documentation and submission is + **optional**; robustness and parameter ranges (section 5); established conditions (section 6.1); + lifecycle management and post-approval change (section 7); multivariate procedures (section 8). +- Adopted alongside Q2(R2) by the ICH Assembly in the same session. +- Licence: as for Q2(R2). + +## Documents identified but not read (paywalled) + +Designation, title, and scope only. **No requirement, threshold, or study design from any of these is +reproduced anywhere in this skill.** Where a numeric criterion is needed, read it from an authorised +copy. + +### USP–NF general chapters + +- Official pages: `<1220>` ; + `<1225>` ; + `<1226>` +- Verified metadata for `<1220>`: incorporated into USP–NF 2022 Issue 1 on **1 November 2021**, + **official 1 May 2022**. It brings the concepts of `<1224>`, `<1225>` and `<1226>` into a single + three-stage lifecycle. `<1225>` covers validation, particularly Stage 2 activities under `<1220>`; + `<1226>` covers verification of compendial procedures. +- Provenance limitation: this metadata came from **secondary sources** (publisher notices and trade + press) rather than from the USP–NF text, which is behind subscription. Marked + **[confirm in USP–NF]**. Confirm the current official text, revision, and any subsequent change. +- Chapters referenced by designation only, not read: `<1224>`, `<1010>`, `<621>`, `<711>`, `<1092>`. + +### CLSI EP series + +- Publisher: +- Designations and subjects recorded in `references/compendial-and-clsi.md`: EP05, EP06, EP07, EP09, + EP15, EP17, EP25, EP28 (formerly C28), plus the EP17IG and EP28IG implementation guides. +- Provenance limitation: designations, titles and edition numbers were taken from **clsi.org product + listings and secondary sources** on the research date, not read from the documents. Every edition + number carries **[confirm edition]** in the reference file. Editions change; verify on clsi.org + before designing a study. + +### ISO standards + +- ISO/IEC 17025:2017 — . Edition 3; supersedes the 2005 + edition. Relevant clauses: 7.2 (selection, verification and validation of methods), 7.6 + (measurement uncertainty). Not read; identified by catalogue metadata. +- ISO 15189, ISO 21748, ISO 5725 series — referenced by designation and scope only. +- Provenance limitation: ISO catalogue pages have historically refused automated access. Confirm + edition and status on iso.org or with a national member body. **[confirm on iso.org]** +- See this repository's `iso-standards-readiness` skill and its own source ledger for the + accreditation-level treatment of these standards. + +## Statistical methods + +The statistical procedures in `references/statistics.md` and `scripts/_common.py` are standard +published methods, not requirements of any framework: + +- Incomplete beta and gamma function implementations follow the standard continued-fraction and series + algorithms; the t, chi-square and F distributions are derived from them. +- Lack-of-fit F test against pure error: standard regression ANOVA. +- Wald–Wolfowitz runs test: standard non-parametric test of randomness in a sequence of signs. +- One-way random-effects variance components with the standard unbalanced expected-mean-square + coefficient; Satterthwaite approximation for effective degrees of freedom of the total. +- Deming regression with jackknife standard errors; Passing–Bablok with the rank-based slope interval. +- Bland–Altman bias and limits of agreement. +- Two one-sided tests (TOST) for equivalence. + +Implementations are verified against published quantiles and hand-checkable cases in +`tests/analytical-method-validation/test_scripts.py`. Where a framework prescribes a specific +statistical treatment, the framework governs — these are the general-purpose tools. + +## What is deliberately absent + +- No numeric acceptance criteria are supplied for ICH Q2(R2) work. The guideline does not set them and + neither does this skill; they come from the specification, the analytical target profile, or + development data. +- No text, table, threshold, or study design from any USP, CLSI, or ISO document. +- No claim that a procedure is validated, a run acceptable, or an investigation closed. diff --git a/.agents/skills/analytical-method-validation/references/statistics.md b/.agents/skills/analytical-method-validation/references/statistics.md new file mode 100644 index 0000000..dfa649b --- /dev/null +++ b/.agents/skills/analytical-method-validation/references/statistics.md @@ -0,0 +1,209 @@ +# The Statistics, and Why Each One + +Every method in this file is implemented in `scripts/_common.py` using only the standard library. +Distribution functions are computed from the regularised incomplete beta and gamma functions, and the +implementations are checked against published quantiles in `tests/analytical-method-validation/`. + +## Calibration response + +### r² is not evidence of linearity + +The coefficient of determination measures how much of the variance in response the model explains. It +rises with the width of the calibration range and is nearly insensitive to curvature. A quadratic +response measured over a decade of concentration routinely gives r² > 0.99 while the back-calculated +result at the bottom of the range is 10% wrong. + +ICH Q2(R2) 3.2.2.1 asks for r or r², the slope, the intercept, the plot, **and an analysis of the +deviation of the actual data points from the regression line**. The last item is the one that +detects a bad model. Report r² because the guideline asks for it, not because it demonstrates +anything. + +### Lack-of-fit F test + +The correct test of a linear calibration model, and it requires replicates at some levels. + +Partition the residual sum of squares into **pure error** (scatter among replicates at the same +level, which no model can explain) and **lack of fit** (systematic deviation of level means from the +line): + +``` +F = MS_lack-of-fit / MS_pure-error, df = (k - 2, n - k) +``` + +for `k` distinct levels and `n` total points. A significant F says the straight line fails to +describe the data beyond what replicate scatter explains. Without replicates the partition is +impossible and no linearity test exists — which is a good reason to replicate at least one level, and +a reason `check_response.py` says so explicitly when it cannot run the test. + +### Residual pattern: runs test + +Curvature makes residual signs cluster: all negative at the ends and positive in the middle, or the +reverse. The Wald–Wolfowitz runs test counts sign changes and compares against the number expected +if signs were random. Too few runs is evidence of systematic misfit. It complements the F test and +works when replicates are absent, though it needs at least eight points with both signs present. + +### Heteroscedasticity and weighting + +Chromatographic response variance usually scales with concentration. Unweighted least squares +minimises absolute squared residuals, so the high-concentration points — which have the largest +absolute residuals — dominate the fit. The result is a curve that is accurate at the top of the range +and biased at the bottom, which is exactly where an impurity reporting threshold or an LLOQ sits. + +`check_response.py` compares residual variance in the top and bottom thirds of the range. A ratio +above roughly 10× with an unweighted fit is flagged; `1/x` or `1/x²` weighting is the usual remedy. +State the weighting in the protocol before validation — switching to weighting after seeing the data +to make the low end pass is not a statistical decision. + +### Back-calculated relative error + +The practical criterion: invert the fitted line, compute the concentration each response implies, +and compare against nominal at each level. This is what the procedure will actually report, and it +exposes a bad model in units an analyst and an assessor both understand. Bioanalytical work has +required it for decades; it belongs in small-molecule QC validation too. + +## Precision + +### Estimate within each level, never pooled across levels + +Pooling results from 80%, 100% and 120% levels into one standard deviation makes the range itself +appear as imprecision. The number produced is meaningless and always too large. +`check_accuracy_precision.py` estimates precision within each level, and separately provides a +level-independent view by converting to percent of nominal first. + +### Repeatability and intermediate precision are different quantities + +A one-way random-effects model on the intermediate-precision factor — day, analyst, or instrument: + +``` +observation = grand mean + group effect + residual +``` + +with `MS_within` and `MS_between` from the ANOVA table: + +``` +s²_repeatability = MS_within +s²_between = max(0, (MS_between - MS_within) / n_effective) +s²_intermediate = s²_repeatability + s²_between +``` + +For a balanced design `n_effective` is the replicates per group; unbalanced designs use the standard +expected-mean-square coefficient, which the script reports when it applies. + +The between-group variance is truncated at zero because a negative variance estimate is not +meaningful — it means the data cannot distinguish the groups. The script says so when it happens +rather than silently reporting zero. + +Why this matters: a procedure can show 0.07% RSD within a day and 1.65% RSD across days. The +within-day figure is real, and reporting it as the procedure's precision understates routine +performance by more than twenty-fold. Q2(R2) 3.3.2.2 exists precisely because the between-day +component is the one that bites in routine use. + +### Confidence intervals on a standard deviation + +A precision estimate from six or nine determinations is imprecise, and Q2(R2) 3.3.2.4 asks for an +interval alongside it. For a variance with `ν` degrees of freedom: + +``` +s · sqrt(ν / χ²_{1-α/2, ν}) < σ < s · sqrt(ν / χ²_{α/2, ν}) +``` + +These intervals are wide, and that is the point. With ν = 5 the upper bound is roughly twice the +point estimate. An RSD that lands just inside a limit on six replicates has not demonstrated that the +procedure meets the limit. For the total (intermediate) SD, which is a sum of variance components, +the effective degrees of freedom come from the Satterthwaite approximation. + +## Accuracy + +Report mean percent recovery, or the difference from the accepted true value, **with a confidence +interval** — Q2(R2) 3.3.1.4 is explicit, and a bare mean is not sufficient. The interval is +`mean ± t_{1-α/2, n-1} · s/√n` at each level. + +The stricter reading, available as `--require-ci-within-limit`, asks that the whole interval sit +inside the acceptance limit rather than just the point estimate. Q2(R2) says the observed interval +should be *compatible with* the criterion. Which reading applies is a decision to make and justify in +the protocol, before the data exist. + +### Combined accuracy and precision + +Q2(R2) 3.3.3 permits a single combined criterion assessed with a prediction interval, a tolerance +interval, or a confidence interval, instead of separate accuracy and precision criteria. This is +often the more honest framing — what matters is whether a future reportable result will be close +enough to the truth, which is a tolerance-interval question. If you use it, describe the approach and +supply the individual results as supporting information. + +## Detection and quantitation limits + +The `3.3σ/S` and `10σ/S` formulae are estimates whose value depends entirely on which σ you choose. +On the same calibration data, σ from the residual SD of the regression, from the SD of the +y-intercept, and from the SD of blank responses commonly give limits spanning a factor of two or +more. None is wrong; they answer slightly different questions. + +Consequences for practice: + +- Report the limit **and the approach**, per Q2(R2) 3.2.3.5. A number alone is not reportable. +- Confirm an estimated limit with real determinations at or near it. `3.2.3.4` allows skipping the + estimate entirely and validating the QL directly by accuracy and precision, which is cleaner. +- For impurity procedures, the QL must be at or below the reporting threshold. +- Signal-to-noise scaling assumes noise is constant with concentration. It usually is not; confirm at + the resulting level. +- CLSI's limit of blank / limit of detection / limit of quantitation are defined differently again, + with their own protocols. Do not translate between the schemes casually. + +## Method comparison and transfer + +### Ordinary least squares is the wrong regression here + +OLS assumes the x values are known without error. In a method comparison both procedures have +measurement error, and ignoring the error in x biases the slope toward zero — a regression-dilution +effect that manufactures apparent proportional bias where none exists. + +**Deming regression** accounts for error in both variables given `λ`, the ratio of error variances. +With `λ = 1` (equal precision) it reduces to orthogonal regression. Standard errors here come from a +jackknife, which avoids distributional assumptions about the slope. + +**Passing–Bablok** is non-parametric: the slope is a shifted median of all pairwise slopes, with a +rank-based confidence interval. It assumes no distribution, tolerates outliers, and is the usual +choice in clinical method comparison. Its confidence intervals are wider, honestly reflecting what +the data support. + +Report both. Agreement between them is reassuring; disagreement points to outliers or to a +distributional problem worth understanding before concluding anything. + +### Bland–Altman answers a different question + +Regression asks whether the relationship is proportional. Bland–Altman asks how far apart two +procedures are on the same sample: mean difference (bias) and limits of agreement at +`bias ± 1.96·SD`. Both matter, and neither substitutes for the other. + +Two cautions. The limits of agreement are themselves estimates with confidence intervals, which are +wide for small n — the script reports the half-width. And if the difference trends with +concentration, a single mean bias and its limits are misleading no matter how tight they look; the +script tests for that trend and flags it. + +### Equivalence: TOST, not a t test + +The default reflex at a transfer is a two-sample or paired t test, and `p > 0.05` written up as "no +significant difference, methods equivalent". This inverts the logic. A non-significant result means +the data were insufficient to detect a difference — and on a transfer dataset of ten or twenty +samples, that outcome is close to guaranteed regardless of whether the procedures agree. The test +rewards small studies. + +**Two one-sided tests** invert the hypotheses to match the question. Given a pre-stated margin `δ`, +test both `H01: difference ≤ -δ` and `H02: difference ≥ +δ`. Rejecting both concludes equivalence. +Operationally: the `(1-2α)` confidence interval on the difference must lie entirely inside `±δ`. + +A worked contrast from `compare_methods.py`: a transfer with a consistent +1.46% bias gives a paired +t-test p-value below 0.0001 — a highly significant difference — while TOST establishes equivalence at +a ±2% margin. Both are correct. The difference is real and it is small enough not to matter. Only +TOST answers the question the transfer actually asks. + +The margin must be pre-stated, from the specification or the analytical target profile. A margin +chosen after seeing the data is not an acceptance criterion, and this is the single most common way +equivalence testing gets misused. + +## What none of this does + +These are computations. They do not establish that a procedure is fit for purpose. That conclusion +requires the intended purpose, the specification, product and process knowledge, the laboratory's +history with the technique, and the judgement of people who are accountable for it. A script that +reported "validated" would be lying about what it can know. diff --git a/.agents/skills/analytical-method-validation/scripts/_catalog.py b/.agents/skills/analytical-method-validation/scripts/_catalog.py new file mode 100644 index 0000000..2911554 --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/_catalog.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +"""Framework catalogue for analytical method validation. + +Content sourced 2026-07-27 from the freely published ICH guidelines, which ICH +licenses for reuse with acknowledgement. Compendial (USP) and CLSI documents are +copyrighted and paywalled: they are referenced here by designation, title, and +scope only. No proprietary text is reproduced. + +See ../references/source-ledger.md for the provenance of every entry. +""" + +from __future__ import annotations + +from typing import Any + +RESEARCH_DATE = "2026-07-27" + +# -------------------------------------------------------------------------- +# Frameworks +# -------------------------------------------------------------------------- + +FRAMEWORKS: dict[str, dict[str, Any]] = { + "ich-q2r2": { + "title": "ICH Q2(R2) Validation of Analytical Procedures", + "adopted": "2023-11-01", + "effective_note": ( + "Adopted by the ICH Assembly 1 Nov 2023; an error correction to Table 5 and " + "Tables 6-11 is dated 30 Nov 2023. Confirm the adoption/implementation date for " + "your region with the regional regulator." + ), + "supersedes": "ICH Q2(R1) (2005)", + "url": "https://database.ich.org/sites/default/files/ICH_Q2%28R2%29_Guideline_2023_1130.pdf", + "reproducible": True, + "scope": ( + "Analytical procedures for release and stability testing of commercial drug " + "substances and products; applicable to other control-strategy procedures on a " + "risk basis, and phase-appropriately during clinical development." + ), + "governs": ["assay", "potency", "purity", "impurity-quantitative", + "impurity-limit", "identity", "dissolution", "content-uniformity"], + "companion": "ICH Q14 (analytical procedure development, robustness, lifecycle)", + }, + "ich-m10": { + "title": "ICH M10 Bioanalytical Method Validation and Study Sample Analysis", + "adopted": "2022-05-24", + "effective_note": ( + "Step 4 adopted 24 May 2022. Regional implementation dates differ; confirm with " + "the regional regulator." + ), + "supersedes": ( + "Harmonises region-specific bioanalytical guidance (e.g., FDA 2018 BMV, " + "EMA 2011); what it replaces depends on the region's implementation" + ), + "url": "https://database.ich.org/sites/default/files/M10_Guideline_Step4_2022_0524.pdf", + "reproducible": True, + "scope": ( + "Bioanalytical methods quantifying drug/metabolite concentrations in biological " + "matrices supporting nonclinical and clinical studies, plus study sample analysis." + ), + "governs": ["pk-concentration", "toxicokinetics", "bioequivalence", "biomarker-selected"], + "companion": "Distinct criteria for chromatographic methods vs ligand binding assays", + }, + "usp-1220": { + "title": "USP General Chapter <1220> Analytical Procedure Life Cycle", + "adopted": "official 2022-05-01", + "effective_note": ( + "Incorporated into USP-NF 2022 Issue 1 (1 Nov 2021), official 1 May 2022. " + "Confirm the current official text and any revision in the USP-NF." + ), + "supersedes": "integrates the concepts of <1224>, <1225>, and <1226> into a lifecycle", + "url": "https://doi.usp.org/USPNF/USPNF_M10975_02_01.html", + "reproducible": False, + "scope": ( + "Three-stage lifecycle: procedure design (Stage 1), performance qualification " + "(Stage 2), ongoing performance verification (Stage 3), organised around an " + "analytical target profile." + ), + "governs": ["compendial-lifecycle"], + "companion": "<1225> validation, <1226> verification, <1224> transfer, <1010> data treatment", + }, + "usp-1225": { + "title": "USP General Chapter <1225> Validation of Compendial Procedures", + "adopted": "see current USP-NF", + "effective_note": "Confirm the current official text and revision in the USP-NF.", + "supersedes": "", + "url": "https://doi.usp.org/USPNF/USPNF_M99945_40101_01.html", + "reproducible": False, + "scope": ( + "Validation of non-compendial procedures and of compendial procedures used " + "outside their stated scope; Stage 2 activities under <1220>." + ), + "governs": ["assay", "impurity-quantitative", "impurity-limit", "identity"], + "companion": "<1226> when verifying a compendial procedure as written", + }, + "usp-1226": { + "title": "USP General Chapter <1226> Verification of Compendial Procedures", + "adopted": "see current USP-NF", + "effective_note": "Confirm the current official text and revision in the USP-NF.", + "supersedes": "", + "url": "https://doi.usp.org/USPNF/USPNF_M870_03_01.html", + "reproducible": False, + "scope": ( + "Assessment of selected performance characteristics to show a compendial " + "procedure works under actual conditions of use. Verification is not " + "revalidation and does not repeat the full validation." + ), + "governs": ["compendial-verification"], + "companion": "<1225> when the procedure is used outside its compendial scope", + }, + "clsi": { + "title": "CLSI EP series (clinical laboratory measurement procedures)", + "adopted": "per document", + "effective_note": ( + "Editions change; the designations below were taken from clsi.org listings and " + "secondary sources on the research date and are marked [confirm on clsi.org]. " + "Verify the current edition before designing a study." + ), + "supersedes": "", + "url": "https://clsi.org/standards/products/method-evaluation/", + "reproducible": False, + "scope": ( + "Establishment and user verification of performance for clinical laboratory " + "measurement procedures, under CLIA/CAP and ISO 15189 quality systems." + ), + "governs": ["clinical-verification", "clinical-establishment"], + "companion": "ISO 15189 for the surrounding medical laboratory quality system", + }, + "iso-17025": { + "title": "ISO/IEC 17025:2017 (testing and calibration laboratory competence)", + "adopted": "2017", + "effective_note": "Copyrighted. Obtain an authorised copy from ISO or a national member.", + "supersedes": "ISO/IEC 17025:2005", + "url": "https://www.iso.org/standard/66912.html", + "reproducible": False, + "scope": ( + "Clause 7.2 covers selection, verification and validation of methods; clause 7.6 " + "covers measurement uncertainty. Method validation is required to the extent " + "necessary for the intended use." + ), + "governs": ["nonstandard-method", "lab-developed-method", "modified-standard-method"], + "companion": ( + "The repo's iso-standards-readiness skill covers the surrounding quality system; " + "this skill covers the individual procedure." + ), + }, +} + +# -------------------------------------------------------------------------- +# ICH Q2(R2) Table 1 -- which validation tests for which measured attribute +# Source: ICH Q2(R2), Table 1. "+" normally conducted, "-" not normally conducted. +# -------------------------------------------------------------------------- + +Q2R2_TESTS_BY_ATTRIBUTE: dict[str, dict[str, str]] = { + "identity": { + "specificity": "required", + "response": "not-normally", + "lower-range-limit": "not-normally", + "accuracy": "not-normally", + "repeatability": "not-normally", + "intermediate-precision": "not-normally", + }, + "impurity-quantitative": { + "specificity": "required", + "response": "required", + "lower-range-limit": "required-QL", + "accuracy": "required", + "repeatability": "required", + "intermediate-precision": "required-unless-reproducibility", + }, + "impurity-limit": { + "specificity": "required", + "response": "not-normally", + "lower-range-limit": "required-DL", + "accuracy": "not-normally", + "repeatability": "not-normally", + "intermediate-precision": "not-normally", + }, + "assay": { + "specificity": "required", + "response": "required", + "lower-range-limit": "not-normally", + "accuracy": "required", + "repeatability": "required", + "intermediate-precision": "required-unless-reproducibility", + }, +} + +ATTRIBUTE_ALIASES = { + "content": "assay", + "potency": "assay", + "assay": "assay", + "identification": "identity", + "identity": "identity", + "id": "identity", + "impurity": "impurity-quantitative", + "impurities": "impurity-quantitative", + "related-substances": "impurity-quantitative", + "purity": "impurity-quantitative", + "impurity-quantitative": "impurity-quantitative", + "impurity-limit": "impurity-limit", + "limit-test": "impurity-limit", +} + +# ICH Q2(R2) Table 2 -- examples of reportable ranges. +Q2R2_REPORTABLE_RANGE: dict[str, dict[str, str]] = { + "assay": { + "low": "80% of declared content, or 80% of the lower specification acceptance criterion", + "high": "120% of declared content, or 120% of the upper specification acceptance criterion", + }, + "potency": { + "low": "lowest specification acceptance criterion -20%", + "high": "highest specification acceptance criterion +20%", + }, + "content-uniformity": { + "low": "70% of declared content", + "high": "130% of declared content", + }, + "dissolution-ir-one-point": { + "low": "Q - 45% of the lowest strength specification", + "high": "(per specification; see ICH Q2(R2) Table 2)", + }, + "dissolution-ir-multi-point": { + "low": "lower limit of reportable range as justified by the specification, or QL", + "high": "130% of declared content of the highest strength", + }, + "dissolution-modified-release": { + "low": "lower limit of reportable range as justified by the specification, or QL", + "high": "(per specification; see ICH Q2(R2) Table 2)", + }, + "impurity-quantitative": { + "low": "reporting threshold", + "high": "120% of the specification acceptance criterion", + }, + "purity-area-percent": { + "low": "80% of the lower specification acceptance criterion", + "high": "upper specification acceptance criterion, or 100%", + }, +} + +# ICH Q2(R2) recommended data, section 3. +Q2R2_STUDY_DESIGN: dict[str, dict[str, str]] = { + "response": { + "requirement": "minimum of 5 concentrations appropriately distributed across the range", + "reference": "Q2(R2) 3.2.2.1", + "report": ( + "plot of the data, correlation coefficient or coefficient of determination, " + "y-intercept, slope, and an analysis of deviation of points from the line " + "(residual pattern)" + ), + }, + "accuracy": { + "requirement": ( + "appropriate number of determinations and levels across the reportable range " + "(e.g., 3 concentrations / 3 replicates each of the full procedure)" + ), + "reference": "Q2(R2) 3.3.1.4", + "report": ( + "mean percent recovery of a known added amount, or difference between mean and " + "accepted true value, with a 100(1-alpha)% confidence interval" + ), + }, + "repeatability": { + "requirement": ( + "minimum 9 determinations covering the reportable range (e.g., 3 concentrations " + "/ 3 replicates), or minimum 6 determinations at 100% of the test concentration" + ), + "reference": "Q2(R2) 3.3.2.1", + "report": "standard deviation, relative standard deviation, and a 100(1-alpha)% CI", + }, + "intermediate-precision": { + "requirement": ( + "effects of random events -- typically different days, environmental conditions, " + "analysts, equipment. Studying effects individually is not necessary; DoE is " + "encouraged. Extent justified by development understanding and risk (ICH Q14)" + ), + "reference": "Q2(R2) 3.3.2.2", + "report": "standard deviation, relative standard deviation, and a 100(1-alpha)% CI", + }, + "reproducibility": { + "requirement": ( + "inter-laboratory trial; usually NOT required for a regulatory submission, but " + "consider for pharmacopoeial standardisation or multi-site procedures" + ), + "reference": "Q2(R2) 3.3.2.3", + "report": "standard deviation, relative standard deviation, and a 100(1-alpha)% CI", + }, + "specificity": { + "requirement": ( + "absence of interference, orthogonal procedure comparison, or technology-inherent " + "justification. For a stability-indicating claim, include samples containing " + "relevant degradation products (spiked, stressed, or aged)" + ), + "reference": "Q2(R2) 3.1, 2.4", + "report": "interference data, resolution/peak purity, or orthogonal comparison", + }, + "lower-range-limit": { + "requirement": ( + "DL/QL by visual evaluation, signal-to-noise, standard deviation of the response " + "and slope, or direct accuracy and precision at the lower limit. An estimated " + "limit should then be confirmed with samples at or near that limit" + ), + "reference": "Q2(R2) 3.2.3", + "report": "the limit and the approach used to determine it", + }, + "robustness": { + "requirement": ( + "deliberate variation of procedure parameters plus solution stability. Normally " + "performed during development under ICH Q14; submitted case-by-case or available " + "on request" + ), + "reference": "Q2(R2) 3.4, ICH Q14 section 5", + "report": "parameters varied, ranges, and the effect on the reportable result", + }, +} + +# ICH Q2(R2) 3.2.3.2/3.2.3.3 -- the estimation approaches and their constants. +DL_QL_APPROACHES = { + "visual": { + "dl": "lowest level reliably detected by analysis of known concentrations", + "ql": "lowest level reliably quantitated by analysis of known concentrations", + "note": "acceptable for both non-instrumental and instrumental procedures", + }, + "signal-to-noise": { + "dl": "S/N of 3:1 generally acceptable", + "ql": "S/N of at least 10:1 acceptable", + "note": "only for procedures exhibiting baseline noise; define the noise region", + }, + "sd-and-slope": { + "dl": "DL = 3.3 * sigma / S", + "ql": "QL = 10 * sigma / S", + "note": ( + "sigma from the SD of blank responses, the residual SD of the regression line, " + "or the SD of y-intercepts of regression lines; S is the calibration slope" + ), + }, + "accuracy-precision": { + "dl": "not applicable", + "ql": "QL validated directly by accuracy and precision at the lower range limit", + "note": "avoids relying on an estimate; Q2(R2) 3.2.3.4", + }, +} + +# -------------------------------------------------------------------------- +# ICH M10 acceptance criteria. Verified against the Step 4 guideline text. +# Chromatographic (CC) and ligand binding assay (LBA) criteria differ and are +# the single most commonly conflated pair in bioanalysis. +# -------------------------------------------------------------------------- + +M10_CRITERIA: dict[str, dict[str, Any]] = { + "chromatographic": { + "label": "Chromatographic assays (ICH M10 section 3)", + "calibration_min_levels": 6, + "calibration_tolerance_pct": 15.0, + "calibration_tolerance_lloq_pct": 20.0, + "calibration_tolerance_uloq_pct": 15.0, + "calibration_min_pass_fraction": 0.75, + "accuracy_tolerance_pct": 15.0, + "accuracy_tolerance_lloq_pct": 20.0, + "precision_cv_pct": 15.0, + "precision_cv_lloq_pct": 20.0, + "limit_levels": "LLOQ", + "qc_levels_accuracy_precision": 4, + "qc_levels_routine_run": 3, + "ap_replicates_per_run": 5, + "ap_min_runs": 3, + "ap_min_days": 2, + "qc_run_pass_fraction": 2.0 / 3.0, + "qc_run_pass_fraction_per_level": 0.50, + "qc_run_tolerance_pct": 15.0, + "total_error_pct": None, + "total_error_pct_at_limits": None, + "isr_tolerance_pct": 20.0, + "isr_pass_fraction": 2.0 / 3.0, + "carryover_blank_pct_of_lloq": 20.0, + "carryover_blank_pct_of_is": 5.0, + "selectivity_min_sources": 6, + "dilution_tolerance_pct": 15.0, + "stability_tolerance_pct": 15.0, + "notes": ( + "Accuracy/precision validation QCs at a minimum of 4 levels: LLOQ, low QC within " + "3x the LLOQ, medium QC around 30-50% of the calibration range, and high QC at " + "least 75% of the ULOQ. Within-run uses at least 5 replicates per level per run; " + "between-run uses each level in at least 3 runs over at least 2 days. Routine " + "(non-accuracy-and-precision) runs may use low, medium and high QCs in duplicate. " + "M10 states no explicit total-error criterion for chromatographic assays." + ), + }, + "lba": { + "label": "Ligand binding assays (ICH M10 section 4)", + "calibration_min_levels": 6, + "calibration_tolerance_pct": 20.0, + "calibration_tolerance_lloq_pct": 25.0, + "calibration_tolerance_uloq_pct": 25.0, + "calibration_min_pass_fraction": 0.75, + "accuracy_tolerance_pct": 20.0, + "accuracy_tolerance_lloq_pct": 25.0, + "precision_cv_pct": 20.0, + "precision_cv_lloq_pct": 25.0, + "limit_levels": "LLOQ and ULOQ", + "qc_levels_accuracy_precision": 5, + "qc_levels_routine_run": 3, + "ap_replicates_per_run": 3, + "ap_min_runs": 6, + "ap_min_days": 2, + "qc_run_pass_fraction": 2.0 / 3.0, + "qc_run_pass_fraction_per_level": 0.50, + "qc_run_tolerance_pct": 20.0, + "total_error_pct": 30.0, + "total_error_pct_at_limits": 40.0, + "isr_tolerance_pct": 30.0, + "isr_pass_fraction": 2.0 / 3.0, + "carryover_blank_pct_of_lloq": None, + "carryover_blank_pct_of_is": None, + "selectivity_min_sources": 6, + "dilution_tolerance_pct": 20.0, + "stability_tolerance_pct": 20.0, + "notes": ( + "Anchor points outside the quantitation range are excluded from the calibration " + "pass count. Accuracy and precision are evaluated at 5 QC levels (LLOQ, low, " + "medium, high, ULOQ) with at least 3 replicates per level per run in at least 6 " + "runs over 2 or more days. LBAs carry an additional total-error criterion: the " + "sum of absolute accuracy (%) and precision (%) must not exceed 30%, or 40% at " + "the LLOQ and ULOQ. Chromatographic assays have no such criterion." + ), + }, +} + +# -------------------------------------------------------------------------- +# Technique notes distilled from ICH Q2(R2) Annex 2 (illustrative, not mandatory) +# -------------------------------------------------------------------------- + +TECHNIQUE_NOTES: dict[str, dict[str, str]] = { + "hplc": { + "annex_table": "Table 3 (quantitative separation techniques)", + "robustness": ( + "extraction volume/time/temperature, dilution, column or capillary lot, mobile " + "phase and buffer composition and pH, column temperature, flow rate, detection " + "wavelength; plus stability of sample and reference preparations" + ), + "special": ( + "Relative response factors: if the RRF falls outside 0.8-1.2, apply a correction " + "factor. If an impurity is overestimated it may be acceptable to omit the " + "correction. Determine RRF under final procedure conditions and document it." + ), + }, + "gc": {"annex_table": "Table 3 (quantitative separation techniques)", + "robustness": "as for HPLC, plus inlet temperature, split ratio, carrier flow, oven ramp", + "special": "same relative response factor 0.8-1.2 consideration as HPLC"}, + "ce": {"annex_table": "Table 3 (quantitative separation techniques)", + "robustness": "capillary lot, buffer composition and pH, capillary temperature, voltage", + "special": "same relative response factor consideration as HPLC"}, + "icp": {"annex_table": "Table 4 (elemental impurities by ICP-OES or ICP-MS)", + "robustness": "plasma conditions, sample introduction, internal standard, matrix matching", + "special": "spectral and non-spectral interference; ICH Q3D drives which elements matter"}, + "dissolution": {"annex_table": "Table 5 (dissolution with HPLC as product performance test)", + "robustness": "medium composition and volume, deaeration, agitation, sinker, filter", + "special": ( + "Table 5 was corrected on 30 Nov 2023 (reportable range linearity " + "formulae). Use the corrected text." + )}, + "qnmr": {"annex_table": "Table 6 (quantitative 1H-NMR for assay of a drug substance)", + "robustness": "pulse angle, relaxation delay, number of scans, temperature, shimming", + "special": "internal standard purity and signal selection dominate accuracy"}, + "bioassay": {"annex_table": "Table 7 (biological assays)", + "robustness": "cell passage, incubation time and temperature, reagent lot, plate layout", + "special": ( + "Non-linear (4- or 5-parameter logistic) response is expected. Linearity " + "of the concentration-response relationship is NOT required; evaluate " + "proportionality of results to expected values instead." + )}, + "qpcr": {"annex_table": "Table 8 (quantitative PCR)", + "robustness": "primer/probe lot, master mix, cycling parameters, template input", + "special": "amplification efficiency and specificity of amplicon detection"}, + "particle-size": {"annex_table": "Table 9 (particle size measurement)", + "robustness": "dispersion medium, sonication, pump speed, obscuration", + "special": "technology-inherent justification may substitute for some characteristics"}, + "nir": {"annex_table": "Table 10 (NIR analytical procedure)", + "robustness": "instrument, probe, sample presentation, temperature, humidity", + "special": ( + "Multivariate: validate in two phases (calibration plus internal testing, " + "then an independent validation set). Report RMSEP against RMSEC. Reference " + "procedure performance must equal or exceed the multivariate procedure's." + )}, + "lcms": {"annex_table": "Table 11 (quantitative LC/MS)", + "robustness": "source conditions, mobile phase additives, column lot, matrix lots", + "special": ( + "Matrix effects and ion suppression need explicit evaluation. For a " + "bioanalytical purpose, ICH M10 governs instead of Q2(R2)." + )}, +} + + +def resolve_attribute(name: str) -> str: + key = name.strip().lower().replace("_", "-") + if key in ATTRIBUTE_ALIASES: + return ATTRIBUTE_ALIASES[key] + raise KeyError( + f"unknown attribute {name!r}; choose from: {', '.join(sorted(set(ATTRIBUTE_ALIASES)))}" + ) diff --git a/.agents/skills/analytical-method-validation/scripts/_common.py b/.agents/skills/analytical-method-validation/scripts/_common.py new file mode 100644 index 0000000..50332f2 --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/_common.py @@ -0,0 +1,955 @@ +#!/usr/bin/env python3 +"""Shared statistics and I/O for analytical method validation checks. + +Standard library only. Every distribution function here is implemented from the +regularised incomplete beta and gamma functions so the scripts run in any +Python 3.11+ interpreter without numpy or scipy. + +These helpers compute and report. They never decide that a procedure is +validated, fit for purpose, or acceptable to a regulator -- that judgement +belongs to the analyst and the quality unit. +""" + +from __future__ import annotations + +import argparse +import csv +import io +import json +import math +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Sequence + +# -------------------------------------------------------------------------- +# Limits and exit codes +# -------------------------------------------------------------------------- + +MAX_INPUT_BYTES = 5_000_000 +MAX_ROWS = 20_000 + +EXIT_OK = 0 +EXIT_FINDINGS = 1 +EXIT_INPUT_ERROR = 2 + +TINY = 1e-300 + + +class InputError(Exception): + """Raised for malformed or out-of-bounds user input.""" + + +# -------------------------------------------------------------------------- +# Special functions +# -------------------------------------------------------------------------- + + +def _betacf(a: float, b: float, x: float, itmax: int = 400, eps: float = 3e-16) -> float: + """Continued fraction for the incomplete beta function (Lentz's method).""" + qab, qap, qam = a + b, a + 1.0, a - 1.0 + c = 1.0 + d = 1.0 - qab * x / qap + if abs(d) < TINY: + d = TINY + d = 1.0 / d + h = d + for m in range(1, itmax + 1): + m2 = 2 * m + aa = m * (b - m) * x / ((qam + m2) * (a + m2)) + d = 1.0 + aa * d + if abs(d) < TINY: + d = TINY + c = 1.0 + aa / c + if abs(c) < TINY: + c = TINY + d = 1.0 / d + h *= d * c + aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)) + d = 1.0 + aa * d + if abs(d) < TINY: + d = TINY + c = 1.0 + aa / c + if abs(c) < TINY: + c = TINY + d = 1.0 / d + delta = d * c + h *= delta + if abs(delta - 1.0) < eps: + break + return h + + +def betainc(a: float, b: float, x: float) -> float: + """Regularised incomplete beta function I_x(a, b).""" + if x <= 0.0: + return 0.0 + if x >= 1.0: + return 1.0 + log_front = ( + math.lgamma(a + b) + - math.lgamma(a) + - math.lgamma(b) + + a * math.log(x) + + b * math.log1p(-x) + ) + front = math.exp(log_front) + if x < (a + 1.0) / (a + b + 2.0): + return front * _betacf(a, b, x) / a + return 1.0 - front * _betacf(b, a, 1.0 - x) / b + + +def gammainc_lower(a: float, x: float) -> float: + """Regularised lower incomplete gamma P(a, x).""" + if x <= 0.0: + return 0.0 + if x < a + 1.0: + # Series representation. + term = 1.0 / a + total = term + n = a + for _ in range(1000): + n += 1.0 + term *= x / n + total += term + if abs(term) < abs(total) * 1e-16: + break + return total * math.exp(-x + a * math.log(x) - math.lgamma(a)) + # Continued fraction for Q(a, x), then complement. + b = x + 1.0 - a + c = 1.0 / TINY + d = 1.0 / b + h = d + for i in range(1, 1000): + an = -i * (i - a) + b += 2.0 + d = an * d + b + if abs(d) < TINY: + d = TINY + c = b + an / c + if abs(c) < TINY: + c = TINY + d = 1.0 / d + delta = d * c + h *= delta + if abs(delta - 1.0) < 1e-16: + break + q = math.exp(-x + a * math.log(x) - math.lgamma(a)) * h + return 1.0 - q + + +def _bisect_ppf(cdf, target: float, lo: float, hi: float, tol: float = 1e-12) -> float: + """Invert a monotone CDF by bisection.""" + for _ in range(300): + mid = 0.5 * (lo + hi) + if cdf(mid) < target: + lo = mid + else: + hi = mid + if hi - lo < tol * max(1.0, abs(mid)): + break + return 0.5 * (lo + hi) + + +def t_cdf(t: float, df: float) -> float: + """CDF of Student's t with df degrees of freedom.""" + if df <= 0: + raise InputError("t distribution needs df > 0") + x = df / (df + t * t) + half = 0.5 * betainc(0.5 * df, 0.5, x) + return half if t <= 0 else 1.0 - half + + +def t_ppf(p: float, df: float) -> float: + """Quantile of Student's t.""" + if not 0.0 < p < 1.0: + raise InputError("t_ppf needs 0 < p < 1") + return _bisect_ppf(lambda t: t_cdf(t, df), p, -1e4, 1e4) + + +def chi2_cdf(x: float, df: float) -> float: + """CDF of the chi-square distribution.""" + if x <= 0: + return 0.0 + return gammainc_lower(0.5 * df, 0.5 * x) + + +def chi2_ppf(p: float, df: float) -> float: + """Quantile of the chi-square distribution.""" + if not 0.0 < p < 1.0: + raise InputError("chi2_ppf needs 0 < p < 1") + return _bisect_ppf(lambda x: chi2_cdf(x, df), p, 1e-12, 1e6) + + +def f_cdf(x: float, df1: float, df2: float) -> float: + """CDF of the F distribution.""" + if x <= 0: + return 0.0 + return betainc(0.5 * df1, 0.5 * df2, df1 * x / (df1 * x + df2)) + + +def f_sf(x: float, df1: float, df2: float) -> float: + """Upper tail of the F distribution (the p-value for an F test). + + Computed from the complementary incomplete beta rather than as 1 - cdf, + which underflows to exactly 0 for large F and would print a lack-of-fit + p-value of 0 in a validation report. + """ + if x <= 0: + return 1.0 + return betainc(0.5 * df2, 0.5 * df1, df2 / (df1 * x + df2)) + + +def z_ppf(p: float) -> float: + """Standard normal quantile.""" + from statistics import NormalDist + + return NormalDist().inv_cdf(p) + + +# -------------------------------------------------------------------------- +# Descriptive helpers +# -------------------------------------------------------------------------- + + +def mean(values: Sequence[float]) -> float: + if not values: + raise InputError("mean of an empty sequence") + return math.fsum(values) / len(values) + + +def sample_sd(values: Sequence[float]) -> float: + n = len(values) + if n < 2: + return float("nan") + m = mean(values) + return math.sqrt(math.fsum((v - m) ** 2 for v in values) / (n - 1)) + + +def rsd_percent(values: Sequence[float]) -> float: + """Relative standard deviation (%CV). NaN when the mean is ~0.""" + m = mean(values) + if abs(m) < 1e-15: + return float("nan") + return 100.0 * sample_sd(values) / abs(m) + + +def median(values: Sequence[float]) -> float: + if not values: + raise InputError("median of an empty sequence") + s = sorted(values) + n = len(s) + mid = n // 2 + return s[mid] if n % 2 else 0.5 * (s[mid - 1] + s[mid]) + + +def sd_confidence_interval(sd: float, df: float, level: float = 0.90) -> tuple[float, float]: + """Chi-square confidence interval for a standard deviation.""" + if df <= 0 or not math.isfinite(sd): + return (float("nan"), float("nan")) + alpha = 1.0 - level + lo_chi = chi2_ppf(1.0 - alpha / 2.0, df) + hi_chi = chi2_ppf(alpha / 2.0, df) + return (sd * math.sqrt(df / lo_chi), sd * math.sqrt(df / hi_chi)) + + +# -------------------------------------------------------------------------- +# Regression +# -------------------------------------------------------------------------- + + +@dataclass +class LinearFit: + """Weighted least-squares straight-line fit and its diagnostics.""" + + n: int + slope: float + intercept: float + se_slope: float + se_intercept: float + residual_sd: float + r_squared: float + r: float + df: int + residuals: list[float] = field(default_factory=list) + fitted: list[float] = field(default_factory=list) + weights: list[float] = field(default_factory=list) + + def predict(self, x: float) -> float: + return self.intercept + self.slope * x + + def slope_ci(self, level: float = 0.95) -> tuple[float, float]: + t = t_ppf(0.5 + level / 2.0, self.df) + return (self.slope - t * self.se_slope, self.slope + t * self.se_slope) + + def intercept_ci(self, level: float = 0.95) -> tuple[float, float]: + t = t_ppf(0.5 + level / 2.0, self.df) + return (self.intercept - t * self.se_intercept, self.intercept + t * self.se_intercept) + + +def fit_linear( + xs: Sequence[float], ys: Sequence[float], weights: Sequence[float] | None = None +) -> LinearFit: + """Fit y = a + b*x by (optionally weighted) least squares.""" + n = len(xs) + if n != len(ys): + raise InputError("x and y must be the same length") + if n < 3: + raise InputError("a regression needs at least 3 points") + w = [1.0] * n if weights is None else [float(v) for v in weights] + if len(w) != n: + raise InputError("weights must match the number of points") + if any(v < 0 for v in w): + raise InputError("weights must be non-negative") + + sw = math.fsum(w) + swx = math.fsum(wi * xi for wi, xi in zip(w, xs)) + swy = math.fsum(wi * yi for wi, yi in zip(w, ys)) + swxx = math.fsum(wi * xi * xi for wi, xi in zip(w, xs)) + swxy = math.fsum(wi * xi * yi for wi, xi, yi in zip(w, xs, ys)) + denom = sw * swxx - swx * swx + if abs(denom) < 1e-300: + raise InputError("x values are collinear or identical; slope is undefined") + + slope = (sw * swxy - swx * swy) / denom + intercept = (swy - slope * swx) / sw + fitted = [intercept + slope * xi for xi in xs] + residuals = [yi - fi for yi, fi in zip(ys, fitted)] + df = n - 2 + ss_res = math.fsum(wi * ri * ri for wi, ri in zip(w, residuals)) + residual_sd = math.sqrt(ss_res / df) + se_slope = residual_sd * math.sqrt(sw / denom) + se_intercept = residual_sd * math.sqrt(swxx / denom) + + ybar_w = swy / sw + ss_tot = math.fsum(wi * (yi - ybar_w) ** 2 for wi, yi in zip(w, ys)) + r_squared = 1.0 - ss_res / ss_tot if ss_tot > 0 else float("nan") + r = math.copysign(math.sqrt(max(0.0, r_squared)), slope) + + return LinearFit( + n=n, + slope=slope, + intercept=intercept, + se_slope=se_slope, + se_intercept=se_intercept, + residual_sd=residual_sd, + r_squared=r_squared, + r=r, + df=df, + residuals=residuals, + fitted=fitted, + weights=w, + ) + + +def runs_test(residuals: Sequence[float]) -> dict[str, Any]: + """Wald-Wolfowitz runs test on residual signs. + + ICH Q2(R2) 3.2.2.1 asks for the impact of any non-random pattern in the + residual plot to be assessed. Curvature shows up as too few runs. + """ + signs = [1 if r >= 0 else -1 for r in residuals if r != 0] + n = len(signs) + n_pos = sum(1 for s in signs if s > 0) + n_neg = n - n_pos + runs = 1 + sum(1 for i in range(1, n) if signs[i] != signs[i - 1]) if n else 0 + if n_pos < 1 or n_neg < 1 or n < 8: + return { + "runs": runs, + "n_pos": n_pos, + "n_neg": n_neg, + "z": float("nan"), + "p_value": float("nan"), + "note": "too few points for a meaningful runs test (need n>=8 with both signs)", + } + exp = 2.0 * n_pos * n_neg / n + 1.0 + var = (2.0 * n_pos * n_neg * (2.0 * n_pos * n_neg - n)) / (n * n * (n - 1.0)) + if var <= 0: + return {"runs": runs, "n_pos": n_pos, "n_neg": n_neg, "z": float("nan"), + "p_value": float("nan"), "note": "degenerate variance"} + z = (runs - exp) / math.sqrt(var) + from statistics import NormalDist + + p = 2.0 * NormalDist().cdf(-abs(z)) + return {"runs": runs, "n_pos": n_pos, "n_neg": n_neg, "expected_runs": exp, + "z": z, "p_value": p, "note": ""} + + +def lack_of_fit(xs: Sequence[float], ys: Sequence[float], fit: LinearFit) -> dict[str, Any]: + """ANOVA lack-of-fit F test, which needs replicate y at some x levels. + + This is the statistically meaningful test of a linear calibration model. + r-squared is not: it rises with range and is insensitive to curvature. + """ + groups: dict[float, list[float]] = {} + for x, y in zip(xs, ys): + groups.setdefault(round(float(x), 12), []).append(float(y)) + k = len(groups) + n = len(xs) + replicated = sum(1 for vals in groups.values() if len(vals) > 1) + df_pe = n - k + df_lof = k - 2 + if df_pe < 1 or df_lof < 1: + return { + "applicable": False, + "levels": k, + "replicated_levels": replicated, + "reason": "needs replicates at >=1 level and >=3 distinct levels", + } + ss_pe = math.fsum( + math.fsum((v - mean(vals)) ** 2 for v in vals) for vals in groups.values() + ) + ss_res = math.fsum(r * r for r in fit.residuals) + ss_lof = max(0.0, ss_res - ss_pe) + ms_pe = ss_pe / df_pe + ms_lof = ss_lof / df_lof + if ms_pe <= 0: + return {"applicable": False, "levels": k, "replicated_levels": replicated, + "reason": "zero pure-error variance; replicates are identical"} + f_stat = ms_lof / ms_pe + return { + "applicable": True, + "levels": k, + "replicated_levels": replicated, + "df_lack_of_fit": df_lof, + "df_pure_error": df_pe, + "ms_lack_of_fit": ms_lof, + "ms_pure_error": ms_pe, + "f_statistic": f_stat, + "p_value": f_sf(f_stat, df_lof, df_pe), + } + + +def heteroscedasticity(xs: Sequence[float], residuals: Sequence[float]) -> dict[str, Any]: + """Compare residual spread in the lowest and highest thirds of the range. + + A large ratio means unweighted least squares over-weights the top of the + curve, which biases back-calculated results at the bottom -- exactly where + an impurity reporting threshold or an LLOQ lives. + """ + pairs = sorted(zip(xs, residuals), key=lambda p: p[0]) + n = len(pairs) + if n < 6: + return {"applicable": False, "reason": "needs at least 6 points"} + cut = max(2, n // 3) + low = [r for _, r in pairs[:cut]] + high = [r for _, r in pairs[-cut:]] + var_low = math.fsum(r * r for r in low) / len(low) + var_high = math.fsum(r * r for r in high) / len(high) + if var_low <= 0: + return {"applicable": False, "reason": "zero residual variance in the low third"} + ratio = var_high / var_low + return { + "applicable": True, + "n_low": len(low), + "n_high": len(high), + "variance_ratio_high_over_low": ratio, + "sd_ratio": math.sqrt(ratio), + } + + +# -------------------------------------------------------------------------- +# Variance components (precision) +# -------------------------------------------------------------------------- + + +@dataclass +class PrecisionComponents: + """One-way random-effects decomposition of precision.""" + + grand_mean: float + n_total: int + n_groups: int + ms_between: float + ms_within: float + df_between: int + df_within: int + sd_repeatability: float + sd_between: float + sd_intermediate: float + balanced: bool + n_effective: float + + def rsd(self, sd: float) -> float: + if abs(self.grand_mean) < 1e-15: + return float("nan") + return 100.0 * sd / abs(self.grand_mean) + + def satterthwaite_df(self) -> float: + """Effective df for the total (intermediate) SD.""" + var_total = self.sd_intermediate ** 2 + if var_total <= 0: + return float("nan") + n = self.n_effective + c1 = 1.0 / n + c2 = (n - 1.0) / n + num = var_total ** 2 + den = 0.0 + if self.df_between > 0: + den += (c1 * self.ms_between) ** 2 / self.df_between + if self.df_within > 0: + den += (c2 * self.ms_within) ** 2 / self.df_within + return num / den if den > 0 else float("nan") + + +def one_way_components(groups: dict[str, Sequence[float]]) -> PrecisionComponents: + """Decompose precision into within-group and between-group components. + + Groups are the intermediate-precision factor -- day, analyst, instrument, + or a combined run. Within-group scatter estimates repeatability; the total + estimates intermediate precision. + """ + clean = {k: [float(v) for v in vals] for k, vals in groups.items() if len(vals) >= 1} + if len(clean) < 2: + raise InputError("intermediate precision needs at least 2 groups") + if all(len(v) < 2 for v in clean.values()): + raise InputError("at least one group needs >=2 replicates to estimate repeatability") + + counts = [len(v) for v in clean.values()] + n_total = sum(counts) + k = len(clean) + all_values = [v for vals in clean.values() for v in vals] + grand = mean(all_values) + + ss_within = math.fsum( + math.fsum((v - mean(vals)) ** 2 for v in vals) for vals in clean.values() + ) + ss_between = math.fsum(len(vals) * (mean(vals) - grand) ** 2 for vals in clean.values()) + df_within = n_total - k + df_between = k - 1 + ms_within = ss_within / df_within if df_within > 0 else float("nan") + ms_between = ss_between / df_between if df_between > 0 else float("nan") + + balanced = len(set(counts)) == 1 + if balanced: + n_eff = float(counts[0]) + else: + # Standard unbalanced coefficient for the expected mean square. + n_eff = (n_total - math.fsum(c * c for c in counts) / n_total) / (k - 1) + + var_within = max(0.0, ms_within) if math.isfinite(ms_within) else 0.0 + var_between = 0.0 + if math.isfinite(ms_between) and math.isfinite(ms_within) and n_eff > 0: + var_between = max(0.0, (ms_between - ms_within) / n_eff) + + return PrecisionComponents( + grand_mean=grand, + n_total=n_total, + n_groups=k, + ms_between=ms_between, + ms_within=ms_within, + df_between=df_between, + df_within=df_within, + sd_repeatability=math.sqrt(var_within), + sd_between=math.sqrt(var_between), + sd_intermediate=math.sqrt(var_within + var_between), + balanced=balanced, + n_effective=n_eff, + ) + + +# -------------------------------------------------------------------------- +# Method comparison +# -------------------------------------------------------------------------- + + +def deming( + xs: Sequence[float], ys: Sequence[float], lambda_ratio: float = 1.0 +) -> dict[str, Any]: + """Deming regression: errors in both variables. + + `lambda_ratio` is var(error in y) / var(error in x) -- the variance of the + random error in the TEST (y) procedure over that in the COMPARATIVE (x) one. + Check the direction against the limits, which are unambiguous: as + lambda -> infinity the fit converges on the ordinary least-squares slope of + y on x (x treated as error-free), and as lambda -> 0 it converges on the + inverse regression (y treated as error-free). lambda = 1 means equal error + variances and reduces to orthogonal regression. + + In practice lambda is estimated as (SD of x replicates / SD of y replicates) + squared, so equal-precision procedures give 1. + + Ordinary least squares assumes x is error-free, which is false when + comparing two measurement procedures, and biases the slope toward zero. + """ + n = len(xs) + if n != len(ys): + raise InputError("x and y must be the same length") + if n < 3: + raise InputError("Deming regression needs at least 3 points") + if lambda_ratio <= 0: + raise InputError("lambda_ratio must be > 0") + + def _fit(xv: Sequence[float], yv: Sequence[float]) -> tuple[float, float]: + xb, yb = mean(xv), mean(yv) + sxx = math.fsum((x - xb) ** 2 for x in xv) + syy = math.fsum((y - yb) ** 2 for y in yv) + sxy = math.fsum((x - xb) * (y - yb) for x, y in zip(xv, yv)) + if abs(sxy) < 1e-300: + raise InputError("zero covariance; Deming slope is undefined") + term = syy - lambda_ratio * sxx + slope = (term + math.sqrt(term * term + 4.0 * lambda_ratio * sxy * sxy)) / ( + 2.0 * sxy + ) + return slope, yb - slope * xb + + slope, intercept = _fit(xs, ys) + + # Jackknife standard errors. + slopes, intercepts = [], [] + for i in range(n): + xv = list(xs[:i]) + list(xs[i + 1 :]) + yv = list(ys[:i]) + list(ys[i + 1 :]) + try: + s, a = _fit(xv, yv) + except InputError: + continue + slopes.append(s) + intercepts.append(a) + if len(slopes) > 2: + m = len(slopes) + se_slope = math.sqrt((m - 1) / m * math.fsum((s - mean(slopes)) ** 2 for s in slopes)) + se_int = math.sqrt( + (m - 1) / m * math.fsum((a - mean(intercepts)) ** 2 for a in intercepts) + ) + df = m - 2 + else: + se_slope = se_int = float("nan") + df = 1 + + t = t_ppf(0.975, df) if df > 0 else float("nan") + return { + "n": n, + "lambda_ratio": lambda_ratio, + "slope": slope, + "intercept": intercept, + "se_slope": se_slope, + "se_intercept": se_int, + "slope_ci95": (slope - t * se_slope, slope + t * se_slope), + "intercept_ci95": (intercept - t * se_int, intercept + t * se_int), + "df": df, + } + + +def passing_bablok(xs: Sequence[float], ys: Sequence[float]) -> dict[str, Any]: + """Passing-Bablok regression: non-parametric, no distributional assumption. + + Robust to outliers and does not assume a known error-variance ratio, which + is why CLSI EP09-style method comparison work often prefers it. + """ + n = len(xs) + if n != len(ys): + raise InputError("x and y must be the same length") + if n < 5: + raise InputError("Passing-Bablok needs at least 5 points") + + slopes: list[float] = [] + for i in range(n): + for j in range(i + 1, n): + dx = xs[j] - xs[i] + dy = ys[j] - ys[i] + if dx == 0 and dy == 0: + continue + if dx == 0: + continue # vertical pair carries no finite slope + slopes.append(dy / dx) + if not slopes: + raise InputError("no usable pairwise slopes") + + slopes.sort() + n_slopes = len(slopes) + shift = sum(1 for s in slopes if s < -1.0) + + def _shifted_median(offset: int) -> float: + idx = n_slopes // 2 + offset + if n_slopes % 2: + return slopes[min(max(idx, 0), n_slopes - 1)] + lo = slopes[min(max(idx - 1, 0), n_slopes - 1)] + hi = slopes[min(max(idx, 0), n_slopes - 1)] + return 0.5 * (lo + hi) + + slope = _shifted_median(shift) + intercept = median([y - slope * x for x, y in zip(xs, ys)]) + + # Rank-based 95% CI on the slope. M1 and M2 are 1-based order statistics of + # the shifted slope list, so both convert to 0-based with the same -1. + c = z_ppf(0.975) * math.sqrt(n * (n - 1.0) * (2.0 * n + 5.0) / 18.0) + m1 = int(round((n_slopes - c) / 2.0)) + m2 = n_slopes - m1 + 1 + lo_idx = min(max(m1 + shift - 1, 0), n_slopes - 1) + hi_idx = min(max(m2 + shift - 1, 0), n_slopes - 1) + slope_lo, slope_hi = slopes[lo_idx], slopes[hi_idx] + int_lo = median([y - slope_hi * x for x, y in zip(xs, ys)]) + int_hi = median([y - slope_lo * x for x, y in zip(xs, ys)]) + + return { + "n": n, + "n_slopes": n_slopes, + "slope": slope, + "intercept": intercept, + "slope_ci95": (slope_lo, slope_hi), + "intercept_ci95": (int_lo, int_hi), + } + + +def bland_altman( + xs: Sequence[float], ys: Sequence[float], relative: bool = False +) -> dict[str, Any]: + """Bias and limits of agreement between paired measurements.""" + n = len(xs) + if n != len(ys): + raise InputError("x and y must be the same length") + if n < 3: + raise InputError("Bland-Altman needs at least 3 pairs") + means = [0.5 * (x + y) for x, y in zip(xs, ys)] + if relative: + diffs = [] + for x, y, m in zip(xs, ys, means): + if abs(m) < 1e-15: + raise InputError("relative differences need non-zero pair means") + diffs.append(100.0 * (y - x) / m) + else: + diffs = [y - x for x, y in zip(xs, ys)] + + bias = mean(diffs) + sd = sample_sd(diffs) + t = t_ppf(0.975, n - 1) + se_bias = sd / math.sqrt(n) + loa_lo, loa_hi = bias - 1.96 * sd, bias + 1.96 * sd + se_loa = sd * math.sqrt(1.0 / n + (1.96 ** 2) / (2.0 * (n - 1))) + + # Proportional-bias check: does the difference trend with the mean? + trend = None + try: + tf = fit_linear(means, diffs) + t_stat = tf.slope / tf.se_slope if tf.se_slope > 0 else float("nan") + trend = { + "slope": tf.slope, + "p_value": 2.0 * (1.0 - t_cdf(abs(t_stat), tf.df)) if math.isfinite(t_stat) else float("nan"), + } + except InputError: + trend = None + + return { + "n": n, + "relative": relative, + "bias": bias, + "sd_differences": sd, + "bias_ci95": (bias - t * se_bias, bias + t * se_bias), + "loa_lower": loa_lo, + "loa_upper": loa_hi, + "loa_ci95_halfwidth": t * se_loa, + "proportional_bias": trend, + } + + +def tost_paired( + diffs: Sequence[float], margin: float, alpha: float = 0.05 +) -> dict[str, Any]: + """Two one-sided tests for equivalence on paired differences. + + Absence of a significant difference is not evidence of equivalence. TOST + tests the hypothesis that actually matters at a method transfer: that the + true difference lies inside +/- margin. + """ + n = len(diffs) + if n < 2: + raise InputError("TOST needs at least 2 differences") + if margin <= 0: + raise InputError("margin must be > 0") + d = mean(diffs) + sd = sample_sd(diffs) + se = sd / math.sqrt(n) + df = n - 1 + if se <= 0: + raise InputError("zero variability; TOST is undefined") + t_lower = (d + margin) / se + t_upper = (d - margin) / se + p_lower = 1.0 - t_cdf(t_lower, df) + p_upper = t_cdf(t_upper, df) + p = max(p_lower, p_upper) + t_crit = t_ppf(1.0 - alpha, df) + ci = (d - t_crit * se, d + t_crit * se) + return { + "n": n, + "mean_difference": d, + "sd_difference": sd, + "margin": margin, + "alpha": alpha, + "p_lower": p_lower, + "p_upper": p_upper, + "p_value": p, + "ci_1_minus_2alpha": ci, + "equivalent": bool(ci[0] > -margin and ci[1] < margin), + } + + +# -------------------------------------------------------------------------- +# I/O +# -------------------------------------------------------------------------- + + +def read_input(path: str | None) -> str: + """Read a bounded amount of text from a path or stdin.""" + if path in (None, "-"): + # Reading a terminal would block forever with no indication why, so an + # omitted --input becomes an error rather than an apparent hang. + if path is None and sys.stdin.isatty(): + raise InputError("no input given; pass --input FILE, or '-' to read stdin") + data = sys.stdin.read(MAX_INPUT_BYTES + 1) + else: + p = Path(path) + if not p.is_file(): + raise InputError(f"not a file: {path}") + if p.stat().st_size > MAX_INPUT_BYTES: + raise InputError(f"input larger than {MAX_INPUT_BYTES} bytes") + data = p.read_text(encoding="utf-8", errors="replace") + if len(data) > MAX_INPUT_BYTES: + raise InputError(f"input larger than {MAX_INPUT_BYTES} bytes") + return data + + +def parse_rows(text: str, path_hint: str | None = None) -> list[dict[str, str]]: + """Parse CSV, TSV, or a JSON array of objects into a list of dicts.""" + stripped = text.lstrip() + if stripped.startswith("[") or stripped.startswith("{"): + try: + payload = json.loads(stripped) + except json.JSONDecodeError as exc: + raise InputError(f"invalid JSON: {exc}") from exc + if isinstance(payload, dict): + payload = payload.get("rows", payload.get("data")) + if not isinstance(payload, list): + raise InputError("JSON input must be an array of objects, or {\"rows\": [...]}") + # Refuse rather than truncate: silently dropping validation data would + # produce a clean-looking result computed on part of the study. + if len(payload) > MAX_ROWS: + raise InputError(f"more than {MAX_ROWS} rows") + rows = [] + for item in payload: + if not isinstance(item, dict): + raise InputError("JSON rows must be objects") + rows.append({str(k): "" if v is None else str(v) for k, v in item.items()}) + if not rows: + raise InputError("no data rows found") + return rows + + delimiter = "\t" if (path_hint or "").endswith((".tsv", ".tab")) else None + if delimiter is None: + first = text.splitlines()[0] if text.splitlines() else "" + delimiter = "\t" if first.count("\t") > first.count(",") else "," + reader = csv.DictReader(io.StringIO(text), delimiter=delimiter) + rows = [] + for i, row in enumerate(reader): + if i >= MAX_ROWS: + raise InputError(f"more than {MAX_ROWS} rows") + rows.append({(k or "").strip(): (v or "").strip() for k, v in row.items()}) + if not rows: + raise InputError("no data rows found") + return rows + + +def require_columns(rows: list[dict[str, str]], columns: Iterable[str]) -> None: + present = set(rows[0].keys()) + missing = [c for c in columns if c not in present] + if missing: + raise InputError( + f"missing required column(s): {', '.join(missing)}; found: {', '.join(sorted(present))}" + ) + + +def to_float(value: str, column: str, row_index: int) -> float: + try: + return float(str(value).strip()) + except (TypeError, ValueError) as exc: + raise InputError( + f"row {row_index + 1}: column '{column}' is not numeric: {value!r}" + ) from exc + + +def fmt(value: Any, digits: int = 4) -> str: + """Format a number for a table cell.""" + if value is None: + return "" + if isinstance(value, bool): + return "yes" if value else "no" + if isinstance(value, float): + if math.isnan(value): + return "n/a" + if math.isinf(value): + return "inf" + if value != 0 and (abs(value) < 1e-4 or abs(value) >= 1e6): + return f"{value:.{digits}e}" + return f"{value:.{digits}f}" + return str(value) + + +def emit_table(rows: list[dict[str, Any]], stream=None) -> None: + """Print aligned columns.""" + stream = stream or sys.stdout + if not rows: + print("(no rows)", file=stream) + return + headers = list(rows[0].keys()) + cells = [[fmt(r.get(h)) for h in headers] for r in rows] + widths = [ + max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) + for i, h in enumerate(headers) + ] + print(" ".join(h.ljust(w) for h, w in zip(headers, widths)).rstrip(), file=stream) + for c in cells: + print(" ".join(v.ljust(w) for v, w in zip(c, widths)).rstrip(), file=stream) + + +def emit(rows: list[dict[str, Any]], fmt_name: str, stream=None) -> None: + """Print rows as a table, TSV, or JSON.""" + stream = stream or sys.stdout + if fmt_name == "json": + json.dump(rows, stream, indent=2, default=_json_default) + print(file=stream) + elif fmt_name == "tsv": + if not rows: + return + headers = list(rows[0].keys()) + print("\t".join(headers), file=stream) + for r in rows: + print("\t".join(fmt(r.get(h)) for h in headers), file=stream) + else: + emit_table(rows, stream=stream) + + +def _json_default(obj: Any) -> Any: + if isinstance(obj, float) and (math.isnan(obj) or math.isinf(obj)): + return None + if isinstance(obj, tuple): + return list(obj) + raise TypeError(f"not JSON serialisable: {type(obj)!r}") + + +def note(message: str) -> None: + """Write provenance and caveats to stderr so stdout stays parseable.""" + print(f"note: {message}", file=sys.stderr) + + +def finding(message: str) -> None: + print(f"finding: {message}", file=sys.stderr) + + +def add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--format", + choices=("table", "tsv", "json"), + default="table", + help="output format (default: table)", + ) + + +def run_cli(main_func) -> None: + """Wrap a main() so InputError becomes a clean exit code 2.""" + try: + sys.exit(main_func()) + except InputError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(EXIT_INPUT_ERROR) + except BrokenPipeError: + sys.exit(EXIT_OK) diff --git a/.agents/skills/analytical-method-validation/scripts/check_accuracy_precision.py b/.agents/skills/analytical-method-validation/scripts/check_accuracy_precision.py new file mode 100644 index 0000000..302b965 --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/check_accuracy_precision.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Evaluate accuracy and precision per ICH Q2(R2) 3.3. + +Accuracy is reported as mean recovery with a confidence interval, which is what +Q2(R2) 3.3.1.4 asks for -- a bare mean is not sufficient. Precision is +decomposed into repeatability and intermediate precision by a one-way +random-effects model, because pooling all results into a single standard +deviation understates the day-to-day variability the procedure will actually +show in routine use. + + python3 check_accuracy_precision.py --input ap.csv + python3 check_accuracy_precision.py -i ap.csv --accuracy-limit 2 --rsd-limit 2 + python3 check_accuracy_precision.py -i ap.csv --design-check assay --format json + +Input columns: + level nominal / added concentration (groups the accuracy analysis) + measured measured or recovered concentration + group optional: day, analyst, instrument or run -- the intermediate + precision factor. Without it only repeatability is estimated. + +Exit codes: 0 no findings, 1 findings raised, 2 bad input. +""" + +from __future__ import annotations + +import argparse +import math +import sys + +sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent)) + +from _common import ( # noqa: E402 + EXIT_FINDINGS, + EXIT_OK, + InputError, + add_common_args, + emit, + finding, + mean, + note, + one_way_components, + parse_rows, + read_input, + require_columns, + rsd_percent, + run_cli, + sample_sd, + sd_confidence_interval, + t_ppf, + to_float, +) + +# ICH Q2(R2) 3.3.2.1 minima, used only to comment on the design. Either option +# is sufficient on its own. +DESIGN_MINIMA = { + "assay": { + "range_determinations": 9, + "range_levels": 3, + "single_level_determinations": 6, + "note": "9 determinations across the range (3x3), or 6 at 100% of test concentration", + }, + "impurity": { + "range_determinations": 9, + "range_levels": 3, + "single_level_determinations": 6, + "note": "9 determinations across the range (3x3), or 6 at 100% of test concentration", + }, +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check accuracy and precision from validation data." + ) + parser.add_argument("--input", "-i", help="CSV/TSV/JSON file, or '-' for stdin") + parser.add_argument("--accuracy-limit", type=float, default=None, + help="flag a level whose mean recovery deviates more than this %% " + "from 100%%") + parser.add_argument("--rsd-limit", type=float, default=None, + help="flag repeatability or intermediate precision %%RSD above this") + parser.add_argument("--ci-level", type=float, default=0.95, + help="confidence level for accuracy intervals (default 0.95)") + parser.add_argument("--sd-ci-level", type=float, default=0.90, + help="confidence level for SD intervals (default 0.90)") + parser.add_argument("--design-check", choices=sorted(DESIGN_MINIMA), default=None, + help="comment on the design against Q2(R2) recommended minima") + parser.add_argument("--require-ci-within-limit", action="store_true", + help="require the whole accuracy CI inside the limit, not just the mean") + add_common_args(parser) + args = parser.parse_args() + + if not 0.5 <= args.ci_level < 1.0: + raise InputError("--ci-level must be in [0.5, 1)") + + rows = parse_rows(read_input(args.input), args.input) + require_columns(rows, ["level", "measured"]) + has_group = "group" in rows[0] + + records = [] + for i, r in enumerate(rows): + records.append( + { + "level": to_float(r["level"], "level", i), + "measured": to_float(r["measured"], "measured", i), + "group": (r.get("group") or "all").strip() or "all", + } + ) + + findings: list[str] = [] + + # ---------------- Accuracy ---------------- + levels = sorted({rec["level"] for rec in records}) + accuracy_rows = [] + all_recoveries: list[float] = [] + for level in levels: + vals = [rec["measured"] for rec in records if rec["level"] == level] + if level == 0: + raise InputError("a nominal level of 0 cannot be used for recovery") + recoveries = [100.0 * v / level for v in vals] + all_recoveries.extend(recoveries) + m = mean(recoveries) + n = len(recoveries) + if n >= 2: + sd = sample_sd(recoveries) + half = t_ppf(0.5 + args.ci_level / 2.0, n - 1) * sd / math.sqrt(n) + lo, hi = m - half, m + half + else: + sd, lo, hi = float("nan"), float("nan"), float("nan") + accuracy_rows.append( + { + "level": level, + "n": n, + "mean_measured": mean(vals), + "mean_recovery_pct": m, + "bias_pct": m - 100.0, + "sd_recovery_pct": sd, + f"ci{int(args.ci_level * 100)}_low": lo, + f"ci{int(args.ci_level * 100)}_high": hi, + } + ) + if args.accuracy_limit is not None: + if abs(m - 100.0) > args.accuracy_limit: + findings.append( + f"level {level:g}: mean recovery {m:.2f}% is {m - 100.0:+.2f}% from nominal, " + f"outside +/-{args.accuracy_limit:g}%" + ) + elif args.require_ci_within_limit and math.isfinite(lo): + if lo < 100.0 - args.accuracy_limit or hi > 100.0 + args.accuracy_limit: + findings.append( + f"level {level:g}: mean recovery {m:.2f}% is inside " + f"+/-{args.accuracy_limit:g}% but its " + f"{int(args.ci_level * 100)}% CI ({lo:.2f}, {hi:.2f}) is not -- the data " + "do not demonstrate accuracy at this limit" + ) + + # ---------------- Precision ---------------- + # Precision is estimated WITHIN each concentration level. Pooling levels + # together would let the 80/100/120 spread masquerade as imprecision. + pct = int(args.sd_ci_level * 100) + precision_rows: list[dict] = [] + + def precision_block(label: str, groups: dict[str, list[float]], reference: float | None): + """Append rows for one level (or for the normalised all-levels view).""" + try: + comp = one_way_components(groups) + except InputError as exc: + vals = [v for vs in groups.values() for v in vs] + if len(vals) < 2: + note(f"{label}: too few values for a precision estimate") + return None + sd = sample_sd(vals) + base = reference if reference else abs(mean(vals)) + rsd = 100.0 * sd / base if base else float("nan") + lo, hi = sd_confidence_interval(sd, len(vals) - 1, args.sd_ci_level) + precision_rows.append( + {"level": label, "component": "repeatability only", "sd": sd, "rsd_pct": rsd, + "df": len(vals) - 1, f"ci{pct}_low_sd": lo, f"ci{pct}_high_sd": hi} + ) + note(f"{label}: intermediate precision not estimated ({exc})") + return None + + base = reference if reference else abs(comp.grand_mean) + + def as_rsd(sd: float) -> float: + return 100.0 * sd / base if base else float("nan") + + sr_lo, sr_hi = sd_confidence_interval( + comp.sd_repeatability, comp.df_within, args.sd_ci_level + ) + sat_df = comp.satterthwaite_df() + si_lo, si_hi = sd_confidence_interval(comp.sd_intermediate, sat_df, args.sd_ci_level) + precision_rows.extend( + [ + {"level": label, "component": "repeatability (within group)", + "sd": comp.sd_repeatability, "rsd_pct": as_rsd(comp.sd_repeatability), + "df": comp.df_within, f"ci{pct}_low_sd": sr_lo, f"ci{pct}_high_sd": sr_hi}, + {"level": label, "component": "between-group", "sd": comp.sd_between, + "rsd_pct": as_rsd(comp.sd_between), "df": comp.df_between, + f"ci{pct}_low_sd": float("nan"), f"ci{pct}_high_sd": float("nan")}, + {"level": label, "component": "intermediate precision (total)", + "sd": comp.sd_intermediate, "rsd_pct": as_rsd(comp.sd_intermediate), + "df": sat_df, f"ci{pct}_low_sd": si_lo, f"ci{pct}_high_sd": si_hi}, + ] + ) + if comp.sd_between == 0.0: + note( + f"{label}: between-group variance estimated as zero (MS_between <= MS_within); " + "the groups are indistinguishable at this level" + ) + if not comp.balanced: + note(f"{label}: unbalanced design, effective group size {comp.n_effective:.3f}") + if args.rsd_limit is not None: + for name, value in ( + ("repeatability", as_rsd(comp.sd_repeatability)), + ("intermediate precision", as_rsd(comp.sd_intermediate)), + ): + if math.isfinite(value) and value > args.rsd_limit: + findings.append( + f"{label}: {name} {value:.3f}% RSD exceeds the stated " + f"{args.rsd_limit:g}% limit" + ) + return comp + + if has_group: + for level in levels: + groups: dict[str, list[float]] = {} + for rec in records: + if rec["level"] == level: + groups.setdefault(rec["group"], []).append(rec["measured"]) + precision_block(f"{level:g}", groups, reference=abs(level)) + + # Level-independent view: recovery as % of nominal, pooled across levels. + norm_groups: dict[str, list[float]] = {} + for rec in records: + norm_groups.setdefault(rec["group"], []).append( + 100.0 * rec["measured"] / rec["level"] + ) + if len(norm_groups) >= 2: + precision_block("all (% of nominal)", norm_groups, reference=100.0) + else: + for level in levels: + vals = [rec["measured"] for rec in records if rec["level"] == level] + if len(vals) < 2: + continue + sd = sample_sd(vals) + lo, hi = sd_confidence_interval(sd, len(vals) - 1, args.sd_ci_level) + rsd = 100.0 * sd / abs(level) + precision_rows.append( + {"level": f"{level:g}", "component": "repeatability (no group column)", + "sd": sd, "rsd_pct": rsd, "df": len(vals) - 1, + f"ci{pct}_low_sd": lo, f"ci{pct}_high_sd": hi} + ) + if args.rsd_limit is not None and rsd > args.rsd_limit: + findings.append( + f"{level:g}: repeatability {rsd:.3f}% RSD exceeds {args.rsd_limit:g}%" + ) + note( + "no `group` column, so only repeatability was estimated. Q2(R2) 3.3.2.2 expects " + "intermediate precision from different days, analysts or equipment" + ) + + naive = rsd_percent([rec["measured"] for rec in records]) + if len(levels) > 1: + note( + f"a single SD over every result regardless of level would report {naive:.3f}% RSD, " + "which is a range effect and not precision. Precision is reported per level below" + ) + + # ---------------- Design commentary ---------------- + if args.design_check: + # Q2(R2) 3.3.2.1 offers two alternatives, and either one is sufficient: + # (a) >=9 determinations covering the reportable range (e.g. 3 levels x 3), or + # (b) >=6 determinations at 100% of the test concentration. + # Flag only when neither holds. Requiring 9 unconditionally would raise a + # finding against a design the guideline explicitly permits. + spec = DESIGN_MINIMA[args.design_check] + option_a = len(records) >= spec["range_determinations"] and len(levels) >= spec["range_levels"] + per_level = {lv: sum(1 for r in records if r["level"] == lv) for lv in levels} + best_single = max(per_level.values()) + option_b = best_single >= spec["single_level_determinations"] + if option_a: + note( + f"design satisfies Q2(R2) 3.3.2.1 option (a): {len(records)} determinations " + f"across {len(levels)} levels" + ) + elif option_b: + note( + f"design satisfies Q2(R2) 3.3.2.1 option (b): {best_single} determinations at a " + "single level. Note that option (b) gives no information about precision " + "across the range" + ) + else: + findings.append( + f"repeatability design meets neither Q2(R2) 3.3.2.1 option: " + f"{len(records)} determinations across {len(levels)} level(s), with at most " + f"{best_single} at any one level. Option (a) needs " + f"{spec['range_determinations']} across at least {spec['range_levels']} levels; " + f"option (b) needs {spec['single_level_determinations']} at 100% of the test " + "concentration" + ) + + if args.format == "json": + emit([{"accuracy": accuracy_rows, "precision": precision_rows, + "overall_mean_recovery_pct": mean(all_recoveries), + "findings": findings}], "json") + else: + emit(accuracy_rows, args.format) + print() + emit(precision_rows, args.format) + + note( + "Q2(R2) 3.3.1.4: report accuracy as mean percent recovery, or the difference from the " + "accepted true value, with a 100(1-alpha)% confidence interval" + ) + note( + "Q2(R2) 3.3.2.2: intermediate precision covers days, environmental conditions, analysts " + "and equipment; the effects need not be studied individually" + ) + for f in findings: + finding(f) + if not findings: + note("no findings against the checks that were run") + note("this tool does not decide that accuracy or precision is acceptable") + return EXIT_FINDINGS if findings else EXIT_OK + + +if __name__ == "__main__": + run_cli(main) diff --git a/.agents/skills/analytical-method-validation/scripts/check_bioanalytical_run.py b/.agents/skills/analytical-method-validation/scripts/check_bioanalytical_run.py new file mode 100644 index 0000000..aecd2e7 --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/check_bioanalytical_run.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Apply ICH M10 acceptance criteria to a bioanalytical run or an ISR dataset. + +Chromatographic assays and ligand binding assays carry DIFFERENT numeric +criteria in ICH M10, and conflating them is the most common error in this area. +--modality is therefore mandatory: nothing here has a default. + + # calibration standards and QCs from one analytical run + python3 check_bioanalytical_run.py --modality chromatographic --run run1.csv + + # incurred sample reanalysis + python3 check_bioanalytical_run.py --modality lba --isr isr.csv + + # LBA total-error criterion from accuracy/precision validation data + python3 check_bioanalytical_run.py --modality lba --total-error ap.csv + +Input for --run: columns `type` (calibrator|qc), `nominal`, `measured`, and an +optional `label` (e.g. LLOQ, low, medium, high, ULOQ, or ANCHOR for LBA anchor +points, which are excluded from the calibration pass count). + +Input for --isr: columns `original` and `repeat`. + +Input for --total-error: columns `label`, `accuracy_pct`, `precision_pct`. + +Exit codes: 0 all applied criteria met, 1 criteria not met, 2 bad input. +""" + +from __future__ import annotations + +import argparse +import math +import sys + +sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent)) + +from _catalog import M10_CRITERIA # noqa: E402 +from _common import ( # noqa: E402 + EXIT_FINDINGS, + EXIT_OK, + InputError, + add_common_args, + emit, + finding, + note, + parse_rows, + read_input, + require_columns, + run_cli, + to_float, +) + +LIMIT_LABELS = {"lloq", "uloq"} + + +def calibrator_tolerance(label: str, crit: dict) -> float: + """Tolerance for a calibration standard at this label. + + Routine-run QCs are not routed through here: M10 applies one flat tolerance + to every QC level for run acceptance, which check_run uses directly. + """ + low = label.strip().lower() + if low == "lloq": + return crit["calibration_tolerance_lloq_pct"] + if low == "uloq": + return crit["calibration_tolerance_uloq_pct"] + return crit["calibration_tolerance_pct"] + + +def check_run(path: str, crit: dict) -> tuple[list[dict], list[str]]: + rows = parse_rows(read_input(path), path) + require_columns(rows, ["type", "nominal", "measured"]) + findings: list[str] = [] + detail: list[dict] = [] + + cal_pass = cal_total = 0 + cal_levels: set[float] = set() + qc_by_level: dict[str, list[bool]] = {} + qc_pass = qc_total = 0 + + for i, r in enumerate(rows): + kind = (r["type"] or "").strip().lower() + label = (r.get("label") or "").strip() + nominal = to_float(r["nominal"], "nominal", i) + measured = to_float(r["measured"], "measured", i) + if nominal == 0: + raise InputError(f"row {i + 1}: nominal of 0 cannot be used") + dev = 100.0 * (measured - nominal) / nominal + + if kind in ("calibrator", "cal", "standard", "std"): + if label.lower() == "anchor": + detail.append({"type": "calibrator", "label": "ANCHOR", "nominal": nominal, + "measured": measured, "deviation_pct": dev, + "tolerance_pct": float("nan"), "within": "excluded"}) + continue + tol = calibrator_tolerance(label, crit) + ok = abs(dev) <= tol + cal_total += 1 + cal_pass += int(ok) + cal_levels.add(round(nominal, 12)) + detail.append({"type": "calibrator", "label": label or "-", "nominal": nominal, + "measured": measured, "deviation_pct": dev, + "tolerance_pct": tol, "within": ok}) + elif kind in ("qc", "quality-control"): + tol = crit["qc_run_tolerance_pct"] + ok = abs(dev) <= tol + key = label or f"{nominal:g}" + qc_by_level.setdefault(key, []).append(ok) + qc_total += 1 + qc_pass += int(ok) + detail.append({"type": "qc", "label": key, "nominal": nominal, + "measured": measured, "deviation_pct": dev, + "tolerance_pct": tol, "within": ok}) + else: + raise InputError( + f"row {i + 1}: type must be calibrator or qc, got {r['type']!r}" + ) + + # Calibration curve criteria. + if cal_total: + n_levels = len(cal_levels) + need_levels = crit["calibration_min_levels"] + if n_levels < need_levels: + findings.append( + f"calibration curve has {n_levels} concentration levels; M10 requires a " + f"minimum of {need_levels}" + ) + frac = cal_pass / cal_total + need = crit["calibration_min_pass_fraction"] + if frac < need: + findings.append( + f"{cal_pass}/{cal_total} calibration standards within tolerance " + f"({frac * 100:.1f}%); M10 requires at least {need * 100:.0f}%" + ) + + # Routine run QC criteria: both the overall fraction and per-level fraction. + if qc_total: + n_qc_levels = len(qc_by_level) + if n_qc_levels < crit["qc_levels_routine_run"]: + findings.append( + f"{n_qc_levels} QC levels present; M10 expects at least " + f"{crit['qc_levels_routine_run']} for run acceptance" + ) + frac = qc_pass / qc_total + need = crit["qc_run_pass_fraction"] + if frac < need: + findings.append( + f"{qc_pass}/{qc_total} QCs within +/-{crit['qc_run_tolerance_pct']:.0f}% " + f"({frac * 100:.1f}%); M10 requires at least {need * 100:.0f}% of the total" + ) + for level, flags in sorted(qc_by_level.items()): + level_frac = sum(flags) / len(flags) + if level_frac < crit["qc_run_pass_fraction_per_level"]: + findings.append( + f"QC level {level}: {sum(flags)}/{len(flags)} within tolerance " + f"({level_frac * 100:.0f}%); M10 requires at least " + f"{crit['qc_run_pass_fraction_per_level'] * 100:.0f}% at each level" + ) + return detail, findings + + +def check_isr(path: str, crit: dict) -> tuple[list[dict], list[str]]: + rows = parse_rows(read_input(path), path) + require_columns(rows, ["original", "repeat"]) + tol = crit["isr_tolerance_pct"] + detail, findings = [], [] + passes = 0 + for i, r in enumerate(rows): + original = to_float(r["original"], "original", i) + repeat = to_float(r["repeat"], "repeat", i) + mean_val = 0.5 * (original + repeat) + if mean_val == 0: + raise InputError(f"row {i + 1}: mean of original and repeat is zero") + # M10 defines the ISR percent difference against the mean of the two. + diff = 100.0 * (repeat - original) / mean_val + ok = abs(diff) <= tol + passes += int(ok) + detail.append({"sample": r.get("sample", str(i + 1)), "original": original, + "repeat": repeat, "mean": mean_val, "percent_difference": diff, + "tolerance_pct": tol, "within": ok}) + frac = passes / len(rows) + if frac < crit["isr_pass_fraction"]: + findings.append( + f"ISR: {passes}/{len(rows)} repeats within +/-{tol:.0f}% ({frac * 100:.1f}%); " + f"M10 requires at least {crit['isr_pass_fraction'] * 100:.0f}%" + ) + return detail, findings + + +def check_total_error(path: str, crit: dict) -> tuple[list[dict], list[str]]: + if crit["total_error_pct"] is None: + raise InputError( + "ICH M10 states a total-error criterion for ligand binding assays only; " + "use --modality lba, or omit --total-error for a chromatographic assay" + ) + rows = parse_rows(read_input(path), path) + require_columns(rows, ["label", "accuracy_pct", "precision_pct"]) + detail, findings = [], [] + for i, r in enumerate(rows): + label = (r["label"] or "").strip() + acc = to_float(r["accuracy_pct"], "accuracy_pct", i) + prec = to_float(r["precision_pct"], "precision_pct", i) + total = abs(acc) + abs(prec) + limit = ( + crit["total_error_pct_at_limits"] + if label.lower() in LIMIT_LABELS + else crit["total_error_pct"] + ) + ok = total <= limit + detail.append({"label": label or "-", "accuracy_pct": acc, "precision_pct": prec, + "total_error_pct": total, "limit_pct": limit, "within": ok}) + if not ok: + findings.append( + f"{label or 'level ' + str(i + 1)}: total error {total:.2f}% exceeds " + f"{limit:.0f}%" + ) + return detail, findings + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Apply ICH M10 acceptance criteria to bioanalytical data." + ) + parser.add_argument("--modality", required=True, choices=sorted(M10_CRITERIA), + help="chromatographic or lba -- the criteria differ and there is no default") + parser.add_argument("--run", help="calibrators and QCs from an analytical run") + parser.add_argument("--isr", help="incurred sample reanalysis data") + parser.add_argument("--total-error", dest="total_error", + help="accuracy/precision per level (ligand binding assays only)") + parser.add_argument("--criteria", action="store_true", + help="print the criteria that would be applied and exit") + add_common_args(parser) + args = parser.parse_args() + + crit = M10_CRITERIA[args.modality] + + if args.criteria: + rows = [ + {"criterion": k, "value": ("n/a" if v is None else v)} + for k, v in crit.items() + if k not in ("label", "notes") + ] + emit(rows, args.format) + note(crit["label"]) + note(crit["notes"]) + return EXIT_OK + + if not any((args.run, args.isr, args.total_error)): + raise InputError("supply at least one of --run, --isr, --total-error, or --criteria") + + detail: list[dict] = [] + findings: list[str] = [] + sections: dict[str, list[dict]] = {} + + if args.run: + d, f = check_run(args.run, crit) + sections["run"] = d + findings += f + if args.isr: + d, f = check_isr(args.isr, crit) + sections["isr"] = d + findings += f + if args.total_error: + d, f = check_total_error(args.total_error, crit) + sections["total_error"] = d + findings += f + + if args.format == "json": + emit([{"modality": args.modality, **sections, "findings": findings}], "json") + else: + for name, rows in sections.items(): + print(f"[{name}]") + emit(rows, args.format) + print() + + note(crit["label"]) + note(crit["notes"]) + note( + "chromatographic and ligand binding assay criteria differ throughout M10; this run was " + f"assessed as: {args.modality}" + ) + for f in findings: + finding(f) + if not findings: + note("all applied criteria met") + note( + "run acceptance is a documented decision by the analyst; this tool applies stated " + "criteria and does not accept or reject a run" + ) + return EXIT_FINDINGS if findings else EXIT_OK + + +if __name__ == "__main__": + run_cli(main) diff --git a/.agents/skills/analytical-method-validation/scripts/check_detection_limits.py b/.agents/skills/analytical-method-validation/scripts/check_detection_limits.py new file mode 100644 index 0000000..7deebcb --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/check_detection_limits.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Estimate DL and QL by every approach ICH Q2(R2) 3.2.3 allows, and compare them. + +The four approaches routinely disagree by a factor of two or more on the same +data. Reporting one number without naming the approach is the finding an +assessor raises, so this script computes all of the applicable ones side by side +and checks the answer against the reporting threshold it has to serve. + + # sigma from the calibration line, slope from the same fit + python3 check_detection_limits.py --calibration calib.csv + + # sigma from blank responses + python3 check_detection_limits.py --calibration calib.csv --blanks blanks.csv + + # confirm an estimated QL with real data at that level + python3 check_detection_limits.py --calibration calib.csv \ + --confirm-ql 0.05 --confirm-data ql_check.csv --reporting-threshold 0.05 + +Input: + --calibration CSV with `level` and `response` (low-range calibration curve) + --blanks CSV with `response` (blank measurements) + --confirm-data CSV with `measured` (results at or near the claimed QL) + +Exit codes: 0 no findings, 1 findings raised, 2 bad input. +""" + +from __future__ import annotations + +import argparse +import math +import sys + +sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent)) + +from _catalog import DL_QL_APPROACHES # noqa: E402 +from _common import ( # noqa: E402 + EXIT_FINDINGS, + EXIT_OK, + InputError, + add_common_args, + emit, + finding, + fit_linear, + mean, + note, + parse_rows, + read_input, + require_columns, + rsd_percent, + run_cli, + sample_sd, + to_float, +) + +DL_FACTOR = 3.3 +QL_FACTOR = 10.0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Estimate detection and quantitation limits per ICH Q2(R2) 3.2.3." + ) + parser.add_argument("--calibration", required=True, + help="CSV/TSV/JSON with `level` and `response`") + parser.add_argument("--blanks", help="CSV/TSV/JSON with `response` for blank samples") + parser.add_argument("--signal-to-noise", type=float, default=None, + help="measured S/N at a stated concentration (use with --sn-level)") + parser.add_argument("--sn-level", type=float, default=None, + help="the concentration at which --signal-to-noise was measured") + parser.add_argument("--confirm-ql", type=float, default=None, + help="the QL being claimed, to be confirmed with --confirm-data") + parser.add_argument("--confirm-data", help="CSV/TSV/JSON with `measured` at/near the QL") + parser.add_argument("--confirm-accuracy-limit", type=float, default=20.0, + help="max %% bias allowed when confirming the QL (default 20)") + parser.add_argument("--confirm-rsd-limit", type=float, default=20.0, + help="max %%RSD allowed when confirming the QL (default 20)") + parser.add_argument("--reporting-threshold", type=float, default=None, + help="impurity reporting threshold the QL must be at or below") + parser.add_argument("--weight", choices=("none", "1/x", "1/x2"), default="none") + add_common_args(parser) + args = parser.parse_args() + + rows = parse_rows(read_input(args.calibration), args.calibration) + require_columns(rows, ["level", "response"]) + xs = [to_float(r["level"], "level", i) for i, r in enumerate(rows)] + ys = [to_float(r["response"], "response", i) for i, r in enumerate(rows)] + weights = None + if args.weight != "none": + if any(x == 0 for x in xs): + raise InputError("weighting needs non-zero levels") + power = 1 if args.weight == "1/x" else 2 + weights = [1.0 / (abs(x) ** power) for x in xs] + fit = fit_linear(xs, ys, weights) + slope = fit.slope + if slope == 0: + raise InputError("calibration slope is zero; DL/QL cannot be computed") + + findings: list[str] = [] + results: list[dict] = [] + + # Approach 3.2.3.3, sigma = residual SD of the regression line. + results.append( + { + "approach": "sd-and-slope (sigma = residual SD of regression)", + "sigma": fit.residual_sd, + "slope": slope, + "DL": DL_FACTOR * fit.residual_sd / abs(slope), + "QL": QL_FACTOR * fit.residual_sd / abs(slope), + "reference": "Q2(R2) 3.2.3.3", + } + ) + + # Approach 3.2.3.3, sigma = SD of the y-intercept. + results.append( + { + "approach": "sd-and-slope (sigma = SD of y-intercept)", + "sigma": fit.se_intercept, + "slope": slope, + "DL": DL_FACTOR * fit.se_intercept / abs(slope), + "QL": QL_FACTOR * fit.se_intercept / abs(slope), + "reference": "Q2(R2) 3.2.3.3", + } + ) + + # Approach 3.2.3.3, sigma = SD of blank responses. + if args.blanks: + brows = parse_rows(read_input(args.blanks), args.blanks) + require_columns(brows, ["response"]) + blanks = [to_float(r["response"], "response", i) for i, r in enumerate(brows)] + if len(blanks) < 3: + raise InputError("blank SD needs at least 3 blank measurements") + bsd = sample_sd(blanks) + results.append( + { + "approach": f"sd-and-slope (sigma = SD of {len(blanks)} blanks)", + "sigma": bsd, + "slope": slope, + "DL": DL_FACTOR * bsd / abs(slope), + "QL": QL_FACTOR * bsd / abs(slope), + "reference": "Q2(R2) 3.2.3.3", + } + ) + note(f"blank mean response {mean(blanks):.6g}, SD {bsd:.6g}, n={len(blanks)}") + + # Approach 3.2.3.2, signal-to-noise. + if args.signal_to_noise is not None: + if args.sn_level is None: + raise InputError("--signal-to-noise needs --sn-level") + if args.signal_to_noise <= 0 or args.sn_level <= 0: + raise InputError("--signal-to-noise and --sn-level must be > 0") + per_unit = args.signal_to_noise / args.sn_level + results.append( + { + "approach": f"signal-to-noise (S/N {args.signal_to_noise:g} at {args.sn_level:g})", + "sigma": float("nan"), + "slope": slope, + "DL": 3.0 / per_unit, + "QL": 10.0 / per_unit, + "reference": "Q2(R2) 3.2.3.2", + } + ) + note( + "signal-to-noise DL uses the 3:1 ratio and QL the 10:1 ratio from Q2(R2) 3.2.3.2, " + "scaled linearly from the measured S/N. Linear scaling of noise is an assumption -- " + "confirm at the resulting level" + ) + + # Spread across approaches: the point of computing all of them. + qls = [r["QL"] for r in results if math.isfinite(r["QL"])] + spread_note = "" + if len(qls) >= 2: + ratio = max(qls) / min(qls) if min(qls) > 0 else float("inf") + spread_note = ( + f"QL estimates span {min(qls):.6g} to {max(qls):.6g} ({ratio:.2f}x) across " + f"{len(qls)} approaches" + ) + if ratio > 2.0: + findings.append( + spread_note + + ": name the approach used in the report, and confirm the claimed limit with " + "real data at that level (Q2(R2) 3.2.3.5)" + ) + + # QL confirmation with real data (Q2(R2) 3.2.3.4 / 3.2.3.5). + confirm_rows: list[dict] = [] + if args.confirm_data: + if args.confirm_ql is None: + raise InputError("--confirm-data needs --confirm-ql") + crows = parse_rows(read_input(args.confirm_data), args.confirm_data) + require_columns(crows, ["measured"]) + measured = [to_float(r["measured"], "measured", i) for i, r in enumerate(crows)] + if len(measured) < 3: + raise InputError("QL confirmation needs at least 3 determinations") + m = mean(measured) + bias = 100.0 * (m - args.confirm_ql) / args.confirm_ql + rsd = rsd_percent(measured) + confirm_rows = [ + {"metric": "claimed QL", "value": args.confirm_ql}, + {"metric": "n determinations", "value": len(measured)}, + {"metric": "mean measured", "value": m}, + {"metric": "bias vs claimed QL (%)", "value": bias}, + {"metric": "RSD (%)", "value": rsd}, + {"metric": "accuracy limit (%)", "value": args.confirm_accuracy_limit}, + {"metric": "RSD limit (%)", "value": args.confirm_rsd_limit}, + ] + if abs(bias) > args.confirm_accuracy_limit: + findings.append( + f"QL confirmation: bias {bias:+.2f}% at the claimed QL exceeds " + f"+/-{args.confirm_accuracy_limit:g}%" + ) + if math.isfinite(rsd) and rsd > args.confirm_rsd_limit: + findings.append( + f"QL confirmation: {rsd:.2f}% RSD at the claimed QL exceeds " + f"{args.confirm_rsd_limit:g}%" + ) + elif args.confirm_ql is not None: + note( + "a QL was claimed but no confirmation data supplied. Q2(R2) 3.2.3.5 asks that an " + "estimated limit be validated by analysing samples at or near it" + ) + + # The requirement that actually gates an impurity method. + if args.reporting_threshold is not None: + # With no claimed QL, use the LARGEST estimate. Taking the smallest would + # let the check pass on the most flattering choice of sigma, which is the + # wrong direction to err on a compliance requirement. + conservative = max(qls) if qls else float("nan") + claimed = args.confirm_ql if args.confirm_ql is not None else conservative + if args.confirm_ql is None and qls: + note( + f"no --confirm-ql given, so the reporting-threshold check uses the most " + f"conservative estimate ({conservative:.6g}), not the most favourable " + f"({min(qls):.6g})" + ) + if min(qls) <= args.reporting_threshold < conservative: + findings.append( + f"the QL estimates straddle the reporting threshold " + f"{args.reporting_threshold:.6g}: {min(qls):.6g} would pass and " + f"{conservative:.6g} would not. Whether this procedure meets Q2(R2) " + "3.2.3.5 depends on which approach is chosen, so choose it, justify it, " + "and confirm the claimed limit with data at that level" + ) + if math.isfinite(claimed) and claimed > args.reporting_threshold: + findings.append( + f"QL {claimed:.6g} is above the reporting threshold " + f"{args.reporting_threshold:.6g}. Q2(R2) 3.2.3.5 requires the QL for an " + "impurity procedure to be at or below the reporting threshold" + ) + elif math.isfinite(claimed): + ratio = args.reporting_threshold / claimed if claimed > 0 else float("inf") + if math.isclose(ratio, 1.0, rel_tol=1e-9): + note( + f"QL {claimed:.6g} sits exactly at the reporting threshold " + f"{args.reporting_threshold:.6g}, which satisfies Q2(R2) 3.2.3.5 with no " + "margin -- any drift puts the procedure out of compliance" + ) + else: + note( + f"QL {claimed:.6g} is {ratio:.1f}x below the reporting threshold " + f"{args.reporting_threshold:.6g}" + ) + if ratio >= 10: + note( + "the QL is roughly 10x or more below the reporting limit, so Q2(R2) 3.2.3.5 " + "allows the confirmatory validation to be omitted with justification" + ) + + if args.format == "json": + emit( + [ + { + "calibration": { + "n": fit.n, + "slope": slope, + "intercept": fit.intercept, + "residual_sd": fit.residual_sd, + "weighting": args.weight, + }, + "estimates": results, + "confirmation": confirm_rows, + "findings": findings, + } + ], + "json", + ) + else: + emit(results, args.format) + if confirm_rows: + print() + emit(confirm_rows, args.format) + + if not args.confirm_data: + note( + f"approach 'accuracy-precision' ({DL_QL_APPROACHES['accuracy-precision']['note']}) " + "validates the QL directly rather than estimating it -- supply --confirm-data " + "to use it" + ) + if spread_note and not any(spread_note in f for f in findings): + note(spread_note) + note("Q2(R2) 3.2.3.5: report the limit AND the approach used to determine it") + for f in findings: + finding(f) + if not findings: + note("no findings against the checks that were run") + note("this tool does not decide that a detection or quantitation limit is acceptable") + return EXIT_FINDINGS if findings else EXIT_OK + + +if __name__ == "__main__": + run_cli(main) diff --git a/.agents/skills/analytical-method-validation/scripts/check_response.py b/.agents/skills/analytical-method-validation/scripts/check_response.py new file mode 100644 index 0000000..9b7c041 --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/check_response.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Evaluate a calibration response (linearity) the way ICH Q2(R2) 3.2.2 asks. + +Reports what the guideline asks to be reported -- slope, intercept, coefficient +of determination, and an analysis of the deviation of points from the line -- +and adds the diagnostics that actually detect an unsuitable model: a lack-of-fit +F test against pure error, a runs test on residual signs, back-calculated +relative error per level, and a heteroscedasticity check that tells you whether +weighting is needed. + + python3 check_response.py --input calib.csv + python3 check_response.py --input calib.csv --weight 1/x2 --levels-required 5 + python3 check_response.py --input calib.csv --max-back-calc-error 5 --format json + +Input columns: `level` (nominal concentration or %) and `response` (signal). +An optional `replicate` column is ignored -- replicates are simply repeated rows +at the same level, which is what enables the lack-of-fit test. + +Exit codes: 0 no findings, 1 findings raised, 2 bad input. +""" + +from __future__ import annotations + +import argparse +import math +import sys + +sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent)) + +from _common import ( # noqa: E402 + EXIT_FINDINGS, + EXIT_OK, + InputError, + add_common_args, + emit, + finding, + fit_linear, + heteroscedasticity, + lack_of_fit, + mean, + note, + parse_rows, + read_input, + require_columns, + run_cli, + runs_test, + to_float, +) + +WEIGHT_SCHEMES = { + "none": lambda x: 1.0, + "1/x": lambda x: 1.0 / abs(x) if x != 0 else 0.0, + "1/x2": lambda x: 1.0 / (x * x) if x != 0 else 0.0, +} + + +def build_weights(xs: list[float], scheme: str) -> list[float] | None: + if scheme == "none": + return None + fn = WEIGHT_SCHEMES[scheme] + if any(x == 0 for x in xs): + raise InputError(f"weighting {scheme} needs non-zero levels; a zero level was supplied") + return [fn(x) for x in xs] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Check a calibration response for linearity.") + parser.add_argument("--input", "-i", help="CSV/TSV/JSON file, or '-' for stdin") + parser.add_argument("--weight", choices=sorted(WEIGHT_SCHEMES), default="none", + help="calibration weighting (default: none)") + parser.add_argument("--levels-required", type=int, default=5, + help="minimum distinct levels expected (Q2(R2) recommends 5)") + parser.add_argument("--max-back-calc-error", type=float, default=None, + help="flag any level whose mean back-calculated error exceeds this %%") + parser.add_argument("--alpha", type=float, default=0.05, + help="significance level for lack-of-fit and runs tests") + parser.add_argument("--through-origin-tolerance", type=float, default=None, + help="flag when the intercept exceeds this %% of the response at the " + "highest level (a proxy for the y-intercept significance check)") + add_common_args(parser) + args = parser.parse_args() + + rows = parse_rows(read_input(args.input), args.input) + require_columns(rows, ["level", "response"]) + xs = [to_float(r["level"], "level", i) for i, r in enumerate(rows)] + ys = [to_float(r["response"], "response", i) for i, r in enumerate(rows)] + + weights = build_weights(xs, args.weight) + fit = fit_linear(xs, ys, weights) + distinct = sorted({round(x, 12) for x in xs}) + + findings: list[str] = [] + + # Q2(R2) 3.2.2.1: a minimum of five concentrations is recommended. + if len(distinct) < args.levels_required: + findings.append( + f"{len(distinct)} distinct levels; Q2(R2) 3.2.2.1 recommends at least " + f"{args.levels_required} appropriately distributed across the range" + ) + + lof = lack_of_fit(xs, ys, fit) + runs = runs_test(fit.residuals) + het = heteroscedasticity(xs, fit.residuals) + + if lof.get("applicable") and lof["p_value"] < args.alpha: + findings.append( + f"lack-of-fit F={lof['f_statistic']:.3f} on {lof['df_lack_of_fit']}/" + f"{lof['df_pure_error']} df, p={lof['p_value']:.4g}: the straight line does not " + "describe the data beyond replicate scatter" + ) + if math.isfinite(runs.get("p_value", float("nan"))) and runs["p_value"] < args.alpha: + findings.append( + f"residual signs are non-random (runs={runs['runs']}, expected " + f"{runs['expected_runs']:.1f}, p={runs['p_value']:.4g}): inspect the residual plot " + "for curvature" + ) + if het.get("applicable") and het["variance_ratio_high_over_low"] > 10 and args.weight == "none": + findings.append( + f"residual variance is {het['variance_ratio_high_over_low']:.1f}x larger in the top " + "third of the range than the bottom, and the fit is unweighted: back-calculated " + "results at the low end are biased. Consider 1/x or 1/x2 weighting" + ) + + # Back-calculated relative error per level -- the practical test of the model. + level_rows = [] + by_level: dict[float, list[float]] = {} + for x, y in zip(xs, ys): + by_level.setdefault(round(x, 12), []).append(y) + for level in distinct: + responses = by_level[level] + back = [ + (r - fit.intercept) / fit.slope if fit.slope != 0 else float("nan") + for r in responses + ] + mean_back = mean(back) + rel_err = 100.0 * (mean_back - level) / level if level != 0 else float("nan") + level_rows.append( + { + "level": level, + "n": len(responses), + "mean_response": mean(responses), + "mean_back_calculated": mean_back, + "relative_error_pct": rel_err, + } + ) + if ( + args.max_back_calc_error is not None + and math.isfinite(rel_err) + and abs(rel_err) > args.max_back_calc_error + ): + findings.append( + f"level {level:g}: back-calculated mean deviates {rel_err:+.2f}% from nominal, " + f"outside the stated +/-{args.max_back_calc_error:g}%" + ) + + if args.through_origin_tolerance is not None: + top_response = fit.predict(max(distinct)) + if top_response != 0: + pct = 100.0 * abs(fit.intercept) / abs(top_response) + if pct > args.through_origin_tolerance: + findings.append( + f"intercept is {pct:.2f}% of the response at the highest level, above the " + f"stated {args.through_origin_tolerance:g}% tolerance" + ) + + slope_lo, slope_hi = fit.slope_ci() + int_lo, int_hi = fit.intercept_ci() + summary = [ + {"statistic": "n points", "value": fit.n}, + {"statistic": "distinct levels", "value": len(distinct)}, + {"statistic": "weighting", "value": args.weight}, + {"statistic": "slope", "value": fit.slope}, + {"statistic": "slope 95% CI", "value": f"{slope_lo:.6g} to {slope_hi:.6g}"}, + {"statistic": "intercept", "value": fit.intercept}, + {"statistic": "intercept 95% CI", "value": f"{int_lo:.6g} to {int_hi:.6g}"}, + {"statistic": "intercept CI includes 0", "value": int_lo <= 0.0 <= int_hi}, + {"statistic": "coefficient of determination (r2)", "value": fit.r_squared}, + {"statistic": "correlation coefficient (r)", "value": fit.r}, + {"statistic": "residual SD", "value": fit.residual_sd}, + {"statistic": "residual df", "value": fit.df}, + ] + if lof.get("applicable"): + summary += [ + {"statistic": "lack-of-fit F", "value": lof["f_statistic"]}, + {"statistic": "lack-of-fit p", "value": lof["p_value"]}, + ] + else: + summary.append( + {"statistic": "lack-of-fit test", "value": f"not run ({lof.get('reason', '')})"} + ) + summary += [ + {"statistic": "runs test p", "value": runs.get("p_value", float("nan"))}, + { + "statistic": "residual SD ratio (high/low third)", + "value": het.get("sd_ratio", float("nan")), + }, + ] + + if args.format == "json": + emit( + [ + { + "summary": {r["statistic"]: r["value"] for r in summary}, + "levels": level_rows, + "findings": findings, + } + ], + "json", + ) + else: + emit(summary, args.format) + print() + emit(level_rows, args.format) + + note( + "Q2(R2) 3.2.2.1 asks for the plot, r or r-squared, intercept, slope, and an analysis of " + "the deviation of points from the line" + ) + note( + "r-squared alone does not demonstrate linearity: it rises with range and is insensitive " + "to curvature. The lack-of-fit test and the residual pattern are the evidence" + ) + if args.weight != "none" and lof.get("applicable"): + note( + f"the lack-of-fit F test is computed on unweighted residuals while the fit used " + f"{args.weight} weighting, so its null distribution is approximate here. Read it " + "alongside the back-calculated error per level, which is unaffected" + ) + if not lof.get("applicable"): + note( + "no replicates at any level, so pure error could not be separated from lack of fit. " + "Replicating at least one level makes the linearity test possible" + ) + for f in findings: + finding(f) + if not findings: + note("no findings against the checks that were run") + note("this tool does not decide that the calibration model is acceptable") + return EXIT_FINDINGS if findings else EXIT_OK + + +if __name__ == "__main__": + run_cli(main) diff --git a/.agents/skills/analytical-method-validation/scripts/compare_methods.py b/.agents/skills/analytical-method-validation/scripts/compare_methods.py new file mode 100644 index 0000000..88bc593 --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/compare_methods.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Compare two analytical procedures for a transfer, bridging, or bias study. + +Uses the regressions that belong to method comparison -- Deming and +Passing-Bablok, which allow error in both measurements -- rather than ordinary +least squares, which assumes the comparative procedure is error-free and biases +the slope toward zero. Adds Bland-Altman agreement and, most importantly, a TOST +equivalence test. + +The default reflex at a method transfer is a t test, and "p > 0.05, no +significant difference" is then written up as evidence of equivalence. It is not: +failing to detect a difference is not the same as showing there is none, and with +a small transfer dataset that outcome is close to guaranteed. TOST tests the +hypothesis that matters -- that the true difference lies inside a pre-stated +acceptance margin. + + python3 compare_methods.py --input paired.csv --margin 2 + python3 compare_methods.py -i paired.csv --margin 2 --relative --lambda 1.0 + +Input columns: `reference` and `test` (paired results on the same samples). +An optional `sample` column labels the rows. + +Exit codes: 0 equivalence demonstrated at the stated margin, 1 not demonstrated +or other findings, 2 bad input. +""" + +from __future__ import annotations + +import argparse +import math +import sys + +sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent)) + +from _common import ( # noqa: E402 + EXIT_FINDINGS, + EXIT_OK, + InputError, + add_common_args, + bland_altman, + deming, + emit, + finding, + fit_linear, + mean, + note, + parse_rows, + passing_bablok, + read_input, + require_columns, + run_cli, + t_cdf, + to_float, + tost_paired, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Compare two procedures with the statistics method comparison requires." + ) + parser.add_argument("--input", "-i", help="CSV/TSV/JSON with `reference` and `test`") + parser.add_argument("--margin", type=float, required=True, + help="pre-stated equivalence margin for TOST, in the units of the " + "difference (absolute, or %% when --relative is used)") + parser.add_argument("--relative", action="store_true", + help="work in percent differences relative to the pair mean") + parser.add_argument("--lambda", dest="lambda_ratio", type=float, default=1.0, + help="Deming error-variance ratio var(y error)/var(x error) -- test " + "procedure over comparative procedure. Default 1.0 means equal " + "precision. Estimate it as (SD of x replicates / SD of y " + "replicates) squared") + parser.add_argument("--alpha", type=float, default=0.05, + help="one-sided alpha for TOST (default 0.05)") + parser.add_argument("--slope-tolerance", type=float, default=None, + help="flag when the slope CI excludes 1 +/- this amount") + add_common_args(parser) + args = parser.parse_args() + + rows = parse_rows(read_input(args.input), args.input) + require_columns(rows, ["reference", "test"]) + ref = [to_float(r["reference"], "reference", i) for i, r in enumerate(rows)] + test = [to_float(r["test"], "test", i) for i, r in enumerate(rows)] + n = len(ref) + if n < 3: + raise InputError("method comparison needs at least 3 paired results") + + findings: list[str] = [] + + # Agreement. + ba = bland_altman(ref, test, relative=args.relative) + unit = "%" if args.relative else "units" + + # TOST on the differences that Bland-Altman used. + if args.relative: + diffs = [ + 100.0 * (t - r) / (0.5 * (r + t)) for r, t in zip(ref, test) + ] + else: + diffs = [t - r for r, t in zip(ref, test)] + tost = tost_paired(diffs, args.margin, args.alpha) + + # The naive test, computed only to show what it does not establish. + d_mean = mean(diffs) + sd = ba["sd_differences"] + se = sd / math.sqrt(n) if sd > 0 else float("nan") + t_stat = d_mean / se if se and math.isfinite(se) and se > 0 else float("nan") + p_naive = ( + 2.0 * (1.0 - t_cdf(abs(t_stat), n - 1)) if math.isfinite(t_stat) else float("nan") + ) + + # Regressions. + ols = fit_linear(ref, test) + dem = deming(ref, test, args.lambda_ratio) + try: + pb = passing_bablok(ref, test) + except InputError as exc: + pb = None + note(f"Passing-Bablok not computed: {exc}") + + summary = [ + {"statistic": "n pairs", "value": n}, + {"statistic": f"mean difference ({unit})", "value": ba["bias"]}, + {"statistic": "difference 95% CI", "value": + f"{ba['bias_ci95'][0]:.6g} to {ba['bias_ci95'][1]:.6g}"}, + {"statistic": f"SD of differences ({unit})", "value": sd}, + {"statistic": "limits of agreement", "value": + f"{ba['loa_lower']:.6g} to {ba['loa_upper']:.6g}"}, + {"statistic": "LoA 95% CI half-width", "value": ba["loa_ci95_halfwidth"]}, + {"statistic": "--- equivalence ---", "value": ""}, + {"statistic": "TOST margin", "value": args.margin}, + {"statistic": "TOST p-value", "value": tost["p_value"]}, + {"statistic": f"{100 * (1 - 2 * args.alpha):.0f}% CI (TOST)", "value": + f"{tost['ci_1_minus_2alpha'][0]:.6g} to {tost['ci_1_minus_2alpha'][1]:.6g}"}, + {"statistic": "equivalent at stated margin", "value": tost["equivalent"]}, + {"statistic": "--- for contrast only ---", "value": ""}, + {"statistic": "paired t-test p (NOT equivalence)", "value": p_naive}, + {"statistic": "--- regressions ---", "value": ""}, + {"statistic": "OLS slope (biased here)", "value": ols.slope}, + {"statistic": "Deming slope", "value": dem["slope"]}, + {"statistic": "Deming slope 95% CI", "value": + f"{dem['slope_ci95'][0]:.6g} to {dem['slope_ci95'][1]:.6g}"}, + {"statistic": "Deming intercept", "value": dem["intercept"]}, + ] + if pb: + summary += [ + {"statistic": "Passing-Bablok slope", "value": pb["slope"]}, + {"statistic": "Passing-Bablok slope 95% CI", "value": + f"{pb['slope_ci95'][0]:.6g} to {pb['slope_ci95'][1]:.6g}"}, + {"statistic": "Passing-Bablok intercept", "value": pb["intercept"]}, + ] + + prop = ba.get("proportional_bias") + if prop and math.isfinite(prop.get("p_value", float("nan"))): + summary.append( + {"statistic": "proportional bias p (difference vs mean)", "value": prop["p_value"]} + ) + if prop["p_value"] < 0.05: + findings.append( + f"the difference trends with concentration (slope {prop['slope']:.4g}, " + f"p={prop['p_value']:.4g}): a single mean bias does not describe the " + "disagreement, and limits of agreement are misleading" + ) + + if not tost["equivalent"]: + findings.append( + f"equivalence NOT demonstrated at +/-{args.margin:g} {unit}: the " + f"{100 * (1 - 2 * args.alpha):.0f}% CI " + f"({tost['ci_1_minus_2alpha'][0]:.4g}, {tost['ci_1_minus_2alpha'][1]:.4g}) is not " + f"contained in the margin" + ) + + if args.slope_tolerance is not None: + lo, hi = dem["slope_ci95"] + target_lo, target_hi = 1.0 - args.slope_tolerance, 1.0 + args.slope_tolerance + if lo < target_lo or hi > target_hi: + findings.append( + f"Deming slope 95% CI ({lo:.4g}, {hi:.4g}) is not contained in " + f"({target_lo:g}, {target_hi:g})" + ) + + if args.format == "json": + emit([{"summary": {r["statistic"]: r["value"] for r in summary + if not r["statistic"].startswith("---")}, + "bland_altman": ba, "deming": dem, + "passing_bablok": pb, "tost": tost, + "ols_slope": ols.slope, "paired_t_p_value": p_naive, + "findings": findings}], "json") + else: + emit(summary, args.format) + + if math.isfinite(p_naive) and p_naive > 0.05 and not tost["equivalent"]: + note( + f"the paired t-test gives p={p_naive:.4g}, which would often be written up as " + "'no significant difference'. TOST shows equivalence is NOT established at the " + "stated margin. Absence of a detected difference is not evidence of equivalence" + ) + note( + "ordinary least squares assumes the reference values carry no error, which is false in a " + "method comparison; the OLS slope is shown only for contrast" + ) + note( + "the equivalence margin must be pre-stated from the specification or the analytical " + "target profile, never chosen after seeing the data" + ) + for f in findings: + finding(f) + if not findings: + note("no findings against the checks that were run") + note("this tool does not decide that a transfer or comparison passes") + return EXIT_FINDINGS if findings else EXIT_OK + + +if __name__ == "__main__": + run_cli(main) diff --git a/.agents/skills/analytical-method-validation/scripts/plan_validation.py b/.agents/skills/analytical-method-validation/scripts/plan_validation.py new file mode 100644 index 0000000..d54ce45 --- /dev/null +++ b/.agents/skills/analytical-method-validation/scripts/plan_validation.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +"""Design a validation study: which framework, which tests, what study layout. + +Answers the first question of any validation exercise -- what am I required to +demonstrate, and with how much data -- before any sample is injected. + + python3 plan_validation.py --framework ich-q2r2 --attribute assay --technique hplc + python3 plan_validation.py --framework ich-m10 --modality lba + python3 plan_validation.py --list-frameworks + python3 plan_validation.py --framework ich-q2r2 --attribute impurity --protocol > protocol.md + +Exit codes: 0 plan produced, 2 bad input. +""" + +from __future__ import annotations + +import argparse +import sys + +sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent)) + +from _catalog import ( # noqa: E402 + DL_QL_APPROACHES, + FRAMEWORKS, + M10_CRITERIA, + Q2R2_REPORTABLE_RANGE, + Q2R2_STUDY_DESIGN, + Q2R2_TESTS_BY_ATTRIBUTE, + RESEARCH_DATE, + TECHNIQUE_NOTES, + resolve_attribute, +) +from _common import ( # noqa: E402 + EXIT_OK, + InputError, + add_common_args, + emit, + note, + run_cli, +) + +STATUS_TEXT = { + "required": "conduct", + "required-QL": "conduct (QL; DL too in some complex cases)", + "required-DL": "conduct (DL)", + "required-unless-reproducibility": ( + "conduct, unless intermediate precision can be derived from a reproducibility dataset" + ), + "not-normally": "not normally conducted", +} + + +def plan_q2r2(attribute: str, technique: str | None, range_use: str | None) -> list[dict]: + tests = Q2R2_TESTS_BY_ATTRIBUTE[attribute] + rows = [] + for characteristic, status in tests.items(): + design = Q2R2_STUDY_DESIGN.get(characteristic, {}) + rows.append( + { + "characteristic": characteristic, + "required": STATUS_TEXT[status], + "study_design": design.get("requirement", ""), + "report": design.get("report", ""), + "reference": design.get("reference", ""), + } + ) + # Robustness sits in development under Q14 but belongs in the plan. + rows.append( + { + "characteristic": "robustness", + "required": "development activity (ICH Q14); available on request", + "study_design": Q2R2_STUDY_DESIGN["robustness"]["requirement"], + "report": Q2R2_STUDY_DESIGN["robustness"]["report"], + "reference": Q2R2_STUDY_DESIGN["robustness"]["reference"], + } + ) + if technique: + tn = TECHNIQUE_NOTES.get(technique) + if tn: + rows.append( + { + "characteristic": f"technique note ({technique})", + "required": f"see Q2(R2) Annex 2 {tn['annex_table']}", + "study_design": f"robustness parameters: {tn['robustness']}", + "report": tn["special"], + "reference": "Q2(R2) Annex 2", + } + ) + if range_use: + rr = Q2R2_REPORTABLE_RANGE.get(range_use) + if rr: + rows.append( + { + "characteristic": f"reportable range ({range_use})", + "required": "confirm response, accuracy and precision across this range", + "study_design": f"low: {rr['low']}", + "report": f"high: {rr['high']}", + "reference": "Q2(R2) 2.3, Table 2", + } + ) + return rows + + +def plan_m10(modality: str) -> list[dict]: + crit = M10_CRITERIA[modality] + rows = [ + { + "item": "calibration curve", + "requirement": ( + f"minimum {crit['calibration_min_levels']} concentration levels including " + f"the LLOQ; at least " + f"{crit['calibration_min_pass_fraction'] * 100:.0f}% of standards must pass" + ), + "tolerance": ( + f"+/-{crit['calibration_tolerance_pct']:.0f}% nominal; " + f"+/-{crit['calibration_tolerance_lloq_pct']:.0f}% at LLOQ; " + f"+/-{crit['calibration_tolerance_uloq_pct']:.0f}% at ULOQ" + ), + }, + { + "item": "accuracy and precision QC levels", + "requirement": ( + f"{crit['qc_levels_accuracy_precision']} levels; " + f"{crit['ap_replicates_per_run']} replicates per level per run; " + f"at least {crit['ap_min_runs']} runs over at least {crit['ap_min_days']} days" + ), + "tolerance": ( + f"accuracy +/-{crit['accuracy_tolerance_pct']:.0f}% " + f"(+/-{crit['accuracy_tolerance_lloq_pct']:.0f}% at {crit['limit_levels']}); " + f"precision CV <={crit['precision_cv_pct']:.0f}% " + f"(<={crit['precision_cv_lloq_pct']:.0f}% at {crit['limit_levels']})" + ), + }, + { + "item": "routine run acceptance", + "requirement": ( + f"{crit['qc_levels_routine_run']} QC levels; at least " + f"{crit['qc_run_pass_fraction'] * 100:.0f}% of all QCs and at least " + f"{crit['qc_run_pass_fraction_per_level'] * 100:.0f}% at each level must pass" + ), + "tolerance": f"+/-{crit['qc_run_tolerance_pct']:.0f}% nominal", + }, + { + "item": "selectivity", + "requirement": f"at least {crit['selectivity_min_sources']} individual matrix sources/lots", + "tolerance": ( + f"interference <={crit['carryover_blank_pct_of_lloq']:.0f}% of LLOQ analyte " + f"response and <={crit['carryover_blank_pct_of_is']:.0f}% of IS response" + if crit["carryover_blank_pct_of_lloq"] is not None + else "per guideline; evaluate interference in each source" + ), + }, + { + "item": "dilution integrity", + "requirement": "validate the dilution factors used in study sample analysis", + "tolerance": f"mean within +/-{crit['dilution_tolerance_pct']:.0f}% nominal", + }, + { + "item": "stability", + "requirement": "cover the conditions and durations study samples actually experience", + "tolerance": f"mean at each QC level within +/-{crit['stability_tolerance_pct']:.0f}% nominal", + }, + { + "item": "incurred sample reanalysis", + "requirement": ( + f"repeat a predefined subset in separate runs; at least " + f"{crit['isr_pass_fraction'] * 100:.0f}% of repeats must agree" + ), + "tolerance": f"percent difference within +/-{crit['isr_tolerance_pct']:.0f}%", + }, + ] + if crit["total_error_pct"] is not None: + rows.append( + { + "item": "total error", + "requirement": "sum of absolute accuracy (%) and precision (%)", + "tolerance": ( + f"<={crit['total_error_pct']:.0f}%, " + f"<={crit['total_error_pct_at_limits']:.0f}% at LLOQ and ULOQ" + ), + } + ) + return rows + + +def render_protocol(framework: str, attribute: str | None, modality: str | None, + technique: str | None, range_use: str | None) -> str: + fw = FRAMEWORKS[framework] + lines = [ + "# Analytical Procedure Validation Protocol", + "", + "> Draft skeleton. Every bracketed field is a decision the analyst and quality unit", + "> must make and record BEFORE data collection. Acceptance criteria set after seeing", + "> data are not acceptance criteria.", + "", + "## 1. Identification", + "", + "| Field | Value |", + "| --- | --- |", + "| Protocol number / version | [ ] |", + "| Analytical procedure identifier | [ ] |", + "| Product / analyte / matrix | [ ] |", + "| Governing framework | " + fw["title"] + " |", + "| Regional expectation confirmed | [ ] " + fw["effective_note"] + " |", + "| Author / date | [ ] |", + "| Reviewed / approved by (quality unit) | [ ] |", + "", + "## 2. Intended purpose and analytical target profile", + "", + "- Measurand and reporting unit: [ ]", + "- Decision the result supports (release, stability, in-process, clinical): [ ]", + "- Specification or reporting limits the procedure must serve: [ ]", + "- Required reportable range, derived from the specification: [ ]", + "- Performance characteristics and criteria (the ATP, ICH Q14 section 3): [ ]", + "", + "## 3. Pre-stated acceptance criteria", + "", + "| Characteristic | Criterion | Justification | Source |", + "| --- | --- | --- | --- |", + ] + if framework == "ich-m10" and modality: + for row in plan_m10(modality): + lines.append( + f"| {row['item']} | {row['tolerance']} | guideline default | ICH M10 |" + ) + elif attribute: + for row in plan_q2r2(attribute, technique, range_use): + crit = "[ ] state a numeric criterion" + lines.append( + f"| {row['characteristic']} | {crit} | [ ] | {row['reference'] or 'ICH Q2(R2)'} |" + ) + lines += [ + "", + "> ICH Q2(R2) deliberately does not set numeric acceptance criteria for most", + "> characteristics. A criterion has to come from the specification, the ATP, product", + "> knowledge, or development data -- not from a remembered default.", + "", + "## 4. Study design", + "", + "| Characteristic | Levels | Replicates | Runs / days / analysts / instruments |", + "| --- | --- | --- | --- |", + "| [ ] | [ ] | [ ] | [ ] |", + "", + "- Reference materials and their documented identity/purity: [ ]", + "- Number of replicates matches the routine reportable result: [ ] yes / [ ] justified", + "- Prior knowledge or development data used in place of a test, with justification: [ ]", + "", + "## 5. Sample and solution handling", + "", + "- Preparation, storage, and solution stability window: [ ]", + "- Blank, placebo, and spiked matrix definitions: [ ]", + "", + "## 6. Statistical treatment", + "", + "- Calibration model and weighting, stated in advance: [ ]", + "- Interval to be reported with accuracy and precision (confidence level): [ ]", + "- Software, version, and calculation verification: [ ]", + "", + "## 7. Deviations and data integrity", + "", + "- Deviation handling and reporting: [ ]", + "- All results reported, including out-of-criteria values: [ ] confirmed", + "- Raw data location and audit trail: [ ]", + "", + "## 8. Approvals", + "", + "| Role | Name | Signature | Date |", + "| --- | --- | --- | --- |", + "| Author | | | |", + "| Technical reviewer | | | |", + "| Quality unit | | | |", + "", + f"_Framework metadata researched {RESEARCH_DATE}. Confirm the current guideline text " + f"and regional expectation before use: {fw['url']}_", + ] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Plan an analytical procedure validation study.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--framework", help="governing framework key") + parser.add_argument("--attribute", help="measured attribute (assay, impurity, identity, ...)") + parser.add_argument("--modality", choices=sorted(M10_CRITERIA), + help="ICH M10 only: chromatographic or lba") + parser.add_argument("--technique", choices=sorted(TECHNIQUE_NOTES), + help="analytical technique, for Annex 2 notes") + parser.add_argument("--range-use", choices=sorted(Q2R2_REPORTABLE_RANGE), + help="reportable range example to include") + parser.add_argument("--protocol", action="store_true", + help="emit a validation protocol skeleton in Markdown") + parser.add_argument("--list-frameworks", action="store_true") + parser.add_argument("--list-dl-ql", action="store_true", + help="list the DL/QL estimation approaches") + add_common_args(parser) + args = parser.parse_args() + + if args.list_frameworks: + rows = [ + { + "key": key, + "title": fw["title"], + "adopted": fw["adopted"], + "text_reusable": "yes" if fw["reproducible"] else "no (paywalled)", + "scope": fw["scope"], + } + for key, fw in FRAMEWORKS.items() + ] + emit(rows, args.format) + note(f"framework metadata researched {RESEARCH_DATE}; confirm before relying on a date") + return EXIT_OK + + if args.list_dl_ql: + rows = [ + {"approach": k, "detection_limit": v["dl"], "quantitation_limit": v["ql"], + "note": v["note"]} + for k, v in DL_QL_APPROACHES.items() + ] + emit(rows, args.format) + note("source: ICH Q2(R2) 3.2.3") + return EXIT_OK + + if not args.framework: + raise InputError("--framework is required (or use --list-frameworks)") + if args.framework not in FRAMEWORKS: + raise InputError( + f"unknown framework {args.framework!r}; choose from: {', '.join(FRAMEWORKS)}" + ) + fw = FRAMEWORKS[args.framework] + + attribute = None + if args.attribute: + try: + attribute = resolve_attribute(args.attribute) + except KeyError as exc: + raise InputError(str(exc)) from exc + + if args.framework == "ich-m10": + if not args.modality: + raise InputError("ICH M10 needs --modality chromatographic|lba") + rows = plan_m10(args.modality) + elif args.framework == "ich-q2r2": + if not attribute: + raise InputError("ICH Q2(R2) needs --attribute (assay, impurity, identity, ...)") + rows = plan_q2r2(attribute, args.technique, args.range_use) + else: + note( + f"{fw['title']} is copyrighted and not reproduced here. This skill reports its " + "designation, scope, and where to obtain it; the study design must come from the " + "authorised text." + ) + rows = [ + { + "framework": args.framework, + "title": fw["title"], + "scope": fw["scope"], + "obtain_from": fw["url"], + "companion": fw["companion"], + } + ] + + if args.protocol: + print(render_protocol(args.framework, attribute, args.modality, + args.technique, args.range_use)) + note("protocol skeleton written; every bracketed field needs a decision before data") + return EXIT_OK + + emit(rows, args.format) + note(f"framework: {fw['title']}") + note(f"regional applicability: {fw['effective_note']}") + if fw["companion"]: + note(f"companion: {fw['companion']}") + note(f"catalogue researched {RESEARCH_DATE}; confirm the current text at {fw['url']}") + note("this tool reports requirements; it does not decide fitness for purpose") + return EXIT_OK + + +if __name__ == "__main__": + run_cli(main) diff --git a/.agents/skills/anndata/SKILL.md b/.agents/skills/anndata/SKILL.md new file mode 100644 index 0000000..fb8a68e --- /dev/null +++ b/.agents/skills/anndata/SKILL.md @@ -0,0 +1,431 @@ +--- +name: anndata +description: Data structure for annotated matrices in single-cell analysis. Use when working with .h5ad files or integrating with the scverse ecosystem. This is the data format skill—for analysis workflows use scanpy; for probabilistic models use scvi-tools; for population-scale queries use cellxgene-census. +license: BSD-3-Clause license +allowed-tools: Read Write Edit Bash +compatibility: Requires Python 3.11+ and uv. Examples target AnnData 0.12.16, with experimental APIs clearly marked where used. +metadata: + version: "1.1" + skill-author: K-Dense Inc. +--- + +# AnnData + +## Overview + +AnnData is a Python package for handling annotated data matrices, storing experimental measurements (X) alongside observation metadata (obs), variable metadata (var), and multi-dimensional annotations (obsm, varm, obsp, varp, uns). Originally designed for single-cell genomics through Scanpy, it now serves as a general-purpose framework for any annotated data requiring efficient storage, manipulation, and analysis. + +## When to Use This Skill + +Use this skill when: +- Creating, reading, or writing AnnData objects +- Working with h5ad, zarr, or other genomics data formats +- Performing single-cell RNA-seq analysis +- Managing large datasets with sparse matrices or backed mode +- Concatenating multiple datasets or experimental batches +- Subsetting, filtering, or transforming annotated data +- Integrating with scanpy, scvi-tools, or other scverse ecosystem tools + +## Installation + +Requires Python 3.11+. Current stable release: 0.12.16 (released 2026-05-18). + +```bash +uv pip install "anndata==0.12.16" + +# Lazy I/O and dask-backed operations +uv pip install "anndata[dask,lazy]==0.12.16" + +# Development / docs (contributors) +uv pip install "anndata[dev,test,doc]==0.12.16" +``` + +Use unpinned installs only when intentionally tracking the latest compatible release. + +Current API notes: +- Use `anndata.io` for non-native `read_*` and `write_*` helpers. Top-level `anndata.read_h5ad` and `anndata.read_zarr` remain supported. +- Avoid deprecated APIs: `ad.read`, `AnnData.concatenate()`, `AnnData.*_keys()`, and `anndata.__version__`. Prefer `ad.read_h5ad`, `ad.concat`, mapping `.keys()`, and `importlib.metadata.version("anndata")`. +- Treat `anndata.experimental` APIs as useful but unstable. Prefer them for large-data workflows only when their current caveats are acceptable. + +## Quick Start + +### Creating an AnnData object +```python +import anndata as ad +import numpy as np +import pandas as pd + +# Minimal creation +X = np.random.rand(100, 2000) # 100 cells × 2000 genes +adata = ad.AnnData(X) + +# With metadata +obs = pd.DataFrame({ + 'cell_type': ['T cell', 'B cell'] * 50, + 'sample': ['A', 'B'] * 50 +}, index=[f'cell_{i}' for i in range(100)]) + +var = pd.DataFrame({ + 'gene_name': [f'Gene_{i}' for i in range(2000)] +}, index=[f'ENSG{i:05d}' for i in range(2000)]) + +adata = ad.AnnData(X=X, obs=obs, var=var) +``` + +### Reading data +```python +# Native formats (read_h5ad/read_zarr remain at top-level) +adata = ad.read_h5ad('data.h5ad') +adata = ad.read_h5ad('large_data.h5ad', backed='r') # lazy load for large files +adata = ad.read_zarr('data.zarr') + +# Other formats: prefer anndata.io (top-level imports are deprecated) +from anndata.io import read_csv, read_loom, read_mtx + +adata = read_csv('data.csv') +adata = read_loom('data.loom') + +# 10X Genomics: use scanpy (not anndata) — see scanpy skill +import scanpy as sc +adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5') +adata = sc.read_10x_mtx('filtered_feature_bc_matrix/') +``` + +### Writing data +```python +# Write h5ad file +adata.write_h5ad('output.h5ad') + +# Write with compression +adata.write_h5ad('output.h5ad', compression='gzip') + +# Write other formats +adata.write_zarr('output.zarr') +adata.write_csvs('output_dir/') +``` + +### Basic operations +```python +# Subset by conditions +t_cells = adata[adata.obs['cell_type'] == 'T cell'] + +# Subset by indices +subset = adata[0:50, 0:100] + +# Add metadata +adata.obs['quality_score'] = np.random.rand(adata.n_obs) +adata.var['highly_variable'] = np.random.rand(adata.n_vars) > 0.8 + +# Access dimensions +print(f"{adata.n_obs} observations × {adata.n_vars} variables") +``` + +## Core Capabilities + +### 1. Data Structure + +Understand the AnnData object structure including X, obs, var, layers, obsm, varm, obsp, varp, uns, and raw components. + +**See**: `references/data_structure.md` for comprehensive information on: +- Core components (X, obs, var, layers, obsm, varm, obsp, varp, uns, raw) +- Creating AnnData objects from various sources +- Accessing and manipulating data components +- Memory-efficient practices + +### 2. Input/Output Operations + +Read and write data in various formats with support for compression, backed mode, and cloud storage. + +**See**: `references/io_operations.md` for details on: +- Native formats (h5ad, zarr) +- Alternative formats (CSV, MTX, Loom, 10X, Excel) +- Backed mode for large datasets +- Remote data access +- Format conversion +- Performance optimization + +Common commands: +```python +from anndata.io import read_mtx + +# Read/write h5ad +adata = ad.read_h5ad('data.h5ad', backed='r') +adata.write_h5ad('output.h5ad', compression='gzip') + +# 10X Genomics (via scanpy) +import scanpy as sc +adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5') + +# Read MTX format +adata = read_mtx('matrix.mtx').T +``` + +### 3. Concatenation + +Combine multiple AnnData objects along observations or variables with flexible join strategies. + +**See**: `references/concatenation.md` for comprehensive coverage of: +- Basic concatenation (axis=0 for observations, axis=1 for variables) +- Join types (inner, outer) +- Merge strategies (same, unique, first, only) +- Tracking data sources with labels +- Lazy concatenation (AnnCollection) +- On-disk concatenation for large datasets + +Common commands: +```python +# Concatenate observations (combine samples) +adata = ad.concat( + [adata1, adata2, adata3], + axis=0, + join='inner', + label='batch', + keys=['batch1', 'batch2', 'batch3'] +) + +# Concatenate variables (combine modalities) +adata = ad.concat([adata_rna, adata_protein], axis=1) + +# Lazy collection over backed AnnData objects (experimental) +from anndata.experimental import AnnCollection + +backed_adatas = [ + ad.read_h5ad(path, backed='r') + for path in ['data1.h5ad', 'data2.h5ad'] +] +collection = AnnCollection( + backed_adatas, + join_obs='outer', + join_vars='inner', + label='dataset' +) +``` + +### 4. Data Manipulation + +Transform, subset, filter, and reorganize data efficiently. + +**See**: `references/manipulation.md` for detailed guidance on: +- Subsetting (by indices, names, boolean masks, metadata conditions) +- Transposition +- Copying (full copies vs views) +- Renaming (observations, variables, categories) +- Type conversions (strings to categoricals, sparse/dense) +- Adding/removing data components +- Reordering +- Quality control filtering + +Common commands: +```python +# Subset by metadata +filtered = adata[adata.obs['quality_score'] > 0.8] +hv_genes = adata[:, adata.var['highly_variable']] + +# Transpose +adata_T = adata.T + +# Copy vs view +view = adata[0:100, :] # View (lightweight reference) +copy = adata[0:100, :].copy() # Independent copy + +# Convert strings to categoricals +adata.strings_to_categoricals() +``` + +### 5. Best Practices + +Follow recommended patterns for memory efficiency, performance, and reproducibility. + +**See**: `references/best_practices.md` for guidelines on: +- Memory management (sparse matrices, categoricals, backed mode) +- Views vs copies +- Data storage optimization +- Performance optimization +- Working with raw data +- Metadata management +- Reproducibility +- Error handling +- Integration with other tools +- Common pitfalls and solutions + +Key recommendations: +```python +# Use sparse matrices for sparse data +from scipy.sparse import csr_matrix +adata.X = csr_matrix(adata.X) + +# Convert strings to categoricals +adata.strings_to_categoricals() + +# Use backed mode for large files +adata = ad.read_h5ad('large.h5ad', backed='r') + +# Store raw before filtering +adata.raw = adata.copy() +adata = adata[:, adata.var['highly_variable']] +``` + +## Integration with Scverse Ecosystem + +AnnData serves as the foundational data structure for the scverse ecosystem: + +### Scanpy (Single-cell analysis) +```python +import scanpy as sc + +# Preprocessing +sc.pp.filter_cells(adata, min_genes=200) +sc.pp.normalize_total(adata, target_sum=1e4) +sc.pp.log1p(adata) +sc.pp.highly_variable_genes(adata, n_top_genes=2000) + +# Dimensionality reduction +sc.pp.pca(adata, n_comps=50) +sc.pp.neighbors(adata, n_neighbors=15) +sc.tl.umap(adata) +sc.tl.leiden(adata) + +# Visualization +sc.pl.umap(adata, color=['cell_type', 'leiden']) +``` + +### Muon (Multimodal data) +```python +import muon as mu + +# Combine RNA and protein data +mdata = mu.MuData({'rna': adata_rna, 'protein': adata_protein}) +``` + +### PyTorch integration +```python +from anndata.experimental import AnnLoader + +# Create DataLoader for deep learning +dataloader = AnnLoader(adata, batch_size=128, shuffle=True) + +for batch in dataloader: + X = batch.X + # Train model +``` + +## Common Workflows + +### Single-cell RNA-seq analysis +```python +import anndata as ad +import scanpy as sc + +# 1. Load data (10X via scanpy; anndata handles h5ad/zarr natively) +adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5') + +# 2. Quality control +adata.obs['n_genes'] = (adata.X > 0).sum(axis=1) +adata.obs['n_counts'] = adata.X.sum(axis=1) +adata = adata[adata.obs['n_genes'] > 200] +adata = adata[adata.obs['n_counts'] < 50000] + +# 3. Store raw +adata.raw = adata.copy() + +# 4. Normalize and filter +sc.pp.normalize_total(adata, target_sum=1e4) +sc.pp.log1p(adata) +sc.pp.highly_variable_genes(adata, n_top_genes=2000) +adata = adata[:, adata.var['highly_variable']] + +# 5. Save processed data +adata.write_h5ad('processed.h5ad') +``` + +### Batch integration +```python +# Load multiple batches +adata1 = ad.read_h5ad('batch1.h5ad') +adata2 = ad.read_h5ad('batch2.h5ad') +adata3 = ad.read_h5ad('batch3.h5ad') + +# Concatenate with batch labels +adata = ad.concat( + [adata1, adata2, adata3], + label='batch', + keys=['batch1', 'batch2', 'batch3'], + join='inner' +) + +# Apply batch correction +import scanpy as sc +sc.pp.combat(adata, key='batch') + +# Continue analysis +sc.pp.pca(adata) +sc.pp.neighbors(adata) +sc.tl.umap(adata) +``` + +### Working with large datasets +```python +# Open in backed mode +adata = ad.read_h5ad('100GB_dataset.h5ad', backed='r') + +# Filter based on metadata (no data loading) +high_quality = adata[adata.obs['quality_score'] > 0.8] + +# Load filtered subset +adata_subset = high_quality.to_memory() + +# Process subset +process(adata_subset) + +# Or process in chunks +chunk_size = 1000 +for i in range(0, adata.n_obs, chunk_size): + chunk = adata[i:i+chunk_size, :].to_memory() + process(chunk) +``` + +## Troubleshooting + +### Out of memory errors +Use backed mode or convert to sparse matrices: +```python +# Backed mode +adata = ad.read_h5ad('file.h5ad', backed='r') + +# Sparse matrices +from scipy.sparse import csr_matrix +adata.X = csr_matrix(adata.X) +``` + +### Slow file reading +Use compression and appropriate formats: +```python +# Optimize for storage +adata.strings_to_categoricals() +adata.write_h5ad('file.h5ad', compression='gzip') + +# Use Zarr for cloud storage; v3 writes are opt-in in anndata 0.12 +import anndata as ad + +ad.settings.zarr_write_format = 3 +ad.settings.auto_shard_zarr_v3 = True # experimental; independent of zarr_write_format +adata.write_zarr('file.zarr', chunks=(1000, 1000)) +``` + +### Index alignment issues +Always align external data on index: +```python +# Wrong +adata.obs['new_col'] = external_data['values'] + +# Correct +adata.obs['new_col'] = external_data.set_index('cell_id').loc[adata.obs_names, 'values'] +``` + +## Additional Resources + +- **Official documentation**: https://anndata.readthedocs.io/ +- **Scanpy tutorials**: https://scanpy.readthedocs.io/ +- **Scverse ecosystem**: https://scverse.org/ +- **GitHub repository**: https://github.com/scverse/anndata + diff --git a/.agents/skills/anndata/references/best_practices.md b/.agents/skills/anndata/references/best_practices.md new file mode 100644 index 0000000..c125202 --- /dev/null +++ b/.agents/skills/anndata/references/best_practices.md @@ -0,0 +1,532 @@ +# Best Practices + +Guidelines for efficient and effective use of AnnData. + +## Memory Management + +### Use sparse matrices for sparse data +```python +import numpy as np +from scipy.sparse import csr_matrix, csc_matrix +import anndata as ad + +# Check data sparsity +data = np.random.rand(1000, 2000) +sparsity = 1 - np.count_nonzero(data) / data.size +print(f"Sparsity: {sparsity:.2%}") + +# Convert to sparse if >50% zeros (anndata 0.12+ requires csr or csc) +if sparsity > 0.5: + adata = ad.AnnData(X=csr_matrix(data)) +else: + adata = ad.AnnData(X=data) + +# Benefits: 10-100x memory reduction for sparse genomics data +``` + +### Convert strings to categoricals +```python +# Inefficient: string columns use lots of memory +adata.obs['cell_type'] = ['Type_A', 'Type_B', 'Type_C'] * 333 + ['Type_A'] + +# Efficient: convert to categorical +adata.obs['cell_type'] = adata.obs['cell_type'].astype('category') + +# Convert all string columns +adata.strings_to_categoricals() + +# Benefits: 10-50x memory reduction for repeated strings +``` + +### Use backed mode for large datasets +```python +# Don't load entire dataset into memory +adata = ad.read_h5ad('large_dataset.h5ad', backed='r') + +# Work with metadata +filtered = adata[adata.obs['quality'] > 0.8] + +# Load only filtered subset +adata_subset = filtered.to_memory() + +# Benefits: Work with datasets larger than RAM +``` + +## Views vs Copies + +### Understanding views +```python +# Subsetting creates a view by default +subset = adata[0:100, :] +print(subset.is_view) # True + +# Views don't copy data (memory efficient) +# But modifications can affect original + +# Check if object is a view +if adata.is_view: + adata = adata.copy() # Make independent +``` + +### When to use views +```python +# Good: Read-only operations on subsets +mean_expr = adata[adata.obs['cell_type'] == 'T cell'].X.mean() + +# Good: Temporary analysis +temp_subset = adata[:100, :] +result = analyze(temp_subset.X) +``` + +### When to use copies +```python +# Create independent copy for modifications +adata_filtered = adata[keep_cells, :].copy() + +# Safe to modify without affecting original +adata_filtered.obs['new_column'] = values + +# Always copy when: +# - Storing subset for later use +# - Modifying subset data +# - Passing to function that modifies data +``` + +## Data Storage Best Practices + +### Choose the right format + +**H5AD (HDF5) - Default choice** +```python +adata.write_h5ad('data.h5ad', compression='gzip') +``` +- Fast random access +- Supports backed mode +- Good compression +- Best for: Most use cases + +**Zarr - Cloud and parallel access** +```python +import anndata + +# Default is Zarr v2; opt into v3 for cloud workflows (anndata 0.12+) +anndata.settings.zarr_write_format = 3 +anndata.settings.auto_shard_zarr_v3 = True # experimental; independent of zarr_write_format +adata.write_zarr('data.zarr', chunks=(100, 100)) +``` +- Excellent for cloud storage (S3, GCS) +- Supports parallel I/O and opt-in Zarr v3 sharding (0.12+) +- Good compression +- Best for: Large datasets, cloud workflows, parallel processing + +**CSV - Interoperability** +```python +adata.write_csvs('output_dir/') +``` +- Human readable +- Compatible with all tools +- Large file sizes, slow +- Best for: Sharing with non-Python tools, small datasets + +### Optimize file size +```python +# Before saving, optimize: + +# 1. Convert to sparse if appropriate +from scipy.sparse import csr_matrix, issparse +if not issparse(adata.X): + density = np.count_nonzero(adata.X) / adata.X.size + if density < 0.5: + adata.X = csr_matrix(adata.X) + +# 2. Convert strings to categoricals +adata.strings_to_categoricals() + +# 3. Use compression +adata.write_h5ad('data.h5ad', compression='gzip', compression_opts=9) + +# Typical results: 5-20x file size reduction +``` + +## Backed Mode Strategies + +### Read-only analysis +```python +# Open in read-only backed mode +adata = ad.read_h5ad('data.h5ad', backed='r') + +# Perform filtering without loading data +high_quality = adata[adata.obs['quality_score'] > 0.8] + +# Load only filtered data +adata_filtered = high_quality.to_memory() +``` + +### Read-write modifications +```python +# Open in read-write backed mode +adata = ad.read_h5ad('data.h5ad', backed='r+') + +# Modify X (persisted to disk) +adata.X[0, 0] = 0 + +# Metadata changes are not persisted from backed mode; load and write a new file +adata_memory = adata.to_memory() +adata_memory.obs['new_annotation'] = values +adata_memory.write_h5ad('data_with_annotations.h5ad') +``` + +### Chunked processing +```python +# Process large dataset in chunks +adata = ad.read_h5ad('huge_dataset.h5ad', backed='r') + +results = [] +chunk_size = 1000 + +for i in range(0, adata.n_obs, chunk_size): + chunk = adata[i:i+chunk_size, :].to_memory() + result = process(chunk) + results.append(result) + +final_result = combine(results) +``` + +## Performance Optimization + +### Subsetting performance +```python +# Fast: Boolean indexing with arrays +mask = np.array(adata.obs['quality'] > 0.5) +subset = adata[mask, :] + +# Slow: Boolean indexing with Series (creates view chain) +subset = adata[adata.obs['quality'] > 0.5, :] + +# Fastest: Integer indices +indices = np.where(adata.obs['quality'] > 0.5)[0] +subset = adata[indices, :] +``` + +### Avoid repeated subsetting +```python +# Inefficient: Multiple subset operations +for cell_type in ['A', 'B', 'C']: + subset = adata[adata.obs['cell_type'] == cell_type] + process(subset) + +# Efficient: Group and process +groups = adata.obs.groupby('cell_type').groups +for cell_type, indices in groups.items(): + subset = adata[indices, :] + process(subset) +``` + +### Use chunked operations for large matrices +```python +# Process X in chunks +for chunk in adata.chunked_X(chunk_size=1000): + result = compute(chunk) + +# More memory efficient than loading full X +``` + +## Working with Raw Data + +### Store raw before filtering +```python +# Original data with all genes +adata = ad.AnnData(X=counts) + +# Store raw before filtering +adata.raw = adata.copy() + +# Filter to highly variable genes +adata = adata[:, adata.var['highly_variable']] + +# Later: access original data +original_expression = adata.raw.X +all_genes = adata.raw.var_names +``` + +### When to use raw +```python +# Use raw for: +# - Differential expression on filtered genes +# - Visualization of specific genes not in filtered set +# - Accessing original counts after normalization + +# Access raw data +if adata.raw is not None: + gene_expr = adata.raw[:, 'GENE_NAME'].X +else: + gene_expr = adata[:, 'GENE_NAME'].X +``` + +## Metadata Management + +### Naming conventions +```python +# Consistent naming improves usability + +# Observation metadata (obs): +# - cell_id, sample_id +# - cell_type, tissue, condition +# - n_genes, n_counts, percent_mito +# - cluster, leiden, louvain + +# Variable metadata (var): +# - gene_id, gene_name +# - highly_variable, n_cells +# - mean_expression, dispersion + +# Embeddings (obsm): +# - X_pca, X_umap, X_tsne +# - X_diffmap, X_draw_graph_fr + +# Follow conventions from scanpy/scverse ecosystem +``` + +### Document metadata +```python +# Store metadata descriptions in uns +adata.uns['metadata_descriptions'] = { + 'cell_type': 'Cell type annotation from automated clustering', + 'quality_score': 'QC score from scrublet (0-1, higher is better)', + 'batch': 'Experimental batch identifier' +} + +# Store processing history +adata.uns['processing_steps'] = [ + 'Raw counts loaded from 10X', + 'Filtered: n_genes > 200, n_counts < 50000', + 'Normalized to 10000 counts per cell', + 'Log transformed' +] +``` + +## Reproducibility + +### Set random seeds +```python +import numpy as np + +# Set seed for reproducible results +np.random.seed(42) + +# Document in uns +adata.uns['random_seed'] = 42 +``` + +### Store parameters +```python +# Store analysis parameters in uns +adata.uns['pca'] = { + 'n_comps': 50, + 'svd_solver': 'arpack', + 'random_state': 42 +} + +adata.uns['neighbors'] = { + 'n_neighbors': 15, + 'n_pcs': 50, + 'metric': 'euclidean', + 'method': 'umap' +} +``` + +### Version tracking +```python +import sys +from importlib.metadata import version + +# Store package versions (anndata.__version__ deprecated in 0.12.3) +adata.uns['versions'] = { + 'anndata': version('anndata'), + 'scanpy': version('scanpy'), + 'numpy': version('numpy'), + 'python': sys.version, +} +``` + +## Error Handling + +### Check data validity +```python +# Verify dimensions +assert adata.n_obs == len(adata.obs) +assert adata.n_vars == len(adata.var) +assert adata.X.shape == (adata.n_obs, adata.n_vars) + +# Check for NaN values +has_nan = np.isnan(adata.X.data).any() if issparse(adata.X) else np.isnan(adata.X).any() +if has_nan: + print("Warning: Data contains NaN values") + +# Check for negative values (if counts expected) +has_negative = (adata.X.data < 0).any() if issparse(adata.X) else (adata.X < 0).any() +if has_negative: + print("Warning: Data contains negative values") +``` + +### Validate metadata +```python +# Check for missing values +missing_obs = adata.obs.isnull().sum() +if missing_obs.any(): + print("Missing values in obs:") + print(missing_obs[missing_obs > 0]) + +# Verify indices are unique +assert adata.obs_names.is_unique, "Observation names not unique" +assert adata.var_names.is_unique, "Variable names not unique" + +# Check metadata alignment +assert len(adata.obs) == adata.n_obs +assert len(adata.var) == adata.n_vars +``` + +## Integration with Other Tools + +### Scanpy integration +```python +import scanpy as sc + +# AnnData is native format for scanpy +sc.pp.filter_cells(adata, min_genes=200) +sc.pp.filter_genes(adata, min_cells=3) +sc.pp.normalize_total(adata, target_sum=1e4) +sc.pp.log1p(adata) +sc.pp.highly_variable_genes(adata) +sc.pp.pca(adata) +sc.pp.neighbors(adata) +sc.tl.umap(adata) +``` + +### Pandas integration +```python +import pandas as pd + +# Convert to DataFrame +df = adata.to_df() + +# Create from DataFrame +adata = ad.AnnData(df) + +# Work with metadata as DataFrames +adata.obs = adata.obs.merge(external_metadata, left_index=True, right_index=True) +``` + +### PyTorch integration +```python +from anndata.experimental import AnnLoader + +# Create PyTorch DataLoader +dataloader = AnnLoader(adata, batch_size=128, shuffle=True) + +# Iterate in training loop +for batch in dataloader: + X = batch.X + # Train model on batch +``` + +## Common Pitfalls + +### Pitfall 1: Modifying views +```python +# Wrong: Modifying view can affect original +subset = adata[:100, :] +subset.X = new_data # May modify adata.X! + +# Correct: Copy before modifying +subset = adata[:100, :].copy() +subset.X = new_data # Independent copy +``` + +### Pitfall 2: Index misalignment +```python +# Wrong: Assuming order matches +external_data = pd.read_csv('data.csv') +adata.obs['new_col'] = external_data['values'] # May misalign! + +# Correct: Align on index +adata.obs['new_col'] = external_data.set_index('cell_id').loc[adata.obs_names, 'values'] +``` + +### Pitfall 3: Mixing sparse and dense +```python +# Wrong: Converting sparse to dense uses huge memory +result = adata.X + 1 # Converts sparse to dense! + +# Correct: Use sparse operations +from scipy.sparse import issparse +if issparse(adata.X): + result = adata.X.copy() + result.data += 1 +``` + +### Pitfall 4: Not handling views +```python +# Wrong: Assuming subset is independent +subset = adata[mask, :] +del adata # subset may become invalid! + +# Correct: Copy when needed +subset = adata[mask, :].copy() +del adata # subset remains valid +``` + +### Pitfall 5: Ignoring memory constraints +```python +# Wrong: Loading huge dataset into memory +adata = ad.read_h5ad('100GB_file.h5ad') # OOM error! + +# Correct: Use backed mode +adata = ad.read_h5ad('100GB_file.h5ad', backed='r') +subset = adata[adata.obs['keep']].to_memory() +``` + +## Workflow Example + +Complete best-practices workflow: + +```python +import anndata as ad +import numpy as np +from scipy.sparse import csr_matrix, issparse + +# 1. Load with backed mode if large +adata = ad.read_h5ad('data.h5ad', backed='r') + +# 2. Quick metadata check without loading data +print(f"Dataset: {adata.n_obs} cells × {adata.n_vars} genes") + +# 3. Filter based on metadata +high_quality = adata[adata.obs['quality_score'] > 0.8] + +# 4. Load filtered subset to memory +adata = high_quality.to_memory() + +# 5. Convert to optimal storage types (csr/csc sparse only since 0.12) +adata.strings_to_categoricals() +if not issparse(adata.X): + density = np.count_nonzero(adata.X) / adata.X.size + if density < 0.5: + adata.X = csr_matrix(adata.X) + +# 6. Store raw before filtering genes +adata.raw = adata.copy() + +# 7. Filter to highly variable genes +adata = adata[:, adata.var['highly_variable']].copy() + +# 8. Document processing +adata.uns['processing'] = { + 'filtered': 'quality_score > 0.8', + 'n_hvg': adata.n_vars, + 'date': '2025-11-03' +} + +# 9. Save optimized +adata.write_h5ad('processed.h5ad', compression='gzip') +``` diff --git a/.agents/skills/anndata/references/concatenation.md b/.agents/skills/anndata/references/concatenation.md new file mode 100644 index 0000000..85273a0 --- /dev/null +++ b/.agents/skills/anndata/references/concatenation.md @@ -0,0 +1,399 @@ +# Concatenating AnnData Objects + +Combine multiple AnnData objects along either observations or variables axis. + +## Basic Concatenation + +### Concatenate along observations (stack cells/samples) +```python +import anndata as ad +import numpy as np + +# Create multiple AnnData objects +adata1 = ad.AnnData(X=np.random.rand(100, 50)) +adata2 = ad.AnnData(X=np.random.rand(150, 50)) +adata3 = ad.AnnData(X=np.random.rand(200, 50)) + +# Concatenate along observations (axis=0, default) +adata_combined = ad.concat([adata1, adata2, adata3], axis=0) + +print(adata_combined.shape) # (450, 50) +``` + +### Concatenate along variables (stack genes/features) +```python +# Create objects with same observations, different variables +adata1 = ad.AnnData(X=np.random.rand(100, 50)) +adata2 = ad.AnnData(X=np.random.rand(100, 30)) +adata3 = ad.AnnData(X=np.random.rand(100, 70)) + +# Concatenate along variables (axis=1) +adata_combined = ad.concat([adata1, adata2, adata3], axis=1) + +print(adata_combined.shape) # (100, 150) +``` + +## Join Types + +### Inner join (intersection) +Keep only variables/observations present in all objects. + +```python +import pandas as pd + +# Create objects with different variables +adata1 = ad.AnnData( + X=np.random.rand(100, 50), + var=pd.DataFrame(index=[f'Gene_{i}' for i in range(50)]) +) +adata2 = ad.AnnData( + X=np.random.rand(150, 60), + var=pd.DataFrame(index=[f'Gene_{i}' for i in range(10, 70)]) +) + +# Inner join: only genes 10-49 are kept (overlap) +adata_inner = ad.concat([adata1, adata2], join='inner') +print(adata_inner.n_vars) # 40 genes (overlap) +``` + +### Outer join (union) +Keep all variables/observations, filling missing values. + +```python +# Outer join: all genes are kept +adata_outer = ad.concat([adata1, adata2], join='outer') +print(adata_outer.n_vars) # 70 genes (union) + +# Missing values are filled with appropriate defaults: +# - 0 for sparse matrices +# - NaN for dense matrices +``` + +### Fill values for outer joins +```python +# Specify fill value for missing data +adata_filled = ad.concat([adata1, adata2], join='outer', fill_value=0) +``` + +## Tracking Data Sources + +### Add batch labels +```python +# Label which object each observation came from +adata_combined = ad.concat( + [adata1, adata2, adata3], + label='batch', # Column name for labels + keys=['batch1', 'batch2', 'batch3'] # Labels for each object +) + +print(adata_combined.obs['batch'].value_counts()) +# batch1 100 +# batch2 150 +# batch3 200 +``` + +### Automatic batch labels +```python +# If keys not provided, uses integer indices +adata_combined = ad.concat( + [adata1, adata2, adata3], + label='dataset' +) +# dataset column contains: 0, 1, 2 +``` + +## Merge Strategies + +Control how metadata from different objects is combined using the `merge` parameter. + +### merge=None (default for observations) +Exclude metadata on non-concatenation axis. + +```python +# When concatenating observations, var metadata must match +adata1.var['gene_type'] = 'protein_coding' +adata2.var['gene_type'] = 'protein_coding' + +# var is kept only if identical across all objects +adata_combined = ad.concat([adata1, adata2], merge=None) +``` + +### merge='same' +Keep metadata that is identical across all objects. + +```python +adata1.var['chromosome'] = ['chr1'] * 25 + ['chr2'] * 25 +adata2.var['chromosome'] = ['chr1'] * 25 + ['chr2'] * 25 +adata1.var['type'] = 'protein_coding' +adata2.var['type'] = 'lncRNA' # Different + +# 'chromosome' is kept (same), 'type' is excluded (different) +adata_combined = ad.concat([adata1, adata2], merge='same') +``` + +### merge='unique' +Keep metadata columns where each key has exactly one value. + +```python +adata1.var['gene_id'] = [f'ENSG{i:05d}' for i in range(50)] +adata2.var['gene_id'] = [f'ENSG{i:05d}' for i in range(50)] + +# gene_id is kept (unique values for each key) +adata_combined = ad.concat([adata1, adata2], merge='unique') +``` + +### merge='first' +Take values from the first object containing each key. + +```python +adata1.var['description'] = ['Desc1'] * 50 +adata2.var['description'] = ['Desc2'] * 50 + +# Uses descriptions from adata1 +adata_combined = ad.concat([adata1, adata2], merge='first') +``` + +### merge='only' +Keep metadata that appears in only one object. + +```python +adata1.var['adata1_specific'] = [1] * 50 +adata2.var['adata2_specific'] = [2] * 50 + +# Both metadata columns are kept +adata_combined = ad.concat([adata1, adata2], merge='only') +``` + +## Handling Index Conflicts + +### Make indices unique +```python +import pandas as pd + +# Create objects with overlapping observation names +adata1 = ad.AnnData( + X=np.random.rand(3, 10), + obs=pd.DataFrame(index=['cell_1', 'cell_2', 'cell_3']) +) +adata2 = ad.AnnData( + X=np.random.rand(3, 10), + obs=pd.DataFrame(index=['cell_1', 'cell_2', 'cell_3']) +) + +# Make indices unique by appending batch keys +adata_combined = ad.concat( + [adata1, adata2], + label='batch', + keys=['batch1', 'batch2'], + index_unique='_' # Separator for making indices unique +) + +print(adata_combined.obs_names) +# ['cell_1_batch1', 'cell_2_batch1', 'cell_3_batch1', +# 'cell_1_batch2', 'cell_2_batch2', 'cell_3_batch2'] +``` + +## Concatenating Layers + +```python +# Objects with layers +adata1 = ad.AnnData(X=np.random.rand(100, 50)) +adata1.layers['normalized'] = np.random.rand(100, 50) +adata1.layers['scaled'] = np.random.rand(100, 50) + +adata2 = ad.AnnData(X=np.random.rand(150, 50)) +adata2.layers['normalized'] = np.random.rand(150, 50) +adata2.layers['scaled'] = np.random.rand(150, 50) + +# Layers are concatenated automatically if present in all objects +adata_combined = ad.concat([adata1, adata2]) + +print(adata_combined.layers.keys()) +# dict_keys(['normalized', 'scaled']) +``` + +## Concatenating Multi-dimensional Annotations + +### obsm/varm +```python +# Objects with embeddings +adata1.obsm['X_pca'] = np.random.rand(100, 50) +adata2.obsm['X_pca'] = np.random.rand(150, 50) + +# obsm is concatenated along observation axis +adata_combined = ad.concat([adata1, adata2]) +print(adata_combined.obsm['X_pca'].shape) # (250, 50) +``` + +### obsp/varp (pairwise annotations) +```python +from scipy.sparse import csr_matrix + +# Pairwise matrices +adata1.obsp['connectivities'] = csr_matrix((100, 100)) +adata2.obsp['connectivities'] = csr_matrix((150, 150)) + +# By default, obsp is NOT concatenated (set pairwise=True to include) +adata_combined = ad.concat([adata1, adata2]) +# adata_combined.obsp is empty + +# Include pairwise data (creates block diagonal matrix) +adata_combined = ad.concat([adata1, adata2], pairwise=True) +print(adata_combined.obsp['connectivities'].shape) # (250, 250) +``` + +## Concatenating uns (unstructured) + +Unstructured metadata is merged recursively: + +```python +adata1.uns['experiment'] = {'date': '2025-01-01', 'batch': 'A'} +adata2.uns['experiment'] = {'date': '2025-01-01', 'batch': 'B'} + +# Using merge='unique' for uns +adata_combined = ad.concat([adata1, adata2], uns_merge='unique') +# 'date' is kept (same value), 'batch' might be excluded (different values) +``` + +## Lazy Concatenation (AnnCollection) + +For very large datasets, use `AnnCollection` to lazily concatenate AnnData objects along the observation axis. This API is experimental; use backed AnnData objects when the inputs are stored in `.h5ad` files. + +```python +import anndata as ad +from anndata.experimental import AnnCollection + +files = ['data1.h5ad', 'data2.h5ad', 'data3.h5ad'] +backed_adatas = [ad.read_h5ad(path, backed='r') for path in files] + +collection = AnnCollection( + backed_adatas, + join_obs='outer', + join_vars='inner', + label='dataset', + keys=['dataset1', 'dataset2', 'dataset3'] +) + +# Access data lazily +print(collection.n_obs) # Total observations +print(collection.obs.head()) # Metadata loaded, not X + +# Convert to regular AnnData when needed (loads all data) +adata = collection.to_adata() +``` + +### Working with AnnCollection +```python +# Subset without loading data +subset = collection[collection.obs['cell_type'] == 'T cell'] + +# Iterate through datasets +for adata in collection: + print(adata.shape) + +# Access specific dataset +first_dataset = collection[0] +``` + +## Concatenation on Disk + +For datasets too large for memory, concatenate directly on disk: + +```python +import anndata as ad +from anndata.experimental import concat_on_disk + +# Concatenate without loading into memory +concat_on_disk( + ['data1.h5ad', 'data2.h5ad', 'data3.h5ad'], + 'combined.h5ad', + join='outer' +) + +# Load result in backed mode +adata = ad.read_h5ad('combined.h5ad', backed='r') +``` + +## Common Concatenation Patterns + +### Combine technical replicates +```python +# Multiple runs of the same samples +replicates = [adata_run1, adata_run2, adata_run3] +adata_combined = ad.concat( + replicates, + label='technical_replicate', + keys=['rep1', 'rep2', 'rep3'], + join='inner' # Keep only genes measured in all runs +) +``` + +### Combine batches from experiment +```python +# Different experimental batches +batches = [adata_batch1, adata_batch2, adata_batch3] +adata_combined = ad.concat( + batches, + label='batch', + keys=['batch1', 'batch2', 'batch3'], + join='outer' # Keep all genes +) + +# Later: apply batch correction +``` + +### Merge multi-modal data +```python +# Different measurement modalities (e.g., RNA + protein) +adata_rna = ad.AnnData(X=np.random.rand(100, 2000)) +adata_protein = ad.AnnData(X=np.random.rand(100, 50)) + +# Concatenate along variables to combine modalities +adata_multimodal = ad.concat([adata_rna, adata_protein], axis=1) + +# Add labels to distinguish modalities +adata_multimodal.var['modality'] = ['RNA'] * 2000 + ['protein'] * 50 +``` + +## Best Practices + +1. **Check compatibility before concatenating** +```python +# Verify shapes are compatible +print([adata.n_vars for adata in [adata1, adata2, adata3]]) + +# Check variable names match +print([set(adata.var_names) for adata in [adata1, adata2, adata3]]) +``` + +2. **Use appropriate join type** +- `inner`: When you need the same features across all samples (most stringent) +- `outer`: When you want to preserve all features (most inclusive) + +3. **Track data sources** +Always use `label` and `keys` to track which observations came from which dataset. + +4. **Consider memory usage** +- For large datasets, use `AnnCollection` or `concat_on_disk` +- Consider backed mode for the result + +5. **Handle batch effects** +Concatenation combines data but doesn't correct for batch effects. Apply batch correction after concatenation: +```python +# After concatenation, apply batch correction +import scanpy as sc +sc.pp.combat(adata_combined, key='batch') +``` + +6. **Validate results** +```python +# Check dimensions +print(adata_combined.shape) + +# Check batch distribution +print(adata_combined.obs['batch'].value_counts()) + +# Verify metadata integrity +print(adata_combined.var.head()) +print(adata_combined.obs.head()) +``` diff --git a/.agents/skills/anndata/references/data_structure.md b/.agents/skills/anndata/references/data_structure.md new file mode 100644 index 0000000..c419660 --- /dev/null +++ b/.agents/skills/anndata/references/data_structure.md @@ -0,0 +1,314 @@ +# AnnData Object Structure + +The AnnData object stores a data matrix with associated annotations, providing a flexible framework for managing experimental data and metadata. + +## Core Components + +### X (Data Matrix) +The primary data matrix with shape (n_obs, n_vars) storing experimental measurements. + +```python +import anndata as ad +import numpy as np + +# Create with dense array +adata = ad.AnnData(X=np.random.rand(100, 2000)) + +# Create with sparse matrix (recommended for large, sparse data) +from scipy.sparse import csr_matrix +sparse_data = csr_matrix(np.random.rand(100, 2000)) +adata = ad.AnnData(X=sparse_data) +``` + +Access data: +```python +# Full matrix (caution with large datasets) +full_data = adata.X + +# Single observation +obs_data = adata.X[0, :] + +# Single variable across all observations +var_data = adata.X[:, 0] +``` + +### obs (Observation Annotations) +DataFrame storing metadata about observations (rows). Each row corresponds to one observation in X. + +```python +import pandas as pd + +# Create AnnData with observation metadata +obs_df = pd.DataFrame({ + 'cell_type': ['T cell', 'B cell', 'Monocyte'], + 'treatment': ['control', 'treated', 'control'], + 'timepoint': [0, 24, 24] +}, index=['cell_1', 'cell_2', 'cell_3']) + +adata = ad.AnnData(X=np.random.rand(3, 100), obs=obs_df) + +# Access observation metadata +print(adata.obs['cell_type']) +print(adata.obs.loc['cell_1']) +``` + +### var (Variable Annotations) +DataFrame storing metadata about variables (columns). Each row corresponds to one variable in X. + +```python +# Create AnnData with variable metadata +var_df = pd.DataFrame({ + 'gene_name': ['ACTB', 'GAPDH', 'TP53'], + 'chromosome': ['7', '12', '17'], + 'highly_variable': [True, False, True] +}, index=['ENSG00001', 'ENSG00002', 'ENSG00003']) + +adata = ad.AnnData(X=np.random.rand(100, 3), var=var_df) + +# Access variable metadata +print(adata.var['gene_name']) +print(adata.var.loc['ENSG00001']) +``` + +### layers (Alternative Data Representations) +Dictionary storing alternative matrices with the same dimensions as X. + +```python +# Store raw counts, normalized data, and scaled data +adata = ad.AnnData(X=np.random.rand(100, 2000)) +adata.layers['raw_counts'] = np.random.randint(0, 100, (100, 2000)) +adata.layers['normalized'] = adata.X / np.sum(adata.X, axis=1, keepdims=True) +adata.layers['scaled'] = (adata.X - adata.X.mean()) / adata.X.std() + +# Access layers +raw_data = adata.layers['raw_counts'] +normalized_data = adata.layers['normalized'] +``` + +Common layer uses: +- `raw_counts`: Original count data before normalization +- `normalized`: Log-normalized or TPM values +- `scaled`: Z-scored values for analysis +- `imputed`: Data after imputation + +### obsm (Multi-dimensional Observation Annotations) +Dictionary storing multi-dimensional arrays aligned to observations. + +```python +# Store PCA coordinates and UMAP embeddings +adata.obsm['X_pca'] = np.random.rand(100, 50) # 50 principal components +adata.obsm['X_umap'] = np.random.rand(100, 2) # 2D UMAP coordinates +adata.obsm['X_tsne'] = np.random.rand(100, 2) # 2D t-SNE coordinates + +# Access embeddings +pca_coords = adata.obsm['X_pca'] +umap_coords = adata.obsm['X_umap'] +``` + +Common obsm uses: +- `X_pca`: Principal component coordinates +- `X_umap`: UMAP embedding coordinates +- `X_tsne`: t-SNE embedding coordinates +- `X_diffmap`: Diffusion map coordinates +- `protein_expression`: Protein abundance measurements (CITE-seq) + +### varm (Multi-dimensional Variable Annotations) +Dictionary storing multi-dimensional arrays aligned to variables. + +```python +# Store PCA loadings +adata.varm['PCs'] = np.random.rand(2000, 50) # Loadings for 50 components +adata.varm['gene_modules'] = np.random.rand(2000, 10) # Gene module scores + +# Access loadings +pc_loadings = adata.varm['PCs'] +``` + +Common varm uses: +- `PCs`: Principal component loadings +- `gene_modules`: Gene co-expression module assignments + +### obsp (Pairwise Observation Relationships) +Dictionary storing sparse matrices representing relationships between observations. + +```python +from scipy.sparse import csr_matrix + +# Store k-nearest neighbor graph +n_obs = 100 +knn_graph = csr_matrix(np.random.rand(n_obs, n_obs) > 0.95) +adata.obsp['connectivities'] = knn_graph +adata.obsp['distances'] = csr_matrix(np.random.rand(n_obs, n_obs)) + +# Access graphs +knn_connections = adata.obsp['connectivities'] +distances = adata.obsp['distances'] +``` + +Common obsp uses: +- `connectivities`: Cell-cell neighborhood graph +- `distances`: Pairwise distances between cells + +### varp (Pairwise Variable Relationships) +Dictionary storing sparse matrices representing relationships between variables. + +```python +# Store gene-gene correlation matrix +n_vars = 2000 +gene_corr = csr_matrix(np.random.rand(n_vars, n_vars) > 0.99) +adata.varp['correlations'] = gene_corr + +# Access correlations +gene_correlations = adata.varp['correlations'] +``` + +### uns (Unstructured Annotations) +Dictionary storing arbitrary unstructured metadata. + +```python +# Store analysis parameters and results +adata.uns['experiment_date'] = '2025-11-03' +adata.uns['pca'] = { + 'variance_ratio': [0.15, 0.10, 0.08], + 'params': {'n_comps': 50} +} +adata.uns['neighbors'] = { + 'params': {'n_neighbors': 15, 'method': 'umap'}, + 'connectivities_key': 'connectivities' +} + +# Access unstructured data +exp_date = adata.uns['experiment_date'] +pca_params = adata.uns['pca']['params'] +``` + +Common uns uses: +- Analysis parameters and settings +- Color palettes for plotting +- Cluster information +- Tool-specific metadata + +### raw (Original Data Snapshot) +Optional attribute preserving the original data matrix and variable annotations before filtering. + +```python +# Create AnnData and store raw state +adata = ad.AnnData(X=np.random.rand(100, 5000)) +adata.var['gene_name'] = [f'Gene_{i}' for i in range(5000)] + +# Store raw state before filtering +adata.raw = adata.copy() + +# Filter to highly variable genes +highly_variable_mask = np.random.rand(5000) > 0.5 +adata = adata[:, highly_variable_mask] + +# Access original data +original_matrix = adata.raw.X +original_var = adata.raw.var +``` + +## Object Properties + +```python +# Dimensions +n_observations = adata.n_obs +n_variables = adata.n_vars +shape = adata.shape # (n_obs, n_vars) + +# Index information +obs_names = adata.obs_names # Observation identifiers +var_names = adata.var_names # Variable identifiers + +# Storage mode +is_view = adata.is_view # True if this is a view of another object +is_backed = adata.isbacked # True if backed by on-disk storage +filename = adata.filename # Path to backing file (if backed) +``` + +## Creating AnnData Objects + +### From arrays and DataFrames +```python +import anndata as ad +import numpy as np +import pandas as pd + +# Minimal creation +X = np.random.rand(100, 2000) +adata = ad.AnnData(X) + +# With metadata +obs = pd.DataFrame({'cell_type': ['A', 'B'] * 50}, index=[f'cell_{i}' for i in range(100)]) +var = pd.DataFrame({'gene_name': [f'Gene_{i}' for i in range(2000)]}, index=[f'ENSG{i:05d}' for i in range(2000)]) +adata = ad.AnnData(X=X, obs=obs, var=var) + +# With all components +adata = ad.AnnData( + X=X, + obs=obs, + var=var, + layers={'raw': np.random.randint(0, 100, (100, 2000))}, + obsm={'X_pca': np.random.rand(100, 50)}, + uns={'experiment': 'test'} +) +``` + +### From DataFrame +```python +# Create from pandas DataFrame (genes as columns, cells as rows) +df = pd.DataFrame( + np.random.rand(100, 50), + columns=[f'Gene_{i}' for i in range(50)], + index=[f'Cell_{i}' for i in range(100)] +) +adata = ad.AnnData(df) +``` + +## Data Access Patterns + +### Vector extraction +```python +# Get observation annotation as array +cell_types = adata.obs_vector('cell_type') + +# Get variable values across observations +gene_expression = adata.obs_vector('ACTB') # If ACTB is in var_names + +# Get variable annotation as array +gene_names = adata.var_vector('gene_name') +``` + +### Subsetting +```python +# By index +subset = adata[0:10, 0:100] # First 10 obs, first 100 vars + +# By name +subset = adata[['cell_1', 'cell_2'], ['ACTB', 'GAPDH']] + +# By boolean mask +high_count_cells = adata.obs['total_counts'] > 1000 +subset = adata[high_count_cells, :] + +# By observation metadata +t_cells = adata[adata.obs['cell_type'] == 'T cell'] +``` + +## Memory Considerations + +The AnnData structure is designed for memory efficiency: +- Sparse matrices reduce memory for sparse data +- Views avoid copying data when possible +- Backed mode enables working with data larger than RAM +- Categorical annotations reduce memory for discrete values + +```python +# Convert strings to categoricals (more memory efficient) +adata.obs['cell_type'] = adata.obs['cell_type'].astype('category') +adata.strings_to_categoricals() + +# Check if object is a view (doesn't own data) +if adata.is_view: + adata = adata.copy() # Create independent copy +``` diff --git a/.agents/skills/anndata/references/io_operations.md b/.agents/skills/anndata/references/io_operations.md new file mode 100644 index 0000000..f604def --- /dev/null +++ b/.agents/skills/anndata/references/io_operations.md @@ -0,0 +1,466 @@ +# Input/Output Operations + +AnnData provides comprehensive I/O functionality for reading and writing data in various formats. + +Since anndata 0.11, most `read_*` and `write_*` functions live in `anndata.io`. Top-level `read_h5ad` and `read_zarr` remain at `anndata` without deprecation warnings; other top-level imports still work but emit `FutureWarning`. + +```python +import anndata as ad +from anndata.io import read_csv, read_mtx, read_loom, read_elem, write_elem +``` + +Avoid deprecated I/O aliases such as `ad.read`; use `ad.read_h5ad` or `anndata.io.read_h5ad` explicitly. + +## Native Formats + +### H5AD (HDF5-based) +The recommended native format for AnnData objects, providing efficient storage and fast access. + +#### Writing H5AD files +```python +import anndata as ad + +# Write to file +adata.write_h5ad('data.h5ad') + +# Write with compression +adata.write_h5ad('data.h5ad', compression='gzip') + +# Write with specific compression level (0-9, higher = more compression) +adata.write_h5ad('data.h5ad', compression='gzip', compression_opts=9) +``` + +#### Reading H5AD files +```python +# Read entire file into memory +adata = ad.read_h5ad('data.h5ad') + +# Read in backed mode (lazy loading for large files) +adata = ad.read_h5ad('data.h5ad', backed='r') # Read-only +adata = ad.read_h5ad('data.h5ad', backed='r+') # Read-write for X + +# Backed mode enables working with datasets larger than RAM +# Only accessed data is loaded into memory +# In backed mode, only X updates are persisted; write a new file for obs/var/uns changes. +``` + +#### Backed mode operations +```python +# Open in backed mode +adata = ad.read_h5ad('large_dataset.h5ad', backed='r') + +# Access metadata without loading X into memory +print(adata.obs.head()) +print(adata.var.head()) + +# Subset operations create views +subset = adata[:100, :500] # View, no data loaded + +# Load specific data into memory +X_subset = subset.X[:] # Now loads this subset + +# Convert entire backed object to memory +adata_memory = adata.to_memory() +``` + +### Zarr +Hierarchical array storage format, optimized for cloud storage and parallel I/O. + +#### Writing Zarr +```python +# Write to Zarr store +adata.write_zarr('data.zarr') + +# Write with specific chunks (important for performance) +adata.write_zarr('data.zarr', chunks=(100, 100)) +``` + +#### Reading Zarr +```python +# Read Zarr store +adata = ad.read_zarr('data.zarr') +``` + +#### Zarr v3 (anndata 0.12+) +```python +import anndata + +# Default writes Zarr v2; opt into v3 and optional auto-sharding +anndata.settings.zarr_write_format = 3 +anndata.settings.auto_shard_zarr_v3 = True # experimental; independent of zarr_write_format + +adata.write_zarr('data.zarr', chunks=(1000, 1000)) +``` + +Zarr v3 writing is available in anndata 0.12, with structured-array exceptions and evolving performance guidance. Consolidated metadata is recommended for remote Zarr stores. + +#### Remote Zarr access +Only open remote stores from trusted, expected locations. Prefer allowlisted HTTPS/S3/GCS paths or signed URLs, and avoid asking an agent to fetch arbitrary user-supplied URLs. + +```python +import fsspec + +# Access Zarr from an expected S3 location +store = fsspec.get_mapper('s3://bucket-name/data.zarr') +adata = ad.read_zarr(store) + +# Access Zarr from a trusted HTTPS location +store = fsspec.get_mapper('https://example.com/data.zarr') +adata = ad.read_zarr(store) +``` + +## Alternative Input Formats + +### CSV/TSV +```python +from anndata.io import read_csv + +# Read CSV (genes as columns, cells as rows) +adata = read_csv('data.csv') + +# Read with custom delimiter +adata = read_csv('data.tsv', delimiter='\t') + +# Specify that first column is row names +adata = read_csv('data.csv', first_column_names=True) +``` + +### Excel +```python +from anndata.io import read_excel + +# Read Excel file +adata = read_excel('data.xlsx') + +# Read specific sheet +adata = read_excel('data.xlsx', sheet='Sheet1') +``` + +### Matrix Market (MTX) +Common format for sparse matrices in genomics. + +```python +from anndata.io import read_mtx + +# Read MTX with associated files +# Requires: matrix.mtx, genes.tsv, barcodes.tsv +adata = read_mtx('matrix.mtx') + +# Read with custom gene and barcode files +adata = read_mtx( + 'matrix.mtx', + var_names='genes.tsv', + obs_names='barcodes.tsv' +) + +# Transpose if needed (MTX often has genes as rows) +adata = adata.T +``` + +### 10X Genomics formats +10X readers are provided by **scanpy**, not anndata. After loading, the result is a standard `AnnData` object. + +```python +import scanpy as sc + +# Read 10X h5 format +adata = sc.read_10x_h5('filtered_feature_bc_matrix.h5') + +# Read 10X MTX directory +adata = sc.read_10x_mtx('filtered_feature_bc_matrix/') + +# Specify genome if multiple present +adata = sc.read_10x_h5('data.h5', genome='GRCh38') +``` + +### Loom +```python +from anndata.io import read_loom + +# Read Loom file +adata = read_loom('data.loom') + +# Read with specific observation and variable annotations +adata = read_loom( + 'data.loom', + obs_names='CellID', + var_names='Gene' +) +``` + +### Text files +```python +from anndata.io import read_text + +# Read generic text file +adata = read_text('data.txt', delimiter='\t') + +# Read with custom parameters +adata = read_text( + 'data.txt', + delimiter=',', + first_column_names=True, + dtype='float32' +) +``` + +### UMI tools +```python +from anndata.io import read_umi_tools + +# Read UMI tools format +adata = read_umi_tools('counts.tsv') +``` + +### HDF5 (generic) +```python +from anndata.io import read_hdf + +# Read from HDF5 file (not h5ad format) +adata = read_hdf('data.h5', key='dataset') +``` + +## Alternative Output Formats + +### CSV +```python +# Write to CSV files (creates multiple files) +adata.write_csvs('output_dir/') + +# This creates: +# - output_dir/X.csv (expression matrix) +# - output_dir/obs.csv (observation annotations) +# - output_dir/var.csv (variable annotations) +# - output_dir/uns.csv (unstructured annotations, if possible) + +# Skip certain components +adata.write_csvs('output_dir/', skip_data=True) # Skip X matrix +``` + +### Loom +```python +# Write to Loom format +adata.write_loom('output.loom') +``` + +## Reading Specific Elements + +For fine-grained control, read specific elements from an open store: + +```python +import h5py +from anndata.io import read_elem + +# Read just observation annotations from an h5ad file +with h5py.File('data.h5ad', 'r') as f: + obs = read_elem(f['obs']) + layer = read_elem(f['layers/normalized']) + params = read_elem(f['uns/pca']) +``` + +## Writing Specific Elements + +```python +from anndata.io import write_elem +import h5py + +# Write element to existing file +with h5py.File('data.h5ad', 'a') as f: + write_elem(f, 'new_layer', adata.X.copy()) +``` + +## Lazy Operations + +For very large datasets, use lazy reading to avoid loading entire datasets. `read_lazy` is experimental and is designed for on-disk or in-cloud AnnData stores, including lazy `obs` and `var` access. + +```python +from anndata.experimental import read_lazy + +adata = read_lazy('large_data.zarr') +print(adata.obs.head()) # Does not require loading X +``` + +For element-level control, use `read_elem_lazy` on an open store: + +```python +import h5py +from anndata.experimental import read_elem_lazy + +# Lazy read from an open store (returns dask-backed array) +with h5py.File('large_data.h5ad', 'r') as f: + X_lazy = read_elem_lazy(f['X']) + subset = X_lazy[:100, :100].compute() +``` + +## Common I/O Patterns + +### Convert between formats +```python +from anndata.io import read_mtx, read_csv + +# MTX to H5AD +adata = read_mtx('matrix.mtx').T +adata.write_h5ad('data.h5ad') + +# CSV to H5AD +adata = read_csv('data.csv') +adata.write_h5ad('data.h5ad') + +# H5AD to Zarr +adata = ad.read_h5ad('data.h5ad') +adata.write_zarr('data.zarr') +``` + +### Load metadata without data +```python +# Backed mode allows inspecting metadata without loading X +adata = ad.read_h5ad('large_file.h5ad', backed='r') +print(f"Dataset contains {adata.n_obs} observations and {adata.n_vars} variables") +print(adata.obs.columns) +print(adata.var.columns) +# X is not loaded into memory +``` + +### Update backed data or write a new file +```python +# Open in read-write mode for X updates +adata = ad.read_h5ad('data.h5ad', backed='r+') + +# X updates can be persisted in backed mode +adata.X[0, 0] = 0 + +# Metadata changes are not persisted from backed mode; write a new file instead +adata_memory = adata.to_memory() +adata_memory.obs['new_column'] = values +adata_memory.write_h5ad('data_with_metadata.h5ad') +``` + +### Download from a trusted URL +Validate remote sources before downloading. Prefer local files or vetted object-store paths over arbitrary URLs. + +```python +import anndata as ad +import urllib.request +from urllib.parse import urlparse + +url = 'https://example.org/datasets/reference.h5ad' +parsed = urlparse(url) +trusted_hosts = {'example.org'} + +if parsed.scheme != 'https' or parsed.netloc not in trusted_hosts: + raise ValueError('Refusing to download from an untrusted host') + +urllib.request.urlretrieve(url, 'reference.h5ad') +adata = ad.read_h5ad('reference.h5ad') +``` + +## Performance Tips + +### Reading +- Use `backed='r'` for large files you only need to query +- Use `backed='r+'` only for `X` updates; write a new file for metadata changes +- H5AD format is generally fastest for random access +- Zarr is better for cloud storage and parallel access +- Consider compression for storage, but note it may slow down reading + +### Writing +- Use compression for long-term storage: `compression='gzip'` or `compression='lzf'` +- LZF compression is faster but compresses less than GZIP +- For Zarr, tune chunk sizes based on access patterns: + - Larger chunks for sequential reads + - Smaller chunks for random access +- Convert string columns to categorical before writing (smaller files) + +### Memory management +```python +# Convert strings to categoricals (reduces file size and memory) +adata.strings_to_categoricals() +adata.write_h5ad('data.h5ad') + +# Use sparse matrices for sparse data +from scipy.sparse import csr_matrix +if isinstance(adata.X, np.ndarray): + density = np.count_nonzero(adata.X) / adata.X.size + if density < 0.5: # If more than 50% zeros + adata.X = csr_matrix(adata.X) +``` + +## Handling Large Datasets + +### Strategy 1: Backed mode +```python +# Work with dataset larger than RAM +adata = ad.read_h5ad('100GB_file.h5ad', backed='r') + +# Filter based on metadata (fast, no data loading) +filtered = adata[adata.obs['quality_score'] > 0.8] + +# Load filtered subset into memory +adata_memory = filtered.to_memory() +``` + +### Strategy 2: Chunked processing +```python +# Process data in chunks +adata = ad.read_h5ad('large_file.h5ad', backed='r') + +chunk_size = 1000 +results = [] + +for i in range(0, adata.n_obs, chunk_size): + chunk = adata[i:i+chunk_size, :].to_memory() + # Process chunk + result = process(chunk) + results.append(result) +``` + +### Strategy 3: Use AnnCollection +```python +import anndata as ad +from anndata.experimental import AnnCollection + +# Create backed objects, then lazily concatenate along observations +adatas = [ + ad.read_h5ad(f'dataset_{i}.h5ad', backed='r') + for i in range(10) +] +collection = AnnCollection( + adatas, + join_obs='inner', + join_vars='inner' +) + +# Process collection lazily +# Data is loaded only when accessed +``` + +## Common Issues and Solutions + +### Issue: Out of memory when reading +**Solution**: Use backed mode or read in chunks +```python +adata = ad.read_h5ad('file.h5ad', backed='r') +``` + +### Issue: Slow reading from cloud storage +**Solution**: Use Zarr format with appropriate chunking +```python +adata.write_zarr('data.zarr', chunks=(1000, 1000)) +``` + +### Issue: Large file sizes +**Solution**: Use compression and convert to sparse/categorical +```python +adata.strings_to_categoricals() +from scipy.sparse import csr_matrix +adata.X = csr_matrix(adata.X) +adata.write_h5ad('compressed.h5ad', compression='gzip') +``` + +### Issue: Cannot modify backed metadata +**Solution**: Load to memory and write a new file. Backed mode only persists updates to `X`. +```python +adata = adata.to_memory() +adata.obs['new_column'] = values +adata.write_h5ad('updated_file.h5ad') +``` diff --git a/.agents/skills/anndata/references/manipulation.md b/.agents/skills/anndata/references/manipulation.md new file mode 100644 index 0000000..9266ebd --- /dev/null +++ b/.agents/skills/anndata/references/manipulation.md @@ -0,0 +1,516 @@ +# Data Manipulation + +Operations for transforming, subsetting, and manipulating AnnData objects. + +## Subsetting + +### By indices +```python +import anndata as ad +import numpy as np + +adata = ad.AnnData(X=np.random.rand(1000, 2000)) + +# Integer indices +subset = adata[0:100, 0:500] # First 100 obs, first 500 vars + +# List of indices +obs_indices = [0, 10, 20, 30, 40] +var_indices = [0, 1, 2, 3, 4] +subset = adata[obs_indices, var_indices] + +# Single observation or variable +single_obs = adata[0, :] +single_var = adata[:, 0] +``` + +### By names +```python +import pandas as pd + +# Create with named indices +obs_names = [f'cell_{i}' for i in range(1000)] +var_names = [f'gene_{i}' for i in range(2000)] +adata = ad.AnnData( + X=np.random.rand(1000, 2000), + obs=pd.DataFrame(index=obs_names), + var=pd.DataFrame(index=var_names) +) + +# Subset by observation names +subset = adata[['cell_0', 'cell_1', 'cell_2'], :] + +# Subset by variable names +subset = adata[:, ['gene_0', 'gene_10', 'gene_20']] + +# Both axes +subset = adata[['cell_0', 'cell_1'], ['gene_0', 'gene_1']] +``` + +### By boolean masks +```python +# Create boolean masks +high_count_obs = np.random.rand(1000) > 0.5 +high_var_genes = np.random.rand(2000) > 0.7 + +# Subset using masks +subset = adata[high_count_obs, :] +subset = adata[:, high_var_genes] +subset = adata[high_count_obs, high_var_genes] +``` + +### By metadata conditions +```python +# Add metadata +adata.obs['cell_type'] = np.random.choice(['A', 'B', 'C'], 1000) +adata.obs['quality_score'] = np.random.rand(1000) +adata.var['highly_variable'] = np.random.rand(2000) > 0.8 + +# Filter by cell type +t_cells = adata[adata.obs['cell_type'] == 'A'] + +# Filter by multiple conditions +high_quality_a_cells = adata[ + (adata.obs['cell_type'] == 'A') & + (adata.obs['quality_score'] > 0.7) +] + +# Filter by variable metadata +hv_genes = adata[:, adata.var['highly_variable']] + +# Complex conditions +filtered = adata[ + (adata.obs['quality_score'] > 0.5) & + (adata.obs['cell_type'].isin(['A', 'B'])), + adata.var['highly_variable'] +] +``` + +## Transposition + +```python +# Transpose AnnData object (swap observations and variables) +adata_T = adata.T + +# Shape changes +print(adata.shape) # (1000, 2000) +print(adata_T.shape) # (2000, 1000) + +# obs and var are swapped +print(adata.obs.head()) # Observation metadata +print(adata_T.var.head()) # Same data, now as variable metadata + +# Useful when data is in opposite orientation +# Common with some file formats where genes are rows +``` + +## Copying + +### Full copy +```python +# Create independent copy +adata_copy = adata.copy() + +# Modifications to copy don't affect original +adata_copy.obs['new_column'] = 1 +print('new_column' in adata.obs.columns) # False +``` + +### Shallow copy +```python +# View (doesn't copy data, modifications affect original) +adata_view = adata[0:100, :] + +# Check if object is a view +print(adata_view.is_view) # True + +# Convert view to independent copy +adata_independent = adata_view.copy() +print(adata_independent.is_view) # False +``` + +## Renaming + +### Rename observations and variables +```python +# Rename all observations +adata.obs_names = [f'new_cell_{i}' for i in range(adata.n_obs)] + +# Rename all variables +adata.var_names = [f'new_gene_{i}' for i in range(adata.n_vars)] + +# Make names unique (add suffix to duplicates) +adata.obs_names_make_unique() +adata.var_names_make_unique() +``` + +### Rename categories +```python +# Create categorical column +adata.obs['cell_type'] = pd.Categorical(['A', 'B', 'C'] * 333 + ['A']) + +# Rename categories +adata.rename_categories('cell_type', ['Type_A', 'Type_B', 'Type_C']) + +# Or using dictionary +adata.rename_categories('cell_type', { + 'Type_A': 'T_cell', + 'Type_B': 'B_cell', + 'Type_C': 'Monocyte' +}) +``` + +## Type Conversions + +### Strings to categoricals +```python +# Convert string columns to categorical (more memory efficient) +adata.obs['cell_type'] = ['TypeA', 'TypeB'] * 500 +adata.obs['tissue'] = ['brain', 'liver'] * 500 + +# Convert all string columns to categorical +adata.strings_to_categoricals() + +print(adata.obs['cell_type'].dtype) # category +print(adata.obs['tissue'].dtype) # category +``` + +### Sparse to dense and vice versa +```python +from scipy.sparse import csr_matrix, issparse + +# Dense to sparse +if not issparse(adata.X): + adata.X = csr_matrix(adata.X) + +# Sparse to dense +if issparse(adata.X): + adata.X = adata.X.toarray() + +# Convert layer +adata.layers['normalized'] = csr_matrix(adata.layers['normalized']) +``` + +## Chunked Operations + +Process large datasets in chunks: + +```python +# Iterate through data in chunks +chunk_size = 100 +for chunk in adata.chunked_X(chunk_size): + # Process chunk + result = process_chunk(chunk) +``` + +## Extracting Vectors + +### Get observation vectors +```python +# Get observation metadata as array +cell_types = adata.obs_vector('cell_type') + +# Get gene expression across observations +actb_expression = adata.obs_vector('ACTB') # If ACTB in var_names +``` + +### Get variable vectors +```python +# Get variable metadata as array +gene_names = adata.var_vector('gene_name') +``` + +## Adding/Modifying Data + +### Add observations +```python +# Create new observations +new_obs = ad.AnnData(X=np.random.rand(100, adata.n_vars)) +new_obs.var_names = adata.var_names + +# Concatenate with existing +adata_extended = ad.concat([adata, new_obs], axis=0) +``` + +### Add variables +```python +# Create new variables +new_vars = ad.AnnData(X=np.random.rand(adata.n_obs, 100)) +new_vars.obs_names = adata.obs_names + +# Concatenate with existing +adata_extended = ad.concat([adata, new_vars], axis=1) +``` + +### Add metadata columns +```python +# Add observation annotation +adata.obs['new_score'] = np.random.rand(adata.n_obs) + +# Add variable annotation +adata.var['new_label'] = ['label'] * adata.n_vars + +# Add from external data +external_data = pd.read_csv('metadata.csv', index_col=0) +adata.obs['external_info'] = external_data.loc[adata.obs_names, 'column'] +``` + +### Add layers +```python +# Add new layer +adata.layers['raw_counts'] = np.random.randint(0, 100, adata.shape) +adata.layers['log_transformed'] = np.log1p(adata.X) + +# Replace layer +adata.layers['normalized'] = new_normalized_data +``` + +### Add embeddings +```python +# Add PCA +adata.obsm['X_pca'] = np.random.rand(adata.n_obs, 50) + +# Add UMAP +adata.obsm['X_umap'] = np.random.rand(adata.n_obs, 2) + +# Add multiple embeddings +adata.obsm['X_tsne'] = np.random.rand(adata.n_obs, 2) +adata.obsm['X_diffmap'] = np.random.rand(adata.n_obs, 10) +``` + +### Add pairwise relationships +```python +from scipy.sparse import csr_matrix + +# Add nearest neighbor graph +n_obs = adata.n_obs +knn_graph = csr_matrix(np.random.rand(n_obs, n_obs) > 0.95) +adata.obsp['connectivities'] = knn_graph + +# Add distance matrix +adata.obsp['distances'] = csr_matrix(np.random.rand(n_obs, n_obs)) +``` + +### Add unstructured data +```python +# Add analysis parameters +adata.uns['pca'] = { + 'variance': [0.2, 0.15, 0.1], + 'variance_ratio': [0.4, 0.3, 0.2], + 'params': {'n_comps': 50} +} + +# Add color schemes +adata.uns['cell_type_colors'] = ['#FF0000', '#00FF00', '#0000FF'] +``` + +## Removing Data + +### Remove observations or variables +```python +# Keep only specific observations +keep_obs = adata.obs['quality_score'] > 0.5 +adata = adata[keep_obs, :] + +# Remove specific variables +remove_vars = adata.var['low_count'] +adata = adata[:, ~remove_vars] +``` + +### Remove metadata columns +```python +# Remove observation column +adata.obs.drop('unwanted_column', axis=1, inplace=True) + +# Remove variable column +adata.var.drop('unwanted_column', axis=1, inplace=True) +``` + +### Remove layers +```python +# Remove specific layer +del adata.layers['unwanted_layer'] + +# Remove all layers +adata.layers = {} +``` + +### Remove embeddings +```python +# Remove specific embedding +del adata.obsm['X_tsne'] + +# Remove all embeddings +adata.obsm = {} +``` + +### Remove unstructured data +```python +# Remove specific key +del adata.uns['unwanted_key'] + +# Remove all unstructured data +adata.uns = {} +``` + +## Reordering + +### Sort observations +```python +# Sort by observation metadata +adata = adata[adata.obs.sort_values('quality_score').index, :] + +# Sort by observation names +adata = adata[sorted(adata.obs_names), :] +``` + +### Sort variables +```python +# Sort by variable metadata +adata = adata[:, adata.var.sort_values('gene_name').index] + +# Sort by variable names +adata = adata[:, sorted(adata.var_names)] +``` + +### Reorder to match external list +```python +# Reorder observations to match external list +desired_order = ['cell_10', 'cell_5', 'cell_20', ...] +adata = adata[desired_order, :] + +# Reorder variables +desired_genes = ['TP53', 'ACTB', 'GAPDH', ...] +adata = adata[:, desired_genes] +``` + +## Data Transformations + +### Normalize +```python +# Total count normalization (CPM/TPM-like) +total_counts = adata.X.sum(axis=1) +adata.layers['normalized'] = adata.X / total_counts[:, np.newaxis] * 1e6 + +# Log transformation +adata.layers['log1p'] = np.log1p(adata.X) + +# Z-score normalization +mean = adata.X.mean(axis=0) +std = adata.X.std(axis=0) +adata.layers['scaled'] = (adata.X - mean) / std +``` + +### Filter +```python +# Filter cells by total counts +total_counts = np.array(adata.X.sum(axis=1)).flatten() +adata.obs['total_counts'] = total_counts +adata = adata[adata.obs['total_counts'] > 1000, :] + +# Filter genes by detection rate +detection_rate = (adata.X > 0).sum(axis=0) / adata.n_obs +adata.var['detection_rate'] = np.array(detection_rate).flatten() +adata = adata[:, adata.var['detection_rate'] > 0.01] +``` + +## Working with Views + +Views are lightweight references to subsets of data that don't copy the underlying matrix: + +```python +# Create view +view = adata[0:100, 0:500] +print(view.is_view) # True + +# Views allow read access +data = view.X + +# Modifying view data affects original +# (Be careful!) + +# Convert view to independent copy +independent = view.copy() + +# Force AnnData to be a copy, not a view +adata = adata.copy() +``` + +## Merging Metadata + +```python +# Merge external metadata +external_metadata = pd.read_csv('additional_metadata.csv', index_col=0) + +# Join metadata (inner join on index) +adata.obs = adata.obs.join(external_metadata) + +# Left join (keep all adata observations) +adata.obs = adata.obs.merge( + external_metadata, + left_index=True, + right_index=True, + how='left' +) +``` + +## Common Manipulation Patterns + +### Quality control filtering +```python +# Calculate QC metrics +adata.obs['n_genes'] = (adata.X > 0).sum(axis=1) +adata.obs['total_counts'] = adata.X.sum(axis=1) +adata.var['n_cells'] = (adata.X > 0).sum(axis=0) + +# Filter low-quality cells +adata = adata[adata.obs['n_genes'] > 200, :] +adata = adata[adata.obs['total_counts'] < 50000, :] + +# Filter rarely detected genes +adata = adata[:, adata.var['n_cells'] >= 3] +``` + +### Select highly variable genes +```python +# Mark highly variable genes +gene_variance = np.var(adata.X, axis=0) +adata.var['variance'] = np.array(gene_variance).flatten() +adata.var['highly_variable'] = adata.var['variance'] > np.percentile(gene_variance, 90) + +# Subset to highly variable genes +adata_hvg = adata[:, adata.var['highly_variable']].copy() +``` + +### Downsample +```python +# Random sampling of observations +np.random.seed(42) +n_sample = 500 +sample_indices = np.random.choice(adata.n_obs, n_sample, replace=False) +adata_downsampled = adata[sample_indices, :].copy() + +# Stratified sampling by cell type +from sklearn.model_selection import train_test_split +train_idx, test_idx = train_test_split( + range(adata.n_obs), + test_size=0.2, + stratify=adata.obs['cell_type'] +) +adata_train = adata[train_idx, :].copy() +adata_test = adata[test_idx, :].copy() +``` + +### Split train/test +```python +# Random train/test split +np.random.seed(42) +n_obs = adata.n_obs +train_size = int(0.8 * n_obs) +indices = np.random.permutation(n_obs) +train_indices = indices[:train_size] +test_indices = indices[train_size:] + +adata_train = adata[train_indices, :].copy() +adata_test = adata[test_indices, :].copy() +``` diff --git a/.agents/skills/arbor/SKILL.md b/.agents/skills/arbor/SKILL.md new file mode 100644 index 0000000..6e273d1 --- /dev/null +++ b/.agents/skills/arbor/SKILL.md @@ -0,0 +1,152 @@ +--- +name: arbor +description: Autonomously improve a real artifact (code, training recipe, agent harness, data pipeline, prompt) against an objective and an evaluator, using Hypothesis Tree Refinement (HTR) from the Arbor paper. Use this whenever someone wants to iteratively optimize something over many experiments without overfitting — e.g. "get my model's eval score up", "improve this agent/harness", "tune this pipeline", "beat the baseline on this benchmark", "run a search over approaches and keep the best", "do an MLE-bench / Kaggle-style optimization", or any long-horizon "make this artifact better and don't just memorize the dev set" task. Trigger it even when the user doesn't say "Arbor" or "hypothesis tree" but describes repeated experiment-and-evaluate loops, branching exploration of competing ideas, or worries about a dev/test gap. Runs Claude itself as the coordinator with subagent executors in isolated git worktrees; for the standalone `arbor` CLI tool see references/arbor-upstream.md. +allowed-tools: Read Write Edit Bash Agent +license: MIT license +metadata: + version: "1.1" + skill-author: K-Dense Inc. +--- + +# Arbor — Autonomous Optimization via Hypothesis Tree Refinement + +## Overview + +This skill runs an **Autonomous Optimization (AO)** loop: starting from an existing artifact and a measurable objective, improve it through many rounds of experiment and evaluation — without step-by-step human supervision and without overfitting to the feedback signal. It's the right tool when the bottleneck isn't writing one good change, but *organizing dozens of trials* so that lessons accumulate instead of evaporating. + +It implements **Hypothesis Tree Refinement (HTR)** from *Arbor* (Jin et al., 2026). The key idea: keep the research state in a persistent **hypothesis tree** rather than in conversation history. Each node binds a hypothesis, the distilled insight it produced, and a pointer to the artifact version that realizes it. You play the long-lived **coordinator** that owns this tree and decides where to search; short-lived **executor** subagents test one hypothesis each in isolated git worktrees and report back. A **held-out merge gate** admits a change only when it improves on a *test* evaluator the search never optimized against. This is what turns trial-and-error into cumulative, auditable research. + +Use the `scripts/tree.py` state manager for all the bookkeeping (creating nodes, writing evidence, propagating insights, pruning, the merge gate, the Observe projection). It keeps the state consistent and frees you to spend judgment on what the evidence *means*. + +## When to use this skill + +Reach for Arbor when the task is **iterative improvement of a concrete artifact under an evaluator**: +- Model training: optimizer/architecture/recipe changes to lower loss or hit a target in fewer steps. +- Harness/agent engineering: raising pass rate or accuracy of an agent loop, search harness, or tool-use scaffold. +- Data synthesis: improving a generation/filtering pipeline judged by downstream model behavior. +- Benchmark optimization: MLE-bench / Kaggle-style "improve the submission" tasks. +- Prompt/system optimization where you can score outputs automatically. + +The distinguishing signals: there's an **artifact you can modify**, an **objective**, a way to **score** candidates, and you expect to run **many experiments**. If the user only wants a single fix or a one-shot answer, this is overkill — just do the work directly. If they want open-ended ideation with no evaluator, use `hypothesis-generation` or `scientific-brainstorming` instead. + +## The AO setup — pin this down first + +Before any experiments, establish the task tuple `(M_0, O, E_dev, E_test)`. Getting this right matters more than any later decision, so confirm it explicitly: + +- **M_0 — initial material**: the artifact to improve (a repo, a script, a config, a prompt). Make sure it's under git and currently runs. +- **O — objective**: the natural-language goal and the metric *direction* (maximize accuracy? minimize loss/steps?). +- **E_dev — development evaluator**: a command you can run freely during search to score a candidate. Fast, repeatable. +- **E_test — held-out test evaluator**: a *separate* evaluator (different seeds, different split, or a larger run) used only at the merge gate. It must not be used as a search oracle — that's the whole point. + +If the user hasn't given you a clean dev/test split, **construct one and say so**. The dev/test separation is the mechanism that catches overfitting: a candidate that wins on dev but not on test isn't a success, it's a warning that you're exploiting the feedback signal. Without it, autonomous search reliably overfits. + +Initialize the run: + +```bash +python scripts/tree.py init \ + --objective "Improve BrowseComp answer accuracy on the search harness" \ + --dev-eval "python eval.py --split dev --n 50" \ + --test-eval "python eval.py --split test --n 300" \ + --material "." --metric-direction max --branching 3 --max-depth 2 --budget 12 +``` + +`--branching` is how many sibling hypotheses you propose per parent; `--max-depth 2` keeps directions at depth 1 and concrete interventions at depth 2 (the paper's default); `--budget` is the number of coordinator cycles. Start small (10–20 cycles) — structured search beats brute force, and you can extend if progress is still being made. + +## The coordinator loop + +You run repeated cycles of six steps. This is the heart of HTR; do not collapse it into ad-hoc editing. Run `python scripts/tree.py cycle` once per cycle to track the budget. + +### 1. Observe +Begin every cycle by re-grounding in the tree, not in your memory of the conversation: + +```bash +python scripts/tree.py observe +``` + +This prints the objective, global insights, the active frontier (selectable hypotheses), executed nodes with their evidence, pruned lessons (negative constraints), and the current best artifact. Treating the tree as the source of truth is what keeps you coherent over a long run, after context compression has thrown away the details. + +### 2. Ideate +Pick a promising parent and propose a few child hypotheses under it. **Condition on the tree's evidence** — this is the difference between Arbor and random search: +- Validated insights are assumptions you can build on. +- Pruned nodes are dead ends to avoid. +- A "half-right" result is a *starting point for a sharper hypothesis*, not a reason to abandon the direction. + +Each hypothesis should be a **falsifiable claim about how changing the artifact will move the metric**, not a vague intention. Depth-1 nodes are broad directions ("the search harness loses correct answers it already retrieved"); depth-2 nodes are concrete, executable interventions ("run K=5 independent rollouts and aggregate by evidence dossier instead of majority vote"). + +```bash +python scripts/tree.py add-node --parent n0 --hypothesis "Verification, not retrieval, is the bottleneck: candidates are found but discarded" +python scripts/tree.py add-node --parent n4 --hypothesis "Decompose the question into atomic constraints and verify each independently" +``` + +### 3. Select +Choose which pending leaves to run next. **Selection is not pure score-maximization** — pick a hypothesis because it has strong prior evidence, because it would resolve an ambiguity its siblings exposed, or because its failure would clarify an important assumption. Frontier control under delayed feedback rewards informative experiments, not just promising ones. + +### 4. Dispatch +Run each selected hypothesis as an **executor subagent in an isolated worktree** (use the Agent tool with `isolation: "worktree"`, or have the executor create one with `git worktree add`). Isolation matters: parallel experiments must not clobber each other or the current best, and exploratory changes stay quarantined until they pass the merge gate. + +Dispatch siblings **in parallel** (multiple Agent calls in one message) when they're independent — comparative evidence within one direction is exactly what makes later pruning and abstraction possible. + +Give each executor a tight, **hypothesis-bound** brief. See `references/executor-brief.md` for the full template. The contract that makes HTR work: **the executor may not change the hypothesis when the metric stalls.** It repairs its own code and reruns, but `h_n` is fixed — otherwise the returned score is no longer evidence about the assigned node and the tree's semantics break. The executor returns exactly four things: +- **dev_score** — the dev evaluator result (for selection); +- **result** — a factual summary of what happened; +- **insight** — the distilled, reusable lesson (*why* the result supports, weakens, or bounds the hypothesis); +- **branch_ref** — the git branch/commit/worktree path holding the artifact. + +Mark a node `running` before dispatch (`tree.py set-status --node n5 --status running`) so the Observe projection stays accurate. + +### 5. Backpropagate +When an executor returns, write its report into the node, then **abstract the lesson upward**: + +```bash +python scripts/tree.py set-evidence --node n5 --dev-score 70.0 \ + --result "K=5 dossier aggregation recovers answers in minority rollouts" \ + --insight "Correct answers often appear in a minority of rollouts; aggregation beats majority vote" \ + --branch-ref "wt/n5" + +python scripts/tree.py propagate --node n5 \ + --insight "Candidate coverage, not verification, limits this direction" --to-root +``` + +This is the step that makes the tree more than a log. A leaf-level observation ("data-interface mismatch") should become a direction-level constraint and, if it generalizes, a global prior that shapes future ideation. **Insight propagation is the component that drives most of HTR's gains** — in the paper's MLE-Bench Lite ablation, a tree *without* insight feedback scored even lower than a flat experiment queue with no tree at all (54.5% vs. 63.6% any-medal, against 81.8% for the full system). Hierarchy alone isn't enough: the semantic memory is what matters. So spend real thought on the abstraction; don't just copy the leaf insight upward verbatim. + +### 6. Decide +Decide what to do with the new evidence: keep expanding a direction, prune a falsified subtree, or attempt to merge a candidate. + +- **Prune** dead ends, recording *why* — the reason becomes a negative constraint: + ```bash + python scripts/tree.py prune --node n7 --reason "search-augmented judge overfits dev questions; no test transfer" + ``` +- **Merge gate** — promote a candidate to the new best **only if it improves on `E_test`**. Run the test evaluator in a *fresh* worktree (not the dev worktree, to avoid leakage), then: + ```bash + python scripts/tree.py merge --node n5 --test-score 67.67 --branch-ref "wt/n5" + ``` + If the gate rejects it, that's informative: a high-dev / low-test candidate is evidence the direction may be exploiting the dev signal rather than producing a transferable improvement. Record that lesson; don't quietly promote it anyway. + +Repeat until the budget is spent, the frontier is exhausted, or progress has clearly stalled. + +## Finishing the run + +When you stop, produce a short report (see `references/report-template.md`) covering: +- the final best artifact, its test score, and its delta over `M_0`; +- the tree (`python scripts/tree.py status`) as the audit trail of what was tried; +- the main hypothesis shifts — how task understanding deepened across the run (early nodes test broad mechanisms; later nodes find their limits; ancestor insights compress these into the constraints behind the final design); +- merged vs. explored: many nodes improve dev, far fewer pass the test gate — report that gap honestly rather than overstating dev wins. + +Always leave `M_best` as a real, runnable artifact on a named branch, and tell the user how to check it out. + +## Principles that make this work (not rote rules) + +These come from the paper's analysis; understanding *why* matters more than following them mechanically. + +- **The tree is the memory; conversation is not.** Over a long horizon your context gets compressed. Re-Observe each cycle so decisions rest on durable evidence, not a lossy summary. +- **Structured search, not more sampling.** Arbor's gains come from how the budget is *organized* — maintaining competing hypotheses, comparing siblings, carrying lessons forward — not from spending more tokens. Don't fan out aimlessly; each experiment should be conditioned on what the tree already knows. +- **Dev guides, test admits.** Use dev feedback freely to steer exploration, but never let a dev win into the final artifact without test confirmation. The dev/test disagreement is itself a signal worth reading. +- **Executors are hypothesis-bound.** Local engineering flexibility (edit, debug, rerun) is fine; silently changing the hypothesis to chase a better number is not — it destroys the meaning of the evidence. +- **Failures are constraints, not noise.** A falsified hypothesis tells you what the solution must avoid. Pruned-with-a-reason is more valuable than pruned-and-forgotten. + +## Reference files + +- `references/htr-methodology.md` — deeper explanation of HTR, the node structure, the six steps, and the paper's empirical lessons (ablations, transfer, cost). Read when you want the rationale behind a design choice. +- `references/executor-brief.md` — the template for the brief you hand each executor subagent. +- `references/report-template.md` — the final-report structure. +- `references/arbor-upstream.md` — how to install and run the standalone `arbor` CLI from RUC-NLPIR/Arbor instead of orchestrating it natively, and when to prefer each. diff --git a/.agents/skills/arbor/references/arbor-upstream.md b/.agents/skills/arbor/references/arbor-upstream.md new file mode 100644 index 0000000..f6ae76a --- /dev/null +++ b/.agents/skills/arbor/references/arbor-upstream.md @@ -0,0 +1,91 @@ +# Running the standalone Arbor CLI (upstream tool) + +This skill normally runs HTR **natively** — Claude is the coordinator and +subagents are executors. That's the recommended path: no extra install, no +separate API keys, and you stay in the loop to read evidence between cycles. + +But the paper's authors also ship a full implementation as a CLI. Use it instead +when the user explicitly wants to run *the published system* (e.g. to reproduce +paper results), wants Arbor to run fully unattended for many hours via its own +live dashboard, or wants its built-in report/web-UI tooling. + +Source: https://github.com/RUC-NLPIR/Arbor + +## Install + +Requires Python ≥ 3.10 and Git. + +```bash +git clone https://github.com/RUC-NLPIR/Arbor.git +cd Arbor +python -m venv .venv && source .venv/bin/activate +uv pip install -e . +arbor doctor # verify install, PATH, git, API keys +``` + +## Configure provider/model/keys + +```bash +arbor setup # writes ~/.arbor/config.yaml (provider, model, base URL, keys) +``` + +Supported backends: Anthropic, OpenAI / OpenAI-compatible Responses API, and +LiteLLM (DeepSeek, Gemini, Qwen, vLLM, Ollama, local gateways). Keys can also be +set via environment variables. + +## Run + +1. Prepare a benchmark directory: an initial artifact under a **clean git repo** + plus an evaluation script (your `E_dev` / `E_test`). +2. Author a project `research_config.yaml` (task description, coordinator + settings — max cycles, depth, merge thresholds — executor max turns, UI mode). + See `examples/research_config.example.yaml` in the repo. +3. Start the interactive session: + ```bash + arbor + ``` + Arbor runs an intake conversation, forms a Research Contract, then a live + dashboard takes over. Each experiment runs in an isolated git worktree; + verified improvements merge into a per-run trunk. +4. Outputs land in `.arbor/sessions/` with `REPORT.md`, the event log, and + results. Re-render a past session's report with `arbor report `. + +## Key CLI commands + +| Command | Purpose | +|---|---| +| `arbor` | Start an interactive research session | +| `arbor setup` | Configure provider / model / keys | +| `arbor doctor` | Diagnose install, PATH, git, API keys | +| `arbor report ` | Re-render reports for a past session | +| `arbor version` | Print installed version | + +## Codebase map (for the curious / for debugging) + +The implementation lives under `src/` (a src-layout; the CLI installs as +`arbor`). The package directories are: +- `core/` — ReAct loop, tools, LLM providers, context management +- `coordinator/` — coordinator agent, the tree, orchestrator, coordinator tools +- `executor/` — executor agent and CLI +- `cli/` — intake, live dashboard, setup, doctor, config +- `events/` — typed event bus and payloads +- `report/`, `webui/` — report generation and read-only run monitor +- `search_agent/` — the minimal ReAct search harness (the `M_0` for the + BrowseComp / search-agent tasks) +- `plugins/` — domain plugins (e.g. `mle_kaggle.yaml`) +- `skills/` — on-demand markdown playbooks + +(top-level `src/` also has `dashboard.py`, `run.py`, `review.py`.) + +**Naming note:** the paper and this skill call the persistent state the +**hypothesis tree**; the tool's code and dashboard call the same structure the +**Idea Tree**. They are the same thing. The depth convention also matches the +native skill: root/depth 0 = objective + global insights, depth 1 = research +directions, depth 2+ = concrete tested methods. + +## Native vs. upstream — quick guide + +- **Native (this skill)**: best default. Lower setup, transparent, you read and + steer between cycles, reuses your existing Claude Code session and worktrees. +- **Upstream CLI**: choose for paper reproduction, long unattended runs with the + official dashboard, or when the user specifically asks for the `arbor` tool. diff --git a/.agents/skills/arbor/references/executor-brief.md b/.agents/skills/arbor/references/executor-brief.md new file mode 100644 index 0000000..ea67a78 --- /dev/null +++ b/.agents/skills/arbor/references/executor-brief.md @@ -0,0 +1,68 @@ +# Executor brief template + +Each executor is a short-lived subagent that tests **one** hypothesis in an +isolated git worktree and returns structured evidence. Dispatch it with the +Agent tool (use `isolation: "worktree"` so it gets its own copy of the repo, or +instruct it to run `git worktree add` itself). Dispatch independent siblings in +parallel — multiple Agent calls in one message. + +Fill in the bracketed parts. Keep the brief tight: the executor needs the +hypothesis, the context that lets it implement well, and a crisp contract for +what to return — nothing more. + +--- + +``` +You are an Arbor executor. Test ONE hypothesis in an isolated git worktree and +return structured evidence. Do not change the hypothesis — your job is to give +the coordinator clean evidence about THIS claim, even if it turns out false. + +HYPOTHESIS (h_n): + [the falsifiable claim, e.g. "Aggregating K=5 independent rollouts by an + evidence dossier recovers correct answers that majority vote discards."] + +CURRENT BEST ARTIFACT (M_best): + [path or git ref of the current best, e.g. branch `arbor/best` — start from this] + +RELEVANT INSIGHTS FROM THE TREE (assume these; build on them, don't re-litigate): + [ancestor + sibling insights, e.g. "Verification is not the bottleneck; + candidate coverage is. Search-augmented judging overfits dev questions."] + +OBJECTIVE & METRIC: + [O and direction, e.g. "Maximize BrowseComp answer accuracy."] + +DEVELOPMENT EVALUATOR (E_dev) — run this to score your candidate: + [exact command, e.g. `python eval.py --split dev --n 50`] + +WHAT TO DO: + 1. Create/confirm an isolated worktree from M_best so you don't touch other + experiments or the current best. + 2. Implement the MINIMAL change that realizes the hypothesis. You may edit, + debug, and rerun freely to get a working implementation — but keep the + change bound to this hypothesis. If the metric stalls, fix YOUR code; do + not pivot to a different idea. + 3. Run E_dev and record the score. Run it more than once if it's noisy. + 4. Commit the artifact on a clearly named branch. + +RETURN EXACTLY THIS (your final message IS the data the coordinator reads): + - dev_score: + - result: <1-3 sentences of factual outcome — what the change did> + - insight: + - branch_ref: + +Do NOT run the held-out test evaluator — that is the coordinator's merge gate. +``` + +--- + +After the executor returns, the coordinator records it with: + +```bash +python scripts/tree.py set-evidence --node \ + --dev-score --result "..." --insight "..." --branch-ref "" +``` + +then abstracts the lesson upward with `tree.py propagate`. diff --git a/.agents/skills/arbor/references/htr-methodology.md b/.agents/skills/arbor/references/htr-methodology.md new file mode 100644 index 0000000..1bcc19f --- /dev/null +++ b/.agents/skills/arbor/references/htr-methodology.md @@ -0,0 +1,177 @@ +# Hypothesis Tree Refinement (HTR) — methodology and evidence + +Background reference for the `arbor` skill. Source: *Toward Generalist +Autonomous Research via Hypothesis-Tree Refinement* (Jin et al., 2026, +arXiv:2606.11926; code: github.com/RUC-NLPIR/Arbor). Read this when you want +the reasoning behind a design choice in the main loop. + +## The problem: Autonomous Optimization (AO) + +AO is the operational core of autonomous research. An agent starts from an +initial artifact and a research objective, then improves the artifact through +experimental feedback **without step-level human supervision**. Formally a task +is a tuple `P = (M_0, O, E_dev, E_test)`: + +- `M_0` — mutable initial material (usually a codebase + its data). +- `O` — objective: what "better" means, as a metric direction over the + artifact's output. +- `E_dev` — development evaluator the agent may use freely during search. +- `E_test` — held-out test evaluator. Same objective, different evidence. + +The goal is to return `M* = argmax over candidates of S_test(M')`, subject to +the constraint that hypotheses and implementation decisions are made **without +using `E_test` as an exploration oracle**. A candidate that exploits dev-split +idiosyncrasies may raise `S_dev` but is not a successful AO solution unless the +gain also transfers to `S_test`. + +Why this is hard: feedback is delayed, experiments are expensive, and failed +attempts contain information that should guide later search. If an agent treats +each trial as an independent local attempt, it loses the structure of the +research process — what was tried, what evidence came back, how each result +reshapes the space of future hypotheses. + +## The three design requirements + +HTR is built to satisfy three requirements that ordinary agentic tool use does +not: + +1. **Branching with coherence.** Multiple competing hypotheses can be plausible + at once, so exploration must branch — but unrestricted branching degenerates + into an unstructured log. The frontier must keep competing directions + organized, comparable, and actionable. +2. **Global strategy with local execution.** Strategic decisions depend on + evidence across the whole run; implementing one hypothesis is short-horizon + code editing. Separate the two so low-level traces don't obscure the global + state, and outcomes stay attributable to the hypotheses that produced them. +3. **Exploration with held-out admission.** Dev feedback guides search; + artifact-level progress is admitted only when it transfers beyond that + feedback. The system must distinguish exploratory dev improvement from + verified test improvement. + +## The hypothesis tree as research state + +A rooted tree `T = (V, E)`. Each node is a research unit `n = `: + +- **Hypothesis `h_n`** — a verifiable/falsifiable claim about how changing the + material improves the objective. Granularity tracks depth: nodes near the root + are broad directions; deeper nodes are concrete interventions an executor can + implement and evaluate. This organizes exploration as progressive refinement + rather than a flat sequence of independent trials. +- **Insight `iota_n`** — the reusable interpretation of evidence. For an + executed leaf: what was tried, what happened, and *why* the result supports, + weakens, or constrains the hypothesis. For an internal node: an abstraction + over its children's insights — the current understanding of that direction. + It is **not** an execution transcript; it is compact semantic memory for later + ideation and selection. +- **Metadata `mu_n`** — connects the semantic hypothesis to executable evidence: + node status, dev score, factual result, implementation reference (git branch + or commit), optional background. The material itself is **not** duplicated in + the tree — only references to external artifact states produced in isolated + worktrees. This keeps the state compact while every hypothesis stays grounded + in a verifiable implementation. + +Internal nodes hold abstract directions and accumulated lessons; leaves hold +candidate interventions to dispatch. After a leaf executes, its score, result, +artifact ref, and insight are written back, and the insight is propagated upward +along the path to the root. Through this abstraction, local outcomes become +direction-level lessons and eventually a compact global understanding. + +The tree therefore plays three roles at once: a **search frontier** (which +directions are active/validated/pruned), a **long-term memory** (reusable +evidence from successes *and* failures), and an **auditable record** (each +artifact change linked to the hypothesis and evidence that motivated it). + +## The coordinator–executor split + +- A persistent **coordinator** owns the shared tree and decides where to expand, + which evidence to trust, what to prune, and when to merge. It sees the whole + frontier but does not perform every low-level implementation step. +- Short-lived **executors** are invoked to test one hypothesis each. An executor + gets `h_n`, relevant ancestor insights, and the current best artifact; it + creates an isolated git worktree, implements the minimal change `h_n` requires, + evaluates on `E_dev`, repairs its own broken/inactive code, and returns + structured evidence. + +The boundary is the point: exploratory code changes stay isolated until they +pass the merge gate, and the tree records only decision-relevant evidence +(scores, factual outcomes, artifact refs, distilled insights) rather than a raw +log of tool calls. This is how transient execution traces become persistent +research state. + +### Executors are hypothesis-bound (and why) + +An executor's local loop may involve many edits and reruns, but it stays bound +to the assigned hypothesis: `h_n` is fixed. If an executor were allowed to +change the hypothesis when the metric stalls, the returned score would no longer +be evidence about the assigned node, and ancestor insights built from it would +become impossible to interpret. Keeping executors hypothesis-bound preserves the +semantic meaning of every tree update while still allowing local engineering +flexibility. + +## The six-step cycle (Algorithm 1, HTR) + +Each coordinator cycle is a controlled mutation of the tree through a narrow +interface: + +1. **Observe** — re-ground in a structured projection of the tree (frontier, + root/global insights, ancestor insights, current best). Makes the tree the + authoritative state after context compression, instead of relying on lossy + conversation history. +2. **Ideate** — under a chosen parent, propose `k` child hypotheses, each a + refinement/alternative/correction. Ideation is conditioned on tree evidence: + validated insights are assumptions to build on, pruned nodes are negative + constraints, recent reports suggest what's feasible or under-tested. +3. **Select** — choose pending nodes to execute. Balance expected utility + against the evidence already accumulated around ancestors and siblings. A + node may be selected because it has strong prior evidence, because its + siblings exposed an unresolved ambiguity, or because its failure would + clarify an important assumption. Selection is frontier control under partial, + delayed feedback — not raw score maximization. +4. **Dispatch** — selected hypotheses go to independent executors in fresh + worktrees. Parallel sibling execution yields comparative evidence within one + direction, which feeds later pruning and abstraction. +5. **Backpropagate** — write each executor's evidence into its leaf, then update + insights along the path to the root. The propagated signal is not just a + scalar: it includes causal attributions, applicability conditions, and + reusable lessons. A leaf-level data-interface mismatch can become a + direction-level constraint and then a global prior. +6. **Decide** — continue expanding a direction, prune a falsified subtree, or + attempt a merge. Promotion is guarded by the **held-out merge gate**: the + candidate is evaluated on `E_test` in a fresh worktree and merged into + `M_best` only if it improves under `O`. This separates exploratory success on + `E_dev` from verified artifact-level progress. + +## Empirical lessons (use these to prioritize effort) + +From the paper's experiments across six AO tasks (model training, harness +engineering, data synthesis) plus MLE-Bench Lite: + +- **Insight feedback is the dominant component.** Ablating insight propagation + while *keeping* the tree caused a larger drop than removing the tree entirely + (on MLE-Bench Lite: full 81.82% any-medal vs. 54.54% w/o insight feedback vs. + 63.64% w/o tree). Hierarchy alone is not enough — a tree without propagated + lessons organizes experiments syntactically but provides no semantic memory. + **Invest your judgment in the abstraction at Backpropagate**, not just in + generating more hypotheses. +- **Structured search, not a bigger budget.** Arbor used a comparable token + budget to single-trajectory baselines (~20–43M tokens) yet got larger held-out + gains. The win is in how the budget is *organized*: maintaining competing + hypotheses, isolated execution, comparison, and an updated frontier. +- **The dev/test split exposes overfitting.** Across tasks, many nodes improved + dev but only a subset passed the test gate. On Terminal-Bench, the highest-dev + candidate was *not* the best on test. Always report the merged-vs-explored gap + honestly; a high-dev/low-test result is evidence of feedback exploitation. +- **Refinement deepens task understanding.** Early nodes test whether a broad + mechanism holds; later nodes localize where it stops working; ancestor + insights compress these into the constraints the final design must satisfy. + Successful proposals are usually *evidence-conditioned* responses to earlier + failures, not fresh guesses. +- **Lessons transfer.** A harness optimized only on one task's dev feedback + improved unrelated held-out tasks, indicating HTR discovers generally useful + design changes rather than fitting the source benchmark — when, and only when, + the merge gate is enforced. +- **What HTR does *not* fix.** Arbor is strongest at a sequence of concrete + refinements once a runnable solution exists. It is weaker when progress + requires a genuinely new high-level formulation only weakly connected to the + current tree — that still leans on good human task design (the choice of + `M_0`, evaluator, metric, and interface). diff --git a/.agents/skills/arbor/references/report-template.md b/.agents/skills/arbor/references/report-template.md new file mode 100644 index 0000000..2fd0f9d --- /dev/null +++ b/.agents/skills/arbor/references/report-template.md @@ -0,0 +1,39 @@ +# Final report template + +Produce this when the run ends (budget spent, frontier exhausted, or progress +stalled). The point is an honest, auditable account — not a victory lap. Keep it +concise and grounded in the tree. + +```markdown +# Arbor run report: [objective] + +## Result +- **Best artifact**: [git branch/ref of M_best, e.g. `arbor/best`] +- **Test score**: [S_test of M_best] vs. initial [S_test of M_0] → delta [Δ] +- **How to check it out**: `git checkout [ref]` +- One-line summary of the change that won. + +## What was tried (audit trail) +[Paste `python scripts/tree.py status` — the tree shows every direction, +which were pruned, which merged, with dev/test scores.] + +## How understanding evolved +2-4 bullets tracing the main hypothesis shifts: which early nodes tested broad +mechanisms, what they confirmed or ruled out, and how that reshaped later +hypotheses. The story should explain *why* the final design looks the way it +does — i.e. the constraints the run discovered. + +## Dev vs. test (overfitting check) +- Nodes that improved **dev**: [count] +- Nodes that passed the **test merge gate**: [count] +- Comment on the gap: were there high-dev / low-test candidates? What did + rejecting them tell you? An honest gap here is more trustworthy than a clean + "everything worked". + +## Open directions +What you'd explore with more budget, and any direction that seemed to need a +new high-level formulation rather than further refinement (HTR's known weak +spot — flag it for the human). +``` + +Always leave `M_best` as a real, runnable artifact on a named git branch. diff --git a/.agents/skills/arbor/scripts/tree.py b/.agents/skills/arbor/scripts/tree.py new file mode 100644 index 0000000..fc49274 --- /dev/null +++ b/.agents/skills/arbor/scripts/tree.py @@ -0,0 +1,564 @@ +#!/usr/bin/env python3 +""" +tree.py — persistent hypothesis-tree state manager for Arbor-style +Hypothesis Tree Refinement (HTR). + +The hypothesis tree is the durable research state for an Autonomous +Optimization (AO) run. This script owns the *mechanical* parts of that state +— creating nodes, writing back evidence, propagating insights up the tree, +pruning falsified branches, recording the held-out merge gate, and rendering +an "Observe" projection — so the coordinator (you, the model) can spend its +judgment on what the evidence *means* rather than on bookkeeping. + +Division of labor: + - This script keeps the state consistent and auditable. It never decides + which hypothesis to try or whether one is good. + - The coordinator reads the projection (`observe`), forms hypotheses, + interprets executor reports, and calls the mutating commands to record + those decisions. + +State lives in `.arbor/` under the run directory (default: current dir): + .arbor/tree.json — the hypothesis tree (nodes, edges, evidence, insights) + .arbor/run.json — run-level config: objective, evaluators, budget, M_best + +Node fields mirror the paper's research unit n = : + hypothesis (h) — the falsifiable claim this node tests + insight (iota) — distilled, reusable lesson (filled after execution) + metadata (mu) — status, dev_score, test_score, result, branch_ref, depth + +Run `python tree.py --help` or `python tree.py --help`. +""" + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +# ---------------------------------------------------------------------------- +# Storage helpers +# ---------------------------------------------------------------------------- + +VALID_STATUS = {"pending", "running", "executed", "merged", "pruned", "root"} + + +def _dir(run_dir): + return Path(run_dir) / ".arbor" + + +def _tree_path(run_dir): + return _dir(run_dir) / "tree.json" + + +def _run_path(run_dir): + return _dir(run_dir) / "run.json" + + +def _load(path, what): + if not path.exists(): + sys.exit( + f"error: no {what} found at {path}. Run `tree.py init` first " + f"(from the run directory, or pass --run-dir)." + ) + with open(path) as f: + return json.load(f) + + +def _save(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + with open(tmp, "w") as f: + json.dump(data, f, indent=2) + tmp.replace(path) + + +def _load_tree(run_dir): + return _load(_tree_path(run_dir), "hypothesis tree") + + +def _load_run(run_dir): + return _load(_run_path(run_dir), "run config") + + +def _next_id(tree): + n = tree.get("_counter", 0) + 1 + tree["_counter"] = n + return f"n{n}" + + +def _node(tree, node_id): + node = tree["nodes"].get(node_id) + if node is None: + sys.exit(f"error: no node with id '{node_id}'. Run `tree.py status` to list nodes.") + return node + + +def _children(tree, node_id): + return [nid for nid, n in tree["nodes"].items() if n.get("parent") == node_id] + + +def _ancestors(tree, node_id): + """Path from node up to (and including) root, nearest first.""" + path = [] + cur = tree["nodes"][node_id].get("parent") + while cur is not None: + path.append(cur) + cur = tree["nodes"][cur].get("parent") + return path + + +def _depth(tree, node_id): + return len(_ancestors(tree, node_id)) + + +def _stamp(node): + node["updated_at"] = int(time.time()) + + +# ---------------------------------------------------------------------------- +# Commands +# ---------------------------------------------------------------------------- + + +def cmd_init(args): + d = _dir(args.run_dir) + if _tree_path(args.run_dir).exists() and not args.force: + sys.exit( + f"error: a tree already exists at {_tree_path(args.run_dir)}. " + f"Use --force to overwrite (this erases the current run)." + ) + root = { + "id": "n0", + "parent": None, + "depth": 0, + "status": "root", + "hypothesis": args.objective, + "insight": "", # global insights accumulate here via backpropagation + "metadata": {"dev_score": None, "test_score": None, "result": "", "branch_ref": None}, + "created_at": int(time.time()), + "updated_at": int(time.time()), + } + tree = {"_counter": 0, "root": "n0", "nodes": {"n0": root}} + run = { + "objective": args.objective, + "metric_direction": args.metric_direction, + "dev_eval": args.dev_eval, + "test_eval": args.test_eval, + "material": args.material, + "branching": args.branching, + "max_depth": args.max_depth, + "budget_cycles": args.budget, + "cycles_used": 0, + "best_node": None, # node id of current M_best + "best_test_score": None, # held-out score of M_best + "best_branch_ref": None, # git ref / path of M_best artifact + "created_at": int(time.time()), + } + _save(_tree_path(args.run_dir), tree) + _save(_run_path(args.run_dir), run) + print(f"Initialized Arbor run in {d}") + print(f" objective : {args.objective}") + print(f" metric direction: {args.metric_direction} (higher-is-better after orienting)") + print(f" dev evaluator : {args.dev_eval}") + print(f" test evaluator : {args.test_eval}") + print(f" budget : {args.budget} cycles, branching {args.branching}, max depth {args.max_depth}") + print("\nNext: `tree.py observe` to read the state, then add direction nodes under n0.") + + +def cmd_add_node(args): + tree = _load_tree(args.run_dir) + parent = _node(tree, args.parent) + run = _load_run(args.run_dir) + nid = _next_id(tree) + depth = _depth(tree, args.parent) + 1 + if depth > run["max_depth"]: + print( + f"warning: node depth {depth} exceeds max_depth {run['max_depth']}. " + f"Deep nodes should be concrete, executable interventions.", + file=sys.stderr, + ) + node = { + "id": nid, + "parent": args.parent, + "depth": depth, + "status": "pending", + "hypothesis": args.hypothesis, + "insight": "", + "metadata": {"dev_score": None, "test_score": None, "result": "", "branch_ref": None}, + "created_at": int(time.time()), + "updated_at": int(time.time()), + } + tree["nodes"][nid] = node + _save(_tree_path(args.run_dir), tree) + kind = "direction" if depth == 1 else "intervention" + print(f"Added {kind} node {nid} (depth {depth}) under {args.parent}: {args.hypothesis}") + + +def cmd_set_status(args): + tree = _load_tree(args.run_dir) + node = _node(tree, args.node) + if args.status not in VALID_STATUS: + sys.exit(f"error: status must be one of {sorted(VALID_STATUS)}") + node["status"] = args.status + _stamp(node) + _save(_tree_path(args.run_dir), tree) + print(f"{args.node} -> status={args.status}") + + +def cmd_set_evidence(args): + """Write an executor's report back into its node (the Backpropagate step, + leaf part). Insight propagation upward is a separate, deliberate call.""" + tree = _load_tree(args.run_dir) + node = _node(tree, args.node) + meta = node["metadata"] + if args.dev_score is not None: + meta["dev_score"] = args.dev_score + if args.result is not None: + meta["result"] = args.result + if args.branch_ref is not None: + meta["branch_ref"] = args.branch_ref + if args.insight is not None: + node["insight"] = args.insight + node["status"] = args.status or "executed" + _stamp(node) + _save(_tree_path(args.run_dir), tree) + print(f"Wrote evidence to {args.node}: dev_score={meta['dev_score']} status={node['status']}") + if node["insight"]: + print(f" insight: {node['insight']}") + anc = _ancestors(tree, args.node) + if anc: + print( + "\nReminder: abstract this leaf insight upward. Decide what direction-level " + f"lesson it implies for ancestors {anc} and record it with " + f"`tree.py propagate --node {args.node} --insight \"...\"` (and update n0 " + "global insights if it generalizes)." + ) + + +def cmd_propagate(args): + """Backpropagate a distilled, direction-level lesson up the ancestor path. + + The coordinator decides the abstracted wording; this appends it to the + chosen ancestor(s) so later ideation is conditioned on it. By default it + updates the immediate parent; --to-root also updates global insights.""" + tree = _load_tree(args.run_dir) + _node(tree, args.node) # validate + targets = _ancestors(tree, args.node) + if not targets: + sys.exit("error: node has no ancestors (is it the root?).") + if not args.to_root: + targets = targets[:1] # immediate parent only + for tid in targets: + anc = tree["nodes"][tid] + existing = anc.get("insight", "") + line = f"[from {args.node}] {args.insight}" + anc["insight"] = (existing + "\n" + line).strip() if existing else line + _stamp(anc) + _save(_tree_path(args.run_dir), tree) + print(f"Propagated insight from {args.node} to ancestors {targets}") + + +def cmd_prune(args): + """Mark a node (and its subtree) pruned. Pruned hypotheses become negative + constraints — record *why* so future ideation avoids the dead end.""" + tree = _load_tree(args.run_dir) + _node(tree, args.node) + stack = [args.node] + pruned = [] + while stack: + cur = stack.pop() + node = tree["nodes"][cur] + if node["status"] in ("merged", "root"): + continue + node["status"] = "pruned" + if cur == args.node and args.reason: + node["metadata"]["prune_reason"] = args.reason + _stamp(node) + pruned.append(cur) + stack.extend(_children(tree, cur)) + _save(_tree_path(args.run_dir), tree) + print(f"Pruned {pruned}" + (f" — reason: {args.reason}" if args.reason else "")) + + +def cmd_merge(args): + """Record a held-out merge gate decision. Only call this AFTER evaluating + the candidate on the TEST evaluator in a fresh worktree. Admitting a + candidate that only improved dev defeats the purpose of the split.""" + tree = _load_tree(args.run_dir) + run = _load_run(args.run_dir) + node = _node(tree, args.node) + + direction = run["metric_direction"] + prev = run["best_test_score"] + + def better(new, old): + if old is None: + return True + return new > old if direction == "max" else new < old + + improves = better(args.test_score, prev) + node["metadata"]["test_score"] = args.test_score + if improves: + node["status"] = "merged" + run["best_node"] = args.node + run["best_test_score"] = args.test_score + run["best_branch_ref"] = args.branch_ref or node["metadata"].get("branch_ref") + _stamp(node) + _save(_tree_path(args.run_dir), tree) + _save(_run_path(args.run_dir), run) + print( + f"MERGE GATE PASSED: {args.node} test={args.test_score} " + f"beats previous best={prev}. M_best is now {args.node} " + f"(ref: {run['best_branch_ref']})." + ) + else: + _stamp(node) + _save(_tree_path(args.run_dir), tree) + print( + f"MERGE GATE REJECTED: {args.node} test={args.test_score} does not beat " + f"best={prev} (direction={direction}). This is informative, not a failure: " + "a high-dev / low-test gap means the candidate may be exploiting the dev " + "signal. Record that lesson and keep M_best unchanged." + ) + + +def cmd_cycle(args): + """Increment the cycle counter (call once per Observe->Decide pass).""" + run = _load_run(args.run_dir) + run["cycles_used"] += 1 + _save(_run_path(args.run_dir), run) + left = run["budget_cycles"] - run["cycles_used"] + print(f"Cycle {run['cycles_used']}/{run['budget_cycles']} ({left} remaining).") + if left <= 0: + print("Budget exhausted — finish the run: do a final merge-gate check and report.") + + +# ---------------------------------------------------------------------------- +# Read-only projections +# ---------------------------------------------------------------------------- + + +def _fmt_score(node, run): + s = node["metadata"].get("dev_score") + t = node["metadata"].get("test_score") + bits = [] + if s is not None: + bits.append(f"dev={s}") + if t is not None: + bits.append(f"test={t}") + return (" [" + " ".join(bits) + "]") if bits else "" + + +def cmd_observe(args): + """The Observe step: a compact projection the coordinator re-grounds on at + the start of each cycle, so decisions come from the tree rather than from a + lossy conversation history.""" + tree = _load_tree(args.run_dir) + run = _load_run(args.run_dir) + nodes = tree["nodes"] + root = nodes[tree["root"]] + + print("=" * 72) + print("OBSERVE — current research state") + print("=" * 72) + print(f"Objective : {run['objective']}") + print(f"Metric direction: {run['metric_direction']}") + print(f"Dev evaluator : {run['dev_eval']}") + print(f"Test evaluator : {run['test_eval']}") + print( + f"Budget : cycle {run['cycles_used']}/{run['budget_cycles']}, " + f"branching {run['branching']}, max depth {run['max_depth']}" + ) + print( + f"Current best : " + + ( + f"{run['best_node']} (test={run['best_test_score']}, ref={run['best_branch_ref']})" + if run["best_node"] + else "none yet — M_best is the initial material" + ) + ) + + print("\n-- Global insights (root) --") + print(root["insight"].strip() if root["insight"].strip() else " (none yet)") + + # Active frontier = pending/running leaves + frontier = [ + n + for n in nodes.values() + if n["status"] in ("pending", "running") and not _children(tree, n["id"]) + ] + print("\n-- Active frontier (selectable hypotheses) --") + if not frontier: + print(" (empty — ideate new children under a promising node)") + for n in sorted(frontier, key=lambda x: x["id"]): + anc = _ancestors(tree, n["id"]) + anc_ins = " | ".join( + nodes[a]["insight"].replace("\n", " ")[:80] for a in anc if nodes[a]["insight"].strip() + ) + print(f" {n['id']} (depth {n['depth']}, {n['status']}): {n['hypothesis']}") + if anc_ins: + print(f" ancestor insights: {anc_ins}") + + # Validated / executed leaves with evidence + executed = [n for n in nodes.values() if n["status"] in ("executed", "merged")] + print("\n-- Executed / merged nodes (evidence) --") + if not executed: + print(" (none yet)") + for n in sorted(executed, key=lambda x: x["id"]): + print(f" {n['id']} [{n['status']}]{_fmt_score(n, run)}: {n['hypothesis']}") + if n["insight"].strip(): + print(f" insight: {n['insight'].splitlines()[0][:120]}") + + # Pruned lessons (negative constraints) + pruned = [n for n in nodes.values() if n["status"] == "pruned"] + print("\n-- Pruned lessons (negative constraints — avoid these) --") + if not pruned: + print(" (none yet)") + for n in sorted(pruned, key=lambda x: x["id"]): + reason = n["metadata"].get("prune_reason", "") + print(f" {n['id']}: {n['hypothesis']}" + (f" — {reason}" if reason else "")) + + print("\n" + "=" * 72) + print( + "Next: Ideate children under a promising node, Select one or more frontier\n" + "leaves, Dispatch each to an executor subagent in an isolated worktree." + ) + + +def cmd_status(args): + """ASCII tree render — useful for reports and quick scans.""" + tree = _load_tree(args.run_dir) + run = _load_run(args.run_dir) + nodes = tree["nodes"] + + symbol = { + "root": "*", + "pending": "o", + "running": "~", + "executed": "=", + "merged": "V", + "pruned": "x", + } + + def render(nid, prefix=""): + n = nodes[nid] + sym = symbol.get(n["status"], "?") + best = " <== M_best" if nid == run["best_node"] else "" + print(f"{prefix}[{sym}] {nid} {n['hypothesis'][:70]}{_fmt_score(n, run)}{best}") + kids = sorted(_children(tree, nid)) + for i, c in enumerate(kids): + render(c, prefix + " ") + + print(f"Run: {run['objective']}") + print(f"Cycle {run['cycles_used']}/{run['budget_cycles']} | legend: * root o pending ~ running = executed V merged x pruned\n") + render(tree["root"]) + + +def cmd_validate(args): + """Check invariants — catch a corrupted or inconsistent tree early.""" + tree = _load_tree(args.run_dir) + run = _load_run(args.run_dir) + nodes = tree["nodes"] + problems = [] + if tree["root"] not in nodes: + problems.append("root id not present in nodes") + for nid, n in nodes.items(): + p = n.get("parent") + if p is not None and p not in nodes: + problems.append(f"{nid}: parent {p} missing") + if n["status"] not in VALID_STATUS: + problems.append(f"{nid}: invalid status {n['status']}") + if run["best_node"] and run["best_node"] not in nodes: + problems.append(f"best_node {run['best_node']} missing") + merged = [nid for nid, n in nodes.items() if n["status"] == "merged"] + if run["best_node"] and run["best_node"] not in merged and run["best_node"] != tree["root"]: + problems.append(f"best_node {run['best_node']} is not marked merged") + if problems: + print("INVALID:") + for p in problems: + print(f" - {p}") + sys.exit(1) + print(f"OK — {len(nodes)} nodes, root={tree['root']}, best={run['best_node']}") + + +# ---------------------------------------------------------------------------- +# CLI +# ---------------------------------------------------------------------------- + + +def build_parser(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--run-dir", default=".", help="Run directory holding .arbor/ (default: current dir)") + sub = p.add_subparsers(dest="command", required=True) + + s = sub.add_parser("init", help="Initialize a new AO run / hypothesis tree") + s.add_argument("--objective", required=True, help="Natural-language research objective (root hypothesis)") + s.add_argument("--dev-eval", required=True, help="Command/description of the development evaluator") + s.add_argument("--test-eval", required=True, help="Command/description of the held-out test evaluator") + s.add_argument("--material", default="", help="Path/ref to the initial artifact M_0") + s.add_argument("--metric-direction", choices=["max", "min"], default="max", help="Is higher or lower the better score?") + s.add_argument("--branching", type=int, default=3, help="Max children proposed per parent (k)") + s.add_argument("--max-depth", type=int, default=2, help="Max tree depth (directions at 1, interventions at 2+)") + s.add_argument("--budget", type=int, default=20, help="Coordinator cycle budget B") + s.add_argument("--force", action="store_true", help="Overwrite an existing run") + s.set_defaults(func=cmd_init) + + s = sub.add_parser("observe", help="Print the research-state projection (start of each cycle)") + s.set_defaults(func=cmd_observe) + + s = sub.add_parser("add-node", help="Add a pending child hypothesis under a parent (Ideate)") + s.add_argument("--parent", required=True, help="Parent node id (n0 for a new research direction)") + s.add_argument("--hypothesis", required=True, help="Falsifiable claim this node tests") + s.set_defaults(func=cmd_add_node) + + s = sub.add_parser("set-status", help="Set a node's status manually") + s.add_argument("--node", required=True) + s.add_argument("--status", required=True, help=f"One of {sorted(VALID_STATUS)}") + s.set_defaults(func=cmd_set_status) + + s = sub.add_parser("set-evidence", help="Write an executor report into its node (Backpropagate, leaf)") + s.add_argument("--node", required=True) + s.add_argument("--dev-score", type=float, default=None, help="Dev evaluator score returned by the executor") + s.add_argument("--result", default=None, help="Factual result summary") + s.add_argument("--insight", default=None, help="Distilled, reusable lesson from this experiment") + s.add_argument("--branch-ref", default=None, help="Git branch/commit/worktree path of the artifact") + s.add_argument("--status", default=None, help="Override status (default: executed)") + s.set_defaults(func=cmd_set_evidence) + + s = sub.add_parser("propagate", help="Abstract a leaf insight up to ancestors (Backpropagate, upward)") + s.add_argument("--node", required=True, help="The leaf the lesson came from") + s.add_argument("--insight", required=True, help="Direction-level abstraction of the lesson") + s.add_argument("--to-root", action="store_true", help="Also record as a global insight on the root") + s.set_defaults(func=cmd_propagate) + + s = sub.add_parser("prune", help="Prune a falsified node and its subtree (Decide)") + s.add_argument("--node", required=True) + s.add_argument("--reason", default="", help="Why this direction is a dead end (becomes a negative constraint)") + s.set_defaults(func=cmd_prune) + + s = sub.add_parser("merge", help="Record a held-out merge gate decision (Decide)") + s.add_argument("--node", required=True) + s.add_argument("--test-score", type=float, required=True, help="Score on the TEST evaluator in a fresh worktree") + s.add_argument("--branch-ref", default=None, help="Artifact ref to promote if it passes") + s.set_defaults(func=cmd_merge) + + s = sub.add_parser("cycle", help="Increment the coordinator cycle counter") + s.set_defaults(func=cmd_cycle) + + s = sub.add_parser("status", help="Render the tree as ASCII (for reports)") + s.set_defaults(func=cmd_status) + + s = sub.add_parser("validate", help="Check tree invariants") + s.set_defaults(func=cmd_validate) + + return p + + +def main(): + args = build_parser().parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/arboreto/SKILL.md b/.agents/skills/arboreto/SKILL.md new file mode 100644 index 0000000..b837e2d --- /dev/null +++ b/.agents/skills/arboreto/SKILL.md @@ -0,0 +1,267 @@ +--- +name: arboreto +description: Infer gene regulatory networks (GRNs) from gene expression data using scalable algorithms (GRNBoost2, GENIE3). Use when analyzing transcriptomics data (bulk RNA-seq, single-cell RNA-seq) to identify transcription factor-target gene relationships and regulatory interactions. Supports distributed computation for large-scale datasets. +license: BSD-3-Clause license +metadata: + version: "1.0" + skill-author: K-Dense Inc. +--- + +# Arboreto + +## Overview + +Arboreto is a Python library from [Aerts Lab](https://github.com/aertslab/arboreto) for inferring gene regulatory networks (GRNs) from gene expression data. It parallelizes tree-based ensemble regression (GRNBoost2, GENIE3) with [Dask](https://distributed.dask.org/) across local cores or remote clusters. + +**Core capability**: Identify which transcription factors (TFs) regulate which target genes based on expression patterns across observations (cells, samples, conditions). + +**Upstream**: PyPI **0.1.6** (2021-02-09, latest). Docs: [arboreto.readthedocs.io](https://arboreto.readthedocs.io/en/latest/). Primary downstream consumer: [pySCENIC](https://github.com/aertslab/pySCENIC). + +## Quick Start + +Install arboreto: +```bash +uv pip install arboreto +``` + +Basic GRN inference: +```python +import pandas as pd +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Load expression data (genes as columns) + expression_matrix = pd.read_csv('expression_data.tsv', sep='\t') + + # Infer regulatory network + network = grnboost2(expression_data=expression_matrix) + + # Save results (TF, target, importance) + network.to_csv('network.tsv', sep='\t', index=False, header=False) +``` + +**Critical**: Always use `if __name__ == '__main__':` guard because Dask spawns new processes. + +## Core Capabilities + +### 1. Basic GRN Inference + +For standard GRN inference workflows including: +- Input data preparation (Pandas DataFrame or NumPy array) +- Running inference with GRNBoost2 or GENIE3 +- Filtering by transcription factors +- Output format and interpretation + +**See**: `references/basic_inference.md` + +**Use the ready-to-run script**: `scripts/basic_grn_inference.py` for standard inference tasks: +```bash +python scripts/basic_grn_inference.py expression_data.tsv output_network.tsv --tf-file tfs.txt --seed 777 --limit 5000 +``` + +### 2. Algorithm Selection + +Arboreto provides two algorithms: + +**GRNBoost2 (Recommended)**: +- Fast gradient boosting-based inference +- Optimized for large datasets (10k+ observations) +- Default choice for most analyses + +**GENIE3**: +- Random Forest-based inference +- Original multiple regression approach +- Use for comparison or validation + +Quick comparison: +```python +from arboreto.algo import grnboost2, genie3 + +# Fast, recommended +network_grnboost = grnboost2(expression_data=matrix) + +# Classic algorithm +network_genie3 = genie3(expression_data=matrix) +``` + +**For detailed algorithm comparison, parameters, and selection guidance**: `references/algorithms.md` + +### 3. Distributed Computing + +Scale inference from local multi-core to cluster environments: + +**Local (default)** - Uses all available cores automatically: +```python +network = grnboost2(expression_data=matrix) +``` + +**Custom local client** - Control resources: +```python +from distributed import LocalCluster, Client + +local_cluster = LocalCluster(n_workers=10, memory_limit='8GB') +client = Client(local_cluster) + +network = grnboost2(expression_data=matrix, client_or_address=client) + +client.close() +local_cluster.close() +``` + +**Cluster computing** - Connect to remote Dask scheduler: +```python +from distributed import Client + +client = Client('tcp://scheduler:8786') +network = grnboost2(expression_data=matrix, client_or_address=client) +``` + +**For cluster setup, performance optimization, and large-scale workflows**: `references/distributed_computing.md` + +## Installation + +```bash +uv pip install arboreto +``` + +Conda (Bioconda): + +```bash +conda install -c bioconda arboreto +``` + +**Dependencies** (from upstream `requirements.txt`): `dask[complete]`, `distributed`, `numpy`, `pandas`, `scikit-learn`, `scipy` + +**Input formats**: pandas DataFrame, dense `numpy.ndarray`, or sparse `scipy.sparse.csc_matrix` (rows = observations, columns = genes). For array/matrix inputs, pass `gene_names` explicitly. + +## Common Use Cases + +### Single-Cell RNA-seq Analysis +```python +import pandas as pd +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Load single-cell expression matrix (cells x genes) + sc_data = pd.read_csv('scrna_counts.tsv', sep='\t') + + # Infer cell-type-specific regulatory network + network = grnboost2(expression_data=sc_data, seed=42) + + # Filter high-confidence links + high_confidence = network[network['importance'] > 0.5] + high_confidence.to_csv('grn_high_confidence.tsv', sep='\t', index=False) +``` + +### Bulk RNA-seq with TF Filtering +```python +from arboreto.utils import load_tf_names +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Load data + expression_data = pd.read_csv('rnaseq_tpm.tsv', sep='\t') + tf_names = load_tf_names('human_tfs.txt') + + # Infer with TF restriction + network = grnboost2( + expression_data=expression_data, + tf_names=tf_names, + seed=123 + ) + + network.to_csv('tf_target_network.tsv', sep='\t', index=False) +``` + +### Comparative Analysis (Multiple Conditions) +```python +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Infer networks for different conditions + conditions = ['control', 'treatment_24h', 'treatment_48h'] + + for condition in conditions: + data = pd.read_csv(f'{condition}_expression.tsv', sep='\t') + network = grnboost2(expression_data=data, seed=42) + network.to_csv(f'{condition}_network.tsv', sep='\t', index=False) +``` + +## Output Interpretation + +Arboreto returns a DataFrame with regulatory links: + +| Column | Description | +|--------|-------------| +| `TF` | Transcription factor (regulator) | +| `target` | Target gene | +| `importance` | Regulatory importance score (higher = stronger) | + +**Filtering strategy**: +- `limit=N` at inference time (return top N links globally) +- Post-hoc importance threshold (e.g., > 0.5) +- Top links per target via `groupby('target')` +- Statistical significance testing (permutation tests, external tools) + +## Integration with pySCENIC + +Arboreto powers the GRN inference step in [pySCENIC](https://github.com/aertslab/pySCENIC). pySCENIC 0.11+ passes sparse expression matrices to `grnboost2` / `genie3`; pySCENIC 0.12+ defaults to `arboreto_with_multiprocessing.py` (no Dask) for compatibility — use standalone arboreto when you need Dask scaling. + +```python +# Standalone: infer co-expression modules before pySCENIC cisTarget pruning +from arboreto.algo import grnboost2 + +network = grnboost2(expression_data=expression_df, tf_names=tf_list, limit=5000) + +# Downstream: pySCENIC ctx pruning, regulon definition, AUCell (see pySCENIC docs) +``` + +Convert AnnData to a DataFrame for arboreto directly: + +```python +expression_df = adata.to_df() # cells x genes +``` + +## Reproducibility + +Always set a seed for reproducible results: +```python +network = grnboost2(expression_data=matrix, seed=777) +``` + +Run multiple seeds for robustness analysis: +```python +from distributed import LocalCluster, Client + +if __name__ == '__main__': + client = Client(LocalCluster()) + + seeds = [42, 123, 777] + networks = [] + + for seed in seeds: + net = grnboost2(expression_data=matrix, client_or_address=client, seed=seed) + networks.append(net) + + # Consensus: links recurring across runs (example: mean importance per TF-target pair) + import pandas as pd + combined = pd.concat(networks) + consensus = ( + combined.groupby(['TF', 'target'], as_index=False)['importance'] + .mean() + .query('importance > 0.5') + ) +``` + +## Troubleshooting + +**Memory errors**: Reduce dataset size by filtering low-variance genes or use distributed computing + +**Slow performance**: Use GRNBoost2 instead of GENIE3, enable distributed client, filter TF list + +**Dask errors**: Ensure `if __name__ == '__main__':` guard is present in scripts (required on Windows/macOS with spawn-based multiprocessing) + +**Empty results**: Check data format (genes as columns), verify TF names match column names in the expression matrix + +**Sparse data**: Use `scipy.sparse.csc_matrix` and pass matching `gene_names`; supported since arboreto 0.1.6 / pySCENIC 0.11 + diff --git a/.agents/skills/arboreto/references/algorithms.md b/.agents/skills/arboreto/references/algorithms.md new file mode 100644 index 0000000..3a569a5 --- /dev/null +++ b/.agents/skills/arboreto/references/algorithms.md @@ -0,0 +1,152 @@ +# GRN Inference Algorithms + +Arboreto provides two high-level algorithms for gene regulatory network (GRN) inference, both based on the multiple regression approach. + +## Algorithm Overview + +Both algorithms follow the same inference strategy: +1. For each target gene in the dataset, train a regression model +2. Identify the most important features (potential regulators) from the model +3. Emit these features as candidate regulators with importance scores + +The key difference is **computational efficiency** and the underlying regression method. + +## GRNBoost2 (Recommended) + +**Purpose**: Fast GRN inference for large-scale datasets using gradient boosting. + +### When to Use +- **Large datasets**: Tens of thousands of observations (e.g., single-cell RNA-seq) +- **Time-constrained analysis**: Need faster results than GENIE3 +- **Default choice**: GRNBoost2 is the flagship algorithm and recommended for most use cases + +### Technical Details +- **Method**: Stochastic gradient boosting with early-stopping regularization +- **Performance**: Significantly faster than GENIE3 on large datasets +- **Output**: Same format as GENIE3 (TF-target-importance triplets) + +### Usage +```python +from arboreto.algo import grnboost2 + +network = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + seed=42, + limit=5000, +) +``` + +### Parameters (`grnboost2`) +```python +grnboost2( + expression_data, # DataFrame, ndarray, or scipy.sparse.csc_matrix + gene_names=None, # Required for ndarray/sparse inputs + tf_names='all', # TF list, None/'all' → all genes as regulators + client_or_address='local', # 'local', scheduler address, or Dask Client + early_stop_window_length=25, # Early-stopping window (GRNBoost2 only) + limit=None, # Return top N links globally + seed=None, # Random seed; None = non-deterministic + verbose=False, +) +``` + +## GENIE3 + +**Purpose**: Classic Random Forest-based GRN inference, serving as the conceptual blueprint. + +### When to Use +- **Smaller datasets**: When dataset size allows for longer computation +- **Comparison studies**: When comparing with published GENIE3 results +- **Validation**: To validate GRNBoost2 results + +### Technical Details +- **Method**: Random Forest regression (ExtraTrees available via `diy`) +- **Foundation**: Original multiple regression GRN inference strategy +- **Trade-off**: More computationally expensive but well-established + +### Usage +```python +from arboreto.algo import genie3 + +network = genie3( + expression_data=expression_matrix, + tf_names=tf_names, + seed=42, +) +``` + +### Parameters (`genie3`) +```python +genie3( + expression_data, + gene_names=None, + tf_names='all', + client_or_address='local', + limit=None, + seed=None, + verbose=False, +) +``` + +## Algorithm Comparison + +| Feature | GRNBoost2 | GENIE3 | +|---------|-----------|--------| +| **Speed** | Fast (optimized for large data) | Slower | +| **Method** | Gradient boosting (GBM) | Random Forest | +| **Best for** | Large-scale data (10k+ observations) | Small-medium datasets | +| **Output format** | Same | Same | +| **Inference strategy** | Multiple regression | Multiple regression | +| **Recommended** | Yes (default choice) | For comparison/validation | +| **Early stopping** | Yes (`early_stop_window_length`) | No | + +## Advanced: Custom Regressors with `diy` + +For custom scikit-learn regressor settings, use `diy()` (not `grnboost2`/`genie3` kwargs): + +```python +from arboreto.algo import diy +from arboreto.core import SGBM_KWARGS, RF_KWARGS + +# Custom GRNBoost2-style run +custom_gbm = diy( + expression_data=expression_matrix, + regressor_type='GBM', # 'RF', 'GBM', or 'ET' + regressor_kwargs={ + **SGBM_KWARGS, + 'n_estimators': 100, + 'max_depth': 5, + 'learning_rate': 0.1, + }, + tf_names=tf_names, + seed=42, +) + +# Custom GENIE3-style run +custom_rf = diy( + expression_data=expression_matrix, + regressor_type='RF', + regressor_kwargs={ + **RF_KWARGS, + 'n_estimators': 1000, + 'max_features': 'sqrt', + }, + tf_names=tf_names, +) +``` + +Import default kwargs from `arboreto.core` and override only the keys you need. + +## Choosing the Right Algorithm + +**Decision guide**: + +1. **Start with GRNBoost2** — faster and better suited to large single-cell datasets +2. **Use GENIE3 if**: + - Comparing with existing GENIE3 publications + - Dataset is small-medium sized + - Validating GRNBoost2 results +3. **Use `diy()` if** you need non-default regressor hyperparameters + +Both algorithms produce comparable regulatory networks with the same output format. diff --git a/.agents/skills/arboreto/references/basic_inference.md b/.agents/skills/arboreto/references/basic_inference.md new file mode 100644 index 0000000..a31b6a8 --- /dev/null +++ b/.agents/skills/arboreto/references/basic_inference.md @@ -0,0 +1,181 @@ +# Basic GRN Inference with Arboreto + +## Input Data Requirements + +Arboreto requires gene expression data in one of two formats: + +### Pandas DataFrame (Recommended) +- **Rows**: Observations (cells, samples, conditions) +- **Columns**: Genes (with gene names as column headers) +- **Format**: Numeric expression values + +Example: +```python +import pandas as pd + +# Load expression matrix with genes as columns +expression_matrix = pd.read_csv('expression_data.tsv', sep='\t') +# Columns: ['gene1', 'gene2', 'gene3', ...] +# Rows: observation data +``` + +### NumPy Array +- **Shape**: (observations, genes) +- **Requirement**: Separately provide gene names list matching column order + +Example: +```python +import numpy as np + +expression_matrix = np.genfromtxt('expression_data.tsv', delimiter='\t', skip_header=1) +with open('expression_data.tsv') as f: + gene_names = [gene.strip() for gene in f.readline().split('\t')] + +assert expression_matrix.shape[1] == len(gene_names) +``` + +### Sparse CSC Matrix (arboreto 0.1.6+) +- **Format**: `scipy.sparse.csc_matrix` with shape (observations, genes) +- **Requirement**: Provide `gene_names` matching column order (same as NumPy) +- **Use case**: Large single-cell matrices; also used by pySCENIC 0.11+ when `--sparse` is enabled + +Example: +```python +import scipy.sparse as sp +from arboreto.algo import grnboost2 + +# expression_sparse: csc_matrix, cells x genes +network = grnboost2( + expression_data=expression_sparse, + gene_names=gene_names, + tf_names=tf_names, +) +``` + +## Transcription Factors (TFs) + +Optionally provide a list of transcription factor names to restrict regulatory inference: + +```python +from arboreto.utils import load_tf_names + +# Load from file (one TF per line) +tf_names = load_tf_names('transcription_factors.txt') + +# Or define directly +tf_names = ['TF1', 'TF2', 'TF3'] +``` + +If `tf_names` is `None` or `'all'`, all `gene_names` are treated as potential regulators. + +## Basic Inference Workflow + +### Using Pandas DataFrame + +```python +import pandas as pd +from arboreto.utils import load_tf_names +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Load expression data + expression_matrix = pd.read_csv('expression_data.tsv', sep='\t') + + # Load transcription factors (optional) + tf_names = load_tf_names('tf_list.txt') + + # Run GRN inference + network = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names # Optional + ) + + # Save results + network.to_csv('network_output.tsv', sep='\t', index=False, header=False) +``` + +**Critical**: The `if __name__ == '__main__':` guard is required because Dask spawns new processes internally. + +### Using NumPy Array + +```python +import numpy as np +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Load expression matrix + expression_matrix = np.genfromtxt('expression_data.tsv', delimiter='\t', skip_header=1) + + # Extract gene names from header + with open('expression_data.tsv') as f: + gene_names = [gene.strip() for gene in f.readline().split('\t')] + + # Verify dimensions match + assert expression_matrix.shape[1] == len(gene_names) + + # Run inference with explicit gene names + network = grnboost2( + expression_data=expression_matrix, + gene_names=gene_names, + tf_names=tf_names + ) + + network.to_csv('network_output.tsv', sep='\t', index=False, header=False) +``` + +## Output Format + +Arboreto returns a Pandas DataFrame with three columns: + +| Column | Description | +|--------|-------------| +| `TF` | Transcription factor (regulator) gene name | +| `target` | Target gene name | +| `importance` | Regulatory importance score (higher = stronger regulation) | + +Example output: +``` +TF1 gene5 0.856 +TF2 gene12 0.743 +TF1 gene8 0.621 +``` + +## Setting Random Seed + +For reproducible results, pass an explicit `seed` (`None` uses random seeds per regressor): + +```python +network = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + seed=777 +) +``` + +## Limiting Output Size + +Return only the top N regulatory links globally: + +```python +network = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + limit=5000, +) +``` + +## Algorithm Selection + +Use `grnboost2()` for most cases (faster, handles large datasets): +```python +from arboreto.algo import grnboost2 +network = grnboost2(expression_data=expression_matrix) +``` + +Use `genie3()` for comparison or specific requirements: +```python +from arboreto.algo import genie3 +network = genie3(expression_data=expression_matrix) +``` + +See `references/algorithms.md` for detailed algorithm comparison. diff --git a/.agents/skills/arboreto/references/distributed_computing.md b/.agents/skills/arboreto/references/distributed_computing.md new file mode 100644 index 0000000..bb7f2db --- /dev/null +++ b/.agents/skills/arboreto/references/distributed_computing.md @@ -0,0 +1,242 @@ +# Distributed Computing with Arboreto + +Arboreto leverages Dask for parallelized computation, enabling efficient GRN inference from single-machine multi-core processing to multi-node cluster environments. + +## Computation Architecture + +GRN inference is inherently parallelizable: +- Each target gene's regression model can be trained independently +- Arboreto represents computation as a Dask task graph +- Tasks are distributed across available computational resources + +## Local Multi-Core Processing (Default) + +By default, arboreto uses all available CPU cores on the local machine: + +```python +from arboreto.algo import grnboost2 + +# Automatically uses all local cores +network = grnboost2(expression_data=expression_matrix, tf_names=tf_names) +``` + +This is sufficient for most use cases and requires no additional configuration. + +## Custom Local Dask Client + +For fine-grained control over local resources, create a custom Dask client: + +```python +from distributed import LocalCluster, Client +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Configure local cluster + local_cluster = LocalCluster( + n_workers=10, # Number of worker processes + threads_per_worker=1, # Threads per worker + memory_limit='8GB' # Memory limit per worker + ) + + # Create client + custom_client = Client(local_cluster) + + # Run inference with custom client + network = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + client_or_address=custom_client + ) + + # Clean up + custom_client.close() + local_cluster.close() +``` + +### Benefits of Custom Client +- **Resource control**: Limit CPU and memory usage +- **Multiple runs**: Reuse same client for different parameter sets +- **Monitoring**: Access Dask dashboard for performance insights + +## Multiple Inference Runs with Same Client + +Reuse a single Dask client for multiple inference runs with different parameters: + +```python +from distributed import LocalCluster, Client +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Initialize client once + local_cluster = LocalCluster(n_workers=8, threads_per_worker=1) + client = Client(local_cluster) + + # Run multiple inferences + network_seed1 = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + client_or_address=client, + seed=666 + ) + + network_seed2 = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + client_or_address=client, + seed=777 + ) + + # Different algorithms with same client + from arboreto.algo import genie3 + network_genie3 = genie3( + expression_data=expression_matrix, + tf_names=tf_names, + client_or_address=client + ) + + # Clean up once + client.close() + local_cluster.close() +``` + +## Distributed Cluster Computing + +For very large datasets, connect to a remote Dask distributed scheduler running on a cluster: + +### Step 1: Set Up Dask Scheduler (on cluster head node) +```bash +dask-scheduler +# Output: Scheduler at tcp://10.118.224.134:8786 +``` + +### Step 2: Start Dask Workers (on cluster compute nodes) +```bash +dask-worker tcp://10.118.224.134:8786 +``` + +### Step 3: Connect from Client +```python +from distributed import Client +from arboreto.algo import grnboost2 + +if __name__ == '__main__': + # Connect to remote scheduler + scheduler_address = 'tcp://10.118.224.134:8786' + cluster_client = Client(scheduler_address) + + # Run inference on cluster + network = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + client_or_address=cluster_client + ) + + cluster_client.close() +``` + +### Cluster Configuration Best Practices + +**Worker configuration**: +```bash +dask-worker tcp://scheduler:8786 \ + --nprocs 4 \ # Number of processes per node + --nthreads 1 \ # Threads per process + --memory-limit 16GB # Memory per process +``` + +**For large-scale inference**: +- Use more workers with moderate memory rather than fewer workers with large memory +- Set `threads_per_worker=1` to avoid GIL contention in scikit-learn +- Monitor memory usage to prevent workers from being killed + +## Monitoring and Debugging + +### Dask Dashboard + +Access the Dask dashboard for real-time monitoring: + +```python +from distributed import Client + +client = Client() # Prints dashboard URL +# Dashboard available at: http://localhost:8787/status +``` + +The dashboard shows: +- **Task progress**: Number of tasks completed/pending +- **Resource usage**: CPU, memory per worker +- **Task stream**: Real-time visualization of computation +- **Performance**: Bottleneck identification + +### Verbose Output + +Enable verbose logging to track inference progress: + +```python +network = grnboost2( + expression_data=expression_matrix, + tf_names=tf_names, + verbose=True +) +``` + +## Performance Optimization Tips + +### 1. Data Format +- **Use Pandas DataFrame when possible**: More efficient than NumPy for Dask operations +- **Reduce data size**: Filter low-variance genes before inference + +### 2. Worker Configuration +- **CPU-bound tasks**: Set `threads_per_worker=1`, increase `n_workers` +- **Memory-bound tasks**: Increase `memory_limit` per worker + +### 3. Cluster Setup +- **Network**: Ensure high-bandwidth, low-latency network between nodes +- **Storage**: Use shared filesystem or object storage for large datasets +- **Scheduling**: Allocate dedicated nodes to avoid resource contention + +### 4. Transcription Factor Filtering +- **Limit TF list**: Providing specific TF names reduces computation +```python +# Full search (slow) +network = grnboost2(expression_data=matrix) + +# Filtered search (faster) +network = grnboost2(expression_data=matrix, tf_names=known_tfs) +``` + +## Example: Large-Scale Single-Cell Analysis + +Complete workflow for processing single-cell RNA-seq data on a cluster: + +```python +from distributed import Client +from arboreto.algo import grnboost2 +import pandas as pd + +if __name__ == '__main__': + # Connect to cluster + client = Client('tcp://cluster-scheduler:8786') + + # Load large single-cell dataset (50,000 cells x 20,000 genes) + expression_data = pd.read_csv('scrnaseq_data.tsv', sep='\t') + + # Load cell-type-specific TFs + tf_names = pd.read_csv('tf_list.txt', header=None)[0].tolist() + + # Run distributed inference + network = grnboost2( + expression_data=expression_data, + tf_names=tf_names, + client_or_address=client, + verbose=True, + seed=42 + ) + + # Save results + network.to_csv('grn_results.tsv', sep='\t', index=False) + + client.close() +``` + +This approach enables analysis of datasets that would be impractical on a single machine. diff --git a/.agents/skills/arboreto/scripts/basic_grn_inference.py b/.agents/skills/arboreto/scripts/basic_grn_inference.py new file mode 100644 index 0000000..7b50c82 --- /dev/null +++ b/.agents/skills/arboreto/scripts/basic_grn_inference.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +""" +Basic GRN inference example using Arboreto. + +This script demonstrates the standard workflow for inferring gene regulatory +networks from expression data using GRNBoost2. + +Usage: + python basic_grn_inference.py [--tf-file TF_FILE] [--seed SEED] [--limit LIMIT] + +Arguments: + expression_file: Path to expression matrix (TSV format, genes as columns) + output_file: Path for output network (TSV format) + --tf-file: Optional path to transcription factors file (one per line) + --seed: Random seed for reproducibility (default: 777) + --limit: Return only the top N regulatory links (optional) +""" + +import argparse +import pandas as pd +from arboreto.algo import grnboost2 +from arboreto.utils import load_tf_names + + +def run_grn_inference(expression_file, output_file, tf_file=None, seed=777, limit=None): + """ + Run GRN inference using GRNBoost2. + + Args: + expression_file: Path to expression matrix TSV file + output_file: Path for output network file + tf_file: Optional path to TF names file + seed: Random seed for reproducibility + limit: Optional cap on number of regulatory links returned + """ + print(f"Loading expression data from {expression_file}...") + expression_data = pd.read_csv(expression_file, sep='\t') + + print(f"Expression matrix shape: {expression_data.shape}") + print(f"Number of genes: {expression_data.shape[1]}") + print(f"Number of observations: {expression_data.shape[0]}") + + # Load TF names if provided + tf_names = 'all' + if tf_file: + print(f"Loading transcription factors from {tf_file}...") + tf_names = load_tf_names(tf_file) + print(f"Number of TFs: {len(tf_names)}") + + # Run GRN inference + print(f"Running GRNBoost2 with seed={seed}...") + network = grnboost2( + expression_data=expression_data, + tf_names=tf_names, + seed=seed, + limit=limit, + verbose=True + ) + + # Save results + print(f"Saving network to {output_file}...") + network.to_csv(output_file, sep='\t', index=False, header=False) + + print(f"Done! Network contains {len(network)} regulatory links.") + print(f"\nTop 10 regulatory links:") + print(network.head(10).to_string(index=False)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Infer gene regulatory network using GRNBoost2' + ) + parser.add_argument( + 'expression_file', + help='Path to expression matrix (TSV format, genes as columns)' + ) + parser.add_argument( + 'output_file', + help='Path for output network (TSV format)' + ) + parser.add_argument( + '--tf-file', + help='Path to transcription factors file (one per line)', + default=None + ) + parser.add_argument( + '--seed', + help='Random seed for reproducibility (default: 777)', + type=int, + default=777 + ) + parser.add_argument( + '--limit', + help='Return only the top N regulatory links', + type=int, + default=None + ) + + args = parser.parse_args() + + run_grn_inference( + expression_file=args.expression_file, + output_file=args.output_file, + tf_file=args.tf_file, + seed=args.seed, + limit=args.limit + ) diff --git a/.agents/skills/astropy/SKILL.md b/.agents/skills/astropy/SKILL.md new file mode 100644 index 0000000..467fb85 --- /dev/null +++ b/.agents/skills/astropy/SKILL.md @@ -0,0 +1,353 @@ +--- +name: astropy +description: Core Python library for astronomy and astrophysics workflows that need Astropy APIs, including units/quantities, coordinates, FITS I/O, tables, time systems, WCS, and cosmology. Use when implementing or debugging astronomical data analysis code with Astropy. +license: BSD-3-Clause license +compatibility: Requires Python 3.11+ with astropy installed (uv for package installation). Some features (object name resolution, site lookups, remote FITS reads, IERS updates) need network access. +metadata: + version: "1.2" + skill-author: K-Dense Inc. +--- + +# Astropy + +## Overview + +Astropy is the core Python package for astronomy, providing essential functionality for astronomical research and data analysis. Use astropy for coordinate transformations, unit and quantity calculations, FITS file operations, cosmological calculations, precise time handling, tabular data manipulation, and astronomical image processing. + +## When to Use This Skill + +Use astropy when tasks involve: +- Converting between celestial coordinate systems (ICRS, Galactic, FK5, AltAz, etc.) +- Working with physical units and quantities (converting Jy to mJy, parsecs to km, etc.) +- Reading, writing, or manipulating FITS files (images or tables) +- Cosmological calculations (luminosity distance, lookback time, Hubble parameter) +- Precise time handling with different time scales (UTC, TAI, TT, TDB) and formats (JD, MJD, ISO) +- Table operations (reading catalogs, cross-matching, filtering, joining) +- WCS transformations between pixel and world coordinates +- Astronomical constants and calculations + +## Quick Start + +```python +import astropy.units as u +from astropy.coordinates import SkyCoord +from astropy.time import Time +from astropy.io import fits +from astropy.table import Table +from astropy.cosmology import Planck18 + +# Units and quantities +distance = 100 * u.pc +distance_km = distance.to(u.km) + +# Coordinates +coord = SkyCoord(ra=10.5*u.degree, dec=41.2*u.degree, frame='icrs') +coord_galactic = coord.galactic + +# Time +t = Time('2023-01-15 12:30:00') +jd = t.jd # Julian Date + +# FITS files +data = fits.getdata('image.fits') +header = fits.getheader('image.fits') + +# Tables +table = Table.read('catalog.fits') + +# Cosmology +d_L = Planck18.luminosity_distance(z=1.0) +``` + +## Core Capabilities + +### 1. Units and Quantities (`astropy.units`) + +Handle physical quantities with units, perform unit conversions, and ensure dimensional consistency in calculations. + +**Key operations:** +- Create quantities by multiplying values with units +- Convert between units using `.to()` method +- Perform arithmetic with automatic unit handling +- Use equivalencies for domain-specific conversions (spectral, doppler, parallax) +- Work with logarithmic units (magnitudes, decibels) + +**See:** `references/units.md` for comprehensive documentation, unit systems, equivalencies, performance optimization, and unit arithmetic. + +### 2. Coordinate Systems (`astropy.coordinates`) + +Represent celestial positions and transform between different coordinate frames. + +**Key operations:** +- Create coordinates with `SkyCoord` in any frame (ICRS, Galactic, FK5, AltAz, etc.) +- Transform between coordinate systems +- Calculate angular separations and position angles +- Match coordinates to catalogs +- Include distance for 3D coordinate operations +- Handle proper motions and radial velocities +- Query named objects from online databases + +**See:** `references/coordinates.md` for detailed coordinate frame descriptions, transformations, observer-dependent frames (AltAz), catalog matching, and performance tips. + +### 3. Cosmological Calculations (`astropy.cosmology`) + +Perform cosmological calculations using standard cosmological models. + +**Key operations:** +- Use built-in cosmologies (Planck18, WMAP9, etc.) +- Create custom cosmological models +- Calculate distances (luminosity, comoving, angular diameter) +- Compute ages and lookback times +- Determine Hubble parameter at any redshift +- Calculate density parameters and volumes +- Perform inverse calculations (find z for given distance) + +**See:** `references/cosmology.md` for available models, distance calculations, time calculations, density parameters, and neutrino effects. + +### 4. FITS File Handling (`astropy.io.fits`) + +Read, write, and manipulate FITS (Flexible Image Transport System) files. + +**Key operations:** +- Open FITS files with context managers +- Access HDUs (Header Data Units) by index or name +- Read and modify headers (keywords, comments, history) +- Work with image data (NumPy arrays) +- Handle table data (binary and ASCII tables) +- Create new FITS files (single or multi-extension) +- Use memory mapping for large files +- Access remote FITS files (S3, HTTP) + +**See:** `references/fits.md` for comprehensive file operations, header manipulation, image and table handling, multi-extension files, and performance considerations. + +### 5. Table Operations (`astropy.table`) + +Work with tabular data with support for units, metadata, and various file formats. + +**Key operations:** +- Create tables from arrays, lists, or dictionaries +- Read/write tables in multiple formats (FITS, CSV, HDF5, VOTable) +- Access and modify columns and rows +- Sort, filter, and index tables +- Perform database-style operations (join, group, aggregate) +- Stack and concatenate tables +- Work with unit-aware columns (QTable) +- Handle missing data with masking + +**See:** `references/tables.md` for table creation, I/O operations, data manipulation, sorting, filtering, joins, grouping, and performance tips. + +### 6. Time Handling (`astropy.time`) + +Precise time representation and conversion between time scales and formats. + +**Key operations:** +- Create Time objects in various formats (ISO, JD, MJD, Unix, etc.) +- Convert between time scales (UTC, TAI, TT, TDB, etc.) +- Perform time arithmetic with TimeDelta +- Calculate sidereal time for observers +- Compute light travel time corrections (barycentric, heliocentric) +- Work with time arrays efficiently +- Handle masked (missing) times + +**See:** `references/time.md` for time formats, time scales, conversions, arithmetic, observing features, and precision handling. + +### 7. World Coordinate System (`astropy.wcs`) + +Transform between pixel coordinates in images and world coordinates. + +**Key operations:** +- Read WCS from FITS headers +- Convert pixel coordinates to world coordinates (and vice versa) +- Calculate image footprints +- Access WCS parameters (reference pixel, projection, scale) +- Create custom WCS objects + +**See:** `references/wcs_and_other_modules.md` for WCS operations and transformations. + +## Additional Capabilities + +The `references/wcs_and_other_modules.md` file also covers: + +### NDData and CCDData +Containers for n-dimensional datasets with metadata, uncertainty, masking, and WCS information. + +### Modeling +Framework for creating and fitting mathematical models to astronomical data. + +### Visualization +Tools for astronomical image display with appropriate stretching and scaling. + +### Constants +Physical and astronomical constants with proper units (speed of light, solar mass, Planck constant, etc.). + +### Convolution +Image processing kernels for smoothing and filtering. + +### Statistics +Robust statistical functions including sigma clipping and outlier rejection. + +## Installation + +```bash +# Reproducible install against the current stable release +uv pip install "astropy==7.2.0" + +# Recommended optional dependencies for plotting and common workflows +uv pip install "astropy[recommended]==7.2.0" + +# Full optional dependency set for broad astronomy workflows +uv pip install "astropy[all]==7.2.0" +``` + +Astropy 7.2.0 requires Python 3.11+ and depends on NumPy, PyERFA, PyYAML, and packaging. Use an isolated virtual environment; do not install Astropy with elevated privileges. + +Note that the `[recommended]` and `[all]` extras pull in transitive dependencies (matplotlib, scipy, etc.) at unpinned versions. For reproducible production environments, pin the full dependency tree with a lockfile (`uv lock` in a project, or `uv pip compile` for requirements files) and review the resolved versions before deploying. + +## Common Workflows + +### Converting Coordinates Between Systems + +```python +from astropy.coordinates import SkyCoord +import astropy.units as u + +# Create coordinate +c = SkyCoord(ra='05h23m34.5s', dec='-69d45m22s', frame='icrs') + +# Transform to galactic +c_gal = c.galactic +print(f"l={c_gal.l.deg}, b={c_gal.b.deg}") + +# Transform to alt-az (requires time and location) +from astropy.time import Time +from astropy.coordinates import EarthLocation, AltAz + +observing_time = Time('2023-06-15 23:00:00') +observing_location = EarthLocation(lat=40*u.deg, lon=-120*u.deg) +aa_frame = AltAz(obstime=observing_time, location=observing_location) +c_altaz = c.transform_to(aa_frame) +print(f"Alt={c_altaz.alt.deg}, Az={c_altaz.az.deg}") +``` + +### Reading and Analyzing FITS Files + +```python +from astropy.io import fits +import numpy as np + +# Open FITS file +with fits.open('observation.fits') as hdul: + # Display structure + hdul.info() + + # Get image data and header + data = hdul[1].data + header = hdul[1].header + + # Access header values + exptime = header['EXPTIME'] + filter_name = header['FILTER'] + + # Analyze data + mean = np.mean(data) + median = np.median(data) + print(f"Mean: {mean}, Median: {median}") +``` + +### Cosmological Distance Calculations + +```python +from astropy.cosmology import Planck18 +import astropy.units as u +import numpy as np + +# Calculate distances at z=1.5 +z = 1.5 +d_L = Planck18.luminosity_distance(z) +d_A = Planck18.angular_diameter_distance(z) + +print(f"Luminosity distance: {d_L}") +print(f"Angular diameter distance: {d_A}") + +# Age of universe at that redshift +age = Planck18.age(z) +print(f"Age at z={z}: {age.to(u.Gyr)}") + +# Lookback time +t_lookback = Planck18.lookback_time(z) +print(f"Lookback time: {t_lookback.to(u.Gyr)}") +``` + +### Cross-Matching Catalogs + +```python +from astropy.table import Table +from astropy.coordinates import SkyCoord, match_coordinates_sky +import astropy.units as u + +# Read catalogs +cat1 = Table.read('catalog1.fits') +cat2 = Table.read('catalog2.fits') + +# Create coordinate objects +coords1 = SkyCoord(ra=cat1['RA']*u.degree, dec=cat1['DEC']*u.degree) +coords2 = SkyCoord(ra=cat2['RA']*u.degree, dec=cat2['DEC']*u.degree) + +# Find matches +idx, sep, _ = coords1.match_to_catalog_sky(coords2) + +# Filter by separation threshold +max_sep = 1 * u.arcsec +matches = sep < max_sep + +# Create matched catalogs +cat1_matched = cat1[matches] +cat2_matched = cat2[idx[matches]] +print(f"Found {len(cat1_matched)} matches") +``` + +## Best Practices + +1. **Always use units**: Attach units to quantities to avoid errors and ensure dimensional consistency +2. **Use context managers for FITS files**: Ensures proper file closing +3. **Prefer arrays over loops**: Process multiple coordinates/times as arrays for better performance +4. **Check coordinate frames**: Verify the frame before transformations +5. **Use appropriate cosmology**: Choose the right cosmological model for your analysis +6. **Handle missing data**: Use masked columns for tables with missing values +7. **Specify time scales**: Be explicit about time scales (UTC, TT, TDB) for precise timing +8. **Use QTable for unit-aware tables**: When table columns have units +9. **Check WCS validity**: Verify WCS before using transformations +10. **Cache frequently used values**: Expensive calculations (e.g., cosmological distances) can be cached +11. **Be explicit about network access**: `SkyCoord.from_name()`, `EarthLocation.of_site(refresh_cache=True)`, `EarthLocation.of_address()`, `download_file()`, remote FITS reads, and some IERS time/coordinate transforms can contact external services or update local caches. Avoid sending sensitive target names, addresses, URLs, or proprietary file locations to third-party services. When working with potentially sensitive targets or data locations, confirm with the user before making these network calls. +12. **Pin for reproducibility**: Use pinned versions such as `astropy==7.2.0` for shared environments; update pins intentionally after reviewing release notes. + +## Current-Version Notes + +- Current stable release researched: Astropy 7.2.0 (released 2025-11-25; verified current as of 2026-06-10) +- Python requirement: 3.11+ +- **Astropy 8.0 is at release-candidate stage** (8.0.0rc1, 2026-05-26). Key changes to anticipate: + - The deprecated `astropy.cosmology` submodule shims (`astropy.cosmology.flrw`, `.core`, `.funcs`, `.connect`, `.parameter`) are removed — import everything directly from `astropy.cosmology` (e.g., `from astropy.cosmology import FlatLambdaCDM, z_at_value`) + - `astropy.constants` defaults change from CODATA 2018 to CODATA 2022; pin a constants version via the `astropyconst` science states if reproducibility matters + - NumPy 2.0 becomes the minimum supported version; the 7.2.x LTS branch retains NumPy 1.x support for six months after the 8.0 release + - The built-in test runner (`astropy.test()`, `TestRunner`) is formally deprecated — invoke `pytest` directly +- Recent 7.x deprecations to avoid in new code: passing a table index identifier as the first `.loc` element (`t.loc["b", 2]`) — use `t.loc.with_index("b")[2]` instead (removal planned for 9.0); `astropy.utils.isiterable()` — use `numpy.iterable()` +- Recent 7.0 removals: older deprecated FITS APIs such as `(Bin)Table.update`, `_ExtensionHDU`, `_NonstandardExtHDU`, and the `tile_size` argument for `CompImageHDU`; `CompImageHeader` is deprecated. Avoid those legacy patterns in new examples. +- The recommended optional extras are `recommended` for common plotting/scientific dependencies and `all` only when a broad optional feature set is needed. + +## Documentation and Resources + +- Official Astropy Documentation: https://docs.astropy.org/en/stable/ +- Tutorials: https://learn.astropy.org/ +- GitHub: https://github.com/astropy/astropy + +## Reference Files + +For detailed information on specific modules: +- `references/units.md` - Units, quantities, conversions, and equivalencies +- `references/coordinates.md` - Coordinate systems, transformations, and catalog matching +- `references/cosmology.md` - Cosmological models and calculations +- `references/fits.md` - FITS file operations and manipulation +- `references/tables.md` - Table creation, I/O, and operations +- `references/time.md` - Time formats, scales, and calculations +- `references/wcs_and_other_modules.md` - WCS, NDData, modeling, visualization, constants, and utilities + diff --git a/.agents/skills/astropy/references/coordinates.md b/.agents/skills/astropy/references/coordinates.md new file mode 100644 index 0000000..ed75790 --- /dev/null +++ b/.agents/skills/astropy/references/coordinates.md @@ -0,0 +1,280 @@ +# Astronomical Coordinates (astropy.coordinates) + +The `astropy.coordinates` package provides tools for representing celestial coordinates and transforming between different coordinate systems. + +## Creating Coordinates with SkyCoord + +The high-level `SkyCoord` class is the recommended interface: + +```python +from astropy import units as u +from astropy.coordinates import SkyCoord + +# Decimal degrees +c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + +# Sexagesimal strings +c = SkyCoord(ra='00h42m30s', dec='+41d12m00s', frame='icrs') + +# Mixed formats +c = SkyCoord('00h42.5m +41d12m', unit=(u.hourangle, u.deg)) + +# Galactic coordinates +c = SkyCoord(l=120.5*u.degree, b=-23.4*u.degree, frame='galactic') +``` + +## Array Coordinates + +Process multiple coordinates efficiently using arrays: + +```python +# Create array of coordinates +coords = SkyCoord(ra=[10, 11, 12]*u.degree, + dec=[41, -5, 42]*u.degree) + +# Access individual elements +coords[0] +coords[1:3] + +# Array operations +coords.shape +len(coords) +``` + +## Accessing Components + +```python +c = SkyCoord(ra=10.68*u.degree, dec=41.27*u.degree, frame='icrs') + +# Access coordinates +c.ra # +c.dec # +c.ra.hour # Convert to hours +c.ra.hms # Hours, minutes, seconds tuple +c.dec.dms # Degrees, arcminutes, arcseconds tuple +``` + +## String Formatting + +```python +c.to_string('decimal') # '10.68 41.27' +c.to_string('dms') # '10d40m48s 41d16m12s' +c.to_string('hmsdms') # '00h42m43.2s +41d16m12s' + +# Custom formatting +c.ra.to_string(unit=u.hour, sep=':', precision=2) +``` + +## Coordinate Transformations + +Transform between reference frames: + +```python +c_icrs = SkyCoord(ra=10.68*u.degree, dec=41.27*u.degree, frame='icrs') + +# Simple transformations (as attributes) +c_galactic = c_icrs.galactic +c_fk5 = c_icrs.fk5 +c_fk4 = c_icrs.fk4 + +# Explicit transformations +c_icrs.transform_to('galactic') +c_icrs.transform_to(FK5(equinox='J1975')) # Custom frame parameters +``` + +## Common Coordinate Frames + +### Celestial Frames +- **ICRS**: International Celestial Reference System (default, most common) +- **FK5**: Fifth Fundamental Catalogue (equinox J2000.0 by default) +- **FK4**: Fourth Fundamental Catalogue (older, requires equinox specification) +- **GCRS**: Geocentric Celestial Reference System +- **CIRS**: Celestial Intermediate Reference System + +### Galactic Frames +- **Galactic**: IAU 1958 galactic coordinates +- **Supergalactic**: De Vaucouleurs supergalactic coordinates +- **Galactocentric**: Galactic center-based 3D coordinates + +### Horizontal Frames +- **AltAz**: Altitude-azimuth (observer-dependent) +- **HADec**: Hour angle-declination + +### Ecliptic Frames +- **GeocentricMeanEcliptic**: Geocentric mean ecliptic +- **BarycentricMeanEcliptic**: Barycentric mean ecliptic +- **HeliocentricMeanEcliptic**: Heliocentric mean ecliptic + +## Observer-Dependent Transformations + +For altitude-azimuth coordinates, specify observation time and location: + +```python +from astropy.time import Time +from astropy.coordinates import EarthLocation, AltAz + +# Define observer location +observing_location = EarthLocation(lat=40.8*u.deg, lon=-121.5*u.deg, height=1060*u.m) +# Or use named observatory +observing_location = EarthLocation.of_site('Apache Point Observatory') + +# Define observation time +observing_time = Time('2023-01-15 23:00:00') + +# Transform to alt-az +aa_frame = AltAz(obstime=observing_time, location=observing_location) +aa = c_icrs.transform_to(aa_frame) + +print(f"Altitude: {aa.alt}") +print(f"Azimuth: {aa.az}") +``` + +## Working with Distances + +Add distance information for 3D coordinates: + +```python +# With distance +c = SkyCoord(ra=10*u.degree, dec=9*u.degree, distance=770*u.kpc, frame='icrs') + +# Access 3D Cartesian coordinates +c.cartesian.x +c.cartesian.y +c.cartesian.z + +# Distance from origin +c.distance + +# 3D separation +c1 = SkyCoord(ra=10*u.degree, dec=9*u.degree, distance=10*u.pc) +c2 = SkyCoord(ra=11*u.degree, dec=10*u.degree, distance=11.5*u.pc) +sep_3d = c1.separation_3d(c2) # 3D distance +``` + +## Angular Separation + +Calculate on-sky separations: + +```python +c1 = SkyCoord(ra=10*u.degree, dec=9*u.degree, frame='icrs') +c2 = SkyCoord(ra=11*u.degree, dec=10*u.degree, frame='fk5') + +# Angular separation (handles frame conversion automatically) +sep = c1.separation(c2) +print(f"Separation: {sep.arcsec} arcsec") + +# Position angle +pa = c1.position_angle(c2) +``` + +## Catalog Matching + +Match coordinates to catalog sources: + +```python +# Single target matching +catalog = SkyCoord(ra=ra_array*u.degree, dec=dec_array*u.degree) +target = SkyCoord(ra=10.5*u.degree, dec=41.2*u.degree) + +# Find closest match +idx, sep2d, dist3d = target.match_to_catalog_sky(catalog) +matched_coord = catalog[idx] + +# Match with maximum separation constraint +matches = target.separation(catalog) < 1*u.arcsec +``` + +## Named Objects + +Retrieve coordinates from online catalogs: + +**Network note:** `SkyCoord.from_name()` sends the object name to remote name-resolution services such as Sesame/SIMBAD/NED. Do not use it with confidential target names or proprietary survey identifiers; use explicit coordinates when privacy or reproducibility matters. + +```python +# Query by name (requires internet) +m31 = SkyCoord.from_name("M31") +crab = SkyCoord.from_name("Crab Nebula") +psr = SkyCoord.from_name("PSR J1012+5307") +``` + +## Earth Locations + +Define observer locations: + +**Network note:** `EarthLocation.of_site()` normally uses the bundled site registry, but `refresh_cache=True` downloads an updated registry. `EarthLocation.of_address()` sends the address to a geocoding service, so prefer explicit latitude/longitude/height for sensitive sites. + +```python +# By coordinates +location = EarthLocation(lat=40*u.deg, lon=-120*u.deg, height=1000*u.m) + +# By named observatory +keck = EarthLocation.of_site('Keck Observatory') +vlt = EarthLocation.of_site('Paranal Observatory') + +# Force a fresh observatory registry only when network access is acceptable +keck = EarthLocation.of_site('Keck Observatory', refresh_cache=True) + +# By address (requires internet) +location = EarthLocation.of_address('1 Observatory Road, Example City') + +# List available observatories +EarthLocation.get_site_names() +``` + +## Velocity Information + +Include proper motion and radial velocity: + +```python +# Proper motion +c = SkyCoord(ra=10*u.degree, dec=41*u.degree, + pm_ra_cosdec=15*u.mas/u.yr, + pm_dec=5*u.mas/u.yr, + distance=150*u.pc) + +# Radial velocity +c = SkyCoord(ra=10*u.degree, dec=41*u.degree, + radial_velocity=20*u.km/u.s) + +# Both +c = SkyCoord(ra=10*u.degree, dec=41*u.degree, distance=150*u.pc, + pm_ra_cosdec=15*u.mas/u.yr, pm_dec=5*u.mas/u.yr, + radial_velocity=20*u.km/u.s) +``` + +## Representation Types + +Switch between coordinate representations: + +```python +# Cartesian representation +c = SkyCoord(x=1*u.kpc, y=2*u.kpc, z=3*u.kpc, + representation_type='cartesian', frame='icrs') + +# Change representation +c.representation_type = 'cylindrical' +c.rho # Cylindrical radius +c.phi # Azimuthal angle +c.z # Height + +# Spherical (default for most frames) +c.representation_type = 'spherical' +``` + +## Performance Tips + +1. **Use arrays, not loops**: Process multiple coordinates as single array +2. **Pre-compute frames**: Reuse frame objects for multiple transformations +3. **Use broadcasting**: Efficiently transform many positions across many times +4. **Enable interpolation**: For dense time sampling, use ErfaAstromInterpolator + +```python +# Fast approach +coords = SkyCoord(ra=ra_array*u.degree, dec=dec_array*u.degree) +coords_transformed = coords.transform_to('galactic') + +# Slow approach (avoid) +for ra, dec in zip(ra_array, dec_array): + c = SkyCoord(ra=ra*u.degree, dec=dec*u.degree) + c_transformed = c.transform_to('galactic') +``` diff --git a/.agents/skills/astropy/references/cosmology.md b/.agents/skills/astropy/references/cosmology.md new file mode 100644 index 0000000..cade2c7 --- /dev/null +++ b/.agents/skills/astropy/references/cosmology.md @@ -0,0 +1,309 @@ +# Cosmological Calculations (astropy.cosmology) + +The `astropy.cosmology` subpackage provides tools for cosmological calculations based on various cosmological models. + +> **Import paths:** Always import classes and functions directly from `astropy.cosmology` (as shown in all examples below). The old submodule import paths (`astropy.cosmology.flrw`, `.core`, `.funcs`, `.connect`, `.parameter`) were deprecated in v7.1 and removed in Astropy 8.0. + +## Using Built-in Cosmologies + +Preloaded cosmologies based on WMAP and Planck observations: + +```python +from astropy.cosmology import Planck18, Planck15, Planck13 +from astropy.cosmology import WMAP9, WMAP7, WMAP5 +from astropy import units as u + +# Use Planck 2018 cosmology +cosmo = Planck18 + +# Calculate distance to z=4 +d = cosmo.luminosity_distance(4) +print(f"Luminosity distance at z=4: {d}") + +# Age of universe at z=0 +age = cosmo.age(0) +print(f"Current age of universe: {age.to(u.Gyr)}") +``` + +## Creating Custom Cosmologies + +### FlatLambdaCDM (Most Common) + +Flat universe with cosmological constant: + +```python +from astropy.cosmology import FlatLambdaCDM + +# Define cosmology +cosmo = FlatLambdaCDM( + H0=70 * u.km / u.s / u.Mpc, # Hubble constant at z=0 + Om0=0.3, # Matter density parameter at z=0 + Tcmb0=2.725 * u.K # CMB temperature (optional) +) +``` + +### LambdaCDM (Non-Flat) + +Non-flat universe with cosmological constant: + +```python +from astropy.cosmology import LambdaCDM + +cosmo = LambdaCDM( + H0=70 * u.km / u.s / u.Mpc, + Om0=0.3, + Ode0=0.7 # Dark energy density parameter +) +``` + +### wCDM and w0wzCDM + +Dark energy with equation of state parameter: + +```python +from astropy.cosmology import FlatwCDM, w0wzCDM + +# Constant w +cosmo_w = FlatwCDM(H0=70 * u.km/u.s/u.Mpc, Om0=0.3, w0=-0.9) + +# Evolving w(z) = w0 + wz * z +cosmo_wz = w0wzCDM(H0=70 * u.km/u.s/u.Mpc, Om0=0.3, Ode0=0.7, + w0=-1.0, wz=0.1) +``` + +## Distance Calculations + +### Comoving Distance + +Line-of-sight comoving distance: + +```python +d_c = cosmo.comoving_distance(z) +``` + +### Luminosity Distance + +Distance for calculating luminosity from observed flux: + +```python +d_L = cosmo.luminosity_distance(z) + +# Calculate absolute magnitude from apparent magnitude +M = m - 5*np.log10(d_L.to(u.pc).value) + 5 +``` + +### Angular Diameter Distance + +Distance for calculating physical size from angular size: + +```python +d_A = cosmo.angular_diameter_distance(z) + +# Calculate physical size from angular size +theta = 10 * u.arcsec # Angular size +physical_size = d_A * theta.to(u.radian).value +``` + +### Comoving Transverse Distance + +Transverse comoving distance (equals comoving distance in flat universe): + +```python +d_M = cosmo.comoving_transverse_distance(z) +``` + +### Distance Modulus + +```python +dm = cosmo.distmod(z) +# Relates apparent and absolute magnitudes: m - M = dm +``` + +## Scale Calculations + +### kpc per Arcminute + +Physical scale at a given redshift: + +```python +scale = cosmo.kpc_proper_per_arcmin(z) +# e.g., "50 kpc per arcminute at z=1" +``` + +### Comoving Volume + +Volume element for survey volume calculations: + +```python +vol = cosmo.comoving_volume(z) # Total volume to redshift z +vol_element = cosmo.differential_comoving_volume(z) # dV/dz +``` + +## Time Calculations + +### Age of Universe + +Age at a given redshift: + +```python +age = cosmo.age(z) +age_now = cosmo.age(0) # Current age +age_at_z1 = cosmo.age(1) # Age at z=1 +``` + +### Lookback Time + +Time since photons were emitted: + +```python +t_lookback = cosmo.lookback_time(z) +# Time between z and z=0 +``` + +## Hubble Parameter + +Hubble parameter as function of redshift: + +```python +H_z = cosmo.H(z) # H(z) in km/s/Mpc +E_z = cosmo.efunc(z) # E(z) = H(z)/H0 +``` + +## Density Parameters + +Evolution of density parameters with redshift: + +```python +Om_z = cosmo.Om(z) # Matter density at z +Ode_z = cosmo.Ode(z) # Dark energy density at z +Ok_z = cosmo.Ok(z) # Curvature density at z +Ogamma_z = cosmo.Ogamma(z) # Photon density at z +Onu_z = cosmo.Onu(z) # Neutrino density at z +``` + +## Critical and Characteristic Densities + +```python +rho_c = cosmo.critical_density(z) # Critical density at z +rho_m = cosmo.critical_density(z) * cosmo.Om(z) # Matter density +``` + +## Inverse Calculations + +Find redshift corresponding to a specific value: + +```python +from astropy.cosmology import z_at_value + +# Find z at specific lookback time +z = z_at_value(cosmo.lookback_time, 10*u.Gyr) + +# Find z at specific luminosity distance +z = z_at_value(cosmo.luminosity_distance, 1000*u.Mpc) + +# Find z at specific age +z = z_at_value(cosmo.age, 1*u.Gyr) +``` + +## Array Operations + +All methods accept array inputs: + +```python +import numpy as np + +z_array = np.linspace(0, 5, 100) +d_L_array = cosmo.luminosity_distance(z_array) +H_array = cosmo.H(z_array) +age_array = cosmo.age(z_array) +``` + +## Neutrino Effects + +Include massive neutrinos: + +```python +from astropy.cosmology import FlatLambdaCDM + +# With massive neutrinos +cosmo = FlatLambdaCDM( + H0=70 * u.km/u.s/u.Mpc, + Om0=0.3, + Tcmb0=2.725 * u.K, + Neff=3.04, # Effective number of neutrino species + m_nu=[0., 0., 0.06] * u.eV # Neutrino masses +) +``` + +Note: Massive neutrinos reduce performance by 3-4x but provide more accurate results. + +## Cloning and Modifying Cosmologies + +Cosmology objects are immutable. Create modified copies: + +```python +# Clone with different H0 +cosmo_new = cosmo.clone(H0=72 * u.km/u.s/u.Mpc) + +# Clone with modified name +cosmo_named = cosmo.clone(name="My Custom Cosmology") +``` + +## Common Use Cases + +### Calculating Absolute Magnitude + +```python +# From apparent magnitude and redshift +z = 1.5 +m_app = 24.5 # Apparent magnitude +d_L = cosmo.luminosity_distance(z) +M_abs = m_app - cosmo.distmod(z).value +``` + +### Survey Volume Calculations + +```python +# Volume between two redshifts +z_min, z_max = 0.5, 1.5 +volume = cosmo.comoving_volume(z_max) - cosmo.comoving_volume(z_min) + +# Convert to Gpc^3 +volume_gpc3 = volume.to(u.Gpc**3) +``` + +### Physical Size from Angular Size + +```python +theta = 1 * u.arcsec # Angular size +z = 2.0 +d_A = cosmo.angular_diameter_distance(z) +size_kpc = (d_A * theta.to(u.radian)).to(u.kpc) +``` + +### Time Since Big Bang + +```python +# Age at specific redshift +z_formation = 6 +age_at_formation = cosmo.age(z_formation) +time_since_formation = cosmo.age(0) - age_at_formation +``` + +## Comparison of Cosmologies + +```python +# Compare different models +from astropy.cosmology import Planck18, WMAP9 + +z = 1.0 +print(f"Planck18 d_L: {Planck18.luminosity_distance(z)}") +print(f"WMAP9 d_L: {WMAP9.luminosity_distance(z)}") +``` + +## Performance Considerations + +- Calculations are fast for most purposes +- Massive neutrinos reduce speed significantly +- Array operations are vectorized and efficient +- Results valid for z < 5000-6000 (depends on model) diff --git a/.agents/skills/astropy/references/fits.md b/.agents/skills/astropy/references/fits.md new file mode 100644 index 0000000..97a21d7 --- /dev/null +++ b/.agents/skills/astropy/references/fits.md @@ -0,0 +1,398 @@ +# FITS File Handling (astropy.io.fits) + +The `astropy.io.fits` module provides comprehensive tools for reading, writing, and manipulating FITS (Flexible Image Transport System) files. + +## Opening FITS Files + +### Basic File Opening + +```python +from astropy.io import fits + +# Open file (returns HDUList - list of HDUs) +hdul = fits.open('filename.fits') + +# Always close when done +hdul.close() + +# Better: use context manager (automatically closes) +with fits.open('filename.fits') as hdul: + hdul.info() # Display file structure + data = hdul[0].data +``` + +### File Opening Modes + +```python +fits.open('file.fits', mode='readonly') # Read-only (default) +fits.open('file.fits', mode='update') # Read and write +fits.open('file.fits', mode='append') # Add HDUs to file +``` + +### Memory Mapping + +For large files, use memory mapping (default behavior): + +```python +hdul = fits.open('large_file.fits', memmap=True) +# Only loads data chunks as needed +``` + +### Remote Files + +Access cloud-hosted FITS files: + +**Network note:** Remote FITS reads disclose the target URI to the storage provider and may use credentials from the configured filesystem backend. Use anonymous reads only for public data, and prefer local files for proprietary observations. + +```python +uri = "s3://bucket-name/image.fits" +with fits.open(uri, use_fsspec=True, fsspec_kwargs={"anon": True}) as hdul: + # Use .section to get cutouts without downloading entire file + cutout = hdul[1].section[100:200, 100:200] +``` + +## HDU Structure + +FITS files contain Header Data Units (HDUs): +- **Primary HDU** (`hdul[0]`): First HDU, always present +- **Extension HDUs** (`hdul[1:]`): Image or table extensions + +```python +hdul.info() # Display all HDUs +# Output: +# No. Name Ver Type Cards Dimensions Format +# 0 PRIMARY 1 PrimaryHDU 220 () +# 1 SCI 1 ImageHDU 140 (1014, 1014) float32 +# 2 ERR 1 ImageHDU 51 (1014, 1014) float32 +``` + +## Accessing HDUs + +```python +# By index +primary = hdul[0] +extension1 = hdul[1] + +# By name +sci = hdul['SCI'] + +# By name and version number +sci2 = hdul['SCI', 2] # Second SCI extension +``` + +## Working with Headers + +### Reading Header Values + +```python +hdu = hdul[0] +header = hdu.header + +# Get keyword value (case-insensitive) +observer = header['OBSERVER'] +exptime = header['EXPTIME'] + +# Get with default if missing +filter_name = header.get('FILTER', 'Unknown') + +# Access by index +value = header[7] # 8th card's value +``` + +### Modifying Headers + +```python +# Update existing keyword +header['OBSERVER'] = 'Edwin Hubble' + +# Add/update with comment +header['OBSERVER'] = ('Edwin Hubble', 'Name of observer') + +# Add keyword at specific position +header.insert(5, ('NEWKEY', 'value', 'comment')) + +# Add HISTORY and COMMENT +header['HISTORY'] = 'File processed on 2025-01-15' +header['COMMENT'] = 'Note about the data' + +# Delete keyword +del header['OLDKEY'] +``` + +### Header Cards + +Each keyword is stored as a "card" (80-character record): + +```python +# Access full card +card = header.cards[0] +print(f"{card.keyword} = {card.value} / {card.comment}") + +# Iterate over all cards +for card in header.cards: + print(f"{card.keyword}: {card.value}") +``` + +## Working with Image Data + +### Reading Image Data + +```python +# Get data from HDU +data = hdul[1].data # Returns NumPy array + +# Data properties +print(data.shape) # e.g., (1024, 1024) +print(data.dtype) # e.g., float32 +print(data.min(), data.max()) + +# Access specific pixels +pixel_value = data[100, 200] +region = data[100:200, 300:400] +``` + +### Data Operations + +Data is a NumPy array, so use standard NumPy operations: + +```python +import numpy as np + +# Statistics +mean = np.mean(data) +median = np.median(data) +std = np.std(data) + +# Modify data +data[data < 0] = 0 # Clip negative values +data = data * gain + bias # Calibration + +# Mathematical operations +log_data = np.log10(data) +smoothed = scipy.ndimage.gaussian_filter(data, sigma=2) +``` + +### Cutouts and Sections + +Extract regions without loading entire array: + +```python +# Section notation [y_start:y_end, x_start:x_end] +cutout = hdul[1].section[500:600, 700:800] +``` + +## Creating New FITS Files + +### Simple Image File + +```python +# Create data +data = np.random.random((100, 100)) + +# Create HDU +hdu = fits.PrimaryHDU(data=data) + +# Add header keywords +hdu.header['OBJECT'] = 'Test Image' +hdu.header['EXPTIME'] = 300.0 + +# Write to file +hdu.writeto('new_image.fits') + +# Overwrite if exists +hdu.writeto('new_image.fits', overwrite=True) +``` + +### Multi-Extension File + +```python +# Create primary HDU (can have no data) +primary = fits.PrimaryHDU() +primary.header['TELESCOP'] = 'HST' + +# Create image extensions +sci_data = np.ones((100, 100)) +sci = fits.ImageHDU(data=sci_data, name='SCI') + +err_data = np.ones((100, 100)) * 0.1 +err = fits.ImageHDU(data=err_data, name='ERR') + +# Combine into HDUList +hdul = fits.HDUList([primary, sci, err]) + +# Write to file +hdul.writeto('multi_extension.fits') +``` + +## Working with Table Data + +### Reading Tables + +```python +# Open table +with fits.open('table.fits') as hdul: + table = hdul[1].data # BinTableHDU or TableHDU + + # Access columns + ra = table['RA'] + dec = table['DEC'] + mag = table['MAG'] + + # Access rows + first_row = table[0] + subset = table[10:20] + + # Column info + cols = hdul[1].columns + print(cols.names) + cols.info() +``` + +### Creating Tables + +```python +# Define columns +col1 = fits.Column(name='ID', format='K', array=[1, 2, 3, 4]) +col2 = fits.Column(name='RA', format='D', array=[10.5, 11.2, 12.3, 13.1]) +col3 = fits.Column(name='DEC', format='D', array=[41.2, 42.1, 43.5, 44.2]) +col4 = fits.Column(name='Name', format='20A', + array=['Star1', 'Star2', 'Star3', 'Star4']) + +# Create table HDU +table_hdu = fits.BinTableHDU.from_columns([col1, col2, col3, col4]) +table_hdu.name = 'CATALOG' + +# Write to file +table_hdu.writeto('catalog.fits', overwrite=True) +``` + +### Column Formats + +Common FITS table column formats: +- `'A'`: Character string (e.g., '20A' for 20 characters) +- `'L'`: Logical (boolean) +- `'B'`: Unsigned byte +- `'I'`: 16-bit integer +- `'J'`: 32-bit integer +- `'K'`: 64-bit integer +- `'E'`: 32-bit floating point +- `'D'`: 64-bit floating point + +## Modifying Existing Files + +### Update Mode + +```python +with fits.open('file.fits', mode='update') as hdul: + # Modify header + hdul[0].header['NEWKEY'] = 'value' + + # Modify data + hdul[1].data[100, 100] = 999 + + # Changes automatically saved when context exits +``` + +### Append Mode + +```python +# Add new extension to existing file +new_data = np.random.random((50, 50)) +new_hdu = fits.ImageHDU(data=new_data, name='NEW_EXT') + +with fits.open('file.fits', mode='append') as hdul: + hdul.append(new_hdu) +``` + +## Convenience Functions + +For quick operations without managing HDU lists: + +```python +# Get data only +data = fits.getdata('file.fits', ext=1) + +# Get header only +header = fits.getheader('file.fits', ext=0) + +# Get both +data, header = fits.getdata('file.fits', ext=1, header=True) + +# Get single keyword value +exptime = fits.getval('file.fits', 'EXPTIME', ext=0) + +# Set keyword value +fits.setval('file.fits', 'NEWKEY', value='newvalue', ext=0) + +# Write simple file +fits.writeto('output.fits', data, header, overwrite=True) + +# Append to file +fits.append('file.fits', data, header) + +# Display file info +fits.info('file.fits') +``` + +## Comparing FITS Files + +```python +# Print differences between two files +fits.printdiff('file1.fits', 'file2.fits') + +# Compare programmatically +diff = fits.FITSDiff('file1.fits', 'file2.fits') +print(diff.report()) +``` + +## Converting Between Formats + +### FITS to/from Astropy Table + +```python +from astropy.table import Table + +# FITS to Table +table = Table.read('catalog.fits') + +# Table to FITS +table.write('output.fits', format='fits', overwrite=True) +``` + +## Best Practices + +1. **Always use context managers** (`with` statements) for safe file handling +2. **Avoid modifying structural keywords** (SIMPLE, BITPIX, NAXIS, etc.) +3. **Use memory mapping** for large files to conserve RAM +4. **Use .section** for remote files to avoid full downloads +5. **Check HDU structure** with `.info()` before accessing data +6. **Verify data types** before operations to avoid unexpected behavior +7. **Use convenience functions** for simple one-off operations + +## Common Issues + +### Handling Non-Standard FITS + +Some files violate FITS standards: + +```python +# Ignore verification warnings +hdul = fits.open('bad_file.fits', ignore_missing_end=True) + +# Fix non-standard files +hdul = fits.open('bad_file.fits') +hdul.verify('fix') # Try to fix issues +hdul.writeto('fixed_file.fits') +``` + +### Large File Performance + +```python +# Use memory mapping (default) +hdul = fits.open('huge_file.fits', memmap=True) + +# For write operations with large arrays, use Dask +import dask.array as da +large_array = da.random.random((10000, 10000)) +fits.writeto('output.fits', large_array) +``` diff --git a/.agents/skills/astropy/references/tables.md b/.agents/skills/astropy/references/tables.md new file mode 100644 index 0000000..e99dce3 --- /dev/null +++ b/.agents/skills/astropy/references/tables.md @@ -0,0 +1,495 @@ +# Table Operations (astropy.table) + +The `astropy.table` module provides flexible tools for working with tabular data, with support for units, masked values, and various file formats. + +## Creating Tables + +### Basic Table Creation + +```python +from astropy.table import Table, QTable +import astropy.units as u +import numpy as np + +# From column arrays +a = [1, 4, 5] +b = [2.0, 5.0, 8.2] +c = ['x', 'y', 'z'] + +t = Table([a, b, c], names=('id', 'flux', 'name')) + +# With units (use QTable) +flux = [1.2, 2.3, 3.4] * u.Jy +wavelength = [500, 600, 700] * u.nm +t = QTable([flux, wavelength], names=('flux', 'wavelength')) +``` + +### From Lists of Rows + +```python +# List of tuples +rows = [(1, 10.5, 'A'), (2, 11.2, 'B'), (3, 12.3, 'C')] +t = Table(rows=rows, names=('id', 'value', 'name')) + +# List of dictionaries +rows = [{'id': 1, 'value': 10.5}, {'id': 2, 'value': 11.2}] +t = Table(rows) +``` + +### From NumPy Arrays + +```python +# Structured array +arr = np.array([(1, 2.0, 'x'), (4, 5.0, 'y')], + dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'U10')]) +t = Table(arr) + +# 2D array with column names +data = np.random.random((100, 3)) +t = Table(data, names=['col1', 'col2', 'col3']) +``` + +### From Pandas DataFrame + +```python +import pandas as pd + +df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]}) +t = Table.from_pandas(df) +``` + +## Accessing Table Data + +### Basic Access + +```python +# Column access +ra_col = t['ra'] # Returns Column object +dec_col = t['dec'] + +# Row access +first_row = t[0] # Returns Row object +row_slice = t[10:20] # Returns new Table + +# Cell access +value = t['ra'][5] # 6th value in 'ra' column +value = t[5]['ra'] # Same thing + +# Multiple columns +subset = t['ra', 'dec', 'mag'] +``` + +### Table Properties + +```python +len(t) # Number of rows +t.colnames # List of column names +t.dtype # Column data types +t.info # Detailed information +t.meta # Metadata dictionary +``` + +### Iteration + +```python +# Iterate over rows +for row in t: + print(row['ra'], row['dec']) + +# Iterate over columns +for colname in t.colnames: + print(t[colname]) +``` + +## Modifying Tables + +### Adding Columns + +```python +# Add new column +t['new_col'] = [1, 2, 3, 4, 5] +t['calc'] = t['a'] + t['b'] # Calculated column + +# Add column with units +t['velocity'] = [10, 20, 30] * u.km / u.s + +# Add empty column +from astropy.table import Column +t['empty'] = Column(length=len(t), dtype=float) + +# Insert at specific position +t.add_column([7, 8, 9], name='inserted', index=2) +``` + +### Removing Columns + +```python +# Remove single column +t.remove_column('old_col') + +# Remove multiple columns +t.remove_columns(['col1', 'col2']) + +# Delete syntax +del t['col_name'] + +# Keep only specific columns +t.keep_columns(['ra', 'dec', 'mag']) +``` + +### Renaming Columns + +```python +t.rename_column('old_name', 'new_name') + +# Rename multiple +t.rename_columns(['old1', 'old2'], ['new1', 'new2']) +``` + +### Adding Rows + +```python +# Add single row +t.add_row([1, 2.5, 'new']) + +# Add row as dict +t.add_row({'ra': 10.5, 'dec': 41.2, 'mag': 18.5}) + +# Note: Adding rows one at a time is slow! +# Better to collect rows and create table at once +``` + +### Modifying Data + +```python +# Modify column values +t['flux'] = t['flux'] * gain +t['mag'][t['mag'] < 0] = np.nan + +# Modify single cell +t['ra'][5] = 10.5 + +# Modify entire row +t[0] = [new_id, new_ra, new_dec] +``` + +## Sorting and Filtering + +### Sorting + +```python +# Sort by single column +t.sort('mag') + +# Sort descending +t.sort('mag', reverse=True) + +# Sort by multiple columns +t.sort(['priority', 'mag']) + +# Get sorted indices without modifying table +indices = t.argsort('mag') +sorted_table = t[indices] +``` + +### Filtering + +```python +# Boolean indexing +bright = t[t['mag'] < 18] +nearby = t[t['distance'] < 100*u.pc] + +# Multiple conditions +selected = t[(t['mag'] < 18) & (t['dec'] > 0)] + +# Using numpy functions +high_snr = t[np.abs(t['flux'] / t['error']) > 5] +``` + +## Reading and Writing Files + +### Supported Formats + +FITS, HDF5, ASCII (CSV, ECSV, IPAC, etc.), VOTable, Parquet, ASDF + +### Reading Files + +```python +# Automatic format detection +t = Table.read('catalog.fits') +t = Table.read('data.csv') +t = Table.read('table.vot') + +# Specify format explicitly +t = Table.read('data.txt', format='ascii') +t = Table.read('catalog.hdf5', path='/dataset/table') + +# Read specific HDU from FITS +t = Table.read('file.fits', hdu=2) +``` + +### Writing Files + +```python +# Automatic format from extension +t.write('output.fits') +t.write('output.csv') + +# Specify format +t.write('output.txt', format='ascii.csv') +t.write('output.hdf5', path='/data/table', serialize_meta=True) + +# Overwrite existing file +t.write('output.fits', overwrite=True) +``` + +### ASCII Format Options + +```python +# CSV with custom delimiter +t.write('output.csv', format='ascii.csv', delimiter='|') + +# Fixed-width format +t.write('output.txt', format='ascii.fixed_width') + +# IPAC format +t.write('output.tbl', format='ascii.ipac') + +# LaTeX table +t.write('table.tex', format='ascii.latex') +``` + +## Table Operations + +### Stacking Tables (Vertical) + +```python +from astropy.table import vstack + +# Concatenate tables vertically +t1 = Table([[1, 2], [3, 4]], names=('a', 'b')) +t2 = Table([[5, 6], [7, 8]], names=('a', 'b')) +t_combined = vstack([t1, t2]) +``` + +### Joining Tables (Horizontal) + +```python +from astropy.table import hstack + +# Concatenate tables horizontally +t1 = Table([[1, 2]], names=['a']) +t2 = Table([[3, 4]], names=['b']) +t_combined = hstack([t1, t2]) +``` + +### Database-Style Joins + +```python +from astropy.table import join + +# Inner join on common column +t1 = Table([[1, 2, 3], ['a', 'b', 'c']], names=('id', 'data1')) +t2 = Table([[1, 2, 4], ['x', 'y', 'z']], names=('id', 'data2')) +t_joined = join(t1, t2, keys='id') + +# Left/right/outer joins +t_joined = join(t1, t2, join_type='left') +t_joined = join(t1, t2, join_type='outer') +``` + +### Grouping and Aggregating + +```python +# Group by column +g = t.group_by('filter') + +# Aggregate groups +means = g.groups.aggregate(np.mean) + +# Iterate over groups +for group in g.groups: + print(f"Filter: {group['filter'][0]}") + print(f"Mean mag: {np.mean(group['mag'])}") +``` + +### Unique Rows + +```python +# Get unique rows +t_unique = t.unique('id') + +# Multiple columns +t_unique = t.unique(['ra', 'dec']) +``` + +## Units and Quantities + +Use QTable for unit-aware operations: + +```python +from astropy.table import QTable + +# Create table with units +t = QTable() +t['flux'] = [1.2, 2.3, 3.4] * u.Jy +t['wavelength'] = [500, 600, 700] * u.nm + +# Unit conversions +t['flux'].to(u.mJy) +t['wavelength'].to(u.angstrom) + +# Calculations preserve units +t['freq'] = t['wavelength'].to(u.Hz, equivalencies=u.spectral()) +``` + +## Masking Missing Data + +```python +from astropy.table import MaskedColumn +import numpy as np + +# Create masked column +flux = MaskedColumn([1.2, np.nan, 3.4], mask=[False, True, False]) +t = Table([flux], names=['flux']) + +# Operations automatically handle masks +mean_flux = np.ma.mean(t['flux']) + +# Fill masked values +t['flux'].filled(0) # Replace masked with 0 +``` + +## Indexing for Fast Lookup + +Create indices for fast row retrieval: + +```python +# Add index on column +t.add_index('id') + +# Fast lookup by index +row = t.loc[12345] # Find row where id=12345 + +# Range queries +subset = t.loc[100:200] + +# With multiple indices, select which index to use explicitly. +# t.loc["b", 2] (index name as first element) is deprecated since 7.2 +# and slated for removal in 9.0. +t.add_index('name') +row = t.loc.with_index('name')['M31'] +``` + +## Table Metadata + +```python +# Set table-level metadata +t.meta['TELESCOPE'] = 'HST' +t.meta['FILTER'] = 'F814W' +t.meta['EXPTIME'] = 300.0 + +# Set column-level metadata +t['ra'].meta['unit'] = 'deg' +t['ra'].meta['description'] = 'Right Ascension' +t['ra'].description = 'Right Ascension' # Shortcut +``` + +## Performance Tips + +### Fast Table Construction + +```python +# SLOW: Adding rows one at a time +t = Table(names=['a', 'b']) +for i in range(1000): + t.add_row([i, i**2]) + +# FAST: Build from lists +rows = [(i, i**2) for i in range(1000)] +t = Table(rows=rows, names=['a', 'b']) +``` + +### Memory-Mapped FITS Tables + +```python +# Don't load entire table into memory +t = Table.read('huge_catalog.fits', memmap=True) + +# Only loads data when accessed +subset = t[10000:10100] # Efficient +``` + +### Copy vs. View + +```python +# Create view (shares data, fast) +t_view = t['ra', 'dec'] + +# Create copy (independent data) +t_copy = t['ra', 'dec'].copy() +``` + +## Displaying Tables + +```python +# Print to console +print(t) + +# Show in interactive browser +t.show_in_browser() +t.show_in_browser(jsviewer=True) # Interactive sorting/filtering + +# Paginated viewing +t.more() + +# Custom formatting +t['flux'].format = '%.3f' +t['ra'].format = '{:.6f}' +``` + +## Converting to Other Formats + +```python +# To NumPy array +arr = np.array(t) + +# To Pandas DataFrame +df = t.to_pandas() + +# To dictionary +d = {name: t[name] for name in t.colnames} +``` + +## Common Use Cases + +### Cross-Matching Catalogs + +```python +from astropy.coordinates import SkyCoord, match_coordinates_sky + +# Create coordinate objects from table columns +coords1 = SkyCoord(t1['ra'], t1['dec'], unit='deg') +coords2 = SkyCoord(t2['ra'], t2['dec'], unit='deg') + +# Find matches +idx, sep, _ = coords1.match_to_catalog_sky(coords2) + +# Filter by separation +max_sep = 1 * u.arcsec +matches = sep < max_sep +t1_matched = t1[matches] +t2_matched = t2[idx[matches]] +``` + +### Binning Data + +```python +from astropy.table import Table +import numpy as np + +# Bin by magnitude +mag_bins = np.arange(10, 20, 0.5) +binned = t.group_by(np.digitize(t['mag'], mag_bins)) +counts = binned.groups.aggregate(len) +``` diff --git a/.agents/skills/astropy/references/time.md b/.agents/skills/astropy/references/time.md new file mode 100644 index 0000000..c994d5a --- /dev/null +++ b/.agents/skills/astropy/references/time.md @@ -0,0 +1,412 @@ +# Time Handling (astropy.time) + +The `astropy.time` module provides robust tools for manipulating times and dates with support for various time scales and formats. + +## Creating Time Objects + +### Basic Creation + +```python +from astropy.time import Time +import astropy.units as u + +# ISO format (automatically detected) +t = Time('2023-01-15 12:30:45') +t = Time('2023-01-15T12:30:45') + +# Specify format explicitly +t = Time('2023-01-15 12:30:45', format='iso', scale='utc') + +# Julian Date +t = Time(2460000.0, format='jd') + +# Modified Julian Date +t = Time(59945.0, format='mjd') + +# Unix time (seconds since 1970-01-01) +t = Time(1673785845.0, format='unix') +``` + +### Array of Times + +```python +# Multiple times +times = Time(['2023-01-01', '2023-06-01', '2023-12-31']) + +# From arrays +import numpy as np +jd_array = np.linspace(2460000, 2460100, 100) +times = Time(jd_array, format='jd') +``` + +## Time Formats + +### Supported Formats + +```python +# ISO 8601 +t = Time('2023-01-15 12:30:45', format='iso') +t = Time('2023-01-15T12:30:45.123', format='isot') + +# Julian dates +t = Time(2460000.0, format='jd') # Julian Date +t = Time(59945.0, format='mjd') # Modified Julian Date + +# Decimal year +t = Time(2023.5, format='decimalyear') +t = Time(2023.5, format='jyear') # Julian year +t = Time(2023.5, format='byear') # Besselian year + +# Year and day-of-year +t = Time('2023:046', format='yday') # 46th day of 2023 + +# FITS format +t = Time('2023-01-15T12:30:45', format='fits') + +# GPS seconds +t = Time(1000000000.0, format='gps') + +# Unix time +t = Time(1673785845.0, format='unix') + +# Matplotlib dates +t = Time(738521.0, format='plot_date') + +# datetime objects +from datetime import datetime +dt = datetime(2023, 1, 15, 12, 30, 45) +t = Time(dt) +``` + +## Time Scales + +### Available Time Scales + +```python +# UTC - Coordinated Universal Time (default) +t = Time('2023-01-15 12:00:00', scale='utc') + +# TAI - International Atomic Time +t = Time('2023-01-15 12:00:00', scale='tai') + +# TT - Terrestrial Time +t = Time('2023-01-15 12:00:00', scale='tt') + +# TDB - Barycentric Dynamical Time +t = Time('2023-01-15 12:00:00', scale='tdb') + +# TCG - Geocentric Coordinate Time +t = Time('2023-01-15 12:00:00', scale='tcg') + +# TCB - Barycentric Coordinate Time +t = Time('2023-01-15 12:00:00', scale='tcb') + +# UT1 - Universal Time +t = Time('2023-01-15 12:00:00', scale='ut1') +``` + +### Converting Time Scales + +```python +t = Time('2023-01-15 12:00:00', scale='utc') + +# Convert to different scales +t_tai = t.tai +t_tt = t.tt +t_tdb = t.tdb +t_ut1 = t.ut1 + +# Check offset +print(f"TAI - UTC = {(t.tai - t.utc).sec} seconds") +# TAI - UTC = 37 seconds (leap seconds) +``` + +## Format Conversions + +### Change Output Format + +```python +t = Time('2023-01-15 12:30:45') + +# Access in different formats +print(t.jd) # Julian Date +print(t.mjd) # Modified Julian Date +print(t.iso) # ISO format +print(t.isot) # ISO with 'T' +print(t.unix) # Unix time +print(t.decimalyear) # Decimal year + +# Change default format +t.format = 'mjd' +print(t) # Displays as MJD +``` + +### High-Precision Output + +```python +# Use subfmt for precision control +t.to_value('mjd', subfmt='float') # Standard float +t.to_value('mjd', subfmt='long') # Extended precision +t.to_value('mjd', subfmt='decimal') # Decimal (highest precision) +t.to_value('mjd', subfmt='str') # String representation +``` + +## Time Arithmetic + +### TimeDelta Objects + +```python +from astropy.time import TimeDelta + +# Create time difference +dt = TimeDelta(1.0, format='jd') # 1 day +dt = TimeDelta(3600.0, format='sec') # 1 hour + +# Subtract times +t1 = Time('2023-01-15') +t2 = Time('2023-02-15') +dt = t2 - t1 +print(dt.jd) # 31 days +print(dt.sec) # 2678400 seconds +``` + +### Adding/Subtracting Time + +```python +t = Time('2023-01-15 12:00:00') + +# Add TimeDelta +t_future = t + TimeDelta(7, format='jd') # Add 7 days + +# Add Quantity +t_future = t + 1*u.hour +t_future = t + 30*u.day +t_future = t + 1*u.year + +# Subtract +t_past = t - 1*u.week +``` + +### Time Ranges + +```python +# Create range of times +start = Time('2023-01-01') +end = Time('2023-12-31') +times = start + np.linspace(0, 365, 100) * u.day + +# Or using TimeDelta +times = start + TimeDelta(np.linspace(0, 365, 100), format='jd') +``` + +## Observing-Related Features + +Some high-precision time and coordinate operations use IERS Earth-rotation data. Astropy uses `astropy-iers-data` by default, but may auto-download fresher IERS-A data when required for UT1 or polar-motion values. Disable auto-download for offline or privacy-sensitive runs: + +```python +from astropy.utils import iers + +iers.conf.auto_download = False +``` + +### Sidereal Time + +```python +from astropy.coordinates import EarthLocation + +# Define observer location +location = EarthLocation(lat=40*u.deg, lon=-120*u.deg, height=1000*u.m) + +# Create time with location +t = Time('2023-06-15 23:00:00', location=location) + +# Calculate sidereal time +lst_apparent = t.sidereal_time('apparent') +lst_mean = t.sidereal_time('mean') + +print(f"Local Sidereal Time: {lst_apparent}") +``` + +### Light Travel Time Corrections + +```python +from astropy.coordinates import SkyCoord, EarthLocation + +# Define target and observer +target = SkyCoord(ra=10*u.deg, dec=20*u.deg) +location = EarthLocation.of_site('Keck Observatory') + +# Observation times +times = Time(['2023-01-01', '2023-06-01', '2023-12-31'], + location=location) + +# Calculate light travel time to solar system barycenter +ltt_bary = times.light_travel_time(target, kind='barycentric') +ltt_helio = times.light_travel_time(target, kind='heliocentric') + +# Apply correction +times_barycentric = times.tdb + ltt_bary +``` + +### Earth Rotation Angle + +```python +# Earth rotation angle (for celestial to terrestrial transformations) +era = t.earth_rotation_angle() +``` + +## Handling Missing or Invalid Times + +### Masked Times + +```python +import numpy as np + +# Create times with missing values +times = Time(['2023-01-01', '2023-06-01', '2023-12-31']) +times[1] = np.ma.masked # Mark as missing + +# Check for masks +print(times.mask) # [False True False] + +# Get unmasked version +times_clean = times.unmasked + +# Fill masked values +times_filled = times.filled(Time('2000-01-01')) +``` + +## Time Precision and Representation + +### Internal Representation + +Time objects use two 64-bit floats (jd1, jd2) for high precision: + +```python +t = Time('2023-01-15 12:30:45.123456789', format='iso', scale='utc') + +# Access internal representation +print(t.jd1, t.jd2) # Integer and fractional parts + +# This allows sub-nanosecond precision over astronomical timescales +``` + +### Precision + +```python +# High precision for long time intervals +t1 = Time('1900-01-01') +t2 = Time('2100-01-01') +dt = t2 - t1 +print(f"Time span: {dt.sec / (365.25 * 86400)} years") +# Maintains precision throughout +``` + +## Time Formatting + +### Custom String Format + +```python +t = Time('2023-01-15 12:30:45') + +# Strftime-style formatting +t.strftime('%Y-%m-%d %H:%M:%S') # '2023-01-15 12:30:45' +t.strftime('%B %d, %Y') # 'January 15, 2023' + +# ISO format subformats +t.iso # '2023-01-15 12:30:45.000' +t.isot # '2023-01-15T12:30:45.000' +t.to_value('iso', subfmt='date_hms') # '2023-01-15 12:30:45.000' +``` + +## Common Use Cases + +### Converting Between Formats + +```python +# MJD to ISO +t_mjd = Time(59945.0, format='mjd') +iso_string = t_mjd.iso + +# ISO to JD +t_iso = Time('2023-01-15 12:00:00') +jd_value = t_iso.jd + +# Unix to ISO +t_unix = Time(1673785845.0, format='unix') +iso_string = t_unix.iso +``` + +### Time Differences in Various Units + +```python +t1 = Time('2023-01-01') +t2 = Time('2023-12-31') + +dt = t2 - t1 +print(f"Days: {dt.to(u.day)}") +print(f"Hours: {dt.to(u.hour)}") +print(f"Seconds: {dt.sec}") +print(f"Years: {dt.to(u.year)}") +``` + +### Creating Regular Time Series + +```python +# Daily observations for a year +start = Time('2023-01-01') +times = start + np.arange(365) * u.day + +# Hourly observations for a day +start = Time('2023-01-15 00:00:00') +times = start + np.arange(24) * u.hour + +# Observations every 30 seconds +start = Time('2023-01-15 12:00:00') +times = start + np.arange(1000) * 30 * u.second +``` + +### Time Zone Handling + +```python +# UTC to local time (requires datetime) +t = Time('2023-01-15 12:00:00', scale='utc') +dt_utc = t.to_datetime() + +# Convert to specific timezone using pytz +import pytz +eastern = pytz.timezone('US/Eastern') +dt_eastern = dt_utc.replace(tzinfo=pytz.utc).astimezone(eastern) +``` + +### Barycentric Correction Example + +```python +from astropy.coordinates import SkyCoord, EarthLocation + +# Target coordinates +target = SkyCoord(ra='23h23m08.55s', dec='+18d24m59.3s') + +# Observatory location +location = EarthLocation.of_site('Keck Observatory') + +# Observation times (must include location) +times = Time(['2023-01-15 08:30:00', '2023-01-16 08:30:00'], + location=location) + +# Calculate barycentric correction +ltt_bary = times.light_travel_time(target, kind='barycentric') + +# Apply correction to get barycentric times +times_bary = times.tdb + ltt_bary + +# For radial velocity work, use the coordinate helper instead +rv_correction = target.radial_velocity_correction(obstime=times, location=location) +``` + +## Performance Considerations + +1. **Array operations are fast**: Process multiple times as arrays +2. **Format conversions are cached**: Repeated access is efficient +3. **Scale conversions may require IERS data**: Update `astropy-iers-data` before offline runs, or set `iers.conf.auto_download = False` to prevent network access +4. **High precision maintained**: Sub-nanosecond accuracy across astronomical timescales diff --git a/.agents/skills/astropy/references/units.md b/.agents/skills/astropy/references/units.md new file mode 100644 index 0000000..587da9a --- /dev/null +++ b/.agents/skills/astropy/references/units.md @@ -0,0 +1,178 @@ +# Units and Quantities (astropy.units) + +The `astropy.units` module handles defining, converting between, and performing arithmetic with physical quantities. + +## Creating Quantities + +Multiply or divide numeric values by built-in units to create Quantity objects: + +```python +from astropy import units as u +import numpy as np + +# Scalar quantities +distance = 42.0 * u.meter +velocity = 100 * u.km / u.s + +# Array quantities +distances = np.array([1., 2., 3.]) * u.m +wavelengths = [500, 600, 700] * u.nm +``` + +Access components via `.value` and `.unit` attributes: +```python +distance.value # 42.0 +distance.unit # Unit("m") +``` + +## Unit Conversions + +Use `.to()` method for conversions: + +```python +distance = 1.0 * u.parsec +distance.to(u.km) # + +wavelength = 500 * u.nm +wavelength.to(u.angstrom) # +``` + +## Arithmetic Operations + +Quantities support standard arithmetic with automatic unit management: + +```python +# Basic operations +speed = 15.1 * u.meter / (32.0 * u.second) # +area = (5 * u.m) * (3 * u.m) # + +# Units cancel when appropriate +ratio = (10 * u.m) / (5 * u.m) # + +# Decompose complex units +time = (3.0 * u.kilometer / (130.51 * u.meter / u.second)) +time.decompose() # +``` + +## Unit Systems + +Convert between major unit systems: + +```python +# SI to CGS +pressure = 1.0 * u.Pa +pressure.cgs # + +# Find equivalent representations +(u.s ** -1).compose() # [Unit("Bq"), Unit("Hz"), ...] +``` + +## Equivalencies + +Domain-specific conversions require equivalencies: + +```python +# Spectral equivalency (wavelength ↔ frequency) +wavelength = 1000 * u.nm +wavelength.to(u.Hz, equivalencies=u.spectral()) +# + +# Doppler equivalencies +velocity = 1000 * u.km / u.s +velocity.to(u.Hz, equivalencies=u.doppler_optical(500*u.nm)) + +# Other equivalencies +u.brightness_temperature(500*u.GHz) +u.doppler_radio(1.4*u.GHz) +u.mass_energy() +u.parallax() +``` + +## Logarithmic Units + +Special units for magnitudes, decibels, and dex: + +```python +# Magnitudes +flux = -2.5 * u.mag(u.ct / u.s) + +# Decibels +power_ratio = 3 * u.dB(u.W) + +# Dex (base-10 logarithm) +abundance = 8.5 * u.dex(u.cm**-3) +``` + +## Common Units + +### Length +`u.m, u.km, u.cm, u.mm, u.micron, u.angstrom, u.au, u.pc, u.kpc, u.Mpc, u.lyr` + +### Time +`u.s, u.min, u.hour, u.day, u.year, u.Myr, u.Gyr` + +### Mass +`u.kg, u.g, u.M_sun, u.M_earth, u.M_jup` + +### Temperature +`u.K, u.deg_C` + +### Angle +`u.deg, u.arcmin, u.arcsec, u.rad, u.hourangle, u.mas` + +### Energy/Power +`u.J, u.erg, u.eV, u.keV, u.MeV, u.GeV, u.W, u.L_sun` + +### Frequency +`u.Hz, u.kHz, u.MHz, u.GHz` + +### Flux +`u.Jy, u.mJy, u.erg / u.s / u.cm**2` + +## Performance Optimization + +Pre-compute composite units for array operations: + +```python +# Slow (creates intermediate quantities) +result = array * u.m / u.s / u.kg / u.sr + +# Fast (pre-computed composite unit) +UNIT_COMPOSITE = u.m / u.s / u.kg / u.sr +result = array * UNIT_COMPOSITE + +# Fastest (avoid copying with <<) +result = array << UNIT_COMPOSITE # 10000x faster +``` + +## String Formatting + +Format quantities with standard Python syntax: + +```python +velocity = 15.1 * u.meter / (32.0 * u.second) +f"{velocity:0.03f}" # '0.472 m / s' +f"{velocity:.2e}" # '4.72e-01 m / s' +f"{velocity.unit:FITS}" # 'm s-1' +``` + +## Defining Custom Units + +```python +# Create new unit +bakers_fortnight = u.def_unit('bakers_fortnight', 13 * u.day) + +# Enable in string parsing +u.add_enabled_units([bakers_fortnight]) +``` + +## Constants + +Access physical constants with units: + +```python +from astropy.constants import c, G, M_sun, h, k_B + +speed_of_light = c.to(u.km/u.s) +gravitational_constant = G.to(u.m**3 / u.kg / u.s**2) +``` diff --git a/.agents/skills/astropy/references/wcs_and_other_modules.md b/.agents/skills/astropy/references/wcs_and_other_modules.md new file mode 100644 index 0000000..3fc1809 --- /dev/null +++ b/.agents/skills/astropy/references/wcs_and_other_modules.md @@ -0,0 +1,377 @@ +# WCS and Other Astropy Modules + +## World Coordinate System (astropy.wcs) + +The WCS module manages transformations between pixel coordinates in images and world coordinates (e.g., celestial coordinates). + +### Reading WCS from FITS + +```python +from astropy.wcs import WCS +from astropy.io import fits + +# Read WCS from FITS header +with fits.open('image.fits') as hdul: + wcs = WCS(hdul[0].header) +``` + +### Pixel to World Transformations + +```python +# Single pixel to world coordinates +world = wcs.pixel_to_world(100, 200) # Returns SkyCoord +print(f"RA: {world.ra}, Dec: {world.dec}") + +# Arrays of pixels +import numpy as np +x_pixels = np.array([100, 200, 300]) +y_pixels = np.array([150, 250, 350]) +world_coords = wcs.pixel_to_world(x_pixels, y_pixels) +``` + +### World to Pixel Transformations + +```python +from astropy.coordinates import SkyCoord +import astropy.units as u + +# Single coordinate +coord = SkyCoord(ra=10.5*u.degree, dec=41.2*u.degree) +x, y = wcs.world_to_pixel(coord) + +# Array of coordinates +coords = SkyCoord(ra=[10, 11, 12]*u.degree, dec=[41, 42, 43]*u.degree) +x_pixels, y_pixels = wcs.world_to_pixel(coords) +``` + +### WCS Information + +```python +# Print WCS details +print(wcs) + +# Access key properties +print(wcs.wcs.crpix) # Reference pixel +print(wcs.wcs.crval) # Reference value (world coords) +print(wcs.wcs.cd) # CD matrix +print(wcs.wcs.ctype) # Coordinate types + +# Pixel scale +pixel_scale = wcs.proj_plane_pixel_scales() # Returns Quantity array +``` + +### Creating WCS + +```python +from astropy.wcs import WCS + +# Create new WCS +wcs = WCS(naxis=2) +wcs.wcs.crpix = [512.0, 512.0] # Reference pixel +wcs.wcs.crval = [10.5, 41.2] # RA, Dec at reference pixel +wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN'] # Projection type +wcs.wcs.cdelt = [-0.0001, 0.0001] # Pixel scale (degrees/pixel) +wcs.wcs.cunit = ['deg', 'deg'] +``` + +### Footprint and Coverage + +```python +# Calculate image footprint (corner coordinates) +footprint = wcs.calc_footprint() +# Returns array of [RA, Dec] for each corner +``` + +## NDData (astropy.nddata) + +Container for n-dimensional datasets with metadata, uncertainty, and masking. + +### Creating NDData + +```python +from astropy.nddata import NDData +import numpy as np +import astropy.units as u + +# Basic NDData +data = np.random.random((100, 100)) +ndd = NDData(data) + +# With units +ndd = NDData(data, unit=u.electron/u.s) + +# With uncertainty +from astropy.nddata import StdDevUncertainty +uncertainty = StdDevUncertainty(np.sqrt(data)) +ndd = NDData(data, uncertainty=uncertainty, unit=u.electron/u.s) + +# With mask +mask = data < 0.1 # Mask low values +ndd = NDData(data, mask=mask) + +# With WCS +from astropy.wcs import WCS +ndd = NDData(data, wcs=wcs) +``` + +### CCDData for CCD Images + +```python +from astropy.nddata import CCDData + +# Create CCDData +ccd = CCDData(data, unit=u.adu, meta={'object': 'M31'}) + +# Read from FITS +ccd = CCDData.read('image.fits', unit=u.adu) + +# Write to FITS +ccd.write('output.fits', overwrite=True) +``` + +## Modeling (astropy.modeling) + +Framework for creating and fitting models to data. + +### Common Models + +```python +from astropy.modeling import models, fitting +import numpy as np + +# 1D Gaussian +gauss = models.Gaussian1D(amplitude=10, mean=5, stddev=1) +x = np.linspace(0, 10, 100) +y = gauss(x) + +# 2D Gaussian +gauss_2d = models.Gaussian2D(amplitude=10, x_mean=50, y_mean=50, + x_stddev=5, y_stddev=3) + +# Polynomial +poly = models.Polynomial1D(degree=3) + +# Power law +power_law = models.PowerLaw1D(amplitude=10, x_0=1, alpha=2) +``` + +### Fitting Models to Data + +```python +# Generate noisy data +true_model = models.Gaussian1D(amplitude=10, mean=5, stddev=1) +x = np.linspace(0, 10, 100) +y_true = true_model(x) +y_noisy = y_true + np.random.normal(0, 0.5, x.shape) + +# Fit model +fitter = fitting.LevMarLSQFitter() +initial_model = models.Gaussian1D(amplitude=8, mean=4, stddev=1.5) +fitted_model = fitter(initial_model, x, y_noisy) + +print(f"Fitted amplitude: {fitted_model.amplitude.value}") +print(f"Fitted mean: {fitted_model.mean.value}") +print(f"Fitted stddev: {fitted_model.stddev.value}") +``` + +### Compound Models + +```python +# Add models +double_gauss = models.Gaussian1D(amp=5, mean=3, stddev=1) + \ + models.Gaussian1D(amp=8, mean=7, stddev=1.5) + +# Compose models +composite = models.Gaussian1D(amp=10, mean=5, stddev=1) | \ + models.Scale(factor=2) # Scale output +``` + +## Visualization (astropy.visualization) + +Tools for visualizing astronomical images and data. + +### Image Normalization + +```python +from astropy.visualization import simple_norm +import matplotlib.pyplot as plt + +# Load image +from astropy.io import fits +data = fits.getdata('image.fits') + +# Normalize for display +norm = simple_norm(data, 'sqrt', percent=99) + +# Display +plt.imshow(data, norm=norm, cmap='gray', origin='lower') +plt.colorbar() +plt.show() +``` + +### Stretching and Intervals + +```python +from astropy.visualization import (MinMaxInterval, AsinhStretch, + ImageNormalize, ZScaleInterval) + +# Z-scale interval +interval = ZScaleInterval() +vmin, vmax = interval.get_limits(data) + +# Asinh stretch +stretch = AsinhStretch() +norm = ImageNormalize(data, interval=interval, stretch=stretch) + +plt.imshow(data, norm=norm, cmap='gray', origin='lower') +``` + +### PercentileInterval + +```python +from astropy.visualization import PercentileInterval + +# Show data between 5th and 95th percentiles +interval = PercentileInterval(90) # 90% of data +vmin, vmax = interval.get_limits(data) + +plt.imshow(data, vmin=vmin, vmax=vmax, cmap='gray', origin='lower') +``` + +## Constants (astropy.constants) + +Physical and astronomical constants with units. + +Note: through Astropy 7.x the default constants are CODATA 2018 + IAU 2015; Astropy 8.0 switches the default to CODATA 2022. For reproducible results across versions, select an explicit constants version via the science state (e.g., `astropy.physical_constants.set('codata2018')` before other astropy imports). + +```python +from astropy import constants as const + +# Speed of light +c = const.c +print(f"c = {c}") +print(f"c in km/s = {c.to(u.km/u.s)}") + +# Gravitational constant +G = const.G + +# Astronomical constants +M_sun = const.M_sun # Solar mass +R_sun = const.R_sun # Solar radius +L_sun = const.L_sun # Solar luminosity +au = const.au # Astronomical unit +pc = const.pc # Parsec + +# Fundamental constants +h = const.h # Planck constant +hbar = const.hbar # Reduced Planck constant +k_B = const.k_B # Boltzmann constant +m_e = const.m_e # Electron mass +m_p = const.m_p # Proton mass +e = const.e # Elementary charge +N_A = const.N_A # Avogadro constant +``` + +### Using Constants in Calculations + +```python +# Calculate Schwarzschild radius +M = 10 * const.M_sun +r_s = 2 * const.G * M / const.c**2 +print(f"Schwarzschild radius: {r_s.to(u.km)}") + +# Calculate escape velocity +M = const.M_earth +R = const.R_earth +v_esc = np.sqrt(2 * const.G * M / R) +print(f"Earth escape velocity: {v_esc.to(u.km/u.s)}") +``` + +## Convolution (astropy.convolution) + +Convolution kernels for image processing. + +```python +from astropy.convolution import Gaussian2DKernel, convolve + +# Create Gaussian kernel +kernel = Gaussian2DKernel(x_stddev=2) + +# Convolve image +smoothed_image = convolve(data, kernel) + +# Handle NaNs +from astropy.convolution import convolve_fft +smoothed = convolve_fft(data, kernel, nan_treatment='interpolate') +``` + +## Stats (astropy.stats) + +Statistical functions for astronomical data. + +```python +from astropy.stats import sigma_clip, sigma_clipped_stats + +# Sigma clipping +clipped_data = sigma_clip(data, sigma=3, maxiters=5) + +# Get statistics with sigma clipping +mean, median, std = sigma_clipped_stats(data, sigma=3.0) + +# Robust statistics +from astropy.stats import mad_std, biweight_location, biweight_scale +robust_std = mad_std(data) +robust_mean = biweight_location(data) +robust_scale = biweight_scale(data) +``` + +## Utils + +### Data Downloads + +`download_file()` fetches a remote URL and caches it locally. Treat the URL as data you are disclosing to the remote host; do not pass confidential signed URLs or internal file locations unless the workflow explicitly permits it. Use `cache=False` for one-off downloads that should not be retained in Astropy's cache. + +```python +from astropy.utils.data import download_file + +# Download file (caches locally) +url = 'https://example.com/data.fits' +local_file = download_file(url, cache=True) +``` + +### Progress Bars + +```python +from astropy.utils.console import ProgressBar + +with ProgressBar(len(data_list)) as bar: + for item in data_list: + # Process item + bar.update() +``` + +## SAMP (Simple Application Messaging Protocol) + +Interoperability with other astronomy tools. + +```python +from astropy.samp import SAMPIntegratedClient + +# Connect to SAMP hub +client = SAMPIntegratedClient() +client.connect() + +# Broadcast table to other applications +message = { + 'samp.mtype': 'table.load.votable', + 'samp.params': { + 'url': 'file:///path/to/table.xml', + 'table-id': 'my_table', + 'name': 'My Catalog' + } +} +client.notify_all(message) + +# Disconnect +client.disconnect() +``` diff --git a/.agents/skills/autoskill/.gitignore b/.agents/skills/autoskill/.gitignore new file mode 100644 index 0000000..75c6182 --- /dev/null +++ b/.agents/skills/autoskill/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/.agents/skills/autoskill/SKILL.md b/.agents/skills/autoskill/SKILL.md new file mode 100644 index 0000000..6e74977 --- /dev/null +++ b/.agents/skills/autoskill/SKILL.md @@ -0,0 +1,233 @@ +--- +name: autoskill +description: Observe the user's screen via screenpipe, detect repeated research workflows, match them against existing scientific-agent-skills, and draft new skills (or composition recipes that chain existing ones) for the patterns not yet covered. Use when the user asks to analyze their recent work and propose skills based on what they actually do. Requires the screenpipe daemon (https://github.com/screenpipe/screenpipe) running locally on port 3030 — the skill has no other data source and will refuse to run if screenpipe is unreachable. All detection runs locally; only redacted cluster summaries reach the LLM. +allowed-tools: Read Write Edit Bash +license: MIT license +metadata: + version: "1.3" + skill-author: K-Dense Inc. + openclaw: + requires: + bins: + - screenpipe + primaryEnv: SCREENPIPE_TOKEN + envVars: + - name: SCREENPIPE_TOKEN + required: true + description: Auth token for the local screenpipe daemon. + - name: ANTHROPIC_API_KEY + required: false + description: For Claude API calls during skill drafting. + - name: FOUNDRY_API_KEY + required: false + description: Optional Foundry access for drafting. +--- + +# autoskill + +> **Requires a running [screenpipe](https://github.com/screenpipe/screenpipe) daemon.** This skill has no alternate data source — it reads exclusively from the local screenpipe HTTP API (default `http://localhost:3030`). If the daemon isn't running, `run()` raises `ScreenpipeUnreachable` with install instructions. + +> **Network access & environment variables.** This skill makes authenticated HTTP requests to (a) the user's local screenpipe daemon on loopback, and (b) the user-configured LLM backend — one of `http://localhost:1234/v1` (LM Studio, default), `https://api.anthropic.com` (opt-in Claude), or a user-supplied BYOK Foundry gateway. The skill reads three environment variables — `SCREENPIPE_TOKEN`, `ANTHROPIC_API_KEY`, `FOUNDRY_API_KEY` — and uses each only to authenticate to the single endpoint its name implies. No other network destinations, no telemetry, no data egress to any third party. + +## Overview + +Turn the user's own workflow history — captured passively by the local [screenpipe](https://github.com/screenpipe/screenpipe) daemon — into new skills. This skill is on-demand: the user invokes it with a time window, it queries screenpipe's local HTTP API, clusters repeated workflow patterns, compares each pattern against the existing skills in this repo, and produces a staged folder of proposals the user can review, edit, and promote. + +## When to Use This Skill + +Invoke this skill when the user asks to: +- "Analyze my last 4 hours / day / week and propose new skills." +- "Look at what I've been doing and tell me what's not covered yet." +- "Draft a skill from my recent workflow." +- "Find composition recipes for workflows I repeat." + +Do **not** invoke it for one-off questions about screenpipe itself, for real-time screen queries, or without an explicit user request — the skill analyzes sensitive local content and must stay explicitly user-triggered. + +## Privacy Posture + +- **Screenpipe handles app/window filtering at capture time.** Install a starter deny-list by copying `references/screenpipe-config.yaml` into the user's screenpipe config. Sensitive apps (password managers, messaging, banking) are never OCR'd in the first place. +- **Raw OCR never leaves the machine.** `scripts/fetch_window.py` pulls data over localhost HTTP. `scripts/cluster.py` reduces the timeline to app/duration/title summaries. `scripts/redact.py` strips emails, API keys, bearer tokens, and phone numbers as defense-in-depth before any cluster summary reaches the LLM. +- **LLM backend defaults to `local`.** The recommended setup is [LM Studio](https://lmstudio.ai/) running `Gemma-4-31B-it` — strong reasoning at a size that fits on most workstation GPUs, and no data ever leaves your machine. Cloud backends (`claude`, `foundry`) are opt-in and documented in `config.yaml` for users who explicitly want them. Detection and embeddings always run locally regardless of backend choice. +- **Dry-run mode** (`--plan`) prints the exact timeline that will be analyzed before any LLM call. +- **TLS for localhost** (optional, for corporate policy): see `references/https-proxy.md` for the Caddy pattern. + +## Prerequisites + +### 1. Screenpipe daemon + +Either install the official release or build from source. Either way the daemon binds HTTP on `localhost:3030` by default. + +**From source** (recommended if you want the CLI daemon without the desktop GUI): + +```bash +git clone --depth 1 https://github.com/mediar-ai/screenpipe.git +cd screenpipe +cargo build -p screenpipe-engine --release +# System deps (macOS): cmake + full Xcode.app (not just Command Line Tools). +# brew install cmake +# # if xcodebuild plug-ins error: sudo xcodebuild -runFirstLaunch +./target/release/screenpipe doctor # confirm permissions + ffmpeg +./target/release/screenpipe record --disable-audio --use-pii-removal +``` + +First run will prompt for macOS Screen Recording permission. Grant it and relaunch. + +### 2. Screenpipe API token + +The local API now requires bearer auth. Retrieve your token and export it: + +```bash +export SCREENPIPE_TOKEN=$(screenpipe auth token) +``` + +(Or set `screenpipe.token` directly in `config.yaml` — env var is preferred since it keeps secrets out of version control.) + +### 3. Python environment + +Via `pipenv` from the repo root: + +```bash +pipenv install httpx pyyaml sentence-transformers +``` + +The embedding model (`sentence-transformers/all-MiniLM-L6-v2`, ~80 MB) downloads on first run. + +### 4. Local LLM (default path) — LM Studio + +- Install [LM Studio](https://lmstudio.ai/). +- Download `Gemma-4-31B-it` (or another strong reasoning model; adjust `local.model` in `config.yaml`). +- Load it via the CLI for headless use (no GUI required): + +```bash +lms load gemma-4-31b-it --context-length 131072 --gpu max -y +lms status # confirm server running on :1234 +``` + +### 5. Cloud LLM backends (optional, opt-in) + +Only if you explicitly opt out of local: +- `claude`: set `ANTHROPIC_API_KEY`, flip `backend: claude` in `config.yaml`. +- `foundry`: set `FOUNDRY_API_KEY`, flip `backend: foundry`, set `foundry.endpoint` to your corporate gateway URL. + +## Architecture + +``` +screenpipe daemon (user-installed) + │ HTTP on localhost:3030 + ▼ +scripts/fetch_window.py → normalized timeline events +scripts/redact.py → regex scrub (defense-in-depth) +scripts/cluster.py → sessions + clusters (local only) +scripts/match_skills.py → top-k vs existing 135 skills (local embeddings) +scripts/synthesize.py → LLM judge: reuse / compose / novel + │ + ▼ +~/.autoskill/proposed// (default; override with --out) + ├── report.md + ├── composition-recipes//SKILL.md + └── new-skills//SKILL.md + +scripts/promote.py → user-approved proposal → skills// +``` + +## Workflow + +The skill ships a unified CLI at `scripts/autoskill.py` with three subcommands: + +```bash +python scripts/autoskill.py doctor --config config.yaml --skills-dir ../ +python scripts/autoskill.py run --start ... --end ... --config config.yaml +python scripts/autoskill.py promote --proposed ~/.autoskill/proposed/ --skills-dir ../ --name +``` + +### 0. Preflight with `doctor` + +Before a full run, verify every dependency in one shot: + +```bash +python scripts/autoskill.py doctor \ + --config skills/autoskill/config.yaml \ + --skills-dir skills +``` + +The report covers `config` (backend choice valid), `skills_dir` (exists), `screenpipe` (reachable + authed), and `llm` (LM Studio serving or API key present). Non-zero exit on any failure, with the offending line marked `error`. + +### 1. Run the pipeline + +```bash +export SCREENPIPE_TOKEN=$(screenpipe auth token) +python scripts/autoskill.py run \ + --start "2026-04-17T00:00:00Z" \ + --end "2026-04-17T23:59:59Z" \ + --config skills/autoskill/config.yaml \ + --skills-dir skills +``` + +Proposals land in `~/.autoskill/proposed//` by default, keeping experimental output out of the skills repo. Pass `--out PATH` to override. + +Internally: +1. **Fetch** — `fetch_window` paginates screenpipe's `/search` endpoint, normalizes events to `{ts, app, window_title, text, content_type}`. +2. **Redact** — `redact` scrubs emails, API keys, bearer tokens, phones from OCR text and window titles as defense-in-depth over screenpipe's own PII removal. +3. **Cluster** — `segment_sessions` splits on idle gaps (default 10 min) and drops short sessions; `cluster_sessions` groups sessions by app-signature and keeps clusters of size `min_cluster_size` (default 2). +4. **Match** — `load_skill_descriptions` reads frontmatter from every `SKILL.md` in `skills/`; `top_k_matches` ranks each cluster against all skills using local `sentence-transformers` embeddings (cosine similarity). +5. **Synthesize** — `synthesize` prompts the configured LLM backend to classify each cluster as `reuse`, `compose`, or `novel` and emit a SKILL.md body where appropriate. +6. **Report** — writes `//report.md`, plus `new-skills//SKILL.md` or `composition-recipes//SKILL.md` for each proposal. + +Add `--dry-run` to stop after clustering; this skips the LLM (and the sentence-transformers load), writing only `plan.md` for inspection. + +### 2. Review and promote + +Open `~/.autoskill/proposed//report.md`, edit drafts in place, delete anything you don't want. Then: + +```bash +python scripts/autoskill.py promote \ + --proposed ~/.autoskill/proposed/2026-04-17T14-30-00 \ + --skills-dir skills \ + --name zotero-pubmed-helper +``` + +`promote` moves the directory into `skills//`, refusing to overwrite an existing skill. Exits non-zero with a friendly error if the proposal isn't found or the target already exists. + +## Configuration + +See `config.yaml` for the full shape. Default values (local-first): + +```yaml +backend: local +local: + endpoint: http://localhost:1234/v1 # LM Studio's Developer server + model: Gemma-4-31B-it + +screenpipe: + url: http://localhost:3030 # or https://screenpipe.local via Caddy + +cluster: + min_session_minutes: 5 + idle_gap_minutes: 10 + min_cluster_size: 2 +``` + +To opt into a cloud backend: + +```yaml +backend: claude # or foundry +claude: + model: claude-opus-4-7 +``` + +## Composition recipes vs new skills + +- **compose**: the LLM judged that chaining existing skills covers the workflow. The emitted SKILL.md is intentionally thin — frontmatter + a "Workflow" section that invokes existing skills in order. The same agent runtime that discovered the skill can then invoke it end-to-end. +- **novel**: no combination of existing skills covers it. A fuller SKILL.md is drafted, still following repo conventions (frontmatter, Overview, When to Use, Workflow). The user should always review new-skill drafts before promoting. + +## Testing + +The skill is covered by a small pytest suite at `tests/autoskill/` in the repository root. Each script is unit-tested in isolation with dependency injection (mock HTTP transport, stub backend, stub embedder): + +```bash +python -m pytest tests/autoskill -v +``` + +## Composition with other skills in this repo + +The autoskill's embedding index covers all 135 sibling skills. Workflows that look like scientific writing will match `scientific-writing` / `literature-review` / `citation-management`; figure work will match `scientific-schematics` / `generate-image` / `infographics`; slide prep matches `scientific-slides` / `pptx`; etc. When a cluster scores high against two or three sibling skills the emitted composition recipe names them explicitly, so the user's future agent invocations use the optimized paths already documented in this repo. diff --git a/.agents/skills/autoskill/config.yaml b/.agents/skills/autoskill/config.yaml new file mode 100644 index 0000000..4a020cc --- /dev/null +++ b/.agents/skills/autoskill/config.yaml @@ -0,0 +1,53 @@ +# autoskill configuration +# +# LLM backend for skill synthesis. Detection/clustering always runs locally; +# only redacted cluster summaries are sent to the LLM. +# +# Local is the default — your screen content never leaves the machine. +# Cloud backends (claude, foundry) are available for users who explicitly +# opt in; see the backend sections below. +backend: local # local | claude | foundry + +# Per-backend settings. Only the selected backend's block is used. +local: + # LM Studio exposes an OpenAI-compatible server. Start it from the + # "Developer" tab; the default port is 1234. + endpoint: http://localhost:1234/v1 + # Gemma-4-31B-it is the recommended default: strong reasoning at a size + # most modern workstation GPUs can run. Swap for any LM Studio model ID. + model: Gemma-4-31B-it + +claude: + model: claude-opus-4-7 + # api_key read from ANTHROPIC_API_KEY env var + +foundry: + endpoint: https://foundry.example.com/anthropic + model: claude-opus-4-7 + # api_key read from FOUNDRY_API_KEY env var + +# Screenpipe HTTP endpoint. For TLS, point this at your local Caddy proxy +# (see references/https-proxy.md). +screenpipe: + url: http://localhost:3030 + # Screenpipe requires a bearer token for its local API. Either set `token` + # here, or export SCREENPIPE_TOKEN in your environment (preferred — keeps + # the token out of version control). Retrieve with: `screenpipe auth token`. + # token: your-token-here + +# Embedding model for matching against existing scientific skills. +# Local only; no API calls. +embeddings: + model: sentence-transformers/all-MiniLM-L6-v2 + +# Clustering thresholds. +cluster: + min_session_minutes: 5 # skip sessions shorter than this + idle_gap_minutes: 10 # new session after this much inactivity + min_cluster_size: 2 # need this many similar sessions before proposing + +# Content redaction regexes applied before any cluster summary leaves +# the local detection layer. Defense-in-depth on top of screenpipe's +# own app/window filtering. +redaction: + enabled: true diff --git a/.agents/skills/autoskill/references/https-proxy.md b/.agents/skills/autoskill/references/https-proxy.md new file mode 100644 index 0000000..9526697 --- /dev/null +++ b/.agents/skills/autoskill/references/https-proxy.md @@ -0,0 +1,62 @@ +# Optional: TLS for localhost screenpipe access + +Screenpipe's HTTP server (Axum, binding `localhost:3030`) speaks plain HTTP. For a Python script running as the same user on the same host, plain HTTP is adequate — loopback traffic never hits a network adapter, so TLS provides no additional confidentiality. + +TLS on localhost is only useful when: + +- A corporate security policy mandates "TLS everywhere" regardless of transport. +- The screenpipe endpoint is tunneled or exposed off-host. +- A browser client requires a "secure context" (Service Workers, WebCrypto). + +If you need it, put a one-line Caddy reverse proxy in front. Caddy's `tls internal` generates and trusts a local CA automatically. + +## Caddy + +Install: + +```bash +brew install caddy # macOS +# or see https://caddyserver.com/docs/install +``` + +Add to your `Caddyfile`: + +```caddyfile +screenpipe.local { + tls internal + reverse_proxy localhost:3030 +} +``` + +Ensure `screenpipe.local` resolves to loopback (add to `/etc/hosts`): + +``` +127.0.0.1 screenpipe.local +``` + +Start Caddy: + +```bash +caddy run +``` + +Then update autoskill's `config.yaml`: + +```yaml +screenpipe: + url: https://screenpipe.local +``` + +No code change is required on the autoskill side. `httpx` handles both HTTP and HTTPS transparently. + +## mkcert (alternative) + +If you prefer managing the cert yourself instead of Caddy's internal CA: + +```bash +brew install mkcert +mkcert -install +mkcert localhost 127.0.0.1 +``` + +Then terminate TLS with nginx, Caddy, or stunnel using the generated cert. diff --git a/.agents/skills/autoskill/references/screenpipe-config.yaml b/.agents/skills/autoskill/references/screenpipe-config.yaml new file mode 100644 index 0000000..5a6c011 --- /dev/null +++ b/.agents/skills/autoskill/references/screenpipe-config.yaml @@ -0,0 +1,61 @@ +# Starter screenpipe configuration for autoskill users. +# +# Copy the relevant sections into your screenpipe config (or pass as CLI +# flags when starting the daemon). Screenpipe handles app/window filtering +# at capture time — sensitive apps on this list will never be OCR'd, so +# their content never reaches autoskill's pipeline. +# +# Reference: https://github.com/screenpipe/screenpipe +# +# Review and edit for your setup. This is a conservative baseline, not +# an exhaustive list. Add any app where you handle secrets. + +ignored_apps: + # Password managers + - "1Password" + - "Bitwarden" + - "Dashlane" + - "Keeper" + - "LastPass" + - "KeePassXC" + + # Private messaging + - "Signal" + - "Telegram" + - "WhatsApp" + - "iMessage" + - "Messages" + + # Mail composition (inbound reading is fine; composition often contains secrets) + # Remove if you explicitly want mail workflows analyzed. + - "Mail" + - "Outlook" + + # Banking / finance apps (add yours) + # - "Bank of America" + # - "Chase" + +# Window title globs to ignore (matched across all apps). +# Useful for catching sensitive URLs in browsers that are otherwise OK to observe. +ignored_windows: + - "*Bitwarden*" + - "*1Password*" + - "*login*" + - "*Sign in*" + - "*Private Browsing*" + - "*Incognito*" + - "*online banking*" + - "*bank*" + - "*account settings*" + +# Optionally restrict capture to specific hours (local time). +# Uncomment to apply. +# hours: +# start: "09:00" +# end: "19:00" + +# Content types to capture. Drop "audio" if you don't want transcripts. +content_types: + - ocr + - ui + # - audio diff --git a/.agents/skills/autoskill/scripts/autoskill.py b/.agents/skills/autoskill/scripts/autoskill.py new file mode 100644 index 0000000..f76c1ac --- /dev/null +++ b/.agents/skills/autoskill/scripts/autoskill.py @@ -0,0 +1,35 @@ +"""Unified CLI for the autoskill skill. + +Subcommands: + run — detect workflows and draft proposed skills + doctor — verify screenpipe + LM Studio + config + skills dir + promote — move an approved proposal into skills/ +""" + +import argparse +import sys + + +def main(argv=None): + parser = argparse.ArgumentParser(prog="autoskill", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("command", choices=["run", "doctor", "promote"], + help="subcommand to run") + parser.add_argument("rest", nargs=argparse.REMAINDER, + help="arguments forwarded to the subcommand") + args = parser.parse_args(argv) + + if args.command == "run": + import run as _run + return _run.main(args.rest) + if args.command == "doctor": + import doctor as _doctor + return _doctor.main(args.rest) + if args.command == "promote": + import promote as _promote + return _promote.main(args.rest) + raise AssertionError(f"unreachable: {args.command!r}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.agents/skills/autoskill/scripts/backends.py b/.agents/skills/autoskill/scripts/backends.py new file mode 100644 index 0000000..95eee87 --- /dev/null +++ b/.agents/skills/autoskill/scripts/backends.py @@ -0,0 +1,116 @@ +import ipaddress +import os +import sys +from urllib.parse import urlparse + +import httpx + + +def _is_loopback(host): + if host in ("localhost", ""): + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def check_remote_endpoint(endpoint, label): + """Reject cleartext transport to a remote host, and name the destination. + + This backend sends summaries derived from the user's screen-capture history. + The endpoint is read from config.yaml, so it is worth being explicit about + where that data is about to go, and refusing to send it -- along with an API + key header -- over plaintext HTTP to anything but the local machine. + """ + parsed = urlparse(endpoint) + host = parsed.hostname or "" + + if parsed.scheme not in ("http", "https"): + raise ValueError( + f"{label} endpoint must be an http:// or https:// URL, got {endpoint!r}" + ) + + if parsed.scheme == "http" and not _is_loopback(host): + raise ValueError( + f"{label} endpoint {endpoint!r} uses plaintext HTTP to a remote host. " + "Screen-derived content and your API key would cross the network " + "unencrypted. Use https://, or point the endpoint at localhost." + ) + + if not _is_loopback(host): + print( + f"[autoskill] sending screen-derived summaries to {parsed.scheme}://{host}", + file=sys.stderr, + ) + + return endpoint + + +class ClaudeBackend: + def __init__(self, api_key, model, client=None): + self.api_key = api_key + self.model = model + self.client = client or httpx.Client(base_url="https://api.anthropic.com", timeout=60.0) + + def __call__(self, prompt): + response = self.client.post( + "/v1/messages", + headers={ + "x-api-key": self.api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + json={ + "model": self.model, + "max_tokens": 4096, + "messages": [{"role": "user", "content": prompt}], + }, + ) + response.raise_for_status() + payload = response.json() + return payload["content"][0]["text"] + + +class LocalBackend: + def __init__(self, endpoint, model, client=None): + self.endpoint = endpoint + self.model = model + self.client = client or httpx.Client(base_url=endpoint, timeout=120.0) + + def __call__(self, prompt): + response = self.client.post( + "/chat/completions", + json={ + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + }, + ) + response.raise_for_status() + payload = response.json() + return payload["choices"][0]["message"]["content"] + + +def make_backend(config): + kind = config.get("backend") + if kind == "claude": + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + raise RuntimeError("ANTHROPIC_API_KEY environment variable not set") + model = config.get("claude", {}).get("model", "claude-opus-4-7") + return ClaudeBackend(api_key=api_key, model=model) + + if kind == "foundry": + api_key = os.environ.get("FOUNDRY_API_KEY") + if not api_key: + raise RuntimeError("FOUNDRY_API_KEY environment variable not set") + f = config.get("foundry", {}) + endpoint = check_remote_endpoint(f["endpoint"], "foundry") + client = httpx.Client(base_url=endpoint, timeout=60.0) + return ClaudeBackend(api_key=api_key, model=f.get("model", "claude-opus-4-7"), client=client) + + if kind == "local": + l = config.get("local", {}) + return LocalBackend(endpoint=check_remote_endpoint(l["endpoint"], "local"), model=l["model"]) + + raise ValueError(f"unknown backend: {kind!r}") diff --git a/.agents/skills/autoskill/scripts/cluster.py b/.agents/skills/autoskill/scripts/cluster.py new file mode 100644 index 0000000..f01e637 --- /dev/null +++ b/.agents/skills/autoskill/scripts/cluster.py @@ -0,0 +1,54 @@ +from collections import defaultdict + + +def segment_sessions(events, idle_gap_seconds, min_session_seconds): + if not events: + return [] + events = sorted(events, key=lambda e: e["ts"]) + groups = [[events[0]]] + for prev, curr in zip(events, events[1:]): + if curr["ts"] - prev["ts"] > idle_gap_seconds: + groups.append([curr]) + else: + groups[-1].append(curr) + + sessions = [] + for group in groups: + duration = group[-1]["ts"] - group[0]["ts"] + if duration < min_session_seconds: + continue + apps, seen = [], set() + for evt in group: + if evt["app"] not in seen: + seen.add(evt["app"]) + apps.append(evt["app"]) + sessions.append({ + "start_ts": group[0]["ts"], + "end_ts": group[-1]["ts"], + "duration_seconds": duration, + "apps": apps, + "window_titles": [e["window_title"] for e in group if e.get("window_title")], + }) + return sessions + + +def cluster_sessions(sessions, min_cluster_size): + buckets = defaultdict(list) + for s in sessions: + buckets[tuple(s["apps"])].append(s) + + clusters = [] + for apps, members in buckets.items(): + if len(members) < min_cluster_size: + continue + example_titles = [] + for m in members: + if m["window_titles"]: + example_titles.append(m["window_titles"][0]) + clusters.append({ + "apps": list(apps), + "session_count": len(members), + "total_duration_seconds": sum(m["duration_seconds"] for m in members), + "example_titles": example_titles, + }) + return clusters diff --git a/.agents/skills/autoskill/scripts/doctor.py b/.agents/skills/autoskill/scripts/doctor.py new file mode 100644 index 0000000..55d3b1e --- /dev/null +++ b/.agents/skills/autoskill/scripts/doctor.py @@ -0,0 +1,108 @@ +import argparse +import os +import sys +from pathlib import Path + +import httpx + +_VALID_BACKENDS = {"local", "claude", "foundry"} + + +def default_screenpipe_probe(config): + sp = config.get("screenpipe", {}) + url = sp.get("url", "http://localhost:3030") + token = sp.get("token") or os.environ.get("SCREENPIPE_TOKEN") + headers = {"Authorization": f"Bearer {token}"} if token else {} + try: + r = httpx.get(f"{url}/health", headers=headers, timeout=5.0) + if r.status_code == 200: + return ("ok", url) + return ("error", f"{url} returned HTTP {r.status_code}") + except httpx.HTTPError as e: + return ("error", f"{url}: {e}") + + +def default_llm_probe(config): + kind = config.get("backend") + if kind == "local": + endpoint = config.get("local", {}).get("endpoint", "http://localhost:1234/v1") + try: + r = httpx.get(f"{endpoint}/models", timeout=5.0) + if r.status_code == 200: + return ("ok", endpoint) + return ("error", f"{endpoint} returned HTTP {r.status_code}") + except httpx.HTTPError as e: + return ("error", f"{endpoint}: {e}") + if kind == "claude": + if not os.environ.get("ANTHROPIC_API_KEY"): + return ("error", "ANTHROPIC_API_KEY not set") + return ("ok", "ANTHROPIC_API_KEY present (not probed)") + if kind == "foundry": + if not os.environ.get("FOUNDRY_API_KEY"): + return ("error", "FOUNDRY_API_KEY not set") + return ("ok", "FOUNDRY_API_KEY present (not probed)") + return ("error", f"unknown backend: {kind!r}") + + +def check(config, *, skills_dir, screenpipe_probe, llm_probe): + result = {} + + kind = config.get("backend") + if kind in _VALID_BACKENDS: + result["config"] = ("ok", f"backend={kind}") + else: + result["config"] = ("error", f"unknown backend: {kind!r}") + + skills_dir = Path(skills_dir) + if skills_dir.is_dir(): + result["skills_dir"] = ("ok", str(skills_dir)) + else: + result["skills_dir"] = ("error", f"not a directory: {skills_dir}") + + result["screenpipe"] = screenpipe_probe(config) + result["llm"] = llm_probe(config) + return result + + +def _format_report(result: dict) -> str: + lines = ["autoskill doctor", "================"] + for key in ("config", "skills_dir", "screenpipe", "llm"): + status, detail = result[key] + lines.append(f" {key:12s}: {status:5s} {detail}") + return "\n".join(lines) + + +def main(argv=None): + import yaml + + parser = argparse.ArgumentParser( + prog="autoskill-doctor", + description="Check that screenpipe, LM Studio, config, and skills dir are ready.", + ) + parser.add_argument("--config", required=True, + help="path to autoskill config.yaml") + parser.add_argument("--skills-dir", required=True, + help="path to skills/") + args = parser.parse_args(argv) + + config = yaml.safe_load(Path(args.config).read_text()) + result = check( + config, + skills_dir=args.skills_dir, + screenpipe_probe=default_screenpipe_probe, + llm_probe=default_llm_probe, + ) + + report = _format_report(result) + print(report) + + any_error = any(status == "error" for status, _ in result.values()) + if any_error: + print("\ndoctor: one or more checks failed", file=sys.stderr) + return 1 + print("\ndoctor: all checks passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.agents/skills/autoskill/scripts/fetch_window.py b/.agents/skills/autoskill/scripts/fetch_window.py new file mode 100644 index 0000000..1d0d6a8 --- /dev/null +++ b/.agents/skills/autoskill/scripts/fetch_window.py @@ -0,0 +1,33 @@ +_MAX_PAGES = 10_000 # bounded exit: hard ceiling so the loop cannot spin forever + + +def fetch_window(client, start_time, end_time, page_size=50, token=None): + events = [] + offset = 0 + headers = {"Authorization": f"Bearer {token}"} if token else {} + for _page in range(_MAX_PAGES): + response = client.get("/search", params={ + "start_time": start_time, + "end_time": end_time, + "limit": page_size, + "offset": offset, + }, headers=headers) + response.raise_for_status() + payload = response.json() + data = payload.get("data", []) + total = payload.get("pagination", {}).get("total", 0) + + for item in data: + content = item.get("content", {}) + events.append({ + "ts": content.get("timestamp"), + "app": content.get("app_name", ""), + "window_title": content.get("window_name", ""), + "text": content.get("text", ""), + "content_type": item.get("type", "").lower(), + }) + + offset += len(data) + if not data or offset >= total: + break + return events diff --git a/.agents/skills/autoskill/scripts/match_skills.py b/.agents/skills/autoskill/scripts/match_skills.py new file mode 100644 index 0000000..6bdb255 --- /dev/null +++ b/.agents/skills/autoskill/scripts/match_skills.py @@ -0,0 +1,46 @@ +import math +from pathlib import Path + + +def _parse_frontmatter(content: str) -> dict: + if not content.startswith("---"): + return {} + _, _, rest = content.partition("---\n") + block, _, _ = rest.partition("\n---") + out = {} + for line in block.splitlines(): + if ":" not in line: + continue + key, _, value = line.partition(":") + out[key.strip()] = value.strip() + return out + + +def load_skill_descriptions(skills_dir): + skills_dir = Path(skills_dir) + skills = [] + for skill_md in sorted(skills_dir.glob("*/SKILL.md")): + fm = _parse_frontmatter(skill_md.read_text()) + if "name" in fm and "description" in fm: + skills.append({"name": fm["name"], "description": fm["description"]}) + return skills + + +def _cosine(a, b): + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + if na == 0 or nb == 0: + return 0.0 + return dot / (na * nb) + + +def top_k_matches(query, skills, embedder, k): + q = embedder(query) + scored = [ + {"name": s["name"], "description": s["description"], + "score": _cosine(q, embedder(s["description"]))} + for s in skills + ] + scored.sort(key=lambda r: r["score"], reverse=True) + return scored[:k] diff --git a/.agents/skills/autoskill/scripts/promote.py b/.agents/skills/autoskill/scripts/promote.py new file mode 100644 index 0000000..c91be75 --- /dev/null +++ b/.agents/skills/autoskill/scripts/promote.py @@ -0,0 +1,58 @@ +import argparse +import shutil +import sys +from pathlib import Path + + +class PromoteError(Exception): + pass + + +_KINDS = ("new-skills", "composition-recipes") + + +def promote(proposed_path, skills_dir, name): + proposed_path = Path(proposed_path) + skills_dir = Path(skills_dir) + + source = None + for kind in _KINDS: + candidate = proposed_path / kind / name + if candidate.is_dir(): + source = candidate + break + if source is None: + raise PromoteError(f"proposed skill {name!r} not found under {proposed_path}") + + target = skills_dir / name + if target.exists(): + raise PromoteError(f"target {target} already exists") + + shutil.move(str(source), str(target)) + return target + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="autoskill-promote", + description="Move a proposed skill from _proposed// into skills/", + ) + parser.add_argument("--proposed", required=True, + help="path to the _proposed// directory") + parser.add_argument("--skills-dir", required=True, + help="path to skills/") + parser.add_argument("--name", required=True, help="skill name to promote") + args = parser.parse_args(argv) + + try: + target = promote(args.proposed, args.skills_dir, args.name) + except PromoteError as e: + print(f"promote failed: {e}", file=sys.stderr) + return 1 + + print(f"promoted: {args.name} -> {target}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.agents/skills/autoskill/scripts/redact.py b/.agents/skills/autoskill/scripts/redact.py new file mode 100644 index 0000000..3c94043 --- /dev/null +++ b/.agents/skills/autoskill/scripts/redact.py @@ -0,0 +1,40 @@ +import re + +# Order matters: multi-line and prefixed patterns run before narrower ones. +_PATTERNS = [ + (re.compile(r"-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+PRIVATE KEY-----"), + "[REDACTED:private_key]"), + + # Known-env-var secret assignments: NAME=value (catches long values only) + (re.compile( + r"\b(?:AWS_SECRET_ACCESS_KEY|AWS_ACCESS_KEY_ID|GITHUB_TOKEN|HF_TOKEN" + r"|ANTHROPIC_API_KEY|OPENAI_API_KEY|FOUNDRY_API_KEY|SCREENPIPE_TOKEN" + r"|GOOGLE_API_KEY|SLACK_TOKEN|DEEPGRAM_API_KEY)" + r"\s*=\s*[^\s\"']+" + ), "[REDACTED:kv_secret]"), + + (re.compile(r"Bearer\s+[A-Za-z0-9_\-\.=]+"), "[REDACTED:bearer]"), + + (re.compile(r"\beyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+"), + "[REDACTED:jwt]"), + + (re.compile(r"\bxox[bpars]-[A-Za-z0-9\-]{10,}"), "[REDACTED:api_key]"), + (re.compile(r"\bhf_[A-Za-z0-9]{32,}"), "[REDACTED:api_key]"), + (re.compile(r"\bsk-[A-Za-z0-9_\-]{20,}"), "[REDACTED:api_key]"), + (re.compile(r"\b(?:sk|pk|rk)_live_[A-Za-z0-9]{24,}"), "[REDACTED:api_key]"), + (re.compile(r"\bghp_[A-Za-z0-9]{36}\b"), "[REDACTED:api_key]"), + (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[REDACTED:api_key]"), + (re.compile(r"\bAIza[A-Za-z0-9_\-]{35}\b"), "[REDACTED:api_key]"), + + (re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}"), + "[REDACTED:email]"), + + (re.compile(r"\(\d{3}\)\s*\d{3}-\d{4}"), "[REDACTED:phone]"), + (re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "[REDACTED:ssn]"), +] + + +def redact(text: str) -> str: + for pattern, placeholder in _PATTERNS: + text = pattern.sub(placeholder, text) + return text diff --git a/.agents/skills/autoskill/scripts/run.py b/.agents/skills/autoskill/scripts/run.py new file mode 100644 index 0000000..b7e9b0f --- /dev/null +++ b/.agents/skills/autoskill/scripts/run.py @@ -0,0 +1,194 @@ +import datetime as _dt +from pathlib import Path + +import httpx + +from cluster import cluster_sessions, segment_sessions +from fetch_window import fetch_window +from match_skills import load_skill_descriptions, top_k_matches +from redact import redact +from synthesize import synthesize + + +class ScreenpipeUnreachable(RuntimeError): + """Raised when the screenpipe daemon cannot be reached. + + The autoskill skill cannot run without screenpipe. Install it from + https://github.com/screenpipe/screenpipe and start the daemon before + invoking this skill. + """ + + +def _default_now() -> str: + return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%S") + + +def _cluster_query(cluster: dict) -> str: + parts = ["apps: " + ", ".join(cluster["apps"])] + if cluster.get("example_titles"): + parts.append("titles: " + "; ".join(cluster["example_titles"])) + return " | ".join(parts) + + +def _write_plan(proposed_path: Path, clusters: list[dict]) -> None: + lines = ["# Dry-run plan", ""] + for i, c in enumerate(clusters, 1): + lines += [ + f"## Cluster {i}", + f"- apps: {', '.join(c['apps'])}", + f"- sessions: {c['session_count']}", + f"- total_duration_seconds: {c['total_duration_seconds']}", + f"- example titles: {'; '.join(c.get('example_titles', []))}", + "", + ] + (proposed_path / "plan.md").write_text("\n".join(lines)) + + +def _write_report(proposed_path: Path, results: list[dict]) -> None: + lines = ["# autoskill report", ""] + if not results: + lines.append("No clusters met the minimum size threshold. Nothing to propose.") + for r in results: + c = r["cluster"] + lines += [ + f"## {', '.join(c['apps'])} — {c['session_count']}× ({c['total_duration_seconds']}s)", + f"- verdict: **{r['verdict']}**", + ] + if r["verdict"] == "reuse": + lines.append(f"- matched skill: `{r['target']}`") + else: + lines.append(f"- draft: `{r['draft_path']}`") + lines.append("- top matches:") + for s in r["top_k"]: + lines.append(f" - `{s['name']}` (score={s['score']:.2f})") + lines.append("") + (proposed_path / "report.md").write_text("\n".join(lines)) + + +def run(config, *, start_time, end_time, out_dir, + screenpipe_client, backend, embedder, skills_dir, + screenpipe_token=None, now=None, dry_run=False): + now = now or _default_now + try: + events = fetch_window(screenpipe_client, start_time, end_time, + token=screenpipe_token) + except (httpx.ConnectError, httpx.ConnectTimeout) as e: + base = getattr(screenpipe_client, "base_url", "http://localhost:3030") + raise ScreenpipeUnreachable( + f"cannot reach screenpipe at {base}: {e}. " + "Install and start the daemon — see " + "https://github.com/screenpipe/screenpipe — " + "or point config.yaml's screenpipe.url at your instance." + ) from e + + for e in events: + e["text"] = redact(e.get("text", "")) + e["window_title"] = redact(e.get("window_title", "")) + + cluster_cfg = config.get("cluster", {}) + idle_gap = cluster_cfg.get("idle_gap_minutes", 10) * 60 + min_session = cluster_cfg.get("min_session_minutes", 5) * 60 + min_cluster = cluster_cfg.get("min_cluster_size", 2) + + # fetch_window returns ts as ISO strings; convert to epoch for segmentation + for e in events: + if isinstance(e["ts"], str): + e["ts"] = int(_dt.datetime.fromisoformat(e["ts"].replace("Z", "+00:00")).timestamp()) + + sessions = segment_sessions(events, idle_gap_seconds=idle_gap, min_session_seconds=min_session) + clusters = cluster_sessions(sessions, min_cluster_size=min_cluster) + + proposed_path = Path(out_dir) / now() + proposed_path.mkdir(parents=True, exist_ok=True) + + if dry_run: + _write_plan(proposed_path, clusters) + return proposed_path + + if not clusters: + _write_report(proposed_path, []) + return proposed_path + + skills = load_skill_descriptions(Path(skills_dir)) + results = [] + for cluster in clusters: + query = _cluster_query(cluster) + top_k = top_k_matches(query, skills, embedder=embedder, k=5) + decision = synthesize(cluster, top_k, backend=backend) + + entry = {"cluster": cluster, "top_k": top_k, "verdict": decision["verdict"]} + if decision["verdict"] == "reuse": + entry["target"] = decision.get("target") + else: + kind = "new-skills" if decision["verdict"] == "novel" else "composition-recipes" + name = decision["name"] + draft_dir = proposed_path / kind / name + draft_dir.mkdir(parents=True) + (draft_dir / "SKILL.md").write_text(decision["skill_body"]) + entry["draft_path"] = str(draft_dir.relative_to(proposed_path)) + results.append(entry) + + _write_report(proposed_path, results) + return proposed_path + + +def main(argv=None): + import argparse + import sys + + import httpx + import yaml + + from backends import make_backend + + parser = argparse.ArgumentParser(prog="autoskill") + parser.add_argument("--start", required=True, help="ISO start time, e.g. 2026-04-17T00:00:00Z") + parser.add_argument("--end", required=True, help="ISO end time") + parser.add_argument("--config", default=str(Path(__file__).resolve().parent.parent / "config.yaml")) + parser.add_argument("--out", default=None, + help="output directory for proposals (default: ~/.autoskill/proposed)") + parser.add_argument("--skills-dir", default=None, + help="path to skills/ (default: parent of this skill's dir)") + parser.add_argument("--dry-run", action="store_true", + help="stop after clustering; do not call the LLM backend") + args = parser.parse_args(argv) + + config = yaml.safe_load(Path(args.config).read_text()) + + here = Path(__file__).resolve() + skills_dir = Path(args.skills_dir) if args.skills_dir else here.parent.parent.parent + out_dir = Path(args.out) if args.out else Path.home() / ".autoskill" / "proposed" + + import os + screenpipe_cfg = config.get("screenpipe", {}) + screenpipe_url = screenpipe_cfg.get("url", "http://localhost:3030") + screenpipe_token = (screenpipe_cfg.get("token") + or os.environ.get("SCREENPIPE_TOKEN")) + screenpipe_client = httpx.Client(base_url=screenpipe_url, timeout=60.0) + + if args.dry_run: + backend = None + embedder = None + else: + backend = make_backend(config) + from sentence_transformers import SentenceTransformer + model = SentenceTransformer( + config.get("embeddings", {}).get("model", "sentence-transformers/all-MiniLM-L6-v2") + ) + + def embedder(text: str): + return list(map(float, model.encode(text))) + + proposed = run( + config, + start_time=args.start, end_time=args.end, out_dir=out_dir, + screenpipe_client=screenpipe_client, backend=backend, embedder=embedder, + skills_dir=skills_dir, screenpipe_token=screenpipe_token, + dry_run=args.dry_run, + ) + print(f"proposals written to: {proposed}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.agents/skills/autoskill/scripts/synthesize.py b/.agents/skills/autoskill/scripts/synthesize.py new file mode 100644 index 0000000..b7ffae2 --- /dev/null +++ b/.agents/skills/autoskill/scripts/synthesize.py @@ -0,0 +1,72 @@ +import json +import re + +VALID_VERDICTS = {"reuse", "compose", "novel"} + + +class SynthesisError(Exception): + pass + + +def _build_prompt(cluster, top_k_skills): + apps = ", ".join(cluster["apps"]) + titles = "; ".join(cluster.get("example_titles", [])) + candidates = "\n".join( + f"- {s['name']} (score={s['score']:.2f}): {s['description']}" + for s in top_k_skills + ) + return f"""You are classifying an observed user workflow against an existing skill library. + +Cluster: +- apps: {apps} +- sessions: {cluster['session_count']} +- total_duration_seconds: {cluster['total_duration_seconds']} +- example titles: {titles} + +Candidate existing skills (ranked by semantic similarity): +{candidates} + +Decide one of: +- "reuse": an existing skill already covers this workflow. +- "compose": no single skill covers it, but chaining existing skills does. Draft a thin SKILL.md that invokes them in order. +- "novel": not covered; draft a full new SKILL.md. + +Respond with a single JSON object: +- reuse: {{"verdict": "reuse", "target": ""}} +- compose: {{"verdict": "compose", "name": "", "skill_body": ""}} +- novel: {{"verdict": "novel", "name": "", "skill_body": ""}} +""" + + +def _extract_json(text: str) -> dict: + fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL) + if fence: + candidate = fence.group(1) + else: + start = text.find("{") + end = text.rfind("}") + if start == -1 or end == -1 or end <= start: + raise SynthesisError(f"no JSON object found in response: {text!r}") + candidate = text[start:end + 1] + try: + return json.loads(candidate) + except json.JSONDecodeError as e: + raise SynthesisError(f"invalid JSON in response: {e}") from e + + +def synthesize(cluster, top_k_skills, backend): + prompt = _build_prompt(cluster, top_k_skills) + response = backend(prompt) + payload = _extract_json(response) + + verdict = payload.get("verdict") + if verdict not in VALID_VERDICTS: + raise SynthesisError(f"unknown verdict: {verdict!r}") + + result = {"verdict": verdict, "skill_body": None} + if verdict == "reuse": + result["target"] = payload.get("target") + else: + result["name"] = payload.get("name") + result["skill_body"] = payload.get("skill_body") + return result diff --git a/.agents/skills/benchling-integration/SKILL.md b/.agents/skills/benchling-integration/SKILL.md new file mode 100644 index 0000000..82cf7d9 --- /dev/null +++ b/.agents/skills/benchling-integration/SKILL.md @@ -0,0 +1,229 @@ +--- +name: benchling-integration +description: Benchling Python SDK and REST API integration for registry entities, inventory, ELN entries, workflows, Benchling Apps, and Data Warehouse queries. Use when automating lab data with benchling-sdk or the v2 API. +license: MIT +allowed-tools: Read Write Edit Bash +compatibility: Requires a Benchling account, tenant URL, and API key or OAuth app credentials. Install benchling-sdk with uv pip install. +metadata: + version: "1.4" + skill-author: K-Dense Inc. + openclaw: + primaryEnv: BENCHLING_API_KEY + envVars: + - name: BENCHLING_TENANT_URL + required: true + description: Benchling tenant base URL. + - name: BENCHLING_API_KEY + required: false + description: API key auth (alternative to OAuth). + - name: BENCHLING_CLIENT_ID + required: false + description: OAuth app client id. + - name: BENCHLING_CLIENT_SECRET + required: false + description: OAuth app client secret. + - name: BENCHLING_PROD_TENANT_URL + required: false + description: Production tenant URL (multi-env setups). + - name: BENCHLING_PROD_API_KEY + required: false + description: Production API key (multi-env setups). + - name: BENCHLING_STAGING_TENANT_URL + required: false + description: Staging tenant URL (multi-env setups). + - name: BENCHLING_STAGING_API_KEY + required: false + description: Staging API key (multi-env setups). +--- + +# Benchling Integration + +## Overview + +Benchling is a cloud platform for life sciences R&D. Access registry entities (DNA, RNA, proteins), inventory, electronic lab notebooks, and workflows programmatically via the Python SDK and REST API. + +**Version note:** Examples target **benchling-sdk 1.25.0** (latest stable on PyPI). Docs: [benchling.com/sdk-docs](https://benchling.com/sdk-docs/). Platform guide: [docs.benchling.com](https://docs.benchling.com/). + +## When to Use This Skill + +This skill should be used when: +- Working with Benchling's Python SDK or REST API +- Managing biological sequences (DNA, RNA, proteins) and registry entities +- Automating inventory operations (samples, containers, locations, transfers) +- Creating or querying electronic lab notebook entries +- Building workflow automations or Benchling Apps +- Syncing data between Benchling and external systems +- Querying the Benchling Data Warehouse for analytics +- Setting up event-driven integrations with AWS EventBridge + +## Core Capabilities + +Seven capability areas, each with code, are in +[references/core_capabilities.md](references/core_capabilities.md): + +1. **Authentication and setup** — API key and OAuth app auth; see + [references/authentication.md](references/authentication.md). +2. **Registry and entity management** — DNA and AA sequences, custom entities, schemas, + and registration. +3. **Inventory management** — containers, boxes, plates, locations, and transfers. +4. **Notebook and documentation** — entries, day-to-day notes, and structured tables. +5. **Workflows and automation** — tasks, flowcharts, and assay runs. +6. **Events and integration** — EventBridge subscriptions; see + [references/eventbridge.md](references/eventbridge.md). +7. **Data warehouse and analytics** — SQL access to the warehouse. + +Endpoint and SDK detail is in +[references/api_endpoints.md](references/api_endpoints.md) and +[references/sdk_reference.md](references/sdk_reference.md). + +## Best Practices + +### Error Handling + +The SDK automatically retries failed requests: +```python +# Automatic retry for 429, 502, 503, 504 status codes +# Up to 5 retries with exponential backoff +# Customize retry behavior if needed +from benchling_sdk.retry import RetryStrategy + +benchling = Benchling( + url=tenant_url, + auth_method=ApiKeyAuth(api_key), + retry_strategy=RetryStrategy(max_retries=3), +) +``` + +### Pagination Efficiency + +Use generators for memory-efficient pagination: +```python +# Generator-based iteration +for page in benchling.dna_sequences.list(): + for sequence in page: + process(sequence) + +# Check estimated count without loading all pages +total = benchling.dna_sequences.list().estimated_count() +``` + +### Schema Fields Helper + +Use the `fields()` helper for custom schema fields: +```python +# Convert dict to Fields object +custom_fields = benchling.models.fields({ + "concentration": "100 ng/μL", + "date_prepared": "2025-10-20", + "notes": "High quality prep" +}) +``` + +### Forward Compatibility + +The SDK handles unknown enum values and types gracefully: +- Unknown enum values are preserved +- Unrecognized polymorphic types return `UnknownType` +- Allows working with newer API versions + +### Security Considerations + +- Never commit API keys or OAuth secrets to version control +- Read only named environment variables (`BENCHLING_TENANT_URL`, `BENCHLING_API_KEY`, etc.) +- Route network calls exclusively to your tenant URL +- Rotate keys if compromised; use OAuth for multi-user production apps +- Grant minimal necessary permissions for apps in the Developer Console + +## Resources + +### references/ + +Detailed reference documentation for in-depth information: + +- **authentication.md** - Comprehensive authentication guide including OIDC, security best practices, and credential management +- **sdk_reference.md** - Detailed Python SDK reference with advanced patterns, examples, and all entity types +- **api_endpoints.md** - REST API endpoint reference for direct HTTP calls without the SDK +- **eventbridge.md** - EventBridge setup, event payload schema, rule examples, Lambda handler, validation, and recovery + +Load these references as needed for specific integration requirements. + +## Common Use Cases + +**1. Bulk Entity Import:** +```python +# Import multiple sequences from FASTA file +from Bio import SeqIO + +for record in SeqIO.parse("sequences.fasta", "fasta"): + benchling.dna_sequences.create( + DnaSequenceCreate( + name=record.id, + bases=str(record.seq), + is_circular=False, + folder_id="fld_abc123" + ) + ) +``` + +**2. Inventory Audit:** +```python +# List all containers in a specific location +containers = benchling.containers.list( + parent_storage_id="box_abc123" +) + +for page in containers: + for container in page: + print(f"{container.name}: {container.barcode}") +``` + +**3. Workflow Automation:** +```python +# Update all pending tasks for a workflow +tasks = benchling.workflow_tasks.list( + workflow_id="wf_abc123", + status="pending" +) + +for page in tasks: + for task in page: + # Perform automated checks + if auto_validate(task): + benchling.workflow_tasks.update( + task_id=task.id, + workflow_task=WorkflowTaskUpdate( + status_id="status_complete" + ) + ) +``` + +**4. Data Export:** +```python +# Export all sequences with specific properties +sequences = benchling.dna_sequences.list() +export_data = [] + +for page in sequences: + for seq in page: + if seq.schema_id == "target_schema_id": + export_data.append({ + "id": seq.id, + "name": seq.name, + "bases": seq.bases, + "length": len(seq.bases) + }) + +# Save to CSV or database +import csv +with open("sequences.csv", "w") as f: + writer = csv.DictWriter(f, fieldnames=export_data[0].keys()) + writer.writeheader() + writer.writerows(export_data) +``` + +## Additional Resources + +- **Official Documentation:** https://docs.benchling.com +- **Python SDK Reference:** https://benchling.com/sdk-docs/ +- **API Reference:** https://benchling.com/api/reference +- **Support:** [email protected] diff --git a/.agents/skills/benchling-integration/references/api_endpoints.md b/.agents/skills/benchling-integration/references/api_endpoints.md new file mode 100644 index 0000000..4e88e45 --- /dev/null +++ b/.agents/skills/benchling-integration/references/api_endpoints.md @@ -0,0 +1,883 @@ +# Benchling REST API Endpoints Reference + +## Base URL + +All API requests use the base URL format: +``` +https://{tenant}.benchling.com/api/v2 +``` + +Replace `{tenant}` with your Benchling tenant name. + +## API Versioning + +Current API version: `v2` + +The API version is specified in the URL path. Stable endpoints follow [Benchling stability guidelines](https://docs.benchling.com/docs/stability); `alpha` and `beta` endpoints may change with shorter notice. + +## Authentication + +All requests require authentication via HTTP headers: + +**API Key (Basic Auth):** +```bash +curl -X GET \ + https://your-tenant.benchling.com/api/v2/dna-sequences \ + -u "your_api_key:" +``` + +**OAuth Bearer Token:** +```bash +curl -X GET \ + https://your-tenant.benchling.com/api/v2/dna-sequences \ + -H "Authorization: Bearer your_access_token" +``` + +## Common Headers + +``` +Authorization: Bearer {token} +Content-Type: application/json +Accept: application/json +``` + +## Response Format + +All responses follow a consistent JSON structure: + +**Single Resource:** +```json +{ + "id": "seq_abc123", + "name": "My Sequence", + "bases": "ATCGATCG", + ... +} +``` + +**List Response:** +```json +{ + "results": [ + {"id": "seq_1", "name": "Sequence 1"}, + {"id": "seq_2", "name": "Sequence 2"} + ], + "nextToken": "token_for_next_page" +} +``` + +## Pagination + +List endpoints support pagination: + +**Query Parameters:** +- `pageSize`: Number of items per page (default: 50, max: 100) +- `nextToken`: Token from previous response for next page + +**Example:** +```bash +curl -X GET \ + "https://your-tenant.benchling.com/api/v2/dna-sequences?pageSize=50&nextToken=abc123" +``` + +## Error Responses + +**Format:** +```json +{ + "error": { + "type": "NotFoundError", + "message": "DNA sequence not found", + "userMessage": "The requested sequence does not exist or you don't have access" + } +} +``` + +**Common Status Codes:** +- `200 OK`: Success +- `201 Created`: Resource created +- `400 Bad Request`: Invalid parameters +- `401 Unauthorized`: Missing or invalid credentials +- `403 Forbidden`: Insufficient permissions +- `404 Not Found`: Resource doesn't exist +- `422 Unprocessable Entity`: Validation error +- `429 Too Many Requests`: Rate limit exceeded +- `500 Internal Server Error`: Server error + +## Core Endpoints + +### DNA Sequences + +**List DNA Sequences:** +```http +GET /api/v2/dna-sequences + +Query Parameters: +- pageSize: integer (default: 50, max: 100) +- nextToken: string +- folderId: string +- schemaId: string +- name: string (filter by name) +- modifiedAt: string (ISO 8601 date) +``` + +**Get DNA Sequence:** +```http +GET /api/v2/dna-sequences/{sequenceId} +``` + +**Create DNA Sequence:** +```http +POST /api/v2/dna-sequences + +Body: +{ + "name": "My Plasmid", + "bases": "ATCGATCG", + "isCircular": true, + "folderId": "fld_abc123", + "schemaId": "ts_abc123", + "fields": { + "gene_name": {"value": "GFP"}, + "resistance": {"value": "Kanamycin"} + }, + "entityRegistryId": "src_abc123", // optional for registration + "namingStrategy": "NEW_IDS" // optional for registration +} +``` + +**Update DNA Sequence:** +```http +PATCH /api/v2/dna-sequences/{sequenceId} + +Body: +{ + "name": "Updated Plasmid", + "fields": { + "gene_name": {"value": "mCherry"} + } +} +``` + +**Archive DNA Sequence:** +```http +POST /api/v2/dna-sequences:archive + +Body: +{ + "dnaSequenceIds": ["seq_abc123"], + "reason": "Deprecated construct" +} +``` + +### RNA Sequences + +**List RNA Sequences:** +```http +GET /api/v2/rna-sequences +``` + +**Get RNA Sequence:** +```http +GET /api/v2/rna-sequences/{sequenceId} +``` + +**Create RNA Sequence:** +```http +POST /api/v2/rna-sequences + +Body: +{ + "name": "gRNA-001", + "bases": "AUCGAUCG", + "folderId": "fld_abc123", + "fields": { + "target_gene": {"value": "TP53"} + } +} +``` + +**Update RNA Sequence:** +```http +PATCH /api/v2/rna-sequences/{sequenceId} +``` + +**Archive RNA Sequence:** +```http +POST /api/v2/rna-sequences:archive +``` + +### Amino Acid (Protein) Sequences + +**List AA Sequences:** +```http +GET /api/v2/aa-sequences +``` + +**Get AA Sequence:** +```http +GET /api/v2/aa-sequences/{sequenceId} +``` + +**Create AA Sequence:** +```http +POST /api/v2/aa-sequences + +Body: +{ + "name": "GFP Protein", + "aminoAcids": "MSKGEELFTGVVPILVELDGDVNGHKF", + "folderId": "fld_abc123" +} +``` + +### Custom Entities + +**List Custom Entities:** +```http +GET /api/v2/custom-entities + +Query Parameters: +- schemaId: string (required to filter by type) +- pageSize: integer +- nextToken: string +``` + +**Get Custom Entity:** +```http +GET /api/v2/custom-entities/{entityId} +``` + +**Create Custom Entity:** +```http +POST /api/v2/custom-entities + +Body: +{ + "name": "HEK293T-Clone5", + "schemaId": "ts_cellline_abc", + "folderId": "fld_abc123", + "fields": { + "passage_number": {"value": "15"}, + "mycoplasma_test": {"value": "Negative"} + } +} +``` + +**Update Custom Entity:** +```http +PATCH /api/v2/custom-entities/{entityId} + +Body: +{ + "fields": { + "passage_number": {"value": "16"} + } +} +``` + +### Mixtures + +**List Mixtures:** +```http +GET /api/v2/mixtures +``` + +**Create Mixture:** +```http +POST /api/v2/mixtures + +Body: +{ + "name": "LB-Amp Media", + "folderId": "fld_abc123", + "schemaId": "ts_mixture_abc", + "ingredients": [ + { + "componentEntityId": "ent_lb_base", + "amount": {"value": "1000", "units": "mL"} + }, + { + "componentEntityId": "ent_ampicillin", + "amount": {"value": "100", "units": "mg"} + } + ] +} +``` + +### Containers + +**List Containers:** +```http +GET /api/v2/containers + +Query Parameters: +- parentStorageId: string (filter by location/box) +- schemaId: string +- barcode: string +``` + +**Get Container:** +```http +GET /api/v2/containers/{containerId} +``` + +**Create Container:** +```http +POST /api/v2/containers + +Body: +{ + "name": "Sample-001", + "schemaId": "cont_schema_abc", + "barcode": "CONT001", + "parentStorageId": "box_abc123", + "fields": { + "concentration": {"value": "100 ng/μL"}, + "volume": {"value": "50 μL"} + } +} +``` + +**Update Container:** +```http +PATCH /api/v2/containers/{containerId} + +Body: +{ + "fields": { + "volume": {"value": "45 μL"} + } +} +``` + +**Transfer Container:** +```http +POST /api/v2/containers:transfer + +Body: +{ + "containerIds": ["cont_abc123"], + "destinationStorageId": "box_xyz789" +} +``` + +**Check Out Container:** +```http +POST /api/v2/containers:checkout + +Body: +{ + "containerIds": ["cont_abc123"], + "comment": "Taking to bench" +} +``` + +**Check In Container:** +```http +POST /api/v2/containers:checkin + +Body: +{ + "containerIds": ["cont_abc123"], + "locationId": "bench_loc_abc" +} +``` + +### Boxes + +**List Boxes:** +```http +GET /api/v2/boxes + +Query Parameters: +- parentStorageId: string +- schemaId: string +``` + +**Get Box:** +```http +GET /api/v2/boxes/{boxId} +``` + +**Create Box:** +```http +POST /api/v2/boxes + +Body: +{ + "name": "Freezer-A-Box-01", + "schemaId": "box_schema_abc", + "parentStorageId": "loc_freezer_a", + "barcode": "BOX001" +} +``` + +### Locations + +**List Locations:** +```http +GET /api/v2/locations +``` + +**Get Location:** +```http +GET /api/v2/locations/{locationId} +``` + +**Create Location:** +```http +POST /api/v2/locations + +Body: +{ + "name": "Freezer A - Shelf 2", + "parentStorageId": "loc_freezer_a", + "barcode": "LOC-A-S2" +} +``` + +### Plates + +**List Plates:** +```http +GET /api/v2/plates +``` + +**Get Plate:** +```http +GET /api/v2/plates/{plateId} +``` + +**Create Plate:** +```http +POST /api/v2/plates + +Body: +{ + "name": "PCR-Plate-001", + "schemaId": "plate_schema_abc", + "barcode": "PLATE001", + "wells": [ + {"position": "A1", "entityId": "ent_abc"}, + {"position": "A2", "entityId": "ent_xyz"} + ] +} +``` + +### Entries (Notebook) + +**List Entries:** +```http +GET /api/v2/entries + +Query Parameters: +- folderId: string +- schemaId: string +- modifiedAt: string +``` + +**Get Entry:** +```http +GET /api/v2/entries/{entryId} +``` + +**Create Entry:** +```http +POST /api/v2/entries + +Body: +{ + "name": "Experiment 2025-10-20", + "folderId": "fld_abc123", + "schemaId": "entry_schema_abc", + "fields": { + "objective": {"value": "Test gene expression"}, + "date": {"value": "2025-10-20"} + } +} +``` + +**Update Entry:** +```http +PATCH /api/v2/entries/{entryId} + +Body: +{ + "fields": { + "results": {"value": "Successful expression"} + } +} +``` + +### Workflow Tasks + +**List Workflow Tasks:** +```http +GET /api/v2/tasks + +Query Parameters: +- workflowId: string +- statusIds: string[] (comma-separated) +- assigneeId: string +``` + +**Get Task:** +```http +GET /api/v2/tasks/{taskId} +``` + +**Create Task:** +```http +POST /api/v2/tasks + +Body: +{ + "name": "PCR Amplification", + "workflowId": "wf_abc123", + "assigneeId": "user_abc123", + "schemaId": "task_schema_abc", + "fields": { + "template": {"value": "seq_abc123"}, + "priority": {"value": "High"} + } +} +``` + +**Update Task:** +```http +PATCH /api/v2/tasks/{taskId} + +Body: +{ + "statusId": "status_complete_abc", + "fields": { + "completion_date": {"value": "2025-10-20"} + } +} +``` + +### Folders + +**List Folders:** +```http +GET /api/v2/folders + +Query Parameters: +- projectId: string +- parentFolderId: string +``` + +**Get Folder:** +```http +GET /api/v2/folders/{folderId} +``` + +**Create Folder:** +```http +POST /api/v2/folders + +Body: +{ + "name": "2025 Experiments", + "parentFolderId": "fld_parent_abc", + "projectId": "proj_abc123" +} +``` + +### Projects + +**List Projects:** +```http +GET /api/v2/projects +``` + +**Get Project:** +```http +GET /api/v2/projects/{projectId} +``` + +### Users + +**Get Current User:** +```http +GET /api/v2/users/me +``` + +**List Users:** +```http +GET /api/v2/users +``` + +**Get User:** +```http +GET /api/v2/users/{userId} +``` + +### Teams + +**List Teams:** +```http +GET /api/v2/teams +``` + +**Get Team:** +```http +GET /api/v2/teams/{teamId} +``` + +### Schemas + +**List Schemas:** +```http +GET /api/v2/schemas + +Query Parameters: +- entityType: string (e.g., "dna_sequence", "custom_entity") +``` + +**Get Schema:** +```http +GET /api/v2/schemas/{schemaId} +``` + +### Registries + +**List Registries:** +```http +GET /api/v2/registries +``` + +**Get Registry:** +```http +GET /api/v2/registries/{registryId} +``` + +## Bulk Operations + +### Batch Archive + +**Archive Multiple Entities:** +```http +POST /api/v2/{entity-type}:archive + +Body: +{ + "{entity}Ids": ["id1", "id2", "id3"], + "reason": "Cleanup" +} +``` + +### Batch Transfer + +**Transfer Multiple Containers:** +```http +POST /api/v2/containers:bulk-transfer + +Body: +{ + "transfers": [ + {"containerId": "cont_1", "destinationId": "box_a"}, + {"containerId": "cont_2", "destinationId": "box_b"} + ] +} +``` + +## Async Operations + +Some operations return task IDs for async processing: + +**Response:** +```json +{ + "taskId": "task_abc123" +} +``` + +**Check Task Status:** +```http +GET /api/v2/tasks/{taskId} + +Response: +{ + "id": "task_abc123", + "status": "RUNNING", // or "SUCCEEDED", "FAILED" + "message": "Processing...", + "response": {...} // Available when status is SUCCEEDED +} +``` + +## Field Value Format + +Custom schema fields use a specific format: + +**Simple Value:** +```json +{ + "field_name": { + "value": "Field Value" + } +} +``` + +**Dropdown:** +```json +{ + "dropdown_field": { + "value": "Option1" // Must match exact option name + } +} +``` + +**Date:** +```json +{ + "date_field": { + "value": "2025-10-20" // Format: YYYY-MM-DD + } +} +``` + +**Entity Link:** +```json +{ + "entity_link_field": { + "value": "seq_abc123" // Entity ID + } +} +``` + +**Numeric:** +```json +{ + "numeric_field": { + "value": "123.45" // String representation + } +} +``` + +## Rate Limiting + +**Limits:** +- Default: 100 requests per 10 seconds per user/app +- Rate limit headers included in responses: + - `X-RateLimit-Limit`: Total allowed requests + - `X-RateLimit-Remaining`: Remaining requests + - `X-RateLimit-Reset`: Unix timestamp when limit resets + +**Handling 429 Responses:** +```json +{ + "error": { + "type": "RateLimitError", + "message": "Rate limit exceeded", + "retryAfter": 5 // Seconds to wait + } +} +``` + +## Filtering and Searching + +**Common Query Parameters:** +- `name`: Partial name match +- `modifiedAt`: ISO 8601 datetime +- `createdAt`: ISO 8601 datetime +- `schemaId`: Filter by schema +- `folderId`: Filter by folder +- `archived`: Boolean (include archived items) + +**Example:** +```bash +curl -X GET \ + "https://tenant.benchling.com/api/v2/dna-sequences?name=plasmid&folderId=fld_abc&archived=false" +``` + +## Best Practices + +### Request Efficiency + +1. **Use appropriate page sizes:** + - Default: 50 items + - Max: 100 items + - Adjust based on needs + +2. **Filter on server-side:** + - Use query parameters instead of client filtering + - Reduces data transfer and processing + +3. **Batch operations:** + - Use bulk endpoints when available + - Archive/transfer multiple items in one request + +### Error Handling + +```javascript +// Example error handling +async function fetchSequence(id) { + try { + const response = await fetch( + `https://tenant.benchling.com/api/v2/dna-sequences/${id}`, + { + headers: { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/json' + } + } + ); + + if (!response.ok) { + if (response.status === 429) { + // Rate limit - retry with backoff + const retryAfter = response.headers.get('Retry-After'); + await sleep(retryAfter * 1000); + return fetchSequence(id); + } else if (response.status === 404) { + return null; // Not found + } else { + throw new Error(`API error: ${response.status}`); + } + } + + return await response.json(); + } catch (error) { + console.error('Request failed:', error); + throw error; + } +} +``` + +### Pagination Loop + +```javascript +async function getAllSequences() { + let allSequences = []; + let nextToken = null; + + do { + const url = new URL('https://tenant.benchling.com/api/v2/dna-sequences'); + if (nextToken) { + url.searchParams.set('nextToken', nextToken); + } + url.searchParams.set('pageSize', '100'); + + const response = await fetch(url, { + headers: { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/json' + } + }); + + const data = await response.json(); + allSequences = allSequences.concat(data.results); + nextToken = data.nextToken; + } while (nextToken); + + return allSequences; +} +``` + +## References + +- **API Documentation:** https://benchling.com/api/reference +- **Interactive API Explorer:** https://your-tenant.benchling.com/api/reference (requires authentication) +- **Changelog:** https://docs.benchling.com/changelog diff --git a/.agents/skills/benchling-integration/references/authentication.md b/.agents/skills/benchling-integration/references/authentication.md new file mode 100644 index 0000000..252436b --- /dev/null +++ b/.agents/skills/benchling-integration/references/authentication.md @@ -0,0 +1,390 @@ +# Benchling Authentication Reference + +## Authentication Methods + +Benchling supports three authentication methods, each suited for different use cases. + +### 1. API Key Authentication (Basic Auth) + +**Best for:** Personal scripts, prototyping, single-user integrations + +**How it works:** +- Use your API key as the username in HTTP Basic authentication +- Leave the password field empty +- All requests must use HTTPS + +**Obtaining an API Key:** +1. Log in to your Benchling account +2. Navigate to Profile Settings +3. Find the API Key section +4. Generate a new API key +5. Store it securely (it will only be shown once) + +**Python SDK Usage:** +```python +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +benchling = Benchling( + url="https://your-tenant.benchling.com", + auth_method=ApiKeyAuth("your_api_key_here") +) +``` + +**Direct HTTP Usage:** +```bash +curl -X GET \ + https://your-tenant.benchling.com/api/v2/dna-sequences \ + -u "your_api_key_here:" +``` + +Note the colon after the API key with no password. + +**Environment Variable Pattern:** +```python +import os +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +api_key = os.environ.get("BENCHLING_API_KEY") +tenant_url = os.environ.get("BENCHLING_TENANT_URL") + +benchling = Benchling( + url=tenant_url, + auth_method=ApiKeyAuth(api_key) +) +``` + +### 2. OAuth 2.0 Client Credentials + +**Best for:** Multi-user applications, service accounts, production integrations + +**How it works:** +1. Register an application in Benchling's Developer Console +2. Obtain client ID and client secret +3. Exchange credentials for an access token +4. Use the access token for API requests +5. Refresh token when expired + +**Registering an App:** +1. Log in to Benchling as an admin +2. Navigate to Developer Console +3. Create a new App +4. Record the client ID and client secret +5. Configure OAuth redirect URIs and permissions + +**Python SDK Usage:** +```python +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2 + +auth_method = ClientCredentialsOAuth2( + client_id="your_client_id", + client_secret="your_client_secret" +) + +benchling = Benchling( + url="https://your-tenant.benchling.com", + auth_method=auth_method +) +``` + +The SDK automatically handles token refresh. + +**Direct HTTP Token Flow:** +```bash +# Get access token +curl -X POST \ + https://your-tenant.benchling.com/api/v2/token \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=your_client_id" \ + -d "client_secret=your_client_secret" + +# Response: +# { +# "access_token": "token_here", +# "token_type": "Bearer", +# "expires_in": 3600 +# } + +# Use access token +curl -X GET \ + https://your-tenant.benchling.com/api/v2/dna-sequences \ + -H "Authorization: Bearer access_token_here" +``` + +### 3. OpenID Connect (OIDC) + +**Best for:** Enterprise integrations with existing identity providers, SSO scenarios + +**How it works:** +- Authenticate users through your identity provider (Okta, Azure AD, etc.) +- Identity provider issues an ID token with email claim +- Benchling verifies the token against the OpenID configuration endpoint +- Matches authenticated user by email + +**Requirements:** +- Enterprise Benchling account +- Configured identity provider (IdP) +- IdP must issue tokens with email claims +- Email in token must match Benchling user email + +**Identity Provider Configuration:** +1. Configure your IdP to issue OpenID Connect tokens +2. Ensure tokens include the `email` claim +3. Provide Benchling with your IdP's OpenID configuration URL +4. Benchling will verify tokens against this configuration + +**Python Usage:** +```python +# Assuming you have an ID token from your IdP +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.oidc_auth import OidcAuth + +auth_method = OidcAuth(id_token="id_token_from_idp") + +benchling = Benchling( + url="https://your-tenant.benchling.com", + auth_method=auth_method +) +``` + +**Direct HTTP Usage:** +```bash +curl -X GET \ + https://your-tenant.benchling.com/api/v2/dna-sequences \ + -H "Authorization: Bearer id_token_here" +``` + +## Security Best Practices + +### Credential Storage + +**DO:** +- Store credentials in environment variables +- Use password managers or secret management services (AWS Secrets Manager, HashiCorp Vault) +- Encrypt credentials at rest +- Use different credentials for dev/staging/production + +**DON'T:** +- Commit credentials to version control +- Hardcode credentials in source files +- Share credentials via email or chat +- Store credentials in plain text files + +**Example with scoped environment variables:** +```python +import os +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +api_key = os.environ.get("BENCHLING_API_KEY") +tenant_url = os.environ.get("BENCHLING_TENANT_URL") + +if not api_key or not tenant_url: + raise ValueError("Set BENCHLING_API_KEY and BENCHLING_TENANT_URL") + +benchling = Benchling( + url=tenant_url, + auth_method=ApiKeyAuth(api_key), +) +``` + +Do not call `load_dotenv()` without filtering, and never iterate over `os.environ` to collect secrets. + +### Credential Rotation + +**API Key Rotation:** +1. Generate a new API key in Profile Settings +2. Update your application to use the new key +3. Verify the new key works +4. Delete the old API key + +**App Secret Rotation:** +1. Navigate to Developer Console +2. Select your app +3. Generate new client secret +4. Update your application configuration +5. Delete the old secret after verifying + +**Best Practice:** Rotate credentials regularly (e.g., every 90 days) and immediately if compromised. + +### Access Control + +**Principle of Least Privilege:** +- Grant only the minimum necessary permissions +- Use service accounts (apps) instead of personal accounts for automation +- Review and audit permissions regularly + +**App Permissions:** +Apps require explicit access grants to: +- Organizations +- Teams +- Projects +- Folders + +Configure these in the Developer Console when setting up your app. + +**User Permissions:** +API access mirrors UI permissions: +- Users can only access data they have permission to view/edit in the UI +- Suspended users lose API access +- Archived apps lose API access until unarchived + +### Network Security + +**HTTPS Only:** +All Benchling API requests must use HTTPS. HTTP requests will be rejected. + +**IP Allowlisting (Enterprise):** +Some enterprise accounts can restrict API access to specific IP ranges. Contact Benchling support to configure. + +**Rate Limiting:** +Benchling implements rate limiting to prevent abuse: +- Default: 100 requests per 10 seconds per user/app +- 429 status code returned when rate limit exceeded +- SDK automatically retries with exponential backoff + +### Audit Logging + +**Tracking API Usage:** +- All API calls are logged with user/app identity +- OAuth apps show proper audit trails with user attribution +- API key calls are attributed to the key owner +- Review audit logs in Benchling's admin console + +**Best Practice for Apps:** +Use OAuth instead of API keys when multiple users interact through your app. This ensures proper audit attribution to the actual user, not just the app. + +## Troubleshooting + +### Common Authentication Errors + +**401 Unauthorized:** +- Invalid or expired credentials +- API key not properly formatted +- Missing "Authorization" header + +**Solution:** +- Verify credentials are correct +- Check API key is not expired or deleted +- Ensure proper header format: `Authorization: Bearer ` + +**403 Forbidden:** +- Valid credentials but insufficient permissions +- User doesn't have access to the requested resource +- App not granted access to the organization/project + +**Solution:** +- Check user/app permissions in Benchling +- Grant necessary access in Developer Console (for apps) +- Verify the resource exists and user has access + +**429 Too Many Requests:** +- Rate limit exceeded +- Too many requests in short time period + +**Solution:** +- Implement exponential backoff +- SDK handles this automatically +- Consider caching results +- Spread requests over time + +### Testing Authentication + +**Quick Test with curl:** +```bash +# Test API key +curl -X GET \ + https://your-tenant.benchling.com/api/v2/users/me \ + -u "your_api_key:" \ + -v + +# Test OAuth token +curl -X GET \ + https://your-tenant.benchling.com/api/v2/users/me \ + -H "Authorization: Bearer your_token" \ + -v +``` + +The `/users/me` endpoint returns the authenticated user's information and is useful for verifying credentials. + +**Python SDK Test:** +```python +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +try: + benchling = Benchling( + url="https://your-tenant.benchling.com", + auth_method=ApiKeyAuth("your_api_key") + ) + + # Test authentication + user = benchling.users.get_me() + print(f"Authenticated as: {user.name} ({user.email})") + +except Exception as e: + print(f"Authentication failed: {e}") +``` + +## Multi-Tenant Considerations + +If working with multiple Benchling tenants, use separate named keys per tenant (for example `BENCHLING_PROD_API_KEY` and `BENCHLING_STAGING_API_KEY`) rather than reading the entire environment: + +```python +import os +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +tenants = { + "production": { + "url": os.environ.get("BENCHLING_PROD_TENANT_URL"), + "api_key": os.environ.get("BENCHLING_PROD_API_KEY"), + }, + "staging": { + "url": os.environ.get("BENCHLING_STAGING_TENANT_URL"), + "api_key": os.environ.get("BENCHLING_STAGING_API_KEY"), + }, +} + +clients = {} +for name, config in tenants.items(): + if not config["url"] or not config["api_key"]: + raise ValueError(f"Missing credentials for {name} tenant") + clients[name] = Benchling( + url=config["url"], + auth_method=ApiKeyAuth(config["api_key"]), + ) + +prod_sequences = clients["production"].dna_sequences.list() +``` + +## Advanced: Custom HTTPS Clients + +For environments with self-signed certificates or corporate proxies: + +```python +import httpx +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +# Custom httpx client with certificate verification +custom_client = httpx.Client( + verify="/path/to/custom/ca-bundle.crt", + timeout=30.0 +) + +benchling = Benchling( + url="https://your-tenant.benchling.com", + auth_method=ApiKeyAuth("your_api_key"), + http_client=custom_client +) +``` + +## References + +- **Official Authentication Docs:** https://docs.benchling.com/docs/authentication +- **Developer Console:** https://your-tenant.benchling.com/developer +- **SDK Documentation:** https://benchling.com/sdk-docs/ diff --git a/.agents/skills/benchling-integration/references/core_capabilities.md b/.agents/skills/benchling-integration/references/core_capabilities.md new file mode 100644 index 0000000..a6fb0dc --- /dev/null +++ b/.agents/skills/benchling-integration/references/core_capabilities.md @@ -0,0 +1,355 @@ +# Core Capabilities + +The seven capability areas in full, with code: authentication and setup, registry and +entity management, inventory management, notebook and documentation, workflows and +automation, events and integration, and the data warehouse and analytics. + +## Core Capabilities + +### 1. Authentication & Setup + +**Python SDK installation:** + +```bash +uv pip install "benchling-sdk==1.25.0" +``` + +Preview builds (alpha; not for production): + +```bash +uv pip install "benchling-sdk" --prerelease allow +``` + +**Environment variables (scoped reads only):** + +Read only the named keys you need — never dump or iterate over the full environment: + +```python +import os + +tenant_url = os.environ.get("BENCHLING_TENANT_URL") # e.g. https://your-tenant.benchling.com +api_key = os.environ.get("BENCHLING_API_KEY") + +if not tenant_url or not api_key: + raise ValueError("Set BENCHLING_TENANT_URL and BENCHLING_API_KEY") +``` + +Obtain an API key from **Profile Settings** in Benchling. For OAuth apps, use the [Developer Console](https://docs.benchling.com/docs/getting-started-benchling-apps) and store `BENCHLING_CLIENT_ID` / `BENCHLING_CLIENT_SECRET` separately. + +**Authentication methods:** + +API key (scripts and personal automation): + +```python +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +benchling = Benchling( + url=tenant_url, + auth_method=ApiKeyAuth(api_key), +) +``` + +OAuth client credentials (multi-user apps and production integrations): + +```python +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2 + +benchling = Benchling( + url=tenant_url, + auth_method=ClientCredentialsOAuth2( + client_id=os.environ["BENCHLING_CLIENT_ID"], + client_secret=os.environ["BENCHLING_CLIENT_SECRET"], + ), +) +``` + +**Key points:** +- All API requests require HTTPS; network calls must target your tenant URL only +- Authentication permissions mirror UI permissions +- Verify credentials with `benchling.users.get_me()` before bulk operations + +For detailed authentication information including OIDC and security best practices, refer to `references/authentication.md`. + +### 2. Registry & Entity Management + +Registry entities include DNA sequences, RNA sequences, AA sequences, custom entities, and mixtures. The SDK provides typed classes for creating and managing these entities. + +**Creating DNA Sequences:** +```python +from benchling_sdk.models import DnaSequenceCreate + +sequence = benchling.dna_sequences.create( + DnaSequenceCreate( + name="My Plasmid", + bases="ATCGATCG", + is_circular=True, + folder_id="fld_abc123", + schema_id="ts_abc123", # optional + fields=benchling.models.fields({"gene_name": "GFP"}) + ) +) +``` + +**Registry Registration:** + +To register an entity directly upon creation: +```python +sequence = benchling.dna_sequences.create( + DnaSequenceCreate( + name="My Plasmid", + bases="ATCGATCG", + is_circular=True, + folder_id="fld_abc123", + entity_registry_id="src_abc123", # Registry to register in + naming_strategy="NEW_IDS" # or "IDS_FROM_NAMES" + ) +) +``` + +**Important:** Use either `entity_registry_id` OR `naming_strategy`, never both. + +**Updating Entities:** +```python +from benchling_sdk.models import DnaSequenceUpdate + +updated = benchling.dna_sequences.update( + sequence_id="seq_abc123", + dna_sequence=DnaSequenceUpdate( + name="Updated Plasmid Name", + fields=benchling.models.fields({"gene_name": "mCherry"}) + ) +) +``` + +Unspecified fields remain unchanged, allowing partial updates. + +**Listing and Pagination:** +```python +# List all DNA sequences (returns a generator) +sequences = benchling.dna_sequences.list() +for page in sequences: + for seq in page: + print(f"{seq.name} ({seq.id})") + +# Check total count +total = sequences.estimated_count() +``` + +**Key Operations:** +- Create: `benchling..create()` +- Read: `benchling..get_by_id(id)` or `.list()` +- Update: `benchling..update(id, update_object)` +- Archive: `benchling..archive(id)` + +Entity types: `dna_sequences`, `rna_sequences`, `aa_sequences`, `custom_entities`, `mixtures` + +For comprehensive SDK reference and advanced patterns, refer to `references/sdk_reference.md`. + +### 3. Inventory Management + +Manage physical samples, containers, boxes, and locations within the Benchling inventory system. + +**Creating Containers:** +```python +from benchling_sdk.models import ContainerCreate + +container = benchling.containers.create( + ContainerCreate( + name="Sample Tube 001", + schema_id="cont_schema_abc123", + parent_storage_id="box_abc123", # optional + fields=benchling.models.fields({"concentration": "100 ng/μL"}) + ) +) +``` + +**Managing Boxes:** +```python +from benchling_sdk.models import BoxCreate + +box = benchling.boxes.create( + BoxCreate( + name="Freezer Box A1", + schema_id="box_schema_abc123", + parent_storage_id="loc_abc123" + ) +) +``` + +**Transferring Items:** +```python +# Transfer a container to a new location +transfer = benchling.containers.transfer( + container_id="cont_abc123", + destination_id="box_xyz789" +) +``` + +**Key Inventory Operations:** +- Create containers, boxes, locations, plates +- Update inventory item properties +- Transfer items between locations +- Check in/out items +- Batch operations for bulk transfers + +### 4. Notebook & Documentation + +Interact with electronic lab notebook (ELN) entries, protocols, and templates. + +**Creating Notebook Entries:** +```python +from benchling_sdk.models import EntryCreate + +entry = benchling.entries.create( + EntryCreate( + name="Experiment 2025-10-20", + folder_id="fld_abc123", + schema_id="entry_schema_abc123", + fields=benchling.models.fields({"objective": "Test gene expression"}) + ) +) +``` + +**Linking Entities to Entries:** +```python +# Add references to entities in an entry +entry_link = benchling.entry_links.create( + entry_id="entry_abc123", + entity_id="seq_xyz789" +) +``` + +**Key Notebook Operations:** +- Create and update lab notebook entries +- Manage entry templates +- Link entities and results to entries +- Export entries for documentation + +### 5. Workflows & Automation + +Automate laboratory processes using Benchling's workflow system. + +**Creating Workflow Tasks:** +```python +from benchling_sdk.models import WorkflowTaskCreate + +task = benchling.workflow_tasks.create( + WorkflowTaskCreate( + name="PCR Amplification", + workflow_id="wf_abc123", + assignee_id="user_abc123", + fields=benchling.models.fields({"template": "seq_abc123"}) + ) +) +``` + +**Updating Task Status:** +```python +from benchling_sdk.models import WorkflowTaskUpdate + +updated_task = benchling.workflow_tasks.update( + task_id="task_abc123", + workflow_task=WorkflowTaskUpdate( + status_id="status_complete_abc123" + ) +) +``` + +**Asynchronous Operations:** + +Some operations are asynchronous and return tasks. The SDK default `max_wait_seconds` for polling is **600 seconds** (since SDK 1.11.0): + +```python +from benchling_sdk.helpers.tasks import wait_for_task + +result = wait_for_task( + benchling, + task_id="task_abc123", + interval_wait_seconds=2, + max_wait_seconds=300, # override for long-running serverless handlers +) +``` + +**Key Workflow Operations:** +- Create and manage workflow tasks +- Update task statuses and assignments +- Execute bulk operations asynchronously +- Monitor task progress + +### 6. Events & Integration + +Subscribe to Benchling changes via **AWS EventBridge** (customer-owned bus) or **Webhooks** (recommended for new Benchling Apps). EventBridge delivers hydrated v2 API objects; webhooks use thinner payloads. + +**Common EventBridge `detail-type` values:** +- `v2.dnaSequence.created`, `v2.dnaSequence.updated` +- `v2.entity.registered` +- `v2.entry.created`, `v2.entry.updated` +- `v2.workflowTask.updated.status` +- `v2.request.created` + +**Minimal EventBridge rule** (filter request creation by schema name): + +```json +{ + "detail-type": ["v2.request.created"], + "detail": { + "schema": { + "name": ["Validated Request"] + } + } +} +``` + +**Lambda handler skeleton:** + +```python +def handler(event, context): + detail_type = event["detail-type"] + detail = event["detail"] + + if detail.get("deprecated"): + # Alert — migrate before Benchling removes this event type + pass + + if detail.get("excludedProperties"): + # Payload exceeded 256 KB; re-fetch via detail["request"]["apiURL"] + pass + + if detail_type == "v2.request.created": + request_id = (detail.get("request") or {}).get("id") + # Re-fetch authoritative state — events can be late or out of order + # request = benchling.requests.get_by_id(request_id) + return {"request_id": request_id} + + return {"status": "ignored", "detail_type": detail_type} +``` + +**Setup flow:** +1. Tenant admin creates a subscription at `https://your-tenant.benchling.com/event-subscriptions` +2. Associate the AWS partner event source with a dedicated event bus immediately (within ~12 days) +3. Create rules + targets (Lambda, SQS, SNS) and grant invoke permissions +4. Validate with a CloudWatch Logs rule, then trigger a matching Benchling action + +**Recovery:** EventBridge deliveries are not replayed. Use the [List Events API](https://benchling.com/api/reference#/Events/listEvents) for events up to ~2 weeks old after outages. + +For payload schema, CloudFormation templates, SDK list/recovery examples, and validation steps, see `references/eventbridge.md`. + +### 7. Data Warehouse & Analytics + +Query historical Benchling data using SQL through the Data Warehouse. + +**Access Method:** +The Benchling Data Warehouse provides SQL access to Benchling data for analytics and reporting. Connect using standard SQL clients with provided credentials. + +**Common Queries:** +- Aggregate experimental results +- Analyze inventory trends +- Generate compliance reports +- Export data for external analysis + +**Integration with Analysis Tools:** +- Jupyter notebooks for interactive analysis +- BI tools (Tableau, Looker, PowerBI) +- Custom dashboards diff --git a/.agents/skills/benchling-integration/references/eventbridge.md b/.agents/skills/benchling-integration/references/eventbridge.md new file mode 100644 index 0000000..cb5c40f --- /dev/null +++ b/.agents/skills/benchling-integration/references/eventbridge.md @@ -0,0 +1,255 @@ +# Benchling Events via AWS EventBridge + +Real-time integrations that react to Benchling changes (entity registration, inventory transfers, workflow updates, and more). + +**Official docs:** +- [Getting Started with Events](https://docs.benchling.com/docs/events-getting-started) +- [Events Reference (payloads and event types)](https://docs.benchling.com/docs/events-reference) +- [Events FAQs](https://docs.benchling.com/docs/events-faqs) +- [List Events API](https://benchling.com/api/reference#/Events/listEvents) + +**Delivery methods:** Benchling supports **Webhooks** (recommended for new apps) and **AWS EventBridge** (customer-owned event bus). EventBridge payloads are **hydrated** (full v2 API objects in `detail`); webhooks use thinner payloads. See the getting-started guide for trade-offs. + +--- + +## Setup checklist + +1. **Tenant admin** enables Developer Platform access and opens [Event Subscriptions](https://your-tenant.benchling.com/event-subscriptions) (Feature settings → Developer Console → Events). +2. Create a subscription with: + - AWS account ID and region + - Event bus name (e.g. `benchling-integrations`) + - Event types to receive (see [Events Reference](https://docs.benchling.com/docs/events-reference)) +3. **Immediately** associate the partner event source with a new EventBridge bus in AWS (within ~12 days or the source expires). +4. Create EventBridge rules with `detail-type` / `detail` filters and targets (Lambda, SQS, SNS, CloudWatch Logs). +5. Grant invoke permissions (`AWS::Lambda::Permission`, queue policies, etc.). +6. Validate with a CloudWatch Logs rule on the bus `source`, then trigger a test action in Benchling. + +Subscription statuses: `Pending` (needs bus association), `Active`, `Expired` (resubscribe in Benchling). + +--- + +## EventBridge event envelope + +All EventBridge deliveries share this top-level shape. The resource body lives under `detail` under a key that varies by event (for example `entry`, `assayRun`, `dnaSequence`). + +```json +{ + "version": "0", + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "detail-type": "v2.dnaSequence.created", + "source": "aws.partner/benchling.com/your-tenant/your-subscription-name", + "account": "123456789012", + "time": "2025-10-20T14:30:00.000000+00:00", + "region": "us-west-2", + "resources": [], + "detail": { + "id": "evt_abc123", + "eventType": "v2.dnaSequence.created", + "createdAt": "2025-10-20T14:30:00.000000+00:00", + "deprecated": false, + "excludedProperties": [], + "schema": { + "id": "ts_abc123", + "name": "Plasmid" + }, + "dnaSequence": { + "id": "seq_xyz789", + "name": "My Plasmid", + "apiURL": "https://your-tenant.benchling.com/api/v2/dna-sequences/seq_xyz789" + } + } +} +``` + +**Naming:** `detail-type` and `detail.eventType` follow `..` (for example `v2.request.created`, `v2.workflowTask.updated.status`). + +**Do not treat payloads as authoritative.** Events may arrive late or out of order. Re-fetch objects with the SDK/API when you need current state. + +**Oversized events (>256 KB):** Dropped fields appear in `detail.excludedProperties`. Use `apiURL` on the resource object to fetch the full record. + +--- + +## Minimal EventBridge rule (CloudFormation) + +Route `v2.request.created` events for a specific request schema to a Lambda: + +```yaml +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: Benchling request.created → Lambda + +Parameters: + BenchlingEventBusName: + Type: String + Description: Partner event bus name from Benchling subscription + +Resources: + RequestCreatedRule: + Type: AWS::Events::Rule + Properties: + Name: benchling-request-created + EventBusName: !Ref BenchlingEventBusName + State: ENABLED + EventPattern: + detail-type: + - v2.request.created + detail: + schema: + name: + - Validated Request + Targets: + - Id: HandleRequestCreated + Arn: !GetAtt HandleEventLambda.Arn + + HandleEventLambda: + Type: AWS::Serverless::Function + Properties: + Handler: app.handler + Runtime: python3.12 + CodeUri: src/ + Timeout: 30 + + AllowEventBridgeInvoke: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: !Ref HandleEventLambda + Principal: events.amazonaws.com + SourceArn: !GetAtt RequestCreatedRule.Arn +``` + +**Other filter examples** (from Benchling docs): + +```json +{ + "detail-type": ["v2.assayRun.updated"], + "detail": { + "updates": ["my_field"] + } +} +``` + +```json +{ + "detail-type": ["v2.entity.registered"], + "detail": { + "entity": { + "schema": { + "id": ["ts_MySchemaId"] + } + } + } +} +``` + +--- + +## Lambda handler skeleton (Python) + +```python +import json +import logging +import os + +import boto3 + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +# Optional: re-fetch via SDK when payload may be stale or truncated +# from benchling_sdk.benchling import Benchling +# from benchling_sdk.auth.api_key_auth import ApiKeyAuth +# +# benchling = Benchling( +# url=os.environ["BENCHLING_TENANT_URL"], +# auth_method=ApiKeyAuth(os.environ["BENCHLING_API_KEY"]), +# ) + + +def handler(event, context): + """Process a single Benchling EventBridge delivery.""" + detail_type = event.get("detail-type") + detail = event.get("detail") or {} + + logger.info( + "benchling_event", + extra={ + "detail_type": detail_type, + "event_id": detail.get("id"), + "benchling_event_type": detail.get("eventType"), + }, + ) + + if detail.get("deprecated"): + logger.warning("deprecated_event_type: %s", detail_type) + + if detail.get("excludedProperties"): + logger.warning( + "truncated_payload excluded=%s", detail.get("excludedProperties") + ) + + if detail_type == "v2.dnaSequence.created": + sequence = detail.get("dnaSequence") or {} + sequence_id = sequence.get("id") + if not sequence_id: + raise ValueError("missing dnaSequence.id in event detail") + # Prefer API lookup for authoritative data: + # seq = benchling.dna_sequences.get_by_id(sequence_id) + return {"status": "ok", "sequence_id": sequence_id} + + if detail_type == "v2.workflowTask.updated.status": + task = detail.get("workflowTask") or {} + return {"status": "ok", "task_id": task.get("id")} + + logger.info("no_handler_for_detail_type: %s", detail_type) + return {"status": "ignored", "detail_type": detail_type} +``` + +For serverless timeouts: SDK `wait_for_task` defaults to 600s — keep Lambda timeouts and EventBridge retry/DLQ settings aligned with expected processing time. + +--- + +## Validation steps + +1. **Subscription active:** In Benchling, subscription status is `Active` (not `Pending` or `Expired`). +2. **Partner source associated:** In AWS EventBridge → Partner event sources, source is associated with your bus. +3. **Log all events:** Add a catch-all rule targeting a CloudWatch log group, filtering on your bus `source` (shown in Benchling subscription UI). +4. **Trigger a test event:** Create or update an object matching your rule filter (for example register a DNA sequence). +5. **Inspect logs:** Confirm `detail-type`, `detail.id`, and resource IDs match expectations. +6. **Re-fetch check:** Call the SDK/API for the resource ID and confirm it matches your integration logic. + +--- + +## Recovering missed events + +Benchling does **not** replay EventBridge deliveries. After an outage: + +1. Get the affected time window from Benchling support. +2. List historical events with the [List Events API](https://benchling.com/api/reference#/Events/listEvents) (retained ~2 weeks). +3. Re-route recovered events through your own infrastructure. + +SDK example (ISO 8601 timestamp; see API reference for filters): + +```python +events = benchling.events.list( + created_atgte="2025-10-20T00:00:00+00:00", + event_types="v2.dnaSequence.created", +) + +for page in events: + for evt in page: + print(evt.event_type, evt.id) +``` + +--- + +## EventBridge vs Webhooks + +| | EventBridge | Webhooks | +|---|-------------|----------| +| Setup | Benchling console + AWS bus/rules | Benchling App configuration | +| Payload | Hydrated v2 API objects | Thin IDs + metadata | +| Filtering | EventBridge `EventPattern` | App code | +| Permissions | Not permissioned at delivery | Inherited from app | + +For new Benchling Apps, Benchling recommends **webhooks** unless you already standardize on EventBridge in AWS. See [Getting Started with Webhooks](https://docs.benchling.com/docs/getting-started-with-webhooks). diff --git a/.agents/skills/benchling-integration/references/sdk_reference.md b/.agents/skills/benchling-integration/references/sdk_reference.md new file mode 100644 index 0000000..8816820 --- /dev/null +++ b/.agents/skills/benchling-integration/references/sdk_reference.md @@ -0,0 +1,772 @@ +# Benchling Python SDK Reference + +## Installation & Setup + +### Installation + +```bash +# Stable release (recommended) +uv pip install "benchling-sdk==1.25.0" + +# Preview builds — alpha functionality, not for production +uv pip install "benchling-sdk" --prerelease allow +``` + +### Requirements +- Python 3.9+ (3.12 supported since SDK 1.11.0; repo recommends 3.11+) +- API access enabled on your Benchling tenant +- Developer Platform access from your tenant admin (for apps and events) + +### Basic Initialization + +```python +import os +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth + +benchling = Benchling( + url=os.environ["BENCHLING_TENANT_URL"], + auth_method=ApiKeyAuth(os.environ["BENCHLING_API_KEY"]), +) +``` + +## SDK Architecture + +### Main Classes + +**Benchling Client:** +The `benchling_sdk.benchling.Benchling` class is the root of all SDK interactions. It provides access to all resource endpoints: + +```python +benchling.dna_sequences # DNA sequence operations +benchling.rna_sequences # RNA sequence operations +benchling.aa_sequences # Amino acid sequence operations +benchling.custom_entities # Custom entity operations +benchling.mixtures # Mixture operations +benchling.containers # Container operations +benchling.boxes # Box operations +benchling.locations # Location operations +benchling.plates # Plate operations +benchling.entries # Notebook entry operations +benchling.workflow_tasks # Workflow task operations +benchling.requests # Request operations +benchling.folders # Folder operations +benchling.projects # Project operations +benchling.users # User operations +benchling.teams # Team operations +``` + +### Resource Pattern + +All resources follow a consistent CRUD pattern: + +```python +# Create +resource.create(CreateModel(...)) + +# Read (single) +resource.get_by_id("resource_id") + +# Read (list) +resource.list(optional_filters...) + +# Update +resource.update(id="resource_id", UpdateModel(...)) + +# Archive/Delete +resource.archive(id="resource_id") +``` + +## Entity Management + +### DNA Sequences + +**Create:** +```python +from benchling_sdk.models import DnaSequenceCreate + +sequence = benchling.dna_sequences.create( + DnaSequenceCreate( + name="pET28a-GFP", + bases="ATCGATCGATCG", + is_circular=True, + folder_id="fld_abc123", + schema_id="ts_abc123", + fields=benchling.models.fields({ + "gene_name": "GFP", + "resistance": "Kanamycin", + "copy_number": "High" + }) + ) +) +``` + +**Read:** +```python +# Get by ID +seq = benchling.dna_sequences.get_by_id("seq_abc123") +print(f"{seq.name}: {len(seq.bases)} bp") + +# List with filters +sequences = benchling.dna_sequences.list( + folder_id="fld_abc123", + schema_id="ts_abc123", + name="pET28a" # Filter by name +) + +for page in sequences: + for seq in page: + print(f"{seq.id}: {seq.name}") +``` + +**Update:** +```python +from benchling_sdk.models import DnaSequenceUpdate + +updated = benchling.dna_sequences.update( + sequence_id="seq_abc123", + dna_sequence=DnaSequenceUpdate( + name="pET28a-GFP-v2", + fields=benchling.models.fields({ + "gene_name": "eGFP", + "notes": "Codon optimized" + }) + ) +) +``` + +**Archive:** +```python +benchling.dna_sequences.archive( + sequence_id="seq_abc123", + reason="Deprecated construct" +) +``` + +### RNA Sequences + +Similar pattern to DNA sequences: + +```python +from benchling_sdk.models import RnaSequenceCreate, RnaSequenceUpdate + +# Create +rna = benchling.rna_sequences.create( + RnaSequenceCreate( + name="gRNA-target1", + bases="AUCGAUCGAUCG", + folder_id="fld_abc123", + fields=benchling.models.fields({ + "target_gene": "TP53", + "off_target_score": "95" + }) + ) +) + +# Update +updated_rna = benchling.rna_sequences.update( + rna_sequence_id=rna.id, + rna_sequence=RnaSequenceUpdate( + fields=benchling.models.fields({ + "validated": "Yes" + }) + ) +) +``` + +### Amino Acid (Protein) Sequences + +```python +from benchling_sdk.models import AaSequenceCreate + +protein = benchling.aa_sequences.create( + AaSequenceCreate( + name="Green Fluorescent Protein", + amino_acids="MSKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKF", + folder_id="fld_abc123", + fields=benchling.models.fields({ + "molecular_weight": "27000", + "extinction_coefficient": "21000" + }) + ) +) +``` + +### Custom Entities + +Custom entities are defined by your tenant's schemas: + +```python +from benchling_sdk.models import CustomEntityCreate, CustomEntityUpdate + +# Create +cell_line = benchling.custom_entities.create( + CustomEntityCreate( + name="HEK293T-Clone5", + schema_id="ts_cellline_abc123", + folder_id="fld_abc123", + fields=benchling.models.fields({ + "passage_number": "15", + "mycoplasma_test": "Negative", + "freezing_date": "2025-10-15" + }) + ) +) + +# Update +updated_cell_line = benchling.custom_entities.update( + entity_id=cell_line.id, + custom_entity=CustomEntityUpdate( + fields=benchling.models.fields({ + "passage_number": "16", + "notes": "Expanded for experiment" + }) + ) +) +``` + +### Mixtures + +Mixtures combine multiple components: + +```python +from benchling_sdk.models import MixtureCreate, IngredientCreate + +mixture = benchling.mixtures.create( + MixtureCreate( + name="LB-Amp Media", + folder_id="fld_abc123", + schema_id="ts_mixture_abc123", + ingredients=[ + IngredientCreate( + component_entity_id="ent_lb_base", + amount="1000 mL" + ), + IngredientCreate( + component_entity_id="ent_ampicillin", + amount="100 mg" + ) + ], + fields=benchling.models.fields({ + "pH": "7.0", + "sterilized": "Yes" + }) + ) +) +``` + +### Registry Operations + +**Direct Registry Registration:** +```python +# Register entity upon creation +registered_seq = benchling.dna_sequences.create( + DnaSequenceCreate( + name="Construct-001", + bases="ATCG", + is_circular=True, + folder_id="fld_abc123", + entity_registry_id="src_abc123", + naming_strategy="NEW_IDS" # or "IDS_FROM_NAMES" + ) +) +print(f"Registry ID: {registered_seq.registry_id}") +``` + +**Naming Strategies:** +- `NEW_IDS`: Benchling generates new registry IDs +- `IDS_FROM_NAMES`: Use entity names as registry IDs (names must be unique) + +## Inventory Management + +### Containers + +```python +from benchling_sdk.models import ContainerCreate, ContainerUpdate + +# Create +container = benchling.containers.create( + ContainerCreate( + name="Sample-001-Tube", + schema_id="cont_schema_abc123", + barcode="CONT001", + parent_storage_id="box_abc123", # Place in box + fields=benchling.models.fields({ + "concentration": "100 ng/μL", + "volume": "50 μL", + "sample_type": "gDNA" + }) + ) +) + +# Update location +benchling.containers.transfer( + container_id=container.id, + destination_id="box_xyz789" +) + +# Update properties +updated = benchling.containers.update( + container_id=container.id, + container=ContainerUpdate( + fields=benchling.models.fields({ + "volume": "45 μL", + "notes": "Used 5 μL for PCR" + }) + ) +) + +# Check out +benchling.containers.check_out( + container_id=container.id, + comment="Taking to bench" +) + +# Check in +benchling.containers.check_in( + container_id=container.id, + location_id="bench_location_abc" +) +``` + +### Boxes + +```python +from benchling_sdk.models import BoxCreate + +box = benchling.boxes.create( + BoxCreate( + name="Freezer-A-Box-01", + schema_id="box_schema_abc123", + parent_storage_id="loc_freezer_a", + barcode="BOX001", + fields=benchling.models.fields({ + "box_type": "81-place", + "temperature": "-80C" + }) + ) +) + +# List containers in box +containers = benchling.containers.list( + parent_storage_id=box.id +) +``` + +### Locations + +```python +from benchling_sdk.models import LocationCreate + +location = benchling.locations.create( + LocationCreate( + name="Freezer A - Shelf 2", + parent_storage_id="loc_freezer_a", + barcode="LOC-A-S2" + ) +) +``` + +### Plates + +```python +from benchling_sdk.models import PlateCreate, WellCreate + +# Create 96-well plate +plate = benchling.plates.create( + PlateCreate( + name="PCR-Plate-001", + schema_id="plate_schema_abc123", + barcode="PLATE001", + wells=[ + WellCreate( + position="A1", + entity_id="sample_entity_abc" + ), + WellCreate( + position="A2", + entity_id="sample_entity_xyz" + ) + # ... more wells + ] + ) +) +``` + +## Notebook Operations + +### Entries + +```python +from benchling_sdk.models import EntryCreate, EntryUpdate + +# Create entry +entry = benchling.entries.create( + EntryCreate( + name="Cloning Experiment 2025-10-20", + folder_id="fld_abc123", + schema_id="entry_schema_abc123", + fields=benchling.models.fields({ + "objective": "Clone GFP into pET28a", + "date": "2025-10-20", + "experiment_type": "Molecular Biology" + }) + ) +) + +# Update entry +updated_entry = benchling.entries.update( + entry_id=entry.id, + entry=EntryUpdate( + fields=benchling.models.fields({ + "results": "Successful cloning, 10 colonies", + "notes": "Colony 5 shows best fluorescence" + }) + ) +) +``` + +### Linking Entities to Entries + +```python +# Link DNA sequence to entry +link = benchling.entry_links.create( + entry_id="entry_abc123", + entity_id="seq_xyz789" +) + +# List links for an entry +links = benchling.entry_links.list(entry_id="entry_abc123") +``` + +## Workflow Management + +### Tasks + +```python +from benchling_sdk.models import WorkflowTaskCreate, WorkflowTaskUpdate + +# Create task +task = benchling.workflow_tasks.create( + WorkflowTaskCreate( + name="PCR Amplification", + workflow_id="wf_abc123", + assignee_id="user_abc123", + schema_id="task_schema_abc123", + fields=benchling.models.fields({ + "template": "seq_abc123", + "primers": "Forward: ATCG, Reverse: CGAT", + "priority": "High" + }) + ) +) + +# Update status +completed_task = benchling.workflow_tasks.update( + task_id=task.id, + workflow_task=WorkflowTaskUpdate( + status_id="status_complete_abc123", + fields=benchling.models.fields({ + "completion_date": "2025-10-20", + "yield": "500 ng" + }) + ) +) + +# List tasks +tasks = benchling.workflow_tasks.list( + workflow_id="wf_abc123", + status_ids=["status_pending", "status_in_progress"] +) +``` + +## Advanced Features + +### Pagination + +The SDK uses generators for memory-efficient pagination: + +```python +# Automatic pagination +sequences = benchling.dna_sequences.list() + +# Get estimated total count +total = sequences.estimated_count() +print(f"Total sequences: {total}") + +# Iterate through all pages +for page in sequences: + for seq in page: + process(seq) + +# Manual page size control +sequences = benchling.dna_sequences.list(page_size=50) +``` + +### Async Task Handling + +Some operations are asynchronous and return task IDs: + +```python +from benchling_sdk.helpers.tasks import wait_for_task +from benchling_sdk.errors import WaitForTaskExpiredError + +# Start async operation +response = benchling.some_bulk_operation(...) +task_id = response.task_id + +# Wait for completion +try: + result = wait_for_task( + benchling, + task_id=task_id, + interval_wait_seconds=2, # Poll every 2 seconds + max_wait_seconds=600 # Timeout after 10 minutes + ) + print("Task completed successfully") +except WaitForTaskExpiredError: + print("Task timed out") +``` + +### Error Handling + +```python +from benchling_sdk.errors import ( + BenchlingError, + NotFoundError, + ValidationError, + UnauthorizedError +) + +try: + sequence = benchling.dna_sequences.get_by_id("seq_invalid") +except NotFoundError: + print("Sequence not found") +except UnauthorizedError: + print("Insufficient permissions") +except ValidationError as e: + print(f"Invalid data: {e}") +except BenchlingError as e: + print(f"General Benchling error: {e}") +``` + +### Retry Strategy + +Customize retry behavior: + +```python +from benchling_sdk.benchling import Benchling +from benchling_sdk.auth.api_key_auth import ApiKeyAuth +from benchling_sdk.retry import RetryStrategy + +# Custom retry configuration +retry_strategy = RetryStrategy( + max_retries=3, + backoff_factor=0.5, + status_codes_to_retry=[429, 502, 503, 504] +) + +benchling = Benchling( + url="https://your-tenant.benchling.com", + auth_method=ApiKeyAuth("your_api_key"), + retry_strategy=retry_strategy +) + +# Disable retries +benchling = Benchling( + url="https://your-tenant.benchling.com", + auth_method=ApiKeyAuth("your_api_key"), + retry_strategy=RetryStrategy(max_retries=0) +) +``` + +### Custom API Calls + +For unsupported endpoints: + +```python +# GET request with model parsing +from benchling_sdk.models import DnaSequence + +response = benchling.api.get_modeled( + path="/api/v2/dna-sequences/seq_abc123", + response_type=DnaSequence +) + +# POST request +from benchling_sdk.models import DnaSequenceCreate + +response = benchling.api.post_modeled( + path="/api/v2/dna-sequences", + request_body=DnaSequenceCreate(...), + response_type=DnaSequence +) + +# Raw requests +raw_response = benchling.api.get( + path="/api/v2/custom-endpoint", + params={"key": "value"} +) +``` + +### Batch Operations + +Efficiently process multiple items: + +```python +# Bulk create +from benchling_sdk.models import DnaSequenceCreate + +sequences_to_create = [ + DnaSequenceCreate(name=f"Seq-{i}", bases="ATCG", folder_id="fld_abc") + for i in range(100) +] + +# Create in batches +batch_size = 10 +for i in range(0, len(sequences_to_create), batch_size): + batch = sequences_to_create[i:i+batch_size] + for seq in batch: + benchling.dna_sequences.create(seq) +``` + +### Schema Fields Helper + +Convert dictionaries to Fields objects: + +```python +# Using fields helper +fields_dict = { + "concentration": "100 ng/μL", + "volume": "50 μL", + "quality_score": "8.5", + "date_prepared": "2025-10-20" +} + +fields = benchling.models.fields(fields_dict) + +# Use in create/update +container = benchling.containers.create( + ContainerCreate( + name="Sample-001", + schema_id="schema_abc", + fields=fields + ) +) +``` + +### Forward Compatibility + +The SDK handles unknown API values gracefully: + +```python +# Unknown enum values are preserved +entity = benchling.dna_sequences.get_by_id("seq_abc") +# Even if API returns new enum value not in SDK, it's preserved + +# Unknown polymorphic types return UnknownType +from benchling_sdk.models import UnknownType + +if isinstance(entity, UnknownType): + print(f"Unknown type: {entity.type}") + # Can still access raw data + print(entity.raw_data) +``` + +## Best Practices + +### Use Type Hints + +```python +from benchling_sdk.models import DnaSequence, DnaSequenceCreate +from typing import List + +def create_sequences(names: List[str], folder_id: str) -> List[DnaSequence]: + sequences = [] + for name in names: + seq = benchling.dna_sequences.create( + DnaSequenceCreate( + name=name, + bases="ATCG", + folder_id=folder_id + ) + ) + sequences.append(seq) + return sequences +``` + +### Efficient Filtering + +Use API filters instead of client-side filtering: + +```python +# Good - filter on server +sequences = benchling.dna_sequences.list( + folder_id="fld_abc123", + schema_id="ts_abc123" +) + +# Bad - loads everything then filters +all_sequences = benchling.dna_sequences.list() +filtered = [s for page in all_sequences for s in page if s.folder_id == "fld_abc123"] +``` + +### Resource Cleanup + +```python +# Archive old entities +cutoff_date = "2024-01-01" +sequences = benchling.dna_sequences.list() + +for page in sequences: + for seq in page: + if seq.created_at < cutoff_date: + benchling.dna_sequences.archive( + sequence_id=seq.id, + reason="Archiving old sequences" + ) +``` + +## Troubleshooting + +### Common Issues + +**Import paths:** +```python +# Preferred (documented in getting started guide) +from benchling_sdk.benchling import Benchling + +# Also valid in benchling-sdk 1.25+ +from benchling_sdk import Benchling +``` + +**Field Validation:** +```python +# Fields must match schema +# Check schema field types in Benchling UI +fields = benchling.models.fields({ + "numeric_field": "123", # Should be string even for numbers + "date_field": "2025-10-20", # Format: YYYY-MM-DD + "dropdown_field": "Option1" # Must match dropdown options exactly +}) +``` + +**Pagination Exhaustion:** +```python +# Generators can only be iterated once +sequences = benchling.dna_sequences.list() +for page in sequences: # First iteration OK + pass +for page in sequences: # Second iteration returns nothing! + pass + +# Solution: Create new generator +sequences = benchling.dna_sequences.list() # New generator +``` + +## References + +- **SDK Source:** https://github.com/benchling/benchling-sdk +- **SDK Docs:** https://benchling.com/sdk-docs/ +- **API Reference:** https://benchling.com/api/reference +- **Common Examples:** https://docs.benchling.com/docs/common-sdk-interactions-and-examples diff --git a/.agents/skills/bgpt-paper-search/SKILL.md b/.agents/skills/bgpt-paper-search/SKILL.md new file mode 100644 index 0000000..d6f16ef --- /dev/null +++ b/.agents/skills/bgpt-paper-search/SKILL.md @@ -0,0 +1,75 @@ +--- +name: bgpt-paper-search +description: Search scientific papers and retrieve structured experimental data extracted from full-text studies via the BGPT MCP server. Returns 25+ fields per paper including methods, results, sample sizes, quality scores, and conclusions. Use for literature reviews, evidence synthesis, and finding experimental details not available in abstracts alone. +license: MIT +compatibility: Requires the BGPT MCP server configured in the agent host (npx mcp-remote or npx bgpt-mcp), internet access to bgpt.pro, and an optional BGPT API key for paid usage. +metadata: + version: "1.1" + skill-author: BGPT + website: https://bgpt.pro/mcp + github: https://github.com/connerlambden/bgpt-mcp +--- + +# BGPT Paper Search + +## Overview + +BGPT is a remote MCP server that searches a curated database of scientific papers built from raw experimental data extracted from full-text studies. Unlike traditional literature databases that return titles and abstracts, BGPT returns structured data from the actual paper content — methods, quantitative results, sample sizes, quality assessments, and 25+ metadata fields per paper. + +## When to Use This Skill + +Use this skill when: +- Searching for scientific papers with specific experimental details +- Conducting systematic or scoping literature reviews +- Finding quantitative results, sample sizes, or effect sizes across studies +- Comparing methodologies used in different studies +- Looking for papers with quality scores or evidence grading +- Needing structured data from full-text papers (not just abstracts) +- Building evidence tables for meta-analyses or clinical guidelines + +## Setup + +BGPT is a remote MCP server — no local installation required. Configure it in your agent's MCP settings before use; this skill instructs the agent to call the `search_papers` MCP tool and does not enable MCP access by itself. + +### Claude Desktop / Claude Code + +Add to your MCP configuration: + +```json +{ + "mcpServers": { + "bgpt": { + "command": "npx", + "args": ["mcp-remote", "https://bgpt.pro/mcp/sse"] + } + } +} +``` + +### npm (alternative) + +```bash +npx bgpt-mcp +``` + +## Usage + +Once the BGPT MCP server is configured, call its `search_papers` tool via the agent's MCP interface (not via Bash): + +``` +Search for papers about: "CRISPR gene editing efficiency in human cells" +``` + +The server returns structured results including: +- **Title, authors, journal, year, DOI** +- **Methods**: Experimental techniques, models, protocols +- **Results**: Key findings with quantitative data +- **Sample sizes**: Number of subjects/samples +- **Quality scores**: Study quality assessments +- **Conclusions**: Author conclusions and implications + +## Pricing + +- **Free tier**: 50 searches per network, no API key required +- **Paid**: $0.01 per result with an API key from [bgpt.pro/mcp](https://bgpt.pro/mcp) + diff --git a/.agents/skills/bids/SKILL.md b/.agents/skills/bids/SKILL.md new file mode 100644 index 0000000..5f2ab1e --- /dev/null +++ b/.agents/skills/bids/SKILL.md @@ -0,0 +1,237 @@ +--- +name: bids +description: > + Use this skill when working with Brain Imaging Data Structure (BIDS) datasets: + organizing neuroscience and biomedical data (MRI, EEG, MEG, iEEG, PET, microscopy, + NIRS, motion capture, EMG, MR spectroscopy, behavioral), querying BIDS layouts, + validating compliance, converting DICOM to BIDS, writing metadata sidecars, or + creating BIDS derivatives. +license: https://creativecommons.org/licenses/by/4.0/ +metadata: + version: "1.1" + skill-author: Yaroslav Halchenko +--- + +# Brain Imaging Data Structure (BIDS) + +## Overview + +The Brain Imaging Data Structure (BIDS) is a community standard for organizing and describing neuroscience and biomedical research datasets. It defines a consistent file naming convention, directory hierarchy, and metadata schema so that datasets are immediately understandable by humans and software tools alike. BIDS is governed by the BIDS Specification (currently v1.11.x) and is maintained by the community via the BIDS-Standard GitHub organization. + +While BIDS originated for MRI, it has grown well beyond neuroimaging. The specification now covers 11 modalities spanning imaging, electrophysiology, and behavioral data: + +- **Imaging**: MRI (structural, functional, diffusion, fieldmaps, perfusion/ASL), PET, microscopy +- **Electrophysiology**: EEG, MEG, iEEG (intracranial EEG), EMG +- **Other**: NIRS (near-infrared spectroscopy), motion capture, behavioral data (without imaging), MR spectroscopy + +Active BEPs are extending BIDS further — notably BEP032 (microelectrode electrophysiology) will add support for extracellular recordings including Neuropixels probes, bringing BIDS to a prevalent methodology in animal neuroscience research (see also the neuropixels-analysis skill). + +Adoption is required or strongly encouraged by major data repositories (OpenNeuro, DANDI), leading journals (NeuroImage, Human Brain Mapping, Scientific Data), and funding agencies (NIH, ERC). + +The Python ecosystem for BIDS centers on **PyBIDS** (`pybids`) for querying and indexing BIDS datasets, and the **bids-validator** (Deno-based, available as PyPI package `bids-validator-deno` or via Deno directly) for compliance checking. Conversion from DICOM is typically done with **HeuDiConv**, **dcm2bids**, or **BIDScoin**. + +## When to Use This Skill + +Apply this skill when: +- Organizing raw neuroscience data (imaging, electrophysiology, behavioral) into BIDS-compliant directory structures +- Querying an existing BIDS dataset to find specific files by subject, session, task, run, or modality +- Validating a dataset against the BIDS specification before sharing or submission +- Converting DICOM data from scanners into BIDS format +- Writing or editing JSON sidecar metadata files +- Creating BIDS-compliant derivatives (preprocessed data, analysis outputs) +- Setting up a `dataset_description.json` for a new dataset +- Working with BIDS entities (subject, session, task, acquisition, run, etc.) +- Configuring `.bidsignore` to exclude files from validation +- Preparing data for upload to OpenNeuro, DANDI, or other BIDS-aware repositories + +## Installation + +```bash +# Core BIDS querying library +uv pip install pybids + +# BIDS validator (Deno-based, installed via PyPI wrapper) +uv pip install bids-validator-deno +# Alternative: install directly via Deno +# deno install -g -A npm:bids-validator + +# DICOM-to-BIDS converters (install as needed) +uv pip install heudiconv # HeuDiConv - heuristic-based DICOM conversion +uv pip install dcm2bids # dcm2bids - config-file-based conversion +# BIDScoin: uv pip install bidscoin + +# Useful companions +uv pip install nibabel # NIfTI/other neuroimaging file I/O +uv pip install pydicom # DICOM file reading (used by converters) +``` + +## Core Workflows + +Twelve workflow areas, each with worked code, are documented in +[references/core_workflows.md](references/core_workflows.md): + +1. **BIDS directory structure** — the required layout and where each modality belongs. +2. **`dataset_description.json`** — the required fields and how to generate it. +3. **Querying with PyBIDS** — `BIDSLayout`, entity filters, sidecar metadata with + automatic inheritance, and building paths from entities. +4. **Validation** — `bids-validator` via the PyPI wrapper (recommended), via Deno + directly, the legacy Node validator, and using `.bidsignore` to exclude files. +5. **Entities and file naming** — the entity order and naming grammar. +6. **DICOM to BIDS conversion** — HeuDiConv (including the turnkey ReproIn path and the + reconnaissance → heuristic → convert sequence) and dcm2bids (config-file based). +7. **Metadata sidecars** — required and recommended JSON fields per modality. +8. **Events files** — task fMRI event timing and column conventions. +9. **Participants file** — `participants.tsv` and its data dictionary. +10. **Derivatives** — the derivatives layout and its `dataset_description.json`. +11. **Advanced PyBIDS** — index caching, including derivatives, confound regressors, and + DataFrame output. +12. **BIDS-Apps** — the standard invocation pattern, and fMRIPrep, MRIQC, and QSIPrep. + +Validate early and often: PyBIDS validates structure when it indexes a dataset, so an +indexing failure usually means a naming or metadata problem rather than a code bug. + +## Reference Materials + +This skill includes detailed reference documentation: + +- **bids_schema.json**: Machine-readable BIDS schema (from https://bids-specification.readthedocs.io/en/stable/schema.json). This is the authoritative source for entity definitions, ordering rules, filename templates, allowed suffixes per datatype, and metadata field requirements. BEP-specific schemas are at https://github.com/bids-standard/bids-schema/tree/main/BEPs. +- **beps.yml**: Current list of all BIDS Extension Proposals with titles, leads, status, and links (from [bids-website](https://github.com/bids-standard/bids-website/blob/main/data/beps/beps.yml)) +- **bids_specification.md**: Human-readable summary of the entity table, datatype reference, directory structure rules, template spaces, and specification changelog +- **metadata_fields.md**: Required and recommended JSON sidecar fields for every BIDS modality (anat, func, dwi, fmap, eeg, meg, pet, etc.) +- **conversion_tools.md**: Detailed workflows for HeuDiConv, dcm2bids, and BIDScoin including heuristic/config examples and troubleshooting + +Update schema and BEPs with: `python scripts/update_schema.py` + +## Common Issues and Solutions + +### 1. Validator reports "Not a BIDS dataset" +**Cause**: Missing `dataset_description.json` at the root. +**Fix**: Create the file with at minimum `{"Name": "...", "BIDSVersion": "1.10.0"}`. + +### 2. Inconsistent subjects warning +**Cause**: Not all subjects have the same set of files (some missing sessions, runs, etc.). +**Fix**: This is a warning, not an error. Use `--ignoreSubjectConsistency` if intentional. Document missing data in `participants.tsv` or a `scans.tsv`. + +### 3. Missing SliceTiming +**Cause**: `dcm2niix` couldn't extract slice timing from DICOM headers. +**Fix**: Determine slice order from the scan protocol and add manually to the JSON sidecar. Common patterns: ascending, descending, interleaved (odd-first or even-first). + +### 4. Phase encoding direction confusion +**Cause**: Axis labels (i/j/k vs x/y/z vs LR/AP/SI) are confusing. +**Fix**: In BIDS, use NIfTI image axes: `i`=first axis, `j`=second, `k`=third. `-` means negative direction. For standard axial acquisitions: `j` is typically anterior-posterior. Verify with the acquisition protocol. + +### 5. PyBIDS is slow on large datasets +**Cause**: Full filesystem indexing on every `BIDSLayout()` call. +**Fix**: Use `database_path` to cache the index to an SQLite file: +```python +layout = BIDSLayout("/data", database_path="/data/.pybids_cache.db") +``` + +### 6. Derivatives not found by PyBIDS +**Cause**: Derivatives directory missing its own `dataset_description.json`. +**Fix**: Every derivatives directory must have `dataset_description.json` with `"DatasetType": "derivative"`. + +### 7. Events file timing is off +**Cause**: `onset` times are relative to the wrong reference (e.g., trigger time vs first volume). +**Fix**: Onsets must be in seconds relative to the first volume of that run's acquisition. Account for dummy scans if they were discarded. + +### 8. TSV files fail validation +**Cause**: Encoding or delimiter issues (spaces instead of tabs, BOM characters, Windows line endings). +**Fix**: Ensure tab-separated values with UTF-8 encoding and Unix line endings (`\n`). Use `n/a` (not `NA`, `NaN`, or empty) for missing values. + +## Best Practices + +1. **Validate early and often** - Run the BIDS validator after every conversion or modification. Fix errors before they compound. + +2. **Use metadata inheritance** - Place shared metadata (e.g., `TaskName`, scanner parameters) in top-level sidecar files rather than duplicating in every subject's directory. + +3. **Keep sourcedata** - Store the original DICOM (or other raw) data under `sourcedata/` so conversions are reproducible. Add `sourcedata/` to `.bidsignore`. + +4. **Use consistent naming from the start** - Define your BIDS naming scheme before data collection. Use the ReproIn naming convention for scan protocols to enable automatic conversion. + +5. **Document your dataset** - Write a thorough `README` describing the study design, acquisition parameters, known issues, and any deviations from BIDS. + +6. **Use scans.tsv for run-level metadata** - Record per-run acquisition times and quality notes: + ``` + filename acq_time quality + func/sub-01_task-rest_bold.nii.gz 2025-01-15T10:30:00 good + ``` + +7. **Version your dataset** - Use `CHANGES` to document dataset modifications. Consider DataLad for full version control of large datasets. + +8. **Deface anatomical images** - Remove facial features from T1w/T2w images before sharing (e.g., using `pydeface`, `mri_deface`, or `afni_refacer`). Store defaced versions as the primary data or use `_defacemask` files. + +9. **Use BIDS URIs for provenance** - In derivatives, reference source files using BIDS URIs: `bids::sub-01/anat/sub-01_T1w.nii.gz`. + +10. **Prefer community tools** - Use established BIDS-Apps (fMRIPrep, MRIQC, QSIPrep) rather than custom pipelines when possible. They handle BIDS I/O correctly and produce BIDS-compliant derivatives. + +11. **Study bids-examples** - The [bids-examples](https://github.com/bids-standard/bids-examples) repository is the canonical collection of prototypical BIDS datasets covering different modalities and use cases (MRI, fMRI, DWI, EEG, MEG, iEEG, PET, ASL, genetics, derivatives, and more). Use it as a reference when structuring your own dataset, as test data for BIDS tools, or to understand how a specific modality should be organized. Each example passes the BIDS validator. + +## BIDS Extension Proposals (BEPs) + +BEPs are community-driven proposals to extend BIDS to new modalities, derivatives, or metadata. The full list with status, leads, and links is in `references/beps.yml` (fetched from the [bids-website](https://github.com/bids-standard/bids-website/blob/main/data/beps/beps.yml)). BEP-specific schema previews are rendered at https://github.com/bids-standard/bids-schema/tree/main/BEPs. + +**Current BEPs** (as of schema update): + +| BEP | Title | Content | Status | +|-----|-------|---------|--------| +| 004 | Susceptibility Weighted Imaging | raw | Seeking new leader | +| 011 | Structural preprocessing derivatives | derivative | Has PR (#518) | +| 012 | Functional preprocessing derivatives | derivative | Has PR (#519), schema implemented | +| 014 | Affine transforms and nonlinear field warps | derivative | X5 format development | +| 016 | Diffusion weighted imaging derivatives | derivative | Has PR (#2211) | +| 017 | Generic BIDS connectivity data schema | derivative | In development | +| 021 | Common Electrophysiological Derivatives | derivative | In development | +| 023 | PET Preprocessing derivatives | derivative | In development | +| 024 | Computed Tomography scan | raw | Seeking contributors | +| 026 | Microelectrode Recordings | raw | Seeking new leader | +| 028 | Provenance | metadata | Has PR (#2099) | +| 032 | Microelectrode electrophysiology | raw | Has PR (#2307), preview available — covers Neuropixels and other extracellular probes; relates to neuropixels-analysis skill | +| 033 | Advanced Diffusion Weighted Imaging | raw | Seeking contributors | +| 034 | Computational modeling | derivative | Has PR (#967) | +| 035 | Mega-analyses with non-compliant derivatives | derivative | In development | +| 036 | Phenotypic Data Guidelines | raw | Community review | +| 037 | Non-Invasive Brain Stimulation | raw | In development | +| 039 | Dimensionality reduction-based networks | raw | In development | +| 040 | Functional Ultrasound | raw | In development | +| 041 | Statistical Model Derivatives | derivative | Collecting feedback | +| 043 | BIDS Term Mapping | metadata | Collecting feedback | +| 044 | Stimuli | raw | Has PR (#2022), community review | +| 045 | Peripheral Physiological Recordings | raw | Has PR (#2267) | +| 046 | Diffusion Tractography | derivative | In development | +| 047 | Audio/video recordings for behavioral experiments | raw | Has PR (#2231) | + +**Related standards:** +- **BIDS-Stats Models**: JSON specification for defining GLM-based neuroimaging analyses +- **BIDS-Derivatives** (BEP003): Standard for preprocessed/analysis outputs (partially merged into spec) + +## Related Tools Ecosystem + +| Tool | Purpose | +|------|---------| +| **fMRIPrep** | fMRI preprocessing (produces BIDS derivatives) | +| **MRIQC** | MRI quality control (produces BIDS derivatives) | +| **QSIPrep** | Diffusion MRI preprocessing | +| **TemplateFlow** | Neuroimaging templates and atlases with BIDS-like naming | +| **Fitlins** | BIDS Stats Models implementation | +| **DataLad** | Version control for large datasets, integrates with BIDS | +| **OpenNeuro** | Free BIDS dataset repository | +| **DANDI** | Neurophysiology data archive (uses BIDS for some modalities) | +| **HeuDiConv** | DICOM-to-BIDS with heuristic Python files | +| **dcm2bids** | DICOM-to-BIDS with JSON config | +| **BIDScoin** | DICOM-to-BIDS with GUI and YAML config | +| **nwb2bids** | Convert NWB (Neurodata Without Borders) files to BIDS | +| **CuBIDS** | BIDS dataset curation and harmonization | +| **bids2table** | Efficient tabular indexing of BIDS datasets | +| **bids-examples** | Canonical collection of prototypical BIDS datasets for all modalities | + +## Documentation + +- **BIDS Specification**: https://bids-specification.readthedocs.io/ +- **BIDS Website**: https://bids.neuroimaging.io/ +- **PyBIDS Documentation**: https://bids-standard.github.io/pybids/ +- **BIDS Validator**: https://github.com/bids-standard/bids-validator +- **BIDS Starter Kit**: https://bids-standard.github.io/bids-starter-kit/ +- **BIDS Examples**: https://github.com/bids-standard/bids-examples — canonical reference datasets for every BIDS modality; use as templates and test data +- **HeuDiConv Docs**: https://heudiconv.readthedocs.io/ +- **Original BIDS paper**: Gorgolewski et al. (2016) Scientific Data, doi:10.1038/sdata.2016.44 diff --git a/.agents/skills/bids/references/beps.yml b/.agents/skills/bids/references/beps.yml new file mode 100644 index 0000000..012bed7 --- /dev/null +++ b/.agents/skills/bids/references/beps.yml @@ -0,0 +1,637 @@ +--- +# template +# - number: +# title: +# display: +# google_doc: +# pull_request: +# html_preview: +# leads: +# # MUST match given-names and family-names from the bids specification citation.cff +# - given-names: ' ' +# family-names: ' ' +# bids_maintainers: +# - given-names: ' ' +# family-names: ' ' +# status: +# blocking: +# communication_channel: +# pull_request_created: +# pull_request_merged: + +- number: '004' + title: Susceptibility Weighted Imaging + google_doc: https://docs.google.com/document/d/1kyw9mGgacNqeMbp4xZet3RnDhcMmf4_BmRgKaOkO2Sc/ + content: + - raw + leads: + - given-names: ' ' + family-names: ' ' + bids_maintainers: + status: + blocking: + - Looking for a new leader. + google_doc_created: 2017-04 + pull_request_created: + pull_request_merged: + +- number: '011' + title: Structural preprocessing derivatives + google_doc: https://docs.google.com/document/d/1YG2g4UkEio4t_STIBOqYOwneLEs1emHIXbGKynx7V0Y/ + pull_request: https://github.com/bids-standard/bids-specification/pull/518 + html_preview: https://bids-specification--518.org.readthedocs.build/en/518/05-derivatives/04-structural-derivatives.html + content: + - derivative + leads: + - given-names: Viviana + family-names: Siless + bids_maintainers: + - given-names: Christopher J. + family-names: Markiewicz + status: + - Stability! (We haven't touched it in a bit.) + blocking: + - Staleness! (We haven't touched it in a bit.) + - Mostly just need to regroup with other BEPs and make sure we're being consistent. + google_doc_created: 2017-08 + pull_request_created: 2020-06 + pull_request_merged: + +- number: '012' + title: Functional preprocessing derivatives + google_doc: + pull_request: https://github.com/bids-standard/bids-specification/pull/519 + html_preview: https://bids-specification--519.org.readthedocs.build/en/519/derivatives/functional-derivatives.html + content: + - derivative + leads: + - given-names: Christopher J. + family-names: Markiewicz + bids_maintainers: + - given-names: Christopher J. + family-names: Markiewicz + status: + - Moderate reworking, post-BEP23 meeting. "boldmap" suffix. + - Schema implemented; waiting on full schema validation to merge. + blocking: + - Overlap with BEP 39 (decompositions). + - Probably just needs dropping from BEP 12, but need to make sure BEP 39 meets the needs served there. + google_doc_created: 2018-10 + pull_request_created: 2020-06 + pull_request_merged: + +- number: '014' + title: Affine transformations and nonlinear field warps + google_doc: https://docs.google.com/document/d/11gCzXOPUbYyuQx8fErtMO9tnOKC3kTWiL9axWkkILNE/ + content: + - derivative + leads: + - given-names: Oscar + family-names: Esteban + bids_maintainers: + - given-names: Christopher J. + family-names: Markiewicz + status: + - After the kick-off meeting (2019), progress locked on developing a prototype of a new HDF5-based format (X5) + - Minor bug fixes and features to support more transforms + - 'Perspectives: The current draft seems sufficient for storing transforms' + - 'Perspectives: X5 has high promises to enable effortless spatial transforms' + blocking: + - Bandwidth to finalize development + - 'Current blockers: surface transforms & X5 storing' + google_doc_created: 2017-08 + pull_request_created: + pull_request_merged: + +- number: '016' + title: Diffusion weighted imaging derivatives + pull_request: https://github.com/bids-standard/bids-specification/pull/2211 + content: + - derivative + leads: + - given-names: Franco + family-names: Pestilli + - given-names: Oscar + family-names: Esteban + bids_maintainers: + status: + - adapted general spatial derivatives proposal, meaning using map, like `dwimap` + - decided on using `params-` to denote different file types, for example `param-md` or `param-fa` + - updated meta-data + blocking: + google_doc_created: + pull_request_created: + pull_request_merged: + +- number: '017' + title: Generic BIDS connectivity data schema + display: Connectivity schema + google_doc: https://docs.google.com/document/d/1ugBdUF6dhElXdj3u9vw0iWjE6f_Bibsro3ah7sRV0GA/ + content: + - derivative + leads: + - given-names: Eugene P. + family-names: Duff + bids_maintainers: + status: + - specified different formats for dense and sparse matrices + - 'proposed arrays in h5 or zarr to cover multi-dimensional matrices (for example: dynamic connectivity)' + blocking: + google_doc_created: 2017-05 + pull_request_created: + pull_request_merged: + +- number: '021' + title: Common Electrophysiological Derivatives + google_doc: https://docs.google.com/document/d/1PmcVs7vg7Th-cGC-UrX8rAhKUHIzOI-uIOh69_mvdlw/ + content: + - derivative + leads: + - given-names: Arnaud + family-names: Delorme + - given-names: Dora + family-names: Hermes + - given-names: Mainak + family-names: Jas + - given-names: Guiomar + family-names: Niso + - given-names: Robert + family-names: Oostenveld + - given-names: Cyril + family-names: Pernet + - given-names: Christine + family-names: Rogers + status: + - 'Focus on raw-ish data: channels by time: epoching, filtering, interpolation...' + - 'All provenance tracking: outsourced to BEP028 (“provenance”)' + - Reusing existing (raw) data formats where applicable + - 'No additional entities or suffixes: focus on “desc” entity' + - Create new “descriptions.tsv” file to accompany and document the “desc” entity + - Working on examples on GitHub + - 'Perspectives: Derivatives beyond channels by time data will be discussed at a later point' + blocking: + google_doc_created: 2018-05 + pull_request_created: + pull_request_merged: + +- number: '023' + title: PET Preprocessing derivatives + google_doc: https://docs.google.com/document/d/1yzsd1J9GT-aA0DWhdlgNr5LCu6_gvbjLyfvYq2FuxlY/ + content: + - derivative + leads: + - given-names: Martin + family-names: Noergaard + - given-names: Graham + family-names: Searle + - given-names: Melanie + family-names: Ganz-Benjaminsen + bids_maintainers: + - given-names: Anthony + family-names: Galassi + status: + - Defined all the necessary components to be included in the specification + - Alignment with other modalities obtained at Copenhagen BIDS derivatives meeting + - Example PET derivatives datasets available on github + - 'Perspectives: 3rd joint meeting between all PET collaborators in August 2023' + - 'Perspectives: Aim is to finish the specification during the fall of 2023' + blocking: + - Need more example datasets (fore example different tracers) with different preprocessing choices to capture as most of the PET community as possible + - Need to finish alignment with other modalities + - Still need to agree on the level of information going into corresponding json files + google_doc_created: 2018-08 + pull_request_created: + pull_request_merged: + +- number: '024' + title: Computed Tomography scan + google_doc: https://docs.google.com/document/d/1fqnJZ18x5LJC8jiJ8yvPHUGFzNBZ6gW2kywYrUKWtuo/ + content: + - raw + leads: + - given-names: Hugo + family-names: Boniface + bids_maintainers: + status: + - Lead seeking more contributors and experts. + blocking: + google_doc_created: 2018-11 + pull_request_created: + pull_request_merged: + +- number: '026' + title: Microelectrode Recordings + google_doc: https://docs.google.com/document/d/14KC1d5-Lx-7ZSMtwS7pVAAvz-2WR_uoo5FvsNirzqJw/ + content: + - raw + leads: + - given-names: ' ' + family-names: ' ' + bids_maintainers: + status: + - BEP is open to new leadership, see also [BEP032 (animal electrophys)](https://docs.google.com/document/d/1oG-C8T-dWPqfVzL2W8HO3elWK8NIh2cOCPssRGv23n0/) + blocking: + - Searching for a new leader. + google_doc_created: 2018-04 + pull_request_created: + pull_request_merged: + +- number: '028' + title: Provenance + google_doc: https://docs.google.com/document/d/1vw3VNDof5cecv2PkFp7Lw_pNUTUo8-m8V4SIdtGJVKs/ + content: + - metadata + leads: + - given-names: Satrajit S. + family-names: Ghosh + - given-names: Camille + family-names: Maumet + - given-names: Yaroslav O. + family-names: Halchenko + bids_maintainers: + status: + - '[Specification](https://bids.neuroimaging.io/bep028)' + - '[Set of examples](https://github.com/bids-standard/BEP028_BIDSprov)' + - 'Perspectives: Opening up to BIDS community for feedback' + - 'Perspectives: Engaging with software developers (in progress w/ SPM, AFNI)' + blocking: + google_doc_created: 2018-08 + pull_request: https://github.com/bids-standard/bids-specification/pull/2099 + pull_request_created: 2025-04 + pull_request_merged: + +- number: '032' + title: Microelectrode electrophysiology + google_doc: https://docs.google.com/document/d/1oG-C8T-dWPqfVzL2W8HO3elWK8NIh2cOCPssRGv23n0/ + pull_request: https://github.com/bids-standard/bids-specification/pull/2307 + html_preview: https://bids-specification--2307.org.readthedocs.build/en/2307/modality-specific-files/microelectrode-electrophysiology.html + content: + - raw + leads: + - given-names: Sylvain + family-names: Takerkart + - given-names: Benjamin + family-names: Dichter + - given-names: Yaroslav O. + family-names: Halchenko + - given-names: Lyuba + family-names: Zehl + - given-names: Andrew + family-names: Davison + bids_maintainers: + - given-names: Rémi + family-names: Gau + status: + - Decided on the new name (not just "Animal" but "Microelectrode"), modalities, datatypes + - Nearly finalized added metadata + - PR compiles green, preview is available + - 'Target: finalize & merge PR into the BIDS specs in 2025' + blocking: + - Need to prepare example datasets + - Need to start thinking about derived data (spike sorted) + google_doc_created: 2020-12 + pull_request_created: 2022-11 + pull_request_merged: + +- number: '033' + title: Advanced Diffusion Weighted Imaging + google_doc: https://docs.google.com/document/d/1en4ByORlPqwDfZwNOOBTj0GwpYBcS0_2peqreTOvwDU/ + content: + - raw + leads: + - given-names: James + family-names: Gholam + - given-names: Leandro + family-names: Beltrachini + - given-names: Filip + family-names: Szczepankiewicz + bids_maintainers: + status: + - 'New BEP, seeking contributors and collecting community feedback relating to: priority sequences to support, best supported binary structured formats + (e.g. CBOR? HDF5? MsgPack?).' + - Comments may be submitted directly on the document. + - Generating example datasets [here](https://github.com/JAgho/MISP_plot/tree/main) and determining best practice with vendors to record data in-sequence + blocking: + google_doc_created: 2021-04 + pull_request_created: + pull_request_merged: + +- number: '034' + title: Computational modeling + pull_request: https://github.com/bids-standard/bids-specification/pull/967 + html_preview: https://bids-specification--967.org.readthedocs.build/en/967/modality-specific-files/computational-models.html + content: + - derivative + - metadata + leads: + - given-names: Michael + family-names: Schirner + - given-names: Petra + family-names: Ritter + bids_maintainers: + status: + - sim2bids app created + - easier to bring neural simulation data (especially in The Virtual Brain format) into the proposed BIDS Comp Model format + - 'Perspectives: A roadmap that coordinates efforts across BEPs would be appreciated.' + blocking: + - BEPs have overlapping concerns (Comp Models, Spaces and mappings, Generic BIDS connectivity schema, time series, Provenance, Atlases) + - need to separate concerns & provide a roadmap for each BEP + google_doc_created: 2021-02 + pull_request_created: 2021-08 + pull_request_merged: + +- number: '035' + title: Modular extensions for individual participant data mega-analyses with non-compliant derivatives + display: Mega-analyses + google_doc: https://docs.google.com/document/d/1tFRNumQyIgjXBNC3brFDLO9FaikjL84noxK6Om-Ctik/ + content: + - derivative + leads: + - given-names: Giuseppe + family-names: Gallitto + - given-names: Balint + family-names: Kincses + - given-names: Tamas + family-names: Spisak + bids_maintainers: + - given-names: Rémi + family-names: Gau + status: + - Meta-data harmonization with a term-mapper goes to BEP043 + - 'Persperctive: Repeated community review of the simplified proposal' + blocking: + - Term-mapping is a general feature => complex, modular proposal + google_doc_created: 2021-12 + pull_request_created: + pull_request_merged: + +- number: '036' + title: Phenotypic Data Guidelines + google_doc: https://docs.google.com/document/d/1WTkfES8L0vItZVyyR68fc-9cO03jS-kCnMnw6602pbc/ + content: + - raw + leads: + - given-names: Eric + family-names: Earl + - given-names: Samuel + family-names: Guay + - given-names: Sebastian + family-names: Urchs + - given-names: Arshitha + family-names: Basavaraj + bids_maintainers: + - given-names: Chris + family-names: Markiewicz + - given-names: Ross + family-names: Blair + status: + - BEP entering community review period shortly. + - BIDS specification PR 2123. + - BIDS examples PR 465. + blocking: + - A successful community review. + google_doc_created: 2021-10 + pull_request_created: 2025-05 + pull_request_merged: + +- number: '037' + title: Non-Invasive Brain Stimulation + google_doc: https://docs.google.com/document/d/1drYd7kaNbHTcYPR3T_CRDsPcEbFSV7JbJUmhMPeWMqY/ + current_repository: https://github.com/nigelrogasch/nibs-bids/tree/master/nibs-bids-v6/ + current_preprint_editable: https://docs.google.com/document/d/1xfetyFkXh8kqObfCViUvku69pk4ZZw8BC5GL8_Cq9TI/edit?tab=t.0 + current_preprint: + author_list: https://docs.google.com/spreadsheets/d/1oMImk-HXsyLZtTj3yIa1uY9LX3hVpsCiAGZP3oSV0Eo/edit?gid=0#gid=0 + content: + - raw + leads: + - given-names: Giacomo + family-names: Bertazzoli + - given-names: Vittorio + family-names: Iacovella + - given-names: Peter J. + family-names: Fried + - given-names: Marta + family-names: Bortoletto + - given-names: Nigel + family-names: Rogasch + past leads (inactive): + - given-names: Carlo + family-names: Miniussi + bids_maintainers: + - given-names: Rémi + family-names: Gau + status: + - v1.0 2020-11-09 Initial work on specification with vittorio.iacovella@unitn.it carlo.miniussi@unitn.it marta.bortoletto@cognitiveneuroscience.it + - v1.0 2021-04 First example of a NIBS-BIDS dataset https://gin.g-node.org/CIMeC/TMS-EEG_brain_connectivity_BIDS + - V1.0 2021-11 Brainhack @Donosti with eleonora.marcantoni@cognitiveneuroscience.it martinabulgari3@gmail.com and g.guidali@campus.unimib.it + - V1.0 2022-02 First international NIBS-BIDS meeting + - V1.0 2022-11 First update of the BEP + - V2.0 2023-08 BEP 2.0 available with the new NIBS-BIDS logic for describing NIBS experiments + - V3.0 2024-07 BEP 3.0 available with a draft of the final BIDS structure. + - V4.0 2024-11 BEP 4.0 updated with a new, more comprehensive structure. Added int files for offline stimulation. Harmonized parameters with SimNIBS. + Use of events and scans files for online stimulation. + - V4.1 2024-12 BEP 4.1 Comments form December 9th, 2024 5th general meeting implemented, added FAQ section. Met with the BIDS maintenance team in + January 2025. Agreed on closing the open discussions, lock the google doc and start the PR. + - V6.2 2026-03 BEP 6.2 Comments form December 12th, 2025 6th general meeting implemented. Met with the BIDS steering team in March 2026. Agreed on + NSF FAIROS application on NIBS-BIDS implementation https://www.nsf.gov/funding/opportunities/fairos-findable-accessible-interoperable-reusable-open-science. + Agreed on creating a preprint version of the BEP037 (to bi cited as a work-in-progress). + blocking: + google_doc_created: 2022-09 + pull_request_created: + pull_request_merged: + +- number: '039' + title: Dimensionality reduction-based networks + google_doc: https://docs.google.com/document/d/1GTWsj0MFQedXjOaNk6H0or6IDVFyMAysrJ9I4Zmpz2E/ + content: + - raw + leads: + - given-names: Arianna + family-names: Sala + - given-names: Anibal + family-names: Sólon + - given-names: Cyrus + family-names: Eierud + - given-names: Franco + family-names: Pestilli + - given-names: Peer + family-names: Herholz + bids_maintainers: + status: + - adapted general spatial derivatives proposal, meaning using `map`, for example `eegmap` and `boldmap` for spatial components + - decided on `model-` and `items-` as keys to denote the utilized model and component number, if files are 3D, respectively + - updated meta-data and examples + blocking: + google_doc_created: 2021-10 + pull_request_created: + pull_request_merged: + +- number: '040' + title: Functional Ultrasound + google_doc: https://docs.google.com/document/d/1W3z01mf1E8cfg_OY7ZGqeUeOKv659jCHQBXavtmT-T8/ + content: + - raw + leads: + - given-names: Jean-Charles + family-names: Mariani + - given-names: Samuel + family-names: Le Meur-Diebolt + - given-names: Thomas + family-names: Deffieux + bids_maintainers: + - given-names: Rémi + family-names: Gau + status: + - All issues raised on the original BEP have been resolved. + - Regular meetings with contributors scheduled. + - Scanner coordinate system has been proposed to harmonize affine transformations with moving probes. + - 'Perspectives: Starting to bidsify existing datasets to stress test the BEP.' + - 'Perspectives: timing metadata has been copied from the fMRI-BIDS specification, but evolutions could be discussed to handle non-stable sampling + frequencies.' + blocking: + google_doc_created: 2023-03 + pull_request_created: + pull_request_merged: + +- number: '041' + title: Statistical Model Derivatives + google_doc: https://docs.google.com/document/d/1KHzp-yk8KXvkUIhtN71WU0m4P4kKT9C1yvI-i9_kNeY/ + content: + - derivative + leads: + - given-names: Taylor + family-names: Salo + bids_maintainers: + - given-names: Taylor + family-names: Salo + status: + - New BEP, collecting community comments and feedback. + - All collaborators are welcome. + blocking: + google_doc_created: 2022-08 + pull_request_created: + pull_request_merged: + +- number: '043' + title: BIDS Term Mapping + google_doc: https://docs.google.com/document/d/1LACjc5hFXDpa2l_QddBPR41Vce_gglGv9WeuBB7LsBU/ + content: + - metadata + leads: + - given-names: Christopher J. + family-names: Markiewicz + - given-names: Eric A. + family-names: Earl + bids_maintainers: + - given-names: Christopher J. + family-names: Markiewicz + - given-names: Eric A. + family-names: Earl + status: + - For being able to map non-BIDS data as BIDS-compatible. + - Now collecting community comments and feedback. + - All collaborators are welcome. + blocking: + google_doc_created: 2024-03 + pull_request_created: + pull_request_merged: + +- number: '044' + title: Stimuli + pull_request: https://github.com/bids-standard/bids-specification/pull/2022 + html_preview: https://bids-specification--2022.org.readthedocs.build/en/2022/modality-specific-files/stimuli.html + leads: + - given-names: Seyed Yahya + family-names: Shirazi + - given-names: Dora + family-names: Hermes + - given-names: Yaroslav O. + family-names: Halchenko + - given-names: Kay + family-names: Robbins + - given-names: Scott + family-names: Makeig + bids_maintainers: + - given-names: Rémi + family-names: Gau + status: + - Community comments and feedback are being collected (January 2025) + - To harmonize and make more reusable stimuli content under stimuli/ + - Collecting community comments and feedback. All collaborators are welcome. + - 'Original issue: [#153](https://github.com/bids-standard/bids-specification/issues/153)' + content: + - raw + blocking: + google_doc_created: 2023-09 + pull_request_created: 2024-12 + pull_request_merged: + +- number: '045' + title: Peripheral Physiological Recordings + display: Physio + pull_request: https://github.com/bids-standard/bids-specification/pull/2267 + html_preview: https://bids-specification--2267.org.readthedocs.build/en/2267/modality-specific-files/physiological-recordings.html + google_doc: https://docs.google.com/document/d/1oTfjzY5ZnLIYd0kPPWhR81sBmMuy_jC5YYIaqj6OhSA/edit + leads: + - given-names: Mary + family-names: Miedema + - given-names: Stefano + family-names: Moia + - given-names: Sourav + family-names: Kulkarni + bids_maintainers: + - given-names: Seyed Yahya + family-names: Shirazi + status: + - No longer developed in google doc, moved to . + - To update standards for physiological data for improved clarity and a broader range of use cases. + - Now collecting community comments and feedback. All collaborators are welcome. + - 'Original issue: [#1675](https://github.com/bids-standard/bids-specification/issues/1675).' + content: + - raw + blocking: + google_doc_created: 2024-08 + pull_request_created: 2025-11 + pull_request_merged: + +- number: '046' + title: Diffusion Tractography + display: Tractography + google_doc: https://docs.google.com/document/d/1ubDQ2RhgjnfGqoeukzEkPV9YEHhfYMERrj7-3b0c2HI/edit + leads: + - given-names: Robert E. + family-names: Smith + - given-names: Ariel + family-names: Rokem + - given-names: Franco + family-names: Pestilli + status: + - Porting comprehensive description of streamline tractography mechanisms into specification - 10.1016/B978-0-12-817057-1.00023-8 + - Determine appropriate resolution with TRX development - https://tee-ar-ex.github.io/trx-python/ + - Decide on scope of BEP; e.g. whether to include tractometry, complex tract delineation + content: + - derivative + blocking: + google_doc_created: 2022-02 + pull_request_created: + pull_request_merged: + +- number: '047' + title: Audio/video recordings for behavioral experiments + display: Behavioral audio/video recordings + pull_request: https://github.com/bids-standard/bids-specification/pull/2231 + html_preview: https://bids-specification--2231.org.readthedocs.build/en/2231/modality-specific-files/behavioral-experiments.html + content: + - raw + leads: + - given-names: Benjamin + family-names: Dichter + bids_maintainers: + - given-names: Seyed Yahya + family-names: Shirazi + status: + - Adds support for storing audio and video behavioral recordings (new `_audio` and `_video` suffixes) in the `beh/` directory. + blocking: + google_doc_created: + pull_request_created: 2025-10 + pull_request_merged: diff --git a/.agents/skills/bids/references/bids_schema.json b/.agents/skills/bids/references/bids_schema.json new file mode 100644 index 0000000..1770423 --- /dev/null +++ b/.agents/skills/bids/references/bids_schema.json @@ -0,0 +1,21015 @@ +{ + "meta": { + "associations": { + "events": { + "selectors": [ + "extension != '.json'" + ], + "target": { + "suffix": "events", + "extension": ".tsv" + }, + "inherit": true + }, + "aslcontext": { + "selectors": [ + "suffix == 'asl'", + "match(extension, '\\.nii(\\.gz)?$')" + ], + "target": { + "suffix": "aslcontext", + "extension": ".tsv" + }, + "inherit": true + }, + "m0scan": { + "selectors": [ + "suffix == 'asl'", + "match(extension, '\\.nii(\\.gz)?$')" + ], + "target": { + "suffix": "m0scan", + "extension": [ + ".nii", + ".nii.gz" + ] + }, + "inherit": false + }, + "magnitude": { + "selectors": [ + "suffix == 'fieldmap'", + "match(extension, '\\.nii(\\.gz)?$')" + ], + "target": { + "suffix": "magnitude", + "extension": [ + ".nii", + ".nii.gz" + ] + }, + "inherit": false + }, + "magnitude1": { + "selectors": [ + "match(suffix, 'phase(diff|1)$')", + "match(extension, '\\.nii(\\.gz)?$')" + ], + "target": { + "suffix": "magnitude1", + "extension": [ + ".nii", + ".nii.gz" + ] + }, + "inherit": false + }, + "bval": { + "selectors": [ + "intersects([suffix], ['dwi', 'epi'])", + "match(extension, '\\.nii(\\.gz)?$')" + ], + "target": { + "extension": ".bval" + }, + "inherit": true + }, + "bvec": { + "selectors": [ + "intersects([suffix], ['dwi', 'epi'])", + "match(extension, '\\.nii(\\.gz)?$')" + ], + "target": { + "extension": ".bvec" + }, + "inherit": true + }, + "channels": { + "selectors": [ + "intersects([suffix], ['eeg', 'emg', 'ieeg', 'meg', 'nirs', 'motion', 'optodes'])", + "extension != '.json'" + ], + "target": { + "suffix": "channels", + "extension": ".tsv" + }, + "inherit": true + }, + "coordsystem": { + "selectors": [ + "intersects([suffix], ['eeg', 'ieeg', 'meg', 'nirs', 'motion', 'electrodes', 'optodes'])", + "extension != '.json'" + ], + "target": { + "suffix": "coordsystem", + "extension": ".json" + }, + "inherit": true + }, + "electrodes": { + "selectors": [ + "intersects([suffix], ['eeg', 'emg', 'ieeg', 'meg'])", + "extension != '.json'" + ], + "target": { + "suffix": "electrodes", + "extension": ".tsv", + "entities": [ + "space" + ] + }, + "inherit": true + }, + "physio": { + "selectors": [ + "extension != '.json'" + ], + "target": { + "suffix": "physio", + "extension": ".tsv.gz" + }, + "inherit": false + }, + "coordsystems": { + "selectors": [ + "datatype == 'emg'", + "intersects([suffix], ['emg', 'electrodes'])", + "extension != '.json'" + ], + "target": { + "suffix": "coordsystem", + "extension": ".json", + "entities": [ + "space" + ] + }, + "inherit": true + }, + "atlas_description": { + "selectors": [ + "entities.atlas", + "extension != '.json'" + ], + "target": { + "suffix": "description", + "extension": ".json" + } + } + }, + "context": { + "type": "object", + "required": [ + "schema", + "dataset", + "path", + "size", + "sidecar", + "associations" + ], + "additionalProperties": false, + "properties": { + "schema": { + "description": "The BIDS specification schema", + "type": "object" + }, + "dataset": { + "description": "Properties and contents of the entire dataset", + "type": "object", + "required": [ + "dataset_description", + "tree", + "ignored", + "datatypes", + "modalities", + "subjects" + ], + "additionalProperties": false, + "properties": { + "dataset_description": { + "description": "Contents of /dataset_description.json", + "type": "object" + }, + "tree": { + "description": "Tree view of all files in dataset", + "type": "object" + }, + "ignored": { + "description": "Set of ignored files", + "type": "array", + "items": { + "type": "string" + } + }, + "datatypes": { + "description": "Data types present in the dataset", + "type": "array", + "items": { + "type": "string" + } + }, + "modalities": { + "description": "Modalities present in the dataset", + "type": "array", + "items": { + "type": "string" + } + }, + "subjects": { + "description": "Collections of subjects in dataset", + "type": "object", + "required": [ + "sub_dirs" + ], + "additionalProperties": false, + "properties": { + "sub_dirs": { + "description": "Subjects as determined by sub-* directories", + "type": "array", + "items": { + "type": "string" + } + }, + "participant_id": { + "description": "The participant_id column of participants.tsv", + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "subject": { + "description": "Properties and contents of the current subject", + "type": "object", + "required": [ + "sessions" + ], + "additionalProperties": false, + "properties": { + "sessions": { + "description": "Collections of sessions in subject", + "type": "object", + "required": [ + "ses_dirs" + ], + "additionalProperties": false, + "properties": { + "ses_dirs": { + "description": "Sessions as determined by ses-* directories", + "type": "array", + "items": { + "type": "string" + } + }, + "session_id": { + "description": "The session_id column of sessions.tsv", + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "path": { + "description": "Path of the current file", + "type": "string" + }, + "size": { + "description": "Length of the current file in bytes", + "type": "integer" + }, + "entities": { + "description": "Entities parsed from the current filename", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "datatype": { + "description": "Datatype of current file, for examples, anat", + "type": "string" + }, + "suffix": { + "description": "Suffix of current file", + "type": "string" + }, + "extension": { + "description": "Extension of current file including initial dot", + "type": "string" + }, + "modality": { + "description": "Modality of current file, for examples, MRI", + "type": "string" + }, + "sidecar": { + "description": "Sidecar metadata constructed via the inheritance principle", + "type": "object" + }, + "associations": { + "description": "Associated files, indexed by suffix, selected according to the inheritance principle\n", + "type": "object", + "additionalProperties": false, + "properties": { + "events": { + "description": "Events file", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated events file", + "type": "string" + }, + "onset": { + "description": "Contents of the onset column", + "type": "array", + "items": { + "type": "string" + } + }, + "sidecar": { + "description": "Sidecar metadata constructed via the inheritance principle", + "type": "object" + } + } + }, + "aslcontext": { + "description": "ASL context file", + "type": "object", + "required": [ + "path", + "n_rows" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated aslcontext file", + "type": "string" + }, + "n_rows": { + "description": "Number of rows in aslcontext.tsv", + "type": "integer" + }, + "volume_type": { + "description": "Contents of the volume_type column", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "m0scan": { + "description": "M0 scan file", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated M0 scan file", + "type": "string" + } + } + }, + "magnitude": { + "description": "Magnitude image file", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated magnitude file", + "type": "string" + } + } + }, + "magnitude1": { + "description": "Magnitude1 image file", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated magnitude1 file", + "type": "string" + } + } + }, + "bval": { + "description": "B value file", + "type": "object", + "required": [ + "path", + "n_cols", + "n_rows", + "values" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated bval file", + "type": "string" + }, + "n_cols": { + "description": "Number of columns in bval file", + "type": "integer" + }, + "n_rows": { + "description": "Number of rows in bval file", + "type": "integer" + }, + "values": { + "description": "B-values contained in bval file", + "type": "array", + "items": { + "type": "number" + } + } + } + }, + "bvec": { + "description": "B vector file", + "type": "object", + "required": [ + "path", + "n_cols", + "n_rows" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated bvec file", + "type": "string" + }, + "n_cols": { + "description": "Number of columns in bvec file", + "type": "integer" + }, + "n_rows": { + "description": "Number of rows in bvec file", + "type": "integer" + } + } + }, + "channels": { + "description": "Channels file", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated channels file", + "type": "string" + }, + "type": { + "description": "Contents of the type column", + "type": "array", + "items": { + "type": "string" + } + }, + "short_channel": { + "description": "Contents of the short_channel column", + "type": "array", + "items": { + "type": "string" + } + }, + "sampling_frequency": { + "description": "Contents of the sampling_frequency column", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "electrodes": { + "description": "Electrodes file", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated electrodes.tsv file", + "type": "string" + } + } + }, + "coordsystem": { + "description": "Coordinate system file (first found)", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated coordsystem file", + "type": "string" + } + } + }, + "coordsystems": { + "description": "Coordinate system files (all)", + "type": "object", + "required": [ + "paths", + "spaces", + "ParentCoordinateSystems" + ], + "additionalProperties": false, + "properties": { + "paths": { + "description": "Paths to associated coordsystem files", + "type": "array", + "items": { + "type": "string" + } + }, + "spaces": { + "description": "Space entity labels extracted from file names", + "type": "array", + "items": { + "type": "string" + } + }, + "ParentCoordinateSystems": { + "description": "ParentCoordinateSystem fields loaded from files", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "physio": { + "description": "Physiological recording file", + "type": "object", + "required": [ + "path" + ], + "properties": { + "path": { + "description": "Path to associated physio file", + "type": "string" + }, + "sidecar": { + "description": "Sidecar metadata constructed via the inheritance principle", + "type": "object" + } + } + }, + "atlas_description": { + "description": "Atlas description file", + "type": "object", + "required": [ + "path" + ], + "additionalProperties": false, + "properties": { + "path": { + "description": "Path to associated atlas description file", + "type": "string" + } + } + } + } + }, + "columns": { + "description": "TSV columns, indexed by column header, values are arrays with column contents", + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "json": { + "description": "Contents of the current JSON file", + "type": "object" + }, + "gzip": { + "description": "Parsed contents of gzip header", + "type": "object", + "required": [ + "timestamp" + ], + "additionalProperties": false, + "properties": { + "timestamp": { + "description": "Modification time, unix timestamp", + "type": "number" + }, + "filename": { + "description": "Filename", + "type": "string" + }, + "comment": { + "description": "Comment", + "type": "string" + } + } + }, + "nifti_header": { + "name": "NIfTI Header", + "description": "Parsed contents of NIfTI header referenced elsewhere in schema.", + "type": "object", + "required": [ + "dim_info", + "dim", + "pixdim", + "shape", + "voxel_sizes", + "xyzt_units", + "qform_code", + "sform_code", + "axis_codes" + ], + "additionalProperties": false, + "properties": { + "dim_info": { + "name": "Dimension Information", + "description": "Metadata about dimensions data.", + "type": "object", + "required": [ + "freq", + "phase", + "slice" + ], + "additionalProperties": false, + "properties": { + "freq": { + "name": "Frequency", + "description": "These fields encode which spatial dimension (1, 2, or 3).", + "type": "integer" + }, + "phase": { + "name": "Phase", + "description": "Corresponds to which acquisition dimension for MRI data.", + "type": "integer" + }, + "slice": { + "name": "Slice", + "description": "Slice dimensions.", + "type": "integer" + } + } + }, + "dim": { + "name": "Data Dimensions", + "description": "Data seq dimensions.", + "type": "array", + "minItems": 8, + "maxItems": 8, + "items": { + "type": "integer" + } + }, + "pixdim": { + "name": "Pixel Dimension", + "description": "Grid spacings (unit per dimension).", + "type": "array", + "minItems": 8, + "maxItems": 8, + "items": { + "type": "number" + } + }, + "shape": { + "name": "Data shape", + "description": "Data array shape, equal to dim[1:dim[0] + 1]", + "type": "array", + "minItems": 0, + "maxItems": 7, + "items": { + "type": "integer" + } + }, + "voxel_sizes": { + "name": "Voxel sizes", + "description": "Voxel sizes, equal to pixdim[1:dim[0] + 1]", + "type": "array", + "minItems": 0, + "maxItems": 7, + "items": { + "type": "number" + } + }, + "xyzt_units": { + "name": "XYZT Units", + "description": "Units of pixdim[1..4]", + "type": "object", + "required": [ + "xyz", + "t" + ], + "additionalProperties": false, + "properties": { + "xyz": { + "name": "XYZ Units", + "description": "String representing the unit of voxel spacing.", + "type": "string", + "enum": [ + "unknown", + "meter", + "mm", + "um" + ] + }, + "t": { + "name": "Time Unit", + "description": "String representing the unit of inter-volume intervals.", + "type": "string", + "enum": [ + "unknown", + "sec", + "msec", + "usec" + ] + } + } + }, + "qform_code": { + "name": "qform code", + "description": "Use of the quaternion fields.", + "type": "integer" + }, + "sform_code": { + "name": "sform code", + "description": "Use of the affine fields.", + "type": "integer" + }, + "axis_codes": { + "name": "axis codes", + "description": "Orientation labels indicating primary direction of data axes defined with respect to the object of interest.\n", + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "string", + "enum": [ + "R", + "L", + "A", + "P", + "S", + "I" + ] + } + }, + "mrs": { + "name": "NIfTI-MRS extension", + "description": "NIfTI-MRS JSON fields", + "type": "object" + } + } + }, + "ome": { + "name": "Open Microscopy Environment fields", + "description": "Parsed contents of OME-XML header, which may be found in OME-TIFF or OME-ZARR files", + "type": "object", + "additionalProperties": false, + "properties": { + "PhysicalSizeX": { + "name": "PhysicalSizeX", + "description": "Pixels / @PhysicalSizeX", + "type": "number" + }, + "PhysicalSizeY": { + "name": "PhysicalSizeY", + "description": "Pixels / @PhysicalSizeY", + "type": "number" + }, + "PhysicalSizeZ": { + "name": "PhysicalSizeZ", + "description": "Pixels / @PhysicalSizeZ", + "type": "number" + }, + "PhysicalSizeXUnit": { + "name": "PhysicalSizeXUnit", + "description": "Pixels / @PhysicalSizeXUnit", + "type": "string" + }, + "PhysicalSizeYUnit": { + "name": "PhysicalSizeYUnit", + "description": "Pixels / @PhysicalSizeYUnit", + "type": "string" + }, + "PhysicalSizeZUnit": { + "name": "PhysicalSizeZUnit", + "description": "Pixels / @PhysicalSizeZUnit", + "type": "string" + } + } + }, + "tiff": { + "name": "TIFF", + "description": "TIFF file format metadata", + "type": "object", + "required": [ + "version" + ], + "additionalProperties": false, + "properties": { + "version": { + "name": "Version", + "description": "TIFF file format version (the second 2-byte block)", + "type": "integer" + } + } + } + } + }, + "expression_tests": [ + { + "expression": "sidecar.MissingValue", + "result": null + }, + { + "expression": "null.anything", + "result": null + }, + { + "expression": "(null)", + "result": null + }, + { + "expression": "null[0]", + "result": null + }, + { + "expression": "null && true", + "result": null + }, + { + "expression": "true && null", + "result": null + }, + { + "expression": "false && null", + "result": false + }, + { + "expression": "true || null", + "result": true + }, + { + "expression": "null || true", + "result": true + }, + { + "expression": "false || null", + "result": null + }, + { + "expression": "!null", + "result": true + }, + { + "expression": "intersects([], null)", + "result": false + }, + { + "expression": "intersects(null, [])", + "result": false + }, + { + "expression": "allequal([], null)", + "result": false + }, + { + "expression": "allequal(null, [])", + "result": false + }, + { + "expression": "match(null, 'pattern')", + "result": null + }, + { + "expression": "match('string', null)", + "result": false + }, + { + "expression": "substr(null, 1, 4)", + "result": null + }, + { + "expression": "substr('string', null, 4)", + "result": null + }, + { + "expression": "substr('string', 1, null)", + "result": null + }, + { + "expression": "min(null)", + "result": null + }, + { + "expression": "max(null)", + "result": null + }, + { + "expression": "length(null)", + "result": null + }, + { + "expression": "unique(null)", + "result": null + }, + { + "expression": "type(null)", + "result": "null" + }, + { + "expression": "null == false", + "result": false + }, + { + "expression": "null == true", + "result": false + }, + { + "expression": "null != false", + "result": true + }, + { + "expression": "null != true", + "result": true + }, + { + "expression": "null != 1.5", + "result": true + }, + { + "expression": "null == null", + "result": true + }, + { + "expression": "null == 1", + "result": false + }, + { + "expression": "\"VolumeTiming\" in null", + "result": null + }, + { + "expression": "exists(null, \"bids-uri\")", + "result": 0 + }, + { + "expression": "exists([], null)", + "result": 0 + }, + { + "expression": "true || sidecar.MissingValue", + "result": true + }, + { + "expression": "1 + 2", + "result": 3 + }, + { + "expression": "1 - 2", + "result": -1 + }, + { + "expression": "3 * 4", + "result": 12 + }, + { + "expression": "3 / 2", + "result": 1.5 + }, + { + "expression": "3 % 2", + "result": 1 + }, + { + "expression": "\"cat\" + \"dog\"", + "result": "catdog" + }, + { + "expression": "match('string', '.*')", + "result": true + }, + { + "expression": "match('', '.')", + "result": false + }, + { + "expression": "substr('string', 1, 4)", + "result": "tri" + }, + { + "expression": "substr('string', 0, 20)", + "result": "string" + }, + { + "expression": "type(1)", + "result": "number" + }, + { + "expression": "type([])", + "result": "array" + }, + { + "expression": "type({})", + "result": "object" + }, + { + "expression": "type(true)", + "result": "boolean" + }, + { + "expression": "intersects([1], [1, 2])", + "result": [ + 1 + ] + }, + { + "expression": "intersects([1], [])", + "result": false + }, + { + "expression": "length([1, 2, 3])", + "result": 3 + }, + { + "expression": "length([])", + "result": 0 + }, + { + "expression": "count([1, 2, 3], 1)", + "result": 1 + }, + { + "expression": "index([\"i\", \"j\", \"k\"], \"i\")", + "result": 0 + }, + { + "expression": "index([\"i\", \"j\", \"k\"], \"j\")", + "result": 1 + }, + { + "expression": "index([\"i\", \"j\", \"k\"], \"x\")", + "result": null + }, + { + "expression": "sorted([3, 2, 1])", + "result": [ + 1, + 2, + 3 + ] + }, + { + "expression": "sorted([1, 2, 5, 10], \"lexical\")", + "result": [ + 1, + 10, + 2, + 5 + ] + }, + { + "expression": "sorted([\"1\", \"2\", \"5\", \"10\"])", + "result": [ + "1", + "10", + "2", + "5" + ] + }, + { + "expression": "sorted([\"1\", \"2\", \"5\", \"10\"], \"numeric\")", + "result": [ + "1", + "2", + "5", + "10" + ] + }, + { + "expression": "sorted([\"1\", \"2\", \"n/a\"], \"numeric\")", + "result": [ + "1", + "2", + "n/a" + ] + }, + { + "expression": "sorted([\"n/a\", \"2\", \"1\"], \"numeric\")", + "result": [ + "n/a", + "1", + "2" + ] + }, + { + "expression": "allequal(sorted([3, 2, 1]), [1, 2, 3])", + "result": true + }, + { + "expression": "sorted([9, 81, 729, 6561])", + "result": [ + 9, + 81, + 729, + 6561 + ] + }, + { + "expression": "min([-1, \"n/a\", 1])", + "result": -1 + }, + { + "expression": "max([-1, \"n/a\", 1])", + "result": 1 + }, + { + "expression": "min(42)", + "result": 42 + }, + { + "expression": "max(42)", + "result": 42 + }, + { + "expression": "[3, 2, 1][0]", + "result": 3 + }, + { + "expression": "\"string\"[0]", + "result": "s" + }, + { + "expression": "unique([])", + "result": [] + }, + { + "expression": "unique([52, -4, 3, 8, -4, 52, 9])", + "result": [ + 52, + -4, + 3, + 8, + 9 + ] + }, + { + "expression": "unique([\"52\", \"-4\", \"3\", \"8\", \"-4\", \"52\", \"9\"])", + "result": [ + "52", + "-4", + "3", + "8", + "9" + ] + }, + { + "expression": "unique([1, 1.0])", + "result": [ + 1 + ] + }, + { + "expression": "unique([1.0, 1])", + "result": [ + 1.0 + ] + } + ], + "templates": { + "raw": { + "base": { + "entities": { + "subject": "required", + "session": "optional" + } + }, + "recording": { + "entities": { + "acquisition": "optional", + "run": "optional", + "subject": "required", + "session": "optional" + } + }, + "task": { + "entities": { + "task": "required", + "acquisition": "optional", + "run": "optional", + "subject": "required", + "session": "optional" + } + }, + "mri": { + "entities": { + "ceagent": "optional", + "reconstruction": "optional", + "chunk": "optional", + "acquisition": "optional", + "run": "optional", + "subject": "required", + "session": "optional" + } + }, + "epi": { + "entities": { + "direction": "optional", + "part": "optional", + "ceagent": "optional", + "reconstruction": "optional", + "chunk": "optional", + "acquisition": "optional", + "run": "optional", + "subject": "required", + "session": "optional" + } + }, + "pet": { + "entities": { + "task": "optional", + "tracer": "optional", + "reconstruction": "optional", + "run": "optional", + "subject": "required", + "session": "optional" + } + }, + "mrs": { + "entities": { + "task": "optional", + "nucleus": "optional", + "volume": "optional", + "reconstruction": "optional", + "acquisition": "optional", + "run": "optional", + "subject": "required", + "session": "optional" + } + } + }, + "deriv": { + "base": { + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ], + "entities": { + "subject": "optional", + "session": "optional", + "description": "optional" + } + }, + "spatial": { + "entities": { + "space": "optional", + "subject": "optional", + "session": "optional", + "description": "optional" + }, + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ] + }, + "volumetric": { + "extensions": [ + ".nii.gz", + ".nii", + ".json" + ], + "entities": { + "resolution": "optional", + "space": "optional", + "subject": "optional", + "session": "optional", + "description": "optional" + }, + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ] + }, + "surface": { + "entities": { + "hemisphere": "optional", + "density": "optional", + "space": "optional", + "subject": "optional", + "session": "optional", + "description": "optional" + }, + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ] + }, + "mask": { + "suffixes": [ + "mask" + ], + "entities": { + "label": "optional", + "resolution": "optional", + "space": "optional", + "subject": "optional", + "session": "optional", + "description": "optional" + }, + "extensions": [ + ".nii.gz", + ".nii", + ".json" + ], + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ] + }, + "dseg": { + "extensions": [ + ".nii.gz", + ".nii", + ".tsv", + ".json" + ], + "suffixes": [ + "dseg" + ], + "entities": { + "atlas": "optional", + "segmentation": "optional", + "scale": "optional", + "resolution": "optional", + "space": "optional", + "subject": "optional", + "session": "optional", + "description": "optional" + }, + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ] + }, + "probseg": { + "extensions": [ + ".nii.gz", + ".nii", + ".json" + ], + "suffixes": [ + "probseg" + ], + "entities": { + "atlas": "optional", + "segmentation": "optional", + "scale": "optional", + "label": "optional", + "resolution": "optional", + "space": "optional", + "subject": "optional", + "session": "optional", + "description": "optional" + }, + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ] + }, + "dseg_surface": { + "extensions": [ + ".label.gii", + ".dlabel.nii", + ".tsv", + ".json" + ], + "suffixes": [ + "dseg" + ], + "entities": { + "atlas": "optional", + "segmentation": "optional", + "scale": "optional", + "hemisphere": "optional", + "density": "optional", + "space": "optional", + "subject": "optional", + "session": "optional", + "description": "optional" + }, + "selectors": [ + "dataset.dataset_description.DatasetType == 'derivative'" + ] + } + }, + "atlas": { + "base": { + "entities": { + "template": "required", + "cohort": "optional" + } + } + } + }, + "versions": [ + "1.11.1", + "1.11.0", + "1.10.1", + "1.10.0", + "1.9.0", + "1.8.0", + "1.7.0", + "1.6.0", + "1.5.0", + "1.4.1", + "1.4.0", + "1.3.0", + "1.2.2", + "1.2.1", + "1.2.0", + "1.1.2", + "1.1.1", + "1.1.0", + "1.0.2", + "1.0.1", + "1.0.0" + ] + }, + "objects": { + "columns": { + "HED": { + "name": "HED", + "display_name": "HED", + "description": "Hierarchical Event Descriptor (HED) tags.\nSee the [HED Appendix](SPEC_ROOT/appendices/hed.md) for details.\n", + "type": "string" + }, + "abbreviation": { + "name": "abbreviation", + "display_name": "Abbreviation", + "description": "The unique label abbreviation\n", + "type": "string" + }, + "acq_time__scans": { + "name": "acq_time", + "display_name": "Scan acquisition time", + "description": "Acquisition time refers to when the first data point in each run was acquired.\nFurthermore, if this header is provided, the acquisition times of all files\nfrom the same recording MUST be identical.\nDatetime format and their deidentification are described in\n[Units](SPEC_ROOT/common-principles.md#units).\n", + "type": "string", + "format": "datetime" + }, + "acq_time__sessions": { + "name": "acq_time", + "display_name": "Session acquisition time", + "description": "Acquisition time refers to when the first data point of the first run was acquired.\nDatetime format and their deidentification are described in\n[Units](SPEC_ROOT/common-principles.md#units).\n", + "type": "string", + "format": "datetime" + }, + "age": { + "name": "age", + "display_name": "Subject age", + "description": "Numeric value in years (float or integer value).\n\nFor privacy purposes, participant ages should be capped at 89.\nUsing \"89+\" for ages above 88 is DEPRECATED.\n", + "definition": { + "LongName": "Subject age", + "Description": "Subject age in postnatal years", + "Format": "number", + "Units": "year", + "Maximum": 89 + } + }, + "cardiac": { + "name": "cardiac", + "display_name": "Cardiac measurement", + "description": "continuous pulse measurement\n", + "definition": { + "Description": "continuous pulse measurement", + "Format": "number", + "Units": "mV" + } + }, + "channel": { + "name": "channel", + "display_name": "Channel", + "description": "Channel(s) associated with an event.\nIf multiple channels are specified, they MUST be separated by a delimiter\nspecified in the `\"Delimiter\"` field describing the `channel` column.\nFor example, channels separated with a comma (`,`) require the `events.json`\nfile to contain `\"channel\": {\"Delimiter\": \",\"}`.\nIn the absence of a delimiter, tools MUST interpret any character as being part\nof a channel name.\n", + "type": "string" + }, + "color": { + "name": "color", + "display_name": "Color label", + "description": "Hexadecimal. Label color for visualization.\n", + "type": "string", + "unit": "hexadecimal" + }, + "component": { + "name": "component", + "display_name": "Component", + "description": "Description of the spatial axis or label of quaternion component associated with the channel.\nFor example, `x`,`y`,`z` for position channels,\nor `quat_x`, `quat_y`, `quat_z`, `quat_w` for quaternion orientation channels.\n", + "type": "string", + "enum": [ + "x", + "y", + "z", + "quat_x", + "quat_y", + "quat_z", + "quat_w" + ] + }, + "coordinate_system": { + "name": "coordinate_system", + "display_name": "Coordinate System", + "description": "Name of the coordinate system defined in `coordsystem.json` in which the electrode's\n`x`, `y`, and (optionally) `z` coordinates are given.\n", + "type": "string" + }, + "detector__channels": { + "name": "detector", + "display_name": "Detector Name", + "description": "Name of the detector as specified in the `*_optodes.tsv` file.\n`n/a` for channels that do not contain NIRS signals (for example, acceleration).\n", + "type": "string" + }, + "detector_type": { + "name": "detector_type", + "display_name": "Detector Type", + "description": "The type of detector. Only to be used if the field `DetectorType` in `*_nirs.json` is set to `mixed`.\n", + "type": "string" + }, + "derived_from": { + "name": "derived_from", + "display_name": "Derived from", + "description": "`sample-