From 6f376ce022d61909f786d8bd2b86fb430d5e040b Mon Sep 17 00:00:00 2001 From: LouisK92 Date: Wed, 16 Jul 2025 17:09:14 +0200 Subject: [PATCH 1/2] Adjust xenium scripts to enable local run with url inputs --- .../loaders/tenx_xenium/config.vsh.yaml | 4 ++-- src/datasets/loaders/tenx_xenium/script.py | 21 +++++++++++++------ .../process_tenx_xenium/config.vsh.yaml | 4 ++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/datasets/loaders/tenx_xenium/config.vsh.yaml b/src/datasets/loaders/tenx_xenium/config.vsh.yaml index 15eb94705..109536ee4 100644 --- a/src/datasets/loaders/tenx_xenium/config.vsh.yaml +++ b/src/datasets/loaders/tenx_xenium/config.vsh.yaml @@ -4,10 +4,10 @@ namespace: datasets/loaders argument_groups: - name: Inputs arguments: - - type: file + - type: string name: --input required: true - description: A 10x xenium directory or zip file + description: A 10x xenium directory or zip file or download url - type: string name: --segmentation_id required: true diff --git a/src/datasets/loaders/tenx_xenium/script.py b/src/datasets/loaders/tenx_xenium/script.py index 0c4af2a7b..3a3ee4633 100644 --- a/src/datasets/loaders/tenx_xenium/script.py +++ b/src/datasets/loaders/tenx_xenium/script.py @@ -8,12 +8,11 @@ ## VIASH START par = { - "input": "temp/datasets/10x_xenium/2023_10x_mouse_brain_xenium_rep1/Xenium_V1_FF_Mouse_Brain_MultiSection_1_outs", + "input": "https://cf.10xgenomics.com/samples/xenium/1.9.0/Xenium_V1_hLiver_cancer_section_FFPE/Xenium_V1_hLiver_cancer_section_FFPE_outs.zip", "segmentation_id": [ "cell", "nucleus", ], - "output": "output.zarr", "dataset_id": "value", "dataset_name": "value", "dataset_url": "value", @@ -21,16 +20,26 @@ "dataset_summary": "value", "dataset_description": "value", "dataset_organism": "value", - "output": "temp/datasets/10x_xenium/2023_10x_mouse_brain_xenium_rep1/Xenium_V1_FF_Mouse_Brain_MultiSection_1_outs.zarr" + "output": "temp/datasets/10x_xenium/liver/liver.zarr" +} +meta = { + "cpus": 1, } -## VIASH END +## VIASH END -# if input is a zip, extract it to a temporary folder +# Download the data if it's a download url, extract the data if it's a zip file par_input = par["input"] with tempfile.TemporaryDirectory() as tmpdirname: + if par_input.startswith("http"): + print(f"Downloading data to {tmpdirname}", flush=True) + file_name = par_input.split("/")[-1] + # download the data + os.system(f"wget {par['input']} -O {tmpdirname}/{file_name}") + par_input = tmpdirname + "/" + file_name + if zipfile.is_zipfile(par_input): - print("Extracting input zip", flush=True) + print(f"Extracting input zip to {tmpdirname}", flush=True) with zipfile.ZipFile(par_input, "r") as zip_ref: zip_ref.extractall(tmpdirname) par_input = tmpdirname diff --git a/src/datasets/workflows/process_tenx_xenium/config.vsh.yaml b/src/datasets/workflows/process_tenx_xenium/config.vsh.yaml index c4fec772b..ca826bb3d 100644 --- a/src/datasets/workflows/process_tenx_xenium/config.vsh.yaml +++ b/src/datasets/workflows/process_tenx_xenium/config.vsh.yaml @@ -4,10 +4,10 @@ namespace: datasets/workflows argument_groups: - name: Inputs arguments: - - type: file + - type: string name: --input required: true - description: A 10x xenium directory or zip file + description: A 10x xenium directory or zip file or download url - type: string name: --segmentation_id required: true From 4bc094d3207184f8673102c2c0f5952562ece147 Mon Sep 17 00:00:00 2001 From: LouisK92 Date: Wed, 16 Jul 2025 17:10:02 +0200 Subject: [PATCH 2/2] Add download script with additional xenium datasets --- .../spatial/process_10x_xenium_more.sh | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 scripts/create_resources/spatial/process_10x_xenium_more.sh diff --git a/scripts/create_resources/spatial/process_10x_xenium_more.sh b/scripts/create_resources/spatial/process_10x_xenium_more.sh new file mode 100644 index 000000000..ea32e4c3c --- /dev/null +++ b/scripts/create_resources/spatial/process_10x_xenium_more.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +publish_dir="s3://openproblems-data/resources/datasets" + +cat > /tmp/params.yaml << HERE +param_list: + + - id: "10x_xenium/2023_10x_human_lung_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.3.0/Xenium_Preview_Human_Non_diseased_Lung_With_Add_on_FFPE/Xenium_Preview_Human_Non_diseased_Lung_With_Add_on_FFPE_outs.zip + dataset_name: "Xenium Preview Human Non diseased Lung With Add on FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/xenium-human-lung-preview-data-1-standard" + dataset_summary: "Preview of Xenium In Situ Gene Expression data for adult human lung sections, using a development version of the Human Lung Gene Expression Panel and custom add-on panel for lung cell types." + dataset_description: "Preview of Xenium In Situ Gene Expression data for adult human lung sections, using a development version of the Human Lung Gene Expression Panel and custom add-on panel for lung cell types. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.3.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_lung_cancer_xenium" + input: https://s3-us-west-2.amazonaws.com/10x.files/samples/xenium/1.3.0/Xenium_Preview_Human_Lung_Cancer_With_Add_on_2_FFPE/Xenium_Preview_Human_Lung_Cancer_With_Add_on_2_FFPE_outs.zip + dataset_name: "Xenium Preview Human Lung Cancer With Add on 2 FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/xenium-human-lung-preview-data-1-standard" + dataset_summary: "Preview of Xenium In Situ Gene Expression data for adult human lung sections, using a development version of the Human Lung Gene Expression Panel and custom add-on panel for lung cell types." + dataset_description: "Preview of Xenium In Situ Gene Expression data for adult human lung sections, using a development version of the Human Lung Gene Expression Panel and custom add-on panel for lung cell types. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.3.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_pancreas_cancer_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.6.0/Xenium_V1_hPancreas_Cancer_Add_on_FFPE/Xenium_V1_hPancreas_Cancer_Add_on_FFPE_outs.zip + dataset_name: "Xenium V1 hPancreas Cancer Add on FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/pancreatic-cancer-with-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Xenium In Situ Gene Expression data for human pancreatic cancer sections using the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "Xenium In Situ Gene Expression data for human pancreatic cancer sections using the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.6.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_brain_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.3.0/Xenium_V1_FFPE_Human_Brain_Healthy_With_Addon/Xenium_V1_FFPE_Human_Brain_Healthy_With_Addon_outs.zip + dataset_name: "Xenium V1 FFPE Human Brain Healthy With Addon" + dataset_url: "https://www.10xgenomics.com/datasets/xenium-human-brain-preview-data-1-standard" + dataset_summary: "Adult human brain cortical section, healthy." + dataset_description: "Preview of Xenium In Situ Gene Expression data for adult human brain cortical sections, using a development version of the Human Brain Gene Expression Panel and two custom add-on panels for brain cell types. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.3.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2024_10x_human_skin_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.9.0/Xenium_V1_hSkin_nondiseased_section_1_FFPE/Xenium_V1_hSkin_nondiseased_section_1_FFPE_outs.zip + dataset_name: "Xenium V1 hSkin nondiseased section 1 FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/human-skin-data-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Xenium In Situ Gene Expression data for adult human skin sections using the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "Xenium In Situ Gene Expression data for adult human skin sections using the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.9.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2024_10x_human_liver_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.9.0/Xenium_V1_hLiver_nondiseased_section_FFPE/Xenium_V1_hLiver_nondiseased_section_FFPE_outs.zip + dataset_name: "Xenium V1 hLiver nondiseased section FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/human-liver-data-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Xenium In Situ Gene Expression data for adult human liver sections using the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "Xenium In Situ Gene Expression data for adult human liver sections using the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.9.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2024_10x_human_liver_cancer_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.9.0/Xenium_V1_hLiver_cancer_section_FFPE/Xenium_V1_hLiver_cancer_section_FFPE_outs.zip + dataset_name: "Xenium V1 hLiver cancer section FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/human-liver-data-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Xenium In Situ Gene Expression data for adult human liver sections using the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "Xenium In Situ Gene Expression data for adult human liver sections using the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.9.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2024_10x_human_heart_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.9.0/Xenium_V1_hHeart_nondiseased_section_FFPE/Xenium_V1_hHeart_nondiseased_section_FFPE_outs.zip + dataset_name: "Xenium V1 hHeart nondiseased section FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/human-heart-data-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Xenium In Situ Gene Expression data for adult human heart section using the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "Xenium In Situ Gene Expression data for adult human heart section using the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.9.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_colon_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.6.0/Xenium_V1_hColon_Non_diseased_Add_on_FFPE/Xenium_V1_hColon_Non_diseased_Add_on_FFPE_outs.zip + dataset_name: "Xenium V1 hColon Non diseased Add on FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/human-colon-preview-data-xenium-human-colon-gene-expression-panel-1-standard" + dataset_summary: "Preview of Xenium In Situ Gene Expression data for adult human colon sections, using a development version of the Xenium Human Colon Gene Expression Panel." + dataset_description: "Preview of Xenium In Situ Gene Expression data for adult human colon sections, using a development version of the Xenium Human Colon Gene Expression Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.6.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_colon_cancer_xenium" + input: https://s3-us-west-2.amazonaws.com/10x.files/samples/xenium/1.6.0/Xenium_V1_hColon_Cancer_Add_on_FFPE/Xenium_V1_hColon_Cancer_Add_on_FFPE_outs.zip + dataset_name: "Xenium V1 hColon Cancer Add on FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/human-colon-preview-data-xenium-human-colon-gene-expression-panel-1-standard" + dataset_summary: "Preview of Xenium In Situ Gene Expression data for adult human colon sections, using a development version of the Xenium Human Colon Gene Expression Panel." + dataset_description: "Cancer; stage 2A adenocarcinoma. Preview of Xenium In Situ Gene Expression data for adult human colon sections, using a development version of the Xenium Human Colon Gene Expression Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.6.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_kidney_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.5.0/Xenium_V1_hKidney_nondiseased_section/Xenium_V1_hKidney_nondiseased_section_outs.zip + dataset_name: "Xenium V1 hKidney nondiseased section" + dataset_url: "https://www.10xgenomics.com/datasets/human-kidney-preview-data-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Preview of Xenium In Situ Gene Expression data for adult human kidney sections, using a development version of the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "Preview of Xenium In Situ Gene Expression data for adult human kidney sections, using a development version of the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.5.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_kidney_cancer_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.5.0/Xenium_V1_hKidney_cancer_section/Xenium_V1_hKidney_cancer_section_outs.zip + dataset_name: "Xenium V1 hKidney cancer section" + dataset_url: "https://www.10xgenomics.com/datasets/human-kidney-preview-data-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Preview of Xenium In Situ Gene Expression data for adult human kidney sections, using a development version of the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "Kidney cancer (papillary renal cell carcinoma, or PRCC). Preview of Xenium In Situ Gene Expression data for adult human kidney sections, using a development version of the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.5.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2024_10x_mouse_colon_xenium" + input: https://cf.10xgenomics.com/samples/xenium/2.0.0/Xenium_V1_mouse_Colon_FF/Xenium_V1_mouse_Colon_FF_outs.zip + dataset_name: "Xenium V1 mouse Colon FF" + dataset_url: "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-colon-with-xenium-multimodal-cell-segmentation-1-standard" + dataset_summary: "Xenium In Situ Gene Expression with Cell Segmentation Staining data for mouse colon tissue using the Xenium Mouse Tissue Atlassing Panel." + dataset_description: "Xenium In Situ Gene Expression with Cell Segmentation Staining data for mouse colon tissue using the Xenium Mouse Tissue Atlassing Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 2.0.0" + dataset_organism: "mus_musculus" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2023_10x_human_lymph_node_xenium" + input: https://cf.10xgenomics.com/samples/xenium/1.5.0/Xenium_V1_hLymphNode_nondiseased_section/Xenium_V1_hLymphNode_nondiseased_section_outs.zip + dataset_name: "Xenium V1 hLymphNode nondiseased section" + dataset_url: "https://www.10xgenomics.com/datasets/human-lymph-node-preview-data-xenium-human-multi-tissue-and-cancer-panel-1-standard" + dataset_summary: "Preview of Xenium In Situ Gene Expression data for adult human lymph node, using a development version of the Xenium Human Multi-Tissue and Cancer Panel." + dataset_description: "The selected section represents a non-diseased lymph node tissue. Preview of Xenium In Situ Gene Expression data for adult human lymph node, using a development version of the Xenium Human Multi-Tissue and Cancer Panel. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 1.5.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + - id: "10x_xenium/2024_10x_human_ovarian_cancer_xenium" + input: https://cf.10xgenomics.com/samples/xenium/2.0.0/Xenium_V1_Human_Ovarian_Cancer_Addon_FFPE/Xenium_V1_Human_Ovarian_Cancer_Addon_FFPE_outs.zip + dataset_name: "Xenium V1 Human Ovarian Cancer Addon FFPE" + dataset_url: "https://www.10xgenomics.com/datasets/ffpe-human-ovarian-cancer-data-with-human-immuno-oncology-profiling-panel-and-custom-add-on-1-standard" + dataset_summary: "Xenium In Situ Gene Expression data for human ovarian cancer tissue using the Xenium Human Immuno-Oncology Profiling Panel with custom add-on." + dataset_description: "Ovary Serous Carcinoma; Stage II-A; Grade 3. Xenium In Situ Gene Expression data for human ovarian cancer tissue using the Xenium Human Immuno-Oncology Profiling Panel with custom add-on. In Situ Gene Expression dataset analyzed using Xenium Onboard Analysis 2.0.0" + dataset_organism: "homo_sapiens" + segmentation_id: [cell, nucleus] + + +output_dataset: "\$id/dataset.zarr" +output_state: "\$id/state.yaml" +publish_dir: "$publish_dir" +HERE + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/datasets/workflows/process_tenx_xenium/main.nf \ + --workspace 53907369739130 \ + --params-file /tmp/params.yaml \ + --config common/nextflow_helpers/labels_tw.config \ + --labels datasets,10x_xenium