diff --git a/examples/freesurfer/README.md b/examples/freesurfer/README.md deleted file mode 100644 index c437c25..0000000 --- a/examples/freesurfer/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Small example using VIP API - -`./freesurfer-recon-all.sh` script launches Freesurfer on VIP server through VIP API. - -## Requirements - -### VIP API token - -Export the API token before executing `./freesurfer-recon-all.sh` - -`export VIP_API_TOKEN=` - -### Freesurfer license - -Add your Freesurfer license in `inputs/LICENSE.txt` - -### Get dataset - -Install [`datalad`](https://handbook.datalad.org/en/latest/intro/installation.html) - -For Linux: - -via apt -``` -sudo apt-get install datalad -``` -or via PyPI -``` -pip install datalad -``` - -Download the T1 image: -``` -datalad get example/freesurfer/inputs/ds001600/sub-1/anat/sub-1_T1w.nii.gz -``` \ No newline at end of file diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/1-girder_download_input_data.py b/examples/freesurfer/freesurfer_longitudinal_processing/1-girder_download_input_data.py deleted file mode 100644 index 603e69f..0000000 --- a/examples/freesurfer/freesurfer_longitudinal_processing/1-girder_download_input_data.py +++ /dev/null @@ -1,46 +0,0 @@ -# Description: -# Author: Frederic Cervenansky < frederic.cervenansky@creatis.insa-lyon.fr> -# -# Licence Cecill-B -# Copyright (C) Creatis 2017-2024 - - -# import -import girder_client -import types -import sys -import click -from pathlib import Path - -# api Rest url of the warehouse -#Windows users: paths use backslashes (\) while Linux/macOS use forward slashes (/) -url='https://insert/your-server-here/api/v1' -#example URLs would be: -# https://srmnopt.creatis.insa-lyon.fr/warehouse/api/v1 --> for NMR and Optics -# https://myriad.creatis.insa-lyon.fr/api/v1 --> for MYRIAD - -# apiKey is a "mechanism" to share authentication and rights on folder. -# User should defined through the web interface (inside user information) an apiKey ith the corresponding privileges -apiKey = 'GIRDER_API_KEY' - -# Generate the warehouse client -gc = girder_client.GirderClient(apiUrl=url) - -# Authentication to the warehouse -gc.authenticate(apiKey=apiKey) - -# Use the ID of the folder you chose put your input MRIs -folderId ='FOLDER_ID' # 3D Flow phantom+Bubbles -#an example of a folder ID: 698de7fe82d062f2aea9619d - -# Local download directory -download_dir = Path('/insert/your/download/path') - -# Download data from Girder -gc.downloadFolderRecursive(folderId, str(download_dir)) - -# Add derivatives/freesurfer folder -derivatives_fs = download_dir / 'derivatives' / 'freesurfer' -derivatives_fs.mkdir(parents=True, exist_ok=True) # will create if not exists, do nothing if exists - -print(f"'derivatives/freesurfer' folder ensured at: {derivatives_fs}") diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/2-FS_CROSS_parallel_jobs.py b/examples/freesurfer/freesurfer_longitudinal_processing/2-FS_CROSS_parallel_jobs.py deleted file mode 100644 index 2db5e39..0000000 --- a/examples/freesurfer/freesurfer_longitudinal_processing/2-FS_CROSS_parallel_jobs.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env -S uv run --script -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "vip-client", -# ] - -from vip_client import VipSession -from pathlib import Path -import os - -# This directory will be uploaded to VIP — ensure no sensitive data -# (e.g., DICOMs, scripts containing API keys) is included. -input_dir = Path("/insert/your/input/path") # make sure license file is copied in this directory -output_dir = Path("/insert/your/output/path/derivatives/freesurfer") - -# Save current working directory and change to /tmp (temporary workaround for VIP bug) -orig_cwd = os.getcwd() -os.chdir('/tmp') - -# Get T1w NIfTIs only from sub-* folders, excluding run-02, run-03, ... -nifti_files = [ - str(f) - for sub in input_dir.iterdir() - if sub.is_dir() and sub.name.startswith("sub-") - for f in sub.rglob("*.nii.gz") - if f.name.endswith("_T1w.nii.gz") - and ("_run-" not in f.name or "_run-01_" in f.name) -] - -# Create subjid from filenames -subjid = [Path(f).name.replace(".nii.gz", "") for f in nifti_files] - -input_settings = { - "nifti": nifti_files, - "license": str(input_dir / "license.txt"), - "subjid": subjid -} - -session = VipSession.init( - api_key="VIP_API_KEY", - input_dir=str(input_dir), - output_dir= str(output_dir), - pipeline_id="FreeSurfer-Recon-all/7.3.1", - input_settings=input_settings -) - -session.run_session() -session.display() - -# Restore original working directory -os.chdir(orig_cwd) - -# Download outputs to the output_dir -session.download_outputs(get_status=['Finished', 'Killed']) - -# Optional cleanup of outputs on VIP -# session.finish() diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/3-tar_sub_TPs.py b/examples/freesurfer/freesurfer_longitudinal_processing/3-tar_sub_TPs.py deleted file mode 100644 index 5703735..0000000 --- a/examples/freesurfer/freesurfer_longitudinal_processing/3-tar_sub_TPs.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -import tarfile -import shutil - -fs_dir = Path("/insert/your/input/path/derivatives/freesurfer") -tmp_dir = fs_dir / "tmp" - -# create a tmp directory if it doesn't exist -tmp_dir.mkdir(exist_ok=True) - -# Collect eligible tarballs (.tar.gz or .tgz) -tarballs = [ - t for t in fs_dir.iterdir() - if t.is_file() - and t.suffixes in ([".tar", ".gz"], [".tgz"]) - and "ses-" in t.name - and ".long." not in t.name -] - -subjects = {} - -for tar_path in tarballs: - subj = tar_path.name.split("_")[0] # sub-XXXX - - # Remove full archive suffix safely - if tar_path.suffixes == [".tar", ".gz"]: - base_name = tar_path.name[:-7] # remove ".tar.gz" - else: # .tgz - base_name = tar_path.stem # removes ".tgz" - - extract_dir = tmp_dir / "extracted" / base_name - extract_dir.mkdir(parents=True, exist_ok=True) - - # Untar (auto-detect compression) - with tarfile.open(tar_path, "r:*") as tar: - tar.extractall(path=extract_dir) - - subjects.setdefault(subj, []).append(extract_dir) - -# Group per subject -for subj, tp_dirs in subjects.items(): - out_tar = tmp_dir / f"{subj}_TPs.tgz" - - with tarfile.open(out_tar, "w:gz") as tar: - for tp_dir in tp_dirs: - for item in tp_dir.iterdir(): - tar.add(item, arcname=item.name) - -# Cleanup extracted files -shutil.rmtree(tmp_dir / "extracted", ignore_errors=True) - -print("✅ Done: grouped longitudinal timepoints per subject.") - diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/4-FS_BASE_parallel_jobs.py b/examples/freesurfer/freesurfer_longitudinal_processing/4-FS_BASE_parallel_jobs.py deleted file mode 100644 index cbf5101..0000000 --- a/examples/freesurfer/freesurfer_longitudinal_processing/4-FS_BASE_parallel_jobs.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env -S uv run --script -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "vip-client", -# ] - -from vip_client import VipSession -from pathlib import Path -import os - -# This directory will be uploaded to VIP — ensure no sensitive data -# (e.g., DICOMs, scripts containing API keys) is included. -input_dir = Path("/insert/your/input/path/derivatives/freesurfer/tmp") #make sure license file is copied in this directory -output_dir = Path("/insert/your/output/path/derivatives/freesurfer") - -# Save current working directory and change to /tmp (temporary workaround for VIP bug) -orig_cwd = os.getcwd() -os.chdir('/tmp') - -# Collect tarballs and BASE_IDs -tp_tarballs = [f for f in input_dir.iterdir() if f.suffix in (".tgz", ".tar.gz") and "_TPs" in f.name] -tp_tarballs.sort() # optional - -base_ids = [f.name.split("_")[0] for f in tp_tarballs] - -# Build batch input settings -input_settings = { - "LICENSE_FILE": str(input_dir / "license.txt"), - "TP_TARBALL": [str(f) for f in tp_tarballs], # list of tarballs - "BASE_ID": base_ids, # list of base IDs -} - -session = VipSession.init( - api_key="VIP_API_KEY", - input_dir=str(input_dir), - output_dir= str(output_dir), - pipeline_id="FreeSurfer-Recon-all-BASE/7.3.1", - input_settings=input_settings, -) - -session.run_session() # VIP will process tarballs in parallel -session.display() - -# Restore original working directory -os.chdir(orig_cwd) - -# Download outputs to the output_dir -session.download_outputs(get_status=['Finished', 'Killed']) - -# Optional cleanup of outputs on VIP -# session.finish() - -# Delete tmp folder locally -# WARNING: This removes the entire directory. -# Make sure that the license file is stored in another safe location. -shutil.rmtree(input_dir) -print(f"Deleted temporary folder: {input_dir}") - - - diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/5-FS_LONG_parallel_jobs.py b/examples/freesurfer/freesurfer_longitudinal_processing/5-FS_LONG_parallel_jobs.py deleted file mode 100644 index 97f5d10..0000000 --- a/examples/freesurfer/freesurfer_longitudinal_processing/5-FS_LONG_parallel_jobs.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "vip-client", -# ] -# /// - -from vip_client import VipSession -from vip_client.utils import vip - -# VIP API Key -API_KEY = "VIP_API_KEY" - -# Initialize VIP connection -VipSession.init(api_key=API_KEY) - -# VIP paths -BASE_DIR = "/vip/Home/API/VipSession-xxxxxx-xxxxxx-bxx/OUTPUTS/YYYY-MM-DD_HHMMSS/" # from the output of BASE pipeline -TP_DIR = "/vip/Home/API/VipSession-xxxxxx-xxxxxx-bxx/INPUTS/" # from the input of BASE pipeline -LICENSE_FILE = "/vip/Home/API/VipSession-xxxxxx-xxxxxx-bxx/INPUTS/license.txt" # from the input of BASE pipeline - -# Local output directory -output_dir = Path("/insert/your/output/path/derivatives/freesurfer") - -# List BASE and TP files from VIP -base_files = [ - item['path'] - for item in vip.list_elements(BASE_DIR) - if not item['isDirectory'] and item['path'].split('/')[-1].startswith("sub-") and "_TPs" not in item['path'] -] - -tp_files = [ - item['path'] - for item in vip.list_elements(TP_DIR) - if not item['isDirectory'] and item['path'].split('/')[-1].startswith("sub-") and "_TPs" in item['path'] -] - -# Match BASE → TP tarball -base_dict = {f.split("/")[-1].split(".")[0]: f for f in base_files} -tp_dict = {f.split("/")[-1].split("_TPs")[0]: f for f in tp_files} -subjects = set(base_dict.keys()) & set(tp_dict.keys()) - -if not subjects: - raise RuntimeError("No matching BASE and TP_TARBALL found!") - -print(f"Submitting {len(subjects)} LONG jobs for subjects: {', '.join(subjects)}") - -# Gather VIP input files per subject -vip_input_files = [] -for sub in subjects: - vip_input_files.append(base_dict[sub]) - vip_input_files.append(tp_dict[sub]) - -# Input settings for the LONG pipeline -input_settings = { - "LICENSE_FILE": LICENSE_FILE, - "BASE_ID": [base_dict[sub] for sub in subjects], - "TP_TARBALL": [tp_dict[sub] for sub in subjects], - "directives": "-all", -} - -# Create VIP session -session_name = "VIP_FS_LONG_parallel" -session = VipSession(session_name) - -# Launch pipeline directly on VIP -session.launch_pipeline( - pipeline_id="FreeSurfer-Recon-all-LONG/7.3.1", - input_settings=input_settings, - output_dir= str(output_dir) -) - -# Monitor progress -session.monitor_workflows() - -# Download outputs to the output_dir -session.download_outputs(get_status=['Finished', 'Killed']) - -# Optional cleanup of outputs on VIP -# session.finish() - diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/6-girder_upload_freesurfer_folder.py b/examples/freesurfer/freesurfer_longitudinal_processing/6-girder_upload_freesurfer_folder.py deleted file mode 100644 index c9dee45..0000000 --- a/examples/freesurfer/freesurfer_longitudinal_processing/6-girder_upload_freesurfer_folder.py +++ /dev/null @@ -1,39 +0,0 @@ -# Description: -# Author: Frederic Cervenansky < frederic.cervenansky@creatis.insa-lyon.fr> -# -# Licence Cecill-B -# Copyright (C) Creatis 2017-2024 - - -# import -import girder_client -import types -import sys -import click - -# api Rest url of the warehouse -#Windows users: paths use backslashes (\) while Linux/macOS use forward slashes (/) -url='https://insert/your-server-here/api/v1' -# https://srmnopt.creatis.insa-lyon.fr/warehouse/api/v1 --> for NMR and Optics -# https://myriad.creatis.insa-lyon.fr/api/v1 --> for MYRIAD - -# apiKey is a "mechanism" to share authentication and rights on folder. -# User should defined through the web interface (inside user information) an apiKey with the corresponding privileges -apiKey = 'GIRDER_API_KEY' - -# Generate the warehouse client -gc = girder_client.GirderClient(apiUrl=url) - -# Authentication to the warehouse -gc.authenticate(apiKey=apiKey) - -# upload local freesurfer folder in the original input folder on Girder -folderId ='FOLDER_ID' # 3D Flow phantom+Bubbles -#an example of a folder ID: 698de7fe82d062f2aea9619d - -# If 'derivatives' folder exists but 'freeSurfer' folder is absent: -# gc.upload('/insert/your/upload/path/derivatives/freesurfer', folderId) - -# Default upload (if 'derivatives' folder is absent on the warehouse): -gc.upload('/insert/your/upload/path/derivatives/', folderId) - diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/README.md b/examples/freesurfer/freesurfer_longitudinal_processing/README.md deleted file mode 100644 index 6788608..0000000 --- a/examples/freesurfer/freesurfer_longitudinal_processing/README.md +++ /dev/null @@ -1,259 +0,0 @@ - -# FreeSurfer Longitudinal Processing Pipeline Using VIP Platform - -## Overview - -This repository contains scripts and instructions to run **FreeSurfer longitudinal pipeline** encompassing three processing steps (cross, base, long) using **VIP Client** and **Girder** for data storage and retrieval. The pipeline supports **3D T1-weighted MRIs** for whole brain segmentation. - -1. **Download MRI Data from Girder** -2. **Run FreeSurfer Longitudinal [CROSS]**: run the standard cross-sectional `recon-all` pipeline -3. **Group Per-Subject Timepoints**: these will be used as inputs for step 4 -4. **Run FreeSurfer Longitudinal [BASE]** : run `recon-all -base` and create within subject template -5. **Run FreeSurfer Longitudinal [LONG]**: run `recon-all -long` and create subject-specific longitudinal segmentations. -6. **Upload FreeSurfer Outputs Back to Girder**: this step re-upload the `directives` folder containing the outputs back to Girder. - -The pipeline is fully automated with Python scripts, integrates VIP job submission, monitoring, and local download, and ensures all derivatives are centralized on Girder. - -## Requirements - -### Software -- Python ≥ 3.11 -- `vip-client` Python package -- `girder_client` Python package - -### VIP -- VIP account and API key -- Access to the VIP **FreeSurfer pipelines**: - - `FreeSurfer-Recon-all/7.3.1` - - `FreeSurfer-Recon-all-BASE/7.3.1` - - `FreeSurfer-Recon-all-LONG/7.3.1` - -### Girder -- Girder account and API key - -### Data -- BIDS-compliant T1-weighted MRI dataset: - -``` -bids/ -├── sub-*/ -│ └── ses-*/ -│ └── anat/ -│ └── *_T1w.nii.gz -└── directives/ - └── freesurfer/ -``` -- FreeSurfer license file (`license.txt`) - -This setup allows **fully automated longitudinal FreeSurfer processing** with reproducible outputs stored both locally and on Girder for sharing or further analysis. - -## Pipeline Workflow - -## 1- Download MRI Data from Girder - -This script authenticates to the Girder warehouse, download the input BIDS-compliant MRI dataset locally, and ensure the FreeSurfer derivatives directory structure exists. - -### ⚙️ Inputs -- `url`: Girder API endpoint -- `folderId`: ID of the Girder folder containing MRI data -- `download_dir`: Local path where data will be downloaded - -### 📤 Outputs -- Local copy of the MRI dataset -- Created directory: `derivatives/freesurfer/` inside the download folder if not already present. - -### ▶️ How to Run -1. Replace placeholders: -- `url` -- `GIRDER_API_KEY` -- `FOLDER_ID` -- `/insert/your/download/path` -2. Run the script: -```bash -python3 1-girder_download_input_data.py -``` - -## 2- Run FreeSurfer Longitudinal [CROSS] - - -This script launches the **standard FreeSurfer`recon-all` cross sectional** pipeline on VIP for all T1-weighted MRIs and download the results locally. The script: - -- Finds all `*_T1w.nii.gz` files in `sub-*` folders -- Keeps only the `run-01` scans (ignores additional runs) --> See "File Selection Rules" Section below for more details -- Builds subject IDs automatically from the filenames (removing `.nii.gz`) -- Submits parallel jobs to the VIP FreeSurfer recon-all pipeline -- Downloads the completed outputs locally to `derivatives/freesurfer/` - -#### File Selection Rules - - **Note**: If two or more T1-weighted sequences were present at the same session, they were labeled `*_run-01_T1w` and `_run-02_T1w`... The **non-gadolinium (no Gado)** scan (if present) was labeled as `*_run-01_T1w`, as it is preferred for segmentation. - -So, the script uses only files ending with: -- `*_T1w.nii.gz` (no run specification) -OR -- `*_run-01_T1w.nii.gz` - -> ⚠️ If processing of `*_run-01_T1w.nii.gz` fails, you can try `*_run-02_T1w.nii.gz` or `*_run-03_T1w.nii.gz` directly on the platform. Make sure to put the results back in the `directives/freesurfer` folder. - -### ⚙️ Inputs -- `nifti`: List of T1-weighted NIfTI files (`*_T1w.nii.gz`) from `sub-*` folders, keeping only `run-01` -- `license`: FreeSurfer license file, placed inside `input_dir` -- `subjid`: utomatically generated subject IDs used to name output folders for each NIfTI file (derived from the filename minus `.nii.gz`) - -### 📤 Outputs -- FreeSurfer cross-sectional results in: - `derivatives/freesurfer/` - -> ⚠️ Important: Review the outputs carefully and note which results to keep or exclude/re-run before proceeding to the next steps. - -### How to Run -1. Replace placeholders: - - `/insert/your/input/path` - - `/insert/your/output/path/derivatives/freesurfer` - - `VIP_API_KEY` -2. Ensure `license.txt` is present in `input_dir`. -3. Run: -```bash -python3 2-FS_CROSS_parallel_jobs.py -``` - -## 3- Group Per-Subject Timepoints - -This script prepares the **subject-specific timepoints tarballs for the BASE step** by grouping cross-sectional timepoints (`ses-*`) for each subject into a single archive. Each archive contains all sessions for one subject, ready for the `recon-all -base` pipeline. The script: - -- Detects all eligible cross-sectional tarballs in `derivatives/freesurfer/` - - Must contain `ses-` - - Must **not** contain `.long.` -- Extracts each archive temporarily -- Groups timepoints by subject (`sub-XXXX`) -- Re-compresses them into one `.tgz` per subject -- Cleans temporary extracted files - -### ⚙️ Inputs - -- `fs_dir`: Path to the `derivatives/freesurfer/` directory containing cross-sectional `.tar.gz` or `.tgz` outputs from downloaded from VIP in step 1. - - -### 📤 Outputs - -- Grouped per-subject archives for longitudinal processing: -`derivatives/freesurfer/` - -### How to Run - -1. Replace placeholders: - - `/insert/your/input/path/derivatives/freesurfer` -2. Run: -```bash -python3 3-tar_sub_TPs.py -``` - -## 4- Run FreeSurfer Longitudinal [BASE] - - -This script runs **FreeSurfer `recon-all -base`** on VIP using the grouped timepoints from Step 3. It generates the **within-subject template** required for the following longitudinal processing stream. - -The script: - -- Finds all grouped archives (`*_TPs.tgz`) in the `tmp/` folder -- Extracts BASE IDs for each subject -- Launches the VIP **recon-all -base** pipeline -- Downloads the completed BASE outputs locally -- Deletes the temporary `tmp/` folder to clean up local storage - - -### ⚙️ Inputs - -- `TP_TARBALL`: List of grouped timepoint archives (`*_TPs.tgz`) for each subject -- `LICENSE_FILE`: FreeSurfer license file (`license.txt`) located in `input_dir` -- `BASE_ID`: ubject IDs corresponding to each BASE template, automatically extracted from the tarball filenames and used to name the output folders for each subject - -### 📤 Outputs - -- Outputs will be downloaded to: -`derivatives/freesurfer` - -**Note:** Keep the outputs on VIP as well, since they will be used directly in the next processing step [LONG]. This avoids the need to re-upload files. -Also, -> ⚠️ The temporary folder (`tmp`) will be deleted locally at the end of execution, since all data will be available on VIP for the next step. Make sure your FreeSurfer license file (`license.txt`) is copied to a safe location is stored in another safe location. - -### How to Run - -1. Replace placeholders: - - `/insert/your/input/path/derivatives/freesurfer/tmp` - - `/insert/your/output/path/derivatives/freesurfer` - - `VIP_API_KEY` -2. Ensure `license.txt` is present in `input_dir`. -3. Run: -```bash -python3 4-FS_BASE_parallel_jobs.py -``` - -## 5- Run FreeSurfer Longitudinal [LONG] - - -This script runs **FreeSurfer `recon-all -long`** on VIP using the BASE templates from Step 4 and the timepoints (TPs) tarballs produced from Step 3 and were used as an input to step 4, producing segmentations more robustly by registering each timepoint to its corresponding subject template. The script: -- Connects to VIP using the API key -- Lists BASE templates and TP tarballs on VIP -- Matches BASE templates with corresponding TP tarballs for each subject -- Prepares VIP input settings for the LONG pipeline -- Launches `recon-all -long` jobs in parallel on VIP -- Monitors workflow progress -- Downloads the completed longitudinal outputs to the local output directory - -### ⚙️ Inputs -- `TP_TARBALL`: List of grouped timepoint archives (`*_TPs.tgz`) for each subject (from the input folder in Step 4) -- `LICENSE_FILE`: FreeSurfer license file (`license.txt`) located in the Step 4 input folder -- `BASE_ID`: Subject-specific BASE template folder (output from Step 4) - -### 📤 Outputs -- Outputs will be downloaded to: -`derivatives/freesurfer` -- Output folders will be named following this pattern: `sub-*_ses-*_T1w.long.sub-*`. - -### How to Run - -1. Replace placeholders: - - `/vip/Home/API/VipSession-xxxxxx-xxxxxx-bxx/OUTPUTS/YYYY-MM-DD_HHMMSS/` - - `/vip/Home/API/VipSession-xxxxxx-xxxxxx-bxx/INPUTS/` - - `VIP_API_KEY` -2. `license.txt` should already be present in `/vip/Home/API/VipSession-xxxxxx-xxxxxx-bxx/INPUTS/`. -3. Run: -```bash -python3 5-FS_LONG_parallel_jobs.py -``` - -## 6- Upload FreeSurfer Outputs Back to Girder - -This script uploads **local FreeSurfer outputs** (from CROSS, BASE, and LONG pipelines) back to the **Girder warehouse** for centralized storage and sharing. The script: - -- Connects to Girder using the API key -- Authenticates the user -- Uploads the local FreeSurfer `derivatives` and/or `derivatives/freesurfer` back to Girder - -### ⚙️ Inputs -- Girder API: - - `url` — API endpoint of the Girder server - Examples: - ``` - https://srmnopt.creatis.insa-lyon.fr/warehouse/api/v1 - https://myriad.creatis.insa-lyon.fr/api/v1 - ``` -- Local FreeSurfer derivatives folder: -`/insert/your/upload/path/derivatives/freesurfer` -- Girder target folder ID: -`folderId = 'FOLDER_ID'` - -### 📤 Outputs -- Uploads all FreeSurfer derivative files (`derivatives/freesurfer/`) to the specified Girder folder - -### How to Run - -1. Replace placeholders: -- `/insert/your/upload/path/derivatives/freesurfer` → local derivatives folder -- `FOLDER_ID` → Girder target folder ID -- `VIP_API_KEY` → your Girder API key -2. Run: -```bash -python3 6-girder_upload_freesurfer_folder.py -``` diff --git a/examples/freesurfer/freesurfer_longitudinal_processing/freesurfer_longitudinal.ipynb b/examples/freesurfer/freesurfer_longitudinal_processing/freesurfer_longitudinal.ipynb new file mode 100644 index 0000000..200c182 --- /dev/null +++ b/examples/freesurfer/freesurfer_longitudinal_processing/freesurfer_longitudinal.ipynb @@ -0,0 +1,734 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "# FreeSurfer Longitudinal Processing Pipeline Using VIP Platform\n", + "\n", + "## Overview\n", + "\n", + "This repository contains scripts and instructions to run **FreeSurfer longitudinal pipeline** encompassing three processing steps (cross, base, long) using **VIP Client** and **Girder** for data storage and retrieval. The pipeline supports **3D T1-weighted MRIs** for whole brain segmentation.\n", + "\n", + "The process is segmented in the following steps :\n", + " 1. **Download MRI Data from Girder**\n", + " 1. **Run FreeSurfer Longitudinal [CROSS]**: run the standard cross-sectional `recon-all` pipeline\n", + " 1. **Group Per-Subject Timepoints**: these will be used as inputs for step 4\n", + " 1. **Run FreeSurfer Longitudinal [BASE]** : run `recon-all -base` and create within subject template\n", + " 1. **Run FreeSurfer Longitudinal [LONG]**: run `recon-all -long` and create subject-specific longitudinal segmentations.\n", + " 1. **Upload FreeSurfer Outputs Back to Girder**: this step re-upload the `directives/freesurfer` folder containing the outputs back to Girder.\n", + " 1. **Cleanup**\n", + "\n", + "The pipeline is fully automated with Python scripts, integrates VIP job submission, monitoring, and local download, and ensures all derivatives are centralized on Girder.\n", + "\n", + "⚠️ Please **DO NOT** run the full notebook at once, but proceed cell by cell to ensure the inputs and outputs are correct after each step, this will allow you to control your execution and revert changes if needed.\n", + "\n", + "⚠️ If you are closing the notebook (pipelines running in background or pausing between steps), remember to run the Libraries and Parameters cells (before step 1) again before continuing your work." + ], + "id": "eaa23e833be5ca5d" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Requirements\n", + "\n", + "### Software\n", + "\n", + "- Python ≥ 3.11\n", + "- `vip-client` Python package\n", + "- `girder_client` Python package\n", + "\n", + "### VIP\n", + "\n", + "- VIP account and API key\n", + "- Access to the VIP **FreeSurfer pipelines**:\n", + " - `FreeSurfer-Recon-all/7.3.1`\n", + " - `FreeSurfer-Recon-all-BASE/7.3.1`\n", + " - `FreeSurfer-Recon-all-LONG/7.3.1`\n", + "\n", + "### Girder\n", + "\n", + "- Girder account and API key\n", + "\n", + "### Data\n", + "\n", + "- BIDS-compliant T1-weighted MRI dataset:\n", + "```\n", + "bids/\n", + "├── sub-*/\n", + "│ └── ses-*/\n", + "│ └── anat/\n", + "│ └── *_T1w.nii.gz\n", + "└── directives/\n", + " └── freesurfer/\n", + "```\n", + "- FreeSurfer license file\n", + "\n", + "This setup allows **fully automated longitudinal FreeSurfer processing** with reproducible outputs stored both locally and on Girder for sharing or further analysis." + ], + "id": "7e9f48d69ce488ce" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Libraries and functions", + "id": "7817023ff03e7dbf" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Built-ins\n", + "import os\n", + "import tarfile\n", + "import shutil\n", + "from pathlib import Path\n", + "\n", + "# VIP\n", + "from vip_client import VipSession\n", + "\n", + "# Girder\n", + "from girder_client import GirderClient\n", + "\n", + "def flatten_folder(folder):\n", + " parent = folder.parent\n", + " for f in folder.iterdir():\n", + " f.rename(parent / f.name)\n", + " shutil.rmtree(folder)\n" + ], + "id": "18aa877afac9a865", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Parameters", + "id": "3b01f462e227d9fa" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "**User variables**: should be checked and supplied at each execution", + "id": "dd7f54bce66f9e9f" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# API keys\n", + "VIP_KEY = os.environ['VIP_API_KEY'] # Your VIP API key (string, file or environment variable)\n", + "GIRDER_KEY = os.environ['GIRDER_API_KEY'] # Your Girder API key (string or environment variable)\n", + "\n", + "# Girder\n", + "GIRDER_CLIENT = GirderClient(apiUrl='GIRDER_URL') # Your Girder API endpoint url (ex : https://srmnopt.creatis.insa-lyon.fr/warehouse/api/v1 or https://myriad.creatis.insa-lyon.fr/api/v1)\n", + "GIRDER_DATASET_PATH = '/collection/COLLECTION_NAME/FOLDER_NAME' # Your Girder path to the dataset, can be in collections or user folders\n", + "GIRDER_OUTPUT_PATH = '/user/USER_NAME/FOLDER_NAME' # Your Girder path to the final outputs folder (must already exist), can be in collections or user folders\n", + "\n", + "# Local\n", + "LOCAL_DATASET_PATH = Path('~/PATH').expanduser() # Your local dataset path, where the Girder dataset will be downloaded, must already exist\n", + "LICENSE_PATH = Path('~/PATH/LICENSE_NAME.txt').expanduser() # Your pipeline license path, will be copied into VIP input_dirs automatically" + ], + "id": "c7f8787950ccc948", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "**Constant variables**: should not be changed unless the dataset structure or the pipeline have been modified\n", + "id": "8303e0953a323b31" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Name of the pipelines\n", + "PIPELINE_ALL_ID = 'FreeSurfer-Recon-all/7.3.1'\n", + "PIPELINE_BASE_ID = 'FreeSurfer-Recon-all-BASE/7.3.1'\n", + "PIPELINE_LONG_ID = 'FreeSurfer-Recon-all-LONG/7.3.1'\n", + "\n", + "# Path to the FreeSurfer (output) folder\n", + "FREESURFER_DIR = Path(LOCAL_DATASET_PATH / 'derivatives' / 'freesurfer')\n", + "\n", + "# FreeSurfer CROSS\n", + "FS_CROSS_ID = 'CROSS'\n", + "FS_CROSS_OUTPUT_DIR = Path(FREESURFER_DIR / FS_CROSS_ID / 'Outputs')\n", + "\n", + "# FreeSurfer BASE\n", + "FS_BASE_ID = 'BASE'\n", + "FS_BASE_INPUT_DIR = Path(FREESURFER_DIR / FS_BASE_ID / 'Inputs')\n", + "FS_BASE_OUTPUT_DIR = Path(FREESURFER_DIR / FS_BASE_ID / 'Outputs')\n", + "\n", + "# FreeSurfer LONG\n", + "FS_LONG_ID = 'LONG'\n", + "FS_LONG_OUTPUT_DIR = Path(FREESURFER_DIR / FS_LONG_ID / 'Outputs')\n", + "\n", + "# VIP\n", + "SESSION_DATA_NAME = 'session_data.json'\n", + "SESSIONS = {FS_CROSS_ID : FS_CROSS_OUTPUT_DIR,\n", + " FS_BASE_ID : FS_BASE_OUTPUT_DIR,\n", + " FS_LONG_ID : FS_LONG_OUTPUT_DIR}\n", + "\n", + "LICENSES = {FS_CROSS_ID : LOCAL_DATASET_PATH / f'license_{FS_CROSS_ID}.txt',\n", + " FS_BASE_ID : FS_BASE_INPUT_DIR / f'license_{FS_BASE_ID}.txt',\n", + " FS_LONG_ID : FS_BASE_OUTPUT_DIR / f'license_{FS_LONG_ID}.txt'}" + ], + "id": "629bf8acf75b0901", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 1. Download MRI Data from Girder\n", + "\n", + "This script authenticates to the Girder warehouse, downloads the input BIDS-compliant MRI dataset locally, and ensure the FreeSurfer derivatives directory structure exists.\n", + "\n", + "### ⚙️ Inputs\n", + "\n", + "- `GIRDER_DATASET_PATH`: Path of the Girder folder containing MRI data\n", + "- `LOCAL_DATASET_PATH`: Local path where the data will be downloaded\n", + "\n", + "### 📤 Outputs\n", + "\n", + "- Local copy of the MRI dataset\n", + "- Created directory: `derivatives/freesurfer/` inside the download folder if not already present." + ], + "id": "bcc9f08d6ed0ced4" + }, + { + "cell_type": "code", + "id": "initial_id", + "metadata": { + "collapsed": true + }, + "source": [ + "# Authentication to Girder\n", + "GIRDER_CLIENT.authenticate(apiKey=GIRDER_KEY)\n", + "\n", + "# Download dataset from Girder\n", + "GIRDER_CLIENT.downloadFolderRecursive(GIRDER_DATASET_PATH, str(LOCAL_DATASET_PATH))\n", + "\n", + "# Create LOCAL_DATASET_PATH, derivatives and freesurfer folders if they don't exist\n", + "FREESURFER_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "print(f\"✅ Done: 'derivatives/freesurfer' folder ensured at: {FREESURFER_DIR}, Girder dataset copied to : {LOCAL_DATASET_PATH}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 2. Run FreeSurfer Longitudinal [CROSS]\n", + "\n", + "This script launches the **standard FreeSurfer** `recon-all` **cross sectional** pipeline on VIP for all T1-weighted MRIs and download the results locally. The script:\n", + "\n", + "- Finds all `*_T1w.nii.gz` files in `sub-*` folders\n", + "- Keeps only the `run-01` scans (ignores additional runs)\n", + "- Builds subject IDs automatically from the filenames (removing `.nii.gz`)\n", + "- Submits parallel jobs to the VIP FreeSurfer recon-all pipeline\n", + "- Downloads the completed outputs locally to `derivatives/freesurfer/CROSS/Outputs`\n", + "\n", + "### File Selection Rules\n", + "\n", + "**Note**: If two or more T1-weighted sequences were present at the same session, they were labeled `*_run-01_T1w` and `_run-02_T1w`. The **non-gadolinium (no Gado)** scan (if present) was labeled as `*_run-01_T1w`, as it is preferred for segmentation.\n", + "\n", + "So, the script uses only files ending with:\n", + "\n", + "- `*_T1w.nii.gz` (no run specification) OR\n", + "- `*_run-01_T1w.nii.gz`\n", + "\n", + "⚠️ If processing of `*_run-01_T1w.nii.gz` fails, you can try `*_run-02_T1w.nii.gz` or `*_run-03_T1w.nii.gz` directly on the platform. Make sure to put the results back in the derivatives/freesurfer folder.\n", + "\n", + "### ⚙️ Inputs\n", + "\n", + "- `nifti`: List of T1-weighted NIfTI files (`*_T1w.nii.gz`) from `sub-*` folders, keeping only `run-01`\n", + "- `license`: FreeSurfer license file\n", + "- `subjid`: automatically generated subject IDs used to name output folders for each NIfTI file (derived from the filename minus `.nii.gz`)\n", + "\n", + "### 📤 Outputs\n", + "\n", + "- FreeSurfer cross-sectional results in `derivatives/freesurfer/CROSS/Outputs`\n", + "\n", + "⚠️ Review the outputs carefully and note which results to keep or exclude/re-run before proceeding to the next steps." + ], + "id": "bcc452c2a9be03c0" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Remove FreeSurfer CROSS output directory if existing\n", + "if FS_CROSS_OUTPUT_DIR.exists():\n", + " shutil.rmtree(FS_CROSS_OUTPUT_DIR)\n", + "\n", + "# Create FreeSurfer CROSS output directory\n", + "FS_CROSS_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "# Get T1w NIfTIs only sub-* folders, excluding run-02, run-03, ...\n", + "nifti_files = [\n", + " str(f)\n", + " for sub in LOCAL_DATASET_PATH.iterdir()\n", + " if sub.is_dir() and sub.name.startswith('sub-')\n", + " for f in sub.rglob('*.nii.gz')\n", + " if f.name.endswith('_T1w.nii.gz')\n", + " and ('_run-' not in f.name or '_run-01_' in f.name)\n", + "]\n", + "\n", + "# Create subjid from filenames\n", + "subjid = [Path(f).name.replace('.nii.gz', '') for f in nifti_files]\n", + "\n", + "input_settings = {\n", + " 'nifti': nifti_files,\n", + " 'license': shutil.copy2(LICENSE_PATH, LICENSES[FS_CROSS_ID]),\n", + " 'subjid': subjid,\n", + "}\n", + "\n", + "session = VipSession.init(\n", + " api_key=VIP_KEY,\n", + " input_dir=str(LOCAL_DATASET_PATH),\n", + " output_dir= str(FS_CROSS_OUTPUT_DIR),\n", + " pipeline_id=PIPELINE_ALL_ID,\n", + " input_settings=input_settings\n", + ")\n", + "\n", + "session.upload_inputs()\n", + "session.launch_pipeline()\n", + "session.monitor_workflows()\n", + "session.display()" + ], + "id": "74b8555be6d4abb8", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Run this cell after the execution ended to download outputs and setup output directory", + "id": "ca06f1b3ede87d90" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Connect back to session if needed\n", + "VipSession.init(api_key=VIP_KEY)\n", + "session = VipSession(output_dir=str(FS_CROSS_OUTPUT_DIR))\n", + "\n", + "# Download outputs to the output_dir\n", + "session.download_outputs(False, get_status=['Finished', 'Killed'])\n", + "\n", + "# Since results may arrive in a VipSession folder (2026-X), when it's the case we flatten this folder to put files directly under FS_CROSS_OUTPUT_DIR\n", + "for f in FS_CROSS_OUTPUT_DIR.iterdir():\n", + " if f.is_dir():\n", + " flatten_folder(Path(f))" + ], + "id": "ae2c997bbc3d32", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 3. Group Per-Subject Timepoints\n", + "\n", + "This script prepares the **subject-specific timepoints tarballs for the BASE step** by grouping cross-sectional timepoints (`ses-*`) for each subject into a single archive. Each archive contains all sessions for one subject, ready for the `recon-all -base` pipeline. The script:\n", + "\n", + "- Detects all eligible cross-sectional tarballs in `derivatives/freesurfer/CROSS/Outputs` :\n", + " - Must contain `ses-`\n", + " - Must **not** contain `.long.`\n", + "- Extracts each archive temporarily\n", + "- Groups timepoints by subject (`sub-XXXX`)\n", + "- Re-compresses them into one `.tgz` per subject\n", + "- Cleans temporary extracted files\n", + "\n", + "### ⚙️ Inputs\n", + "\n", + "- `FS_CROSS_OUTPUT_DIR`: Path to the `derivatives/freesurfer/Cross/Outputs` directory containing cross-sectional `.tar.gz` or `.tgz` outputs from downloaded from VIP in step 1\n", + "\n", + "### 📤 Outputs\n", + "\n", + "- Grouped per-subject archives for longitudinal processing: `derivatives/freesurfer/BASE/Inputs`" + ], + "id": "bb5bec81f891b58b" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Remove FreeSurfer BASE inputs/outputs directories if existing\n", + "if FS_BASE_INPUT_DIR.exists():\n", + " shutil.rmtree(FS_BASE_INPUT_DIR)\n", + "if FS_BASE_OUTPUT_DIR.exists():\n", + " shutil.rmtree(FS_BASE_OUTPUT_DIR)\n", + "\n", + "# Create FreeSurfer BASE inputs/outputs directories\n", + "FS_BASE_INPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "FS_BASE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "# Collect eligible tarballs (.tar.gz or .tgz)\n", + "tarballs = [\n", + " t for t in FS_CROSS_OUTPUT_DIR.iterdir()\n", + " if t.is_file()\n", + " and t.suffixes in ([\".tar\", \".gz\"], [\".tgz\",])\n", + " and \"ses-\" in t.name\n", + " and \".long.\" not in t.name\n", + "]\n", + "\n", + "subjects = {}\n", + "for tar_path in tarballs:\n", + " subj = tar_path.name.split(\"_\")[0] # sub-XXXX\n", + "\n", + " # Remove full archive suffixes safely\n", + " base_name = tar_path.name[:-(sum(len(s) for s in tar_path.suffixes))]\n", + "\n", + " extract_dir = FS_BASE_INPUT_DIR / \"extracted\" / base_name\n", + " extract_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " # Untar (auto-detect compression)\n", + " with tarfile.open(tar_path, \"r:*\") as tar:\n", + " tar.extractall(path=extract_dir)\n", + "\n", + " subjects.setdefault(subj, []).append(extract_dir)\n", + "\n", + "# Group per subject\n", + "for subj, tp_dirs in subjects.items():\n", + " out_tar = FS_BASE_INPUT_DIR / f\"{subj}_TPs.tgz\"\n", + " with tarfile.open(out_tar, \"w:gz\") as tar:\n", + " for tp_dir in tp_dirs:\n", + " for item in tp_dir.iterdir():\n", + " tar.add(item, arcname=item.name)\n", + "\n", + "# Cleanup extracted files\n", + "shutil.rmtree(FS_BASE_INPUT_DIR / \"extracted\", ignore_errors=True)\n", + "\n", + "print(f\"✅ Done: grouped longitudinal timepoints per subject at '{FS_BASE_INPUT_DIR}'.\")" + ], + "id": "e0896ac9ed719d4", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 4. Run FreeSurfer Longitudinal [BASE]\n", + "\n", + "This script runs **FreeSurfer** `recon-all -base` on VIP using the grouped timepoints from Step 3. It generates the **within-subject template** required for the following longitudinal processing stream.\n", + "\n", + "The script:\n", + "\n", + "- Finds all grouped archives (`*_TPs.tgz`) in the `BASE/Inputs` folder\n", + "- Extracts BASE IDs for each subject\n", + "- Launches the VIP **recon-all -base** pipeline\n", + "- Downloads the completed BASE outputs locally\n", + "\n", + "### ⚙️ Inputs\n", + "\n", + "- `TP_TARBALL`: List of grouped timepoint archives (`*_TPs.tgz`) for each subject\n", + "- `LICENSE_FILE`: FreeSurfer license file\n", + "- `BASE_ID`: Object IDs corresponding to each BASE template, automatically extracted from the tarball filenames and used to name the output folders for each subject\n", + "\n", + "### 📤 Outputs\n", + "\n", + "- Outputs will be downloaded to: `derivatives/freesurfer/BASE/Outputs`" + ], + "id": "9a177ebfc08d0c87" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Collect tarballs and BASE_IDs\n", + "tp_tarballs = [f for f in FS_BASE_INPUT_DIR.iterdir() if f.suffix in (\".tgz\", \".tar.gz\") and \"_TPs\" in f.name]\n", + "base_ids = [f.name.split(\"_\")[0] for f in tp_tarballs]\n", + "\n", + "# Build batch input settings\n", + "input_settings = {\n", + " \"LICENSE_FILE\": shutil.copy2(LICENSE_PATH, LICENSES[FS_BASE_ID]),\n", + " \"TP_TARBALL\": [str(f) for f in tp_tarballs], # list of tarballs\n", + " \"BASE_ID\": base_ids, # list of base IDs\n", + "}\n", + "\n", + "session = VipSession.init(\n", + " api_key=VIP_KEY,\n", + " input_dir=str(FS_BASE_INPUT_DIR),\n", + " output_dir= str(FS_BASE_OUTPUT_DIR),\n", + " pipeline_id=PIPELINE_BASE_ID,\n", + " input_settings=input_settings\n", + ")\n", + "\n", + "session.upload_inputs()\n", + "session.launch_pipeline() # VIP will process tarballs in parallel\n", + "session.monitor_workflows()\n", + "session.display()" + ], + "id": "fda29309aa318951", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Run this cell after the execution ended to download outputs and setup output directory", + "id": "26659b5150193e3f" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Connect back to session if needed\n", + "VipSession.init(api_key=VIP_KEY)\n", + "session = VipSession(output_dir=str(FS_BASE_OUTPUT_DIR))\n", + "\n", + "# Download outputs to the output_dir\n", + "session.download_outputs(False, get_status=['Finished', 'Killed'])\n", + "\n", + "# Since results may arrive in a VipSession folder (2026-X), when it's the case we flatten this folder to put files directly under FS_BASE_OUTPUT_DIR\n", + "for f in FS_BASE_OUTPUT_DIR.iterdir():\n", + " if f.is_dir():\n", + " flatten_folder(Path(f))" + ], + "id": "2ca4dbc9998ae826", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 5. Run FreeSurfer Longitudinal [LONG]\n", + "\n", + "This script runs **FreeSurfer** `recon-all -long` on VIP using the BASE templates from Step 4 and the timepoints (TPs) tarballs produced from Step 3 and were used as an input to step 4, producing segmentations more robustly by registering each timepoint to its corresponding subject template. The script:\n", + "\n", + "- Connects to VIP using the API key\n", + "- Lists BASE templates and TP tarballs on VIP\n", + "- Matches BASE templates with corresponding TP tarballs for each subject\n", + "- Prepares VIP input settings for the LONG pipeline\n", + "- Launches `recon-all -long` jobs in parallel on VIP\n", + "- Monitors workflow progress\n", + "- Downloads the completed longitudinal outputs to the local output directory\n", + "\n", + "### ⚙️ Inputs\n", + "\n", + "- `TP_TARBALL`: List of grouped timepoint archives (`*_TPs.tgz`) for each subject (BASE inputs)\n", + "- `LICENSE_FILE`: FreeSurfer license file\n", + "- `BASE_ID`: Subject-specific BASE template folder (BASE outputs)\n", + "\n", + "### 📤 Outputs\n", + "\n", + "- Outputs will be downloaded to: `derivatives/freesurfer/LONG/Outputs`\n", + "- Output folders will be named following this pattern: `sub-*_ses-*_T1w.long.sub-*.`" + ], + "id": "aeb60ae843de2c40" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Remove FreeSurfer LONG output directory if existing\n", + "if FS_LONG_OUTPUT_DIR.exists():\n", + " shutil.rmtree(FS_LONG_OUTPUT_DIR)\n", + "\n", + "# Create FreeSurfer LONG output directory\n", + "FS_LONG_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "# List BASE and TP files\n", + "base_files = [\n", + " f\n", + " for f in FS_BASE_OUTPUT_DIR.iterdir()\n", + " if f.is_file() and f.name.startswith(\"sub-\") and \"_TPs\" not in f.name\n", + "]\n", + "\n", + "tp_files = [\n", + " f\n", + " for f in FS_BASE_INPUT_DIR.iterdir()\n", + " if f.is_file() and f.name.startswith(\"sub-\") and \"_TPs\" in f.name\n", + "]\n", + "\n", + "# Match BASE → TP tarball\n", + "base_dict = {f.name.split(\".\")[0]: str(f) for f in base_files}\n", + "tp_dict = {f.name.split(\"_TPs\")[0]: str(f) for f in tp_files}\n", + "subjects = set(base_dict.keys()) & set(tp_dict.keys())\n", + "if not subjects:\n", + " raise RuntimeError(\"No matching BASE and TP_TARBALL found!\")\n", + "\n", + "# Input settings for the LONG pipeline\n", + "input_settings = {\n", + " \"LICENSE_FILE\": shutil.copy2(LICENSE_PATH, LICENSES[FS_LONG_ID]),\n", + " \"BASE_ID\": [base_dict[sub] for sub in subjects],\n", + " \"TP_TARBALL\": [tp_dict[sub] for sub in subjects],\n", + " \"directives\": \"-all\",\n", + "}\n", + "\n", + "print(f\"Submitting {len(subjects)} LONG jobs for subjects: {', '.join(subjects)}\")\n", + "\n", + "session = VipSession.init(\n", + " api_key=VIP_KEY,\n", + " input_dir=str(FS_BASE_OUTPUT_DIR.parent), # LONG pipeline uses data from BASE inputs and outputs\n", + " output_dir=str(FS_LONG_OUTPUT_DIR),\n", + " pipeline_id=PIPELINE_LONG_ID,\n", + " input_settings=input_settings\n", + ")\n", + "\n", + "session.upload_inputs()\n", + "session.launch_pipeline()\n", + "session.monitor_workflows()\n", + "session.display()" + ], + "id": "ad057014e8115590", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Run this cell after the execution ended to download outputs and setup output directory", + "id": "40aa3d4780a6b382" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Connect back to session if needed\n", + "VipSession.init(api_key=VIP_KEY)\n", + "session = VipSession(output_dir=str(FS_LONG_OUTPUT_DIR))\n", + "\n", + "# Download outputs to the output_dir\n", + "session.download_outputs(False, get_status=['Finished', 'Killed'])\n", + "\n", + "# Since results may arrive in a VipSession folder (2026-X), when it's the case we flatten this folder to put files directly under FS_LONG_OUTPUT_DIR\n", + "for f in FS_LONG_OUTPUT_DIR.iterdir():\n", + " if f.is_dir():\n", + " flatten_folder(Path(f))" + ], + "id": "c2ba7293aebb2ea9", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 6. Upload FreeSurfer outputs back to Girder\n", + "\n", + "This script uploads **local FreeSurfer outputs** (from CROSS, BASE, and LONG pipelines) back to the **Girder warehouse** for centralized storage and sharing. The script:\n", + "\n", + "- Uploads the local FreeSurfer `derivatives/freesurfer` back to Girder, the user needs to have write access to target Girder path\n", + "- Temporarily moves session_data.json files and licenses so they are not uploaded on Girder\n", + "\n", + "### ⚙️ Inputs\n", + "\n", + "- Local FreeSurfer derivatives folder `FREESURFER_DIR`\n", + "- Girder target folder path `GIRDER_OUTPUT_PATH`\n", + "\n", + "### 📤 Outputs\n", + "\n", + "- Uploads all FreeSurfer derivative files (`derivatives/freesurfer/`) to the specified Girder folder" + ], + "id": "657e95d7051b8ffd" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# Restore all session_data.json and licenses files\n", + "def restore_files(files):\n", + " for original_path, temp_path in files:\n", + " if temp_path.exists():\n", + " shutil.move(str(temp_path), str(original_path))\n", + "\n", + "# Authenticate back to Girder\n", + "GIRDER_CLIENT.authenticate(apiKey=GIRDER_KEY)\n", + "\n", + "moved_files = []\n", + "for session_id in SESSIONS:\n", + " session_data_path = SESSIONS[session_id] / SESSION_DATA_NAME\n", + " license_path = LICENSES[session_id]\n", + " if session_data_path.exists():\n", + " # Move VIP session_data jsons so they are not uploaded on Girder\n", + " temp_path = FREESURFER_DIR.parent / f'{session_id}_session_data.json'\n", + " shutil.move(str(session_data_path), str(temp_path))\n", + " moved_files.append((session_data_path, temp_path))\n", + " if license_path.exists():\n", + " # Move VIP pipelines licenses so they are not uploaded on Girder\n", + " temp_path = FREESURFER_DIR.parent / f'license_{session_id}.txt'\n", + " shutil.move(str(license_path), str(temp_path))\n", + " moved_files.append((license_path, temp_path))\n", + "\n", + "try :\n", + " # Upload full freesurfer dir (outputs) to Girder selected path GIRDER_OUTPUT_PATH\n", + " GIRDER_CLIENT.upload(str(FREESURFER_DIR), GIRDER_CLIENT.resourceLookup(GIRDER_OUTPUT_PATH)['_id'], leafFoldersAsItems=True, reuseExisting=True)\n", + "finally:\n", + " restore_files(moved_files)\n", + " print(f\"✅ Done: uploaded FreeSurfer outputs to Girder path '{GIRDER_OUTPUT_PATH}'.\")" + ], + "id": "f7290df3fc476e5", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## 7. CLEANUP\n", + "\n", + "Run to close VIP sessions, deleting inputs and outputs of different steps on VIP. Also removes VIP licenses and session data locally.\n", + "All inputs and outputs can still be found locally and deleted when needed." + ], + "id": "f6fef12ea7c84f89" + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "for session_id in SESSIONS:\n", + " try:\n", + " # Cleanup on VIP, deletes inputs and ouputs\n", + " VipSession(output_dir=str(SESSIONS[session_id])).finish()\n", + " except:\n", + " print(f\"Error closing session {session_id} on VIP\")\n", + " finally:\n", + " # Cleanup locally, deletes session_data jsons and licenses\n", + " session_data_path = SESSIONS[session_id] / SESSION_DATA_NAME\n", + " license_path = LICENSES[session_id]\n", + " if session_data_path.exists():\n", + " session_data_path.unlink()\n", + " if license_path.exists():\n", + " license_path.unlink()\n", + "\n", + "print(\"✅ Done: cleaned up all used sessions on VIP, and removed session data and license files locally.\")" + ], + "id": "c50880f0d7fe5c49", + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}