Skip to content

Assign parameters per context rather than per environment #607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tieneupin merged 13 commits into from
Jun 12, 2025
Merged

Assign parameters per context rather than per environment #607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
f4fc46b
Convert find_extension to return a bool
stephen-riggs Jun 10, 2025
4817ddb
Migrate parameters from environment to context
stephen-riggs Jun 11, 2025
5c68c7f
Stop passing so many parameters around
stephen-riggs Jun 11, 2025
135020a
Fix for eer fractionation from frontend
stephen-riggs Jun 11, 2025
a391919
Restore the lost parameters which may be needed
stephen-riggs Jun 11, 2025
c81d7da
Need to initialise the environments with the right attributes
stephen-riggs Jun 11, 2025
7228715
Set attributes of the environment if they exist
stephen-riggs Jun 11, 2025
9247982
Gain is better as a string in the client
stephen-riggs Jun 11, 2025
31802b5
None of this is now needed
stephen-riggs Jun 11, 2025
02a777d
Remove the form dependencies for old parameters
stephen-riggs Jun 11, 2025
4b73998
No particle diameter is now provided
stephen-riggs Jun 11, 2025
5f7e5c8
Remove dc params which was forgotten about
stephen-riggs Jun 11, 2025
c3e628d
Rename json variable
stephen-riggs Jun 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 22 additions & 49 deletions src/murfey/client/analyser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,29 +21,12 @@
from murfey.client.contexts.tomo import TomographyContext
from murfey.client.instance_environment import MurfeyInstanceEnvironment
from murfey.client.rsync import RSyncerUpdate, TransferResult
from murfey.client.tui.forms import FormDependency
from murfey.util.client import Observer, get_machine_config_client
from murfey.util.mdoc import get_block
from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo

logger = logging.getLogger("murfey.client.analyser")

spa_form_dependencies: dict = {
"use_cryolo": FormDependency(
dependencies={"estimate_particle_diameter": False}, trigger_value=False
),
"estimate_particle_diameter": FormDependency(
dependencies={
"use_cryolo": True,
"boxsize": "None",
"small_boxsize": "None",
"mask_diameter": "None",
"particle_diameter": "None",
},
trigger_value=True,
),
}


class Analyser(Observer):
def __init__(
Expand Down Expand Up @@ -85,7 +68,7 @@ def __init__(
def __repr__(self) -> str:
return f"<Analyser ({self._basepath})>"

def _find_extension(self, file_path: Path):
def _find_extension(self, file_path: Path) -> bool:
"""
Identifies the file extension and stores that information in the class.
"""
Expand All @@ -101,32 +84,30 @@ def _find_extension(self, file_path: Path):
.get(file_path.suffix)
):
if not any(r in file_path.name for r in required_substrings):
return []
return False

# Checks for MRC, TIFF, TIF, and EER files if no extension has been defined
if (
file_path.suffix in (".mrc", ".tiff", ".tif", ".eer")
and not self._extension
):
logger.info(f"File extension determined: {file_path.suffix}")
self._extension = file_path.suffix
# Check for TIFF, TIF, or EER if the file's already been assigned an extension
elif (
file_path.suffix in (".tiff", ".tif", ".eer")
and self._extension != file_path.suffix
):
logger.info(f"File extension re-evaluated: {file_path.suffix}")
self._extension = file_path.suffix
# Checks for MRC, TIFF, TIF, and EER files
if file_path.suffix in (".mrc", ".tiff", ".tif", ".eer"):
if not self._extension:
logger.info(f"File extension determined: {file_path.suffix}")
self._extension = file_path.suffix
elif self._extension != file_path.suffix:
logger.info(f"File extension re-evaluated: {file_path.suffix}")
self._extension = file_path.suffix
return True
# If we see an .mdoc file first, use that to determine the file extensions
elif file_path.suffix == ".mdoc":
with open(file_path, "r") as md:
md.seek(0)
mdoc_data_block = get_block(md)
if subframe_path := mdoc_data_block.get("SubFramePath"):
self._extension = Path(subframe_path).suffix
return True
# Check for LIF files separately
elif file_path.suffix == ".lif":
self._extension = file_path.suffix
return True
return False

def _find_context(self, file_path: Path) -> bool:
"""
Expand Down Expand Up @@ -269,7 +250,10 @@ def _analyse(self):
elif transferred_file.suffix == ".mdoc":
mdoc_for_reading = transferred_file
if not self._context:
self._find_extension(transferred_file)
valid_extension = self._find_extension(transferred_file)
if not valid_extension:
logger.error(f"No extension found for {transferred_file}")
break
found = self._find_context(transferred_file)
if not found:
logger.debug(
Expand Down Expand Up @@ -325,13 +309,6 @@ def _analyse(self):
self.notify(
{
"form": dc_metadata,
"dependencies": (
spa_form_dependencies
if isinstance(
self._context, SPAModularContext
)
else {}
),
}
)

Expand All @@ -344,7 +321,10 @@ def _analyse(self):

# Handle files with tomography and SPA context differently
elif not self._extension or self._unseen_xml:
self._find_extension(transferred_file)
valid_extension = self._find_extension(transferred_file)
if not valid_extension:
logger.error(f"No extension found for {transferred_file}")
break
if self._extension:
logger.info(
f"Extension found successfully for {transferred_file}"
Expand Down Expand Up @@ -384,13 +364,6 @@ def _analyse(self):
self.notify(
{
"form": dc_metadata,
"dependencies": (
spa_form_dependencies
if isinstance(
self._context, SPAModularContext
)
else {}
),
}
)
elif isinstance(
Expand Down
1 change: 1 addition & 0 deletions src/murfey/client/context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class Context:
def __init__(self, name: str, acquisition_software: str):
self._acquisition_software = acquisition_software
self.name = name
self.data_collection_parameters: dict = {}

def post_transfer(
self,
Expand Down
126 changes: 11 additions & 115 deletions src/murfey/client/contexts/spa.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from __future__ import annotations

import logging
from datetime import datetime
from itertools import count
from pathlib import Path
from typing import Any, Dict, List, Optional, OrderedDict, Tuple
Expand Down Expand Up @@ -109,27 +108,9 @@ class SPAModularContext(Context):
"Dose Per Frame [e- / Angstrom^2 / frame] (after EER grouping if relevant)",
default=1,
),
ProcessingParameter(
"estimate_particle_diameter",
"Use crYOLO to Estimate Particle Diameter",
default=True,
),
ProcessingParameter(
"particle_diameter", "Particle Diameter (Angstroms)", default=None
),
ProcessingParameter("use_cryolo", "Use crYOLO Autopicking", default=True),
ProcessingParameter("symmetry", "Symmetry Group", default="C1"),
ProcessingParameter("eer_fractionation", "EER Fractionation", default=20),
ProcessingParameter(
"mask_diameter", "Mask Diameter (2D classification)", default=190
),
ProcessingParameter("boxsize", "Box Size", default=256),
ProcessingParameter("downscale", "Downscale Extracted Particles", default=True),
ProcessingParameter(
"small_boxsize", "Downscaled Extracted Particle Size (pixels)", default=128
),
ProcessingParameter("gain_ref", "Gain Reference"),
ProcessingParameter("gain_ref_superres", "Unbinned Gain Reference"),
]
metadata_params = [
ProcessingParameter("voltage", "Voltage"),
Expand Down Expand Up @@ -164,7 +145,6 @@ def gather_metadata(
return OrderedDict({})
data = xmltodict.parse(for_parsing)
magnification = 0
num_fractions = 1
metadata: OrderedDict = OrderedDict({})
metadata["experiment_type"] = "SPA"
if data.get("Acquisition"):
Expand Down Expand Up @@ -217,14 +197,6 @@ def gather_metadata(
) # convert e / m^2 to e / A^2
except ValueError:
metadata["total_exposed_dose"] = 1
try:
num_fractions = int(
data["MicroscopeImage"]["microscopeData"]["acquisition"]["camera"][
"CameraSpecificInput"
]["a:KeyValueOfstringanyType"][2]["a:Value"]["b:NumberOffractions"]
)
except (KeyError, IndexError):
pass
c2_index = 3
for i, el in enumerate(
data["MicroscopeImage"]["CustomData"]["a:KeyValueOfstringanyType"]
Expand Down Expand Up @@ -293,85 +265,13 @@ def gather_metadata(
metadata["image_size_x"] = str(int(metadata["image_size_x"]) * binning_factor)
metadata["image_size_y"] = str(int(metadata["image_size_y"]) * binning_factor)
metadata["motion_corr_binning"] = 1 if binning_factor_xml == 2 else 2
if environment:
metadata["gain_ref"] = (
environment.data_collection_parameters.get("gain_ref")
if environment
and environment.data_collection_parameters.get("gain_ref")
not in (None, "None")
else f"{datetime.now().year}/{environment.visit}/processing/gain.mrc"
)
metadata["gain_ref_superres"] = (
environment.data_collection_parameters.get("gain_ref_superres")
if environment
and environment.data_collection_parameters.get("gain_ref_superres")
not in (None, "None")
else f"{datetime.now().year}/{environment.visit}/processing/gain_superres.mrc"
)
else:
metadata["gain_ref"] = None
metadata["gain_ref_superres"] = None
if metadata.get("total_exposed_dose"):
metadata["dose_per_frame"] = (
environment.data_collection_parameters.get("dose_per_frame")
if environment
and environment.data_collection_parameters.get("dose_per_frame")
not in (None, "None")
else round(metadata["total_exposed_dose"] / num_fractions, 3)
)
else:
metadata["dose_per_frame"] = (
environment.data_collection_parameters.get("dose_per_frame")
if environment
else None
)

metadata["use_cryolo"] = (
environment.data_collection_parameters.get("use_cryolo")
if environment
else None
) or True
metadata["symmetry"] = (
environment.data_collection_parameters.get("symmetry")
if environment
else None
) or "C1"
metadata["mask_diameter"] = (
environment.data_collection_parameters.get("mask_diameter")
if environment
else None
) or 190
metadata["boxsize"] = (
environment.data_collection_parameters.get("boxsize")
if environment
else None
) or 256
metadata["downscale"] = (
environment.data_collection_parameters.get("downscale")
if environment
else None
) or True
metadata["small_boxsize"] = (
environment.data_collection_parameters.get("small_boxsize")
if environment
else None
) or 128
metadata["gain_ref"] = environment.gain_ref if environment else None
metadata["dose_per_frame"] = environment.dose_per_frame if environment else None
metadata["symmetry"] = (environment.symmetry if environment else None) or "C1"
metadata["eer_fractionation"] = (
environment.data_collection_parameters.get("eer_fractionation")
if environment
else None
environment.eer_fractionation if environment else None
) or 20
metadata["source"] = str(self._basepath)
metadata["particle_diameter"] = (
environment.data_collection_parameters.get("particle_diameter")
if environment
else None
) or 0
metadata["estimate_particle_diameter"] = (
environment.data_collection_parameters.get("estimate_particle_diameter")
if environment
else None
) or True
return metadata

def _position_analysis(
Expand Down Expand Up @@ -584,10 +484,10 @@ def post_transfer(
f"{str(environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=environment.visit, session_id=environment.murfey_session)}",
json={
"eer_path": str(file_transferred_to),
"fractionation": environment.data_collection_parameters[
"fractionation": self.data_collection_parameters[
"eer_fractionation"
],
"dose_per_frame": environment.data_collection_parameters[
"dose_per_frame": self.data_collection_parameters[
"dose_per_frame"
],
"fractionation_file_name": "eer_fractionation_spa.txt",
Expand Down Expand Up @@ -619,20 +519,16 @@ def post_transfer(
"image_number": environment.movies[
file_transferred_to
].movie_number,
"pixel_size": environment.data_collection_parameters.get(
"pixel_size": self.data_collection_parameters.get(
"pixel_size_on_image"
),
"autoproc_program_id": None,
"dose_per_frame": environment.data_collection_parameters.get(
"dose_per_frame"
),
"mc_binning": environment.data_collection_parameters.get(
"dose_per_frame": environment.dose_per_frame,
"mc_binning": self.data_collection_parameters.get(
"motion_corr_binning", 1
),
"gain_ref": environment.data_collection_parameters.get(
"gain_ref"
),
"extract_downscale": environment.data_collection_parameters.get(
"gain_ref": environment.gain_ref,
"extract_downscale": self.data_collection_parameters.get(
"downscale", True
),
"eer_fractionation_file": eer_fractionation_file,
Expand Down
Loading