From ffcdb93392d67842024bae7016d77418d8c5c23a Mon Sep 17 00:00:00 2001 From: Scott Staniewicz Date: Tue, 5 Nov 2024 10:35:39 -0500 Subject: [PATCH] Add a retry to spurt for brokenprocesspools (#480) * Add a retry to spurt for brokenprocesspools * print the full exception --- src/dolphin/unwrap/_unwrap_3d.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/dolphin/unwrap/_unwrap_3d.py b/src/dolphin/unwrap/_unwrap_3d.py index 235dbdf8..c649e27e 100644 --- a/src/dolphin/unwrap/_unwrap_3d.py +++ b/src/dolphin/unwrap/_unwrap_3d.py @@ -31,6 +31,7 @@ def unwrap_spurt( similarity_filename: PathOrStr | None = None, options: SpurtOptions = DEFAULT_OPTIONS, scratchdir: PathOrStr | None = None, + num_retries: int = 3, ) -> tuple[list[Path], list[Path]]: """Perform 3D unwrapping using `spurt` via subprocess call.""" # NOTE: we are working around spurt currently wanting "temporal_coherence.tif", @@ -118,7 +119,22 @@ def unwrap_spurt( ] ) - subprocess.run(cmd, check=True, text=True) + def run_with_retry(cmd: list[str], num_retries: int = 3): + for attempt in range(num_retries): + try: + result = subprocess.run(cmd, check=True, text=True, capture_output=True) + logging.info(f"Command succeeded on attempt {attempt + 1}") + except subprocess.CalledProcessError as e: + logging.warning(f"Attempt {attempt + 1} failed: {e}") + logging.exception(f"Error output:\n{e.stderr}") + else: + return result + + # If we've exhausted all retries + logging.error(f"Command failed after {num_retries} attempts") + raise RuntimeError(f"Command '{cmd}' failed after {num_retries} attempts") + + run_with_retry(cmd, num_retries=num_retries) # Return paths to output files output_path = Path(output_path)