🔨 Refactor Engine args: Create workspace directory from API (#1773)

* Create anomalib path utils * Add name and category to datamodule and dataset * Add name and category to datamodule and dataset * Rename new_dir to new_version_dir * Use convert_to_snake_case in get_models * Remove result_dir arg from CLI * add name property to anomaly module * Remove model checkpoint from get_callbacks function * format cli.py * Remove get_default_root_dir util function * Remove project creation from update_config util function * Add workspace to engine * Add ModelCheckpoint back to checkpoint init * Fix the following error: TypeError: unsupported operand type(s) for /: 'str' and 'str' * Overwrite name propery in folder 3d dataset * Overwrite name propery in folder dataset * Update the example folder config * Remove results_dir from tests * fix folder tests * Default category is an empty string * Fix the synthetic data tests * Add name parameter to Folder initialization * Fix notebook tests * Add ModelCheckpoint to the list of anomalib callbacks and initialize callbacks before trainer * remove visualization from engine * Fix the image saving path * Revert show filename * Partially address the cli tests * Setup workspace in each entrypoint * Add/remove missing/extra parts due to conflicts * Update setup-workspace * update unit tests Signed-off-by: Ashwin Vaidya <[email protected]> * fix test * stash changes Signed-off-by: Ashwin Vaidya <[email protected]> * update unit tests Signed-off-by: Ashwin Vaidya <[email protected]> * update unit tests Signed-off-by: Ashwin Vaidya <[email protected]> * fix path Signed-off-by: Ashwin Vaidya <[email protected]> * Fix ImageVisualizer tests * Skip AiVAD test Signed-off-by: Ashwin Vaidya <[email protected]> --------- Signed-off-by: Ashwin Vaidya <[email protected]> Co-authored-by: Ashwin Vaidya <[email protected]>
openvinotoolkit · Feb 29, 2024 · 7c1e268 · 7c1e268
1 parent 4672219
commit 7c1e268
Show file tree

Hide file tree

Showing 29 changed files with 432 additions and 268 deletions.
diff --git a/configs/data/folder.yaml b/configs/data/folder.yaml
@@ -1,5 +1,6 @@
 class_path: anomalib.data.Folder
 init_args:
+  name: bottle
   root: "datasets/MVTec/bottle"
   normal_dir: "train/good"
   abnormal_dir: "test/broken_large"

diff --git a/notebooks/100_datamodules/103_folder.ipynb b/notebooks/100_datamodules/103_folder.ipynb
@@ -96,6 +96,7 @@
    "outputs": [],
    "source": [
     "folder_datamodule = Folder(\n",
+    "    name=\"hazelnut_toy\",\n",
     "    root=dataset_root,\n",
     "    normal_dir=\"good\",\n",
     "    abnormal_dir=\"crack\",\n",
@@ -331,6 +332,7 @@
    ],
    "source": [
     "folder_dataset_classification_train = FolderDataset(\n",
+    "    name=\"hazelnut_toy\",\n",
     "    normal_dir=dataset_root / \"good\",\n",
     "    abnormal_dir=dataset_root / \"crack\",\n",
     "    split=\"train\",\n",
@@ -476,6 +478,7 @@
    "source": [
     "# Folder Classification Test Set\n",
     "folder_dataset_classification_test = FolderDataset(\n",
+    "    name=\"hazelnut_toy\",\n",
     "    normal_dir=dataset_root / \"good\",\n",
     "    abnormal_dir=dataset_root / \"crack\",\n",
     "    split=\"test\",\n",
@@ -615,6 +618,7 @@
    "source": [
     "# Folder Segmentation Train Set\n",
     "folder_dataset_segmentation_train = FolderDataset(\n",
+    "    name=\"hazelnut_toy\",\n",
     "    normal_dir=dataset_root / \"good\",\n",
     "    abnormal_dir=dataset_root / \"crack\",\n",
     "    split=\"train\",\n",
@@ -727,6 +731,7 @@
    "source": [
     "# Folder Segmentation Test Set\n",
     "folder_dataset_segmentation_test = FolderDataset(\n",
+    "    name=\"hazelnut_toy\",\n",
     "    normal_dir=dataset_root / \"good\",\n",
     "    abnormal_dir=dataset_root / \"crack\",\n",
     "    split=\"test\",\n",

diff --git a/src/anomalib/callbacks/__init__.py b/src/anomalib/callbacks/__init__.py
@@ -20,6 +20,7 @@
 from .timer import TimerCallback
 
 __all__ = [
+    "ModelCheckpoint",
     "GraphLogger",
     "LoadModelCallback",
     "TilerConfigurationCallback",
@@ -43,21 +44,6 @@ def get_callbacks(config: DictConfig | ListConfig | Namespace) -> list[Callback]
 
     callbacks: list[Callback] = []
 
-    monitor_metric = (
-        None if "early_stopping" not in config.model.init_args else config.model.init_args.early_stopping.metric
-    )
-    monitor_mode = "max" if "early_stopping" not in config.model.init_args else config.model.early_stopping.mode
-
-    checkpoint = ModelCheckpoint(
-        dirpath=Path(config.trainer.default_root_dir) / "weights" / "lightning",
-        filename="model",
-        monitor=monitor_metric,
-        mode=monitor_mode,
-        auto_insert_metric_name=False,
-    )
-
-    callbacks.extend([checkpoint, TimerCallback()])
-
     if "ckpt_path" in config.trainer and config.ckpt_path is not None:
         load_model = LoadModelCallback(config.ckpt_path)
         callbacks.append(load_model)

diff --git a/src/anomalib/callbacks/visualizer.py b/src/anomalib/callbacks/visualizer.py
@@ -97,7 +97,18 @@ def on_test_batch_end(
                         if result.file_name is None:
                             msg = "``save`` is set to ``True`` but file name is ``None``"
                             raise ValueError(msg)
-                        save_image(image=result.image, root=self.root, filename=result.file_name)
+
+                        # Get the filename to save the image.
+                        # Filename is split based on the datamodule name and category.
+                        # For example, if the filename is `MVTec/bottle/000.png`, then the
+                        # filename is split based on `MVTec/bottle` and `000.png` is saved.
+                        if trainer.datamodule is not None:
+                            filename = str(result.file_name).split(
+                                sep=f"{trainer.datamodule.name}/{trainer.datamodule.category}",
+                            )[-1]
+                        else:
+                            filename = Path(result.file_name).name
+                        save_image(image=result.image, root=self.root, filename=filename)
                     if self.show:
                         show_image(image=result.image, title=str(result.file_name))
                     if self.log:

diff --git a/src/anomalib/cli/cli.py b/src/anomalib/cli/cli.py
@@ -34,7 +34,6 @@
     from anomalib.metrics.threshold import BaseThreshold
     from anomalib.models import AnomalyModule
     from anomalib.utils.config import update_config
-    from anomalib.utils.visualization.base import BaseVisualizer
 
 except ImportError:
     _LIGHTNING_AVAILABLE = False
@@ -143,15 +142,6 @@ def add_arguments_to_parser(self, parser: ArgumentParser) -> None:
         from anomalib.callbacks.normalization import get_normalization_callback
 
         parser.add_function_arguments(get_normalization_callback, "normalization")
-        # visualization takes task from the project
-        parser.add_argument(
-            "--visualization.visualizers",
-            type=BaseVisualizer | list[BaseVisualizer] | None,
-            default=None,
-        )
-        parser.add_argument("--visualization.save", type=bool, default=False)
-        parser.add_argument("--visualization.log", type=bool, default=False)
-        parser.add_argument("--visualization.show", type=bool, default=False)
         parser.add_argument("--task", type=TaskType | str, default=TaskType.SEGMENTATION)
         parser.add_argument("--metrics.image", type=list[str] | str | None, default=["F1Score", "AUROC"])
         parser.add_argument("--metrics.pixel", type=list[str] | str | None, default=None, required=False)
@@ -160,13 +150,14 @@ def add_arguments_to_parser(self, parser: ArgumentParser) -> None:
         if hasattr(parser, "subcommand") and parser.subcommand not in ("export", "predict"):
             parser.link_arguments("task", "data.init_args.task")
         parser.add_argument(
-            "--results_dir.path",
+            "--default_root_dir",
             type=Path,
             help="Path to save the results.",
             default=Path("./results"),
         )
-        parser.add_argument("--results_dir.unique", type=bool, help="Whether to create a unique folder.", default=False)
-        parser.link_arguments("results_dir.path", "trainer.default_root_dir")
+        parser.link_arguments("default_root_dir", "trainer.default_root_dir")
+        # TODO(ashwinvaidya17): Tiling should also be a category of its own
+        # CVS-122659
 
     def add_trainer_arguments(self, parser: ArgumentParser, subcommand: str) -> None:
         """Add train arguments to the parser."""
@@ -329,7 +320,6 @@ def instantiate_engine(self) -> None:
             "task": self._get(self.config_init, "task"),
             "image_metrics": self._get(self.config_init, "metrics.image"),
             "pixel_metrics": self._get(self.config_init, "metrics.pixel"),
-            **self._get_visualization_parameters(),
         }
         trainer_config = {**self._get(self.config_init, "trainer", default={}), **engine_args}
         key = "callbacks"
@@ -348,16 +338,6 @@ def instantiate_engine(self) -> None:
         trainer_config[key].extend(get_callbacks(self.config[self.subcommand]))
         self.engine = Engine(**trainer_config)
 
-    def _get_visualization_parameters(self) -> dict[str, Any]:
-        """Return visualization parameters."""
-        subcommand = self.config.subcommand
-        return {
-            "visualizers": self.config_init[subcommand].visualization.visualizers,
-            "save_image": self.config[subcommand].visualization.save,
-            "log_image": self.config[subcommand].visualization.log,
-            "show_image": self.config[subcommand].visualization.show,
-        }
-
     def _run_subcommand(self) -> None:
         """Run subcommand depending on the subcommand.
 

diff --git a/src/anomalib/data/base/datamodule.py b/src/anomalib/data/base/datamodule.py
@@ -110,9 +110,15 @@ def __init__(
         self.test_data: AnomalibDataset
 
         self._samples: DataFrame | None = None
+        self._category: str = ""
 
         self._is_setup = False  # flag to track if setup has been called from the trainer
 
+    @property
+    def name(self) -> str:
+        """Name of the datamodule."""
+        return self.__class__.__name__
+
     def setup(self, stage: str | None = None) -> None:
         """Set up train, validation and test data.
 
@@ -144,6 +150,16 @@ def _setup(self, _stage: str | None = None) -> None:
         """
         raise NotImplementedError
 
+    @property
+    def category(self) -> str:
+        """Get the category of the datamodule."""
+        return self._category
+
+    @category.setter
+    def category(self, category: str) -> None:
+        """Set the category of the datamodule."""
+        self._category = category
+
     def _create_test_split(self) -> None:
         """Obtain the test set based on the settings in the config."""
         if self.test_data.has_normal:

diff --git a/src/anomalib/data/base/dataset.py b/src/anomalib/data/base/dataset.py
@@ -67,6 +67,18 @@ def __init__(self, task: TaskType, transform: Transform | None = None) -> None:
         self.task = task
         self.transform = transform
         self._samples: DataFrame | None = None
+        self._category: str | None = None
+
+    @property
+    def name(self) -> str:
+        """Name of the dataset."""
+        class_name = self.__class__.__name__
+
+        # Remove the `_dataset` suffix from the class name
+        if class_name.endswith("Dataset"):
+            class_name = class_name[:-7]
+
+        return class_name
 
     def __len__(self) -> int:
         """Get length of the dataset."""
@@ -113,6 +125,16 @@ def samples(self, samples: DataFrame) -> None:
 
         self._samples = samples.sort_values(by="image_path", ignore_index=True)
 
+    @property
+    def category(self) -> str | None:
+        """Get the category of the dataset."""
+        return self._category
+
+    @category.setter
+    def category(self, category: str) -> None:
+        """Set the category of the dataset."""
+        self._category = category
+
     @property
     def has_normal(self) -> bool:
         """Check if the dataset contains any normal samples."""

diff --git a/src/anomalib/data/depth/folder_3d.py b/src/anomalib/data/depth/folder_3d.py
@@ -185,6 +185,7 @@ class Folder3DDataset(AnomalibDepthDataset):
     """Folder dataset.
 
     Args:
+        name (str): Name of the dataset.
         task (TaskType): Task type. (``classification``, ``detection`` or ``segmentation``).
         transform (Transform): Transforms that should be applied to the input images.
         normal_dir (str | Path): Path to the directory containing normal images.
@@ -222,6 +223,7 @@ class Folder3DDataset(AnomalibDepthDataset):
 
     def __init__(
         self,
+        name: str,
         task: TaskType,
         normal_dir: str | Path,
         root: str | Path | None = None,
@@ -237,6 +239,7 @@ def __init__(
     ) -> None:
         super().__init__(task, transform)
 
+        self._name = name
         self.split = split
         self.root = root
         self.normal_dir = normal_dir
@@ -261,11 +264,20 @@ def __init__(
             extensions=self.extensions,
         )
 
+    @property
+    def name(self) -> str:
+        """Name of the dataset.
+
+        Folder3D dataset overrides the name property to provide a custom name.
+        """
+        return self._name
+
 
 class Folder3D(AnomalibDataModule):
     """Folder DataModule.
 
     Args:
+        name (str): Name of the dataset. This is used to name the datamodule, especially when logging/saving.
         normal_dir (str | Path): Name of the directory containing normal images.
         root (str | Path | None): Path to the root folder containing normal and abnormal dirs.
             Defaults to ``None``.
@@ -318,6 +330,7 @@ class Folder3D(AnomalibDataModule):
 
     def __init__(
         self,
+        name: str,
         normal_dir: str | Path,
         root: str | Path,
         abnormal_dir: str | Path | None = None,
@@ -355,6 +368,7 @@ def __init__(
             val_split_ratio=val_split_ratio,
             seed=seed,
         )
+        self._name = name
         self.task = TaskType(task)
         self.root = Path(root)
         self.normal_dir = normal_dir
@@ -368,6 +382,7 @@ def __init__(
 
     def _setup(self, _stage: str | None = None) -> None:
         self.train_data = Folder3DDataset(
+            name=self.name,
             task=self.task,
             transform=self.train_transform,
             split=Split.TRAIN,
@@ -383,6 +398,7 @@ def _setup(self, _stage: str | None = None) -> None:
         )
 
         self.test_data = Folder3DDataset(
+            name=self.name,
             task=self.task,
             transform=self.eval_transform,
             split=Split.TEST,
@@ -396,3 +412,11 @@ def _setup(self, _stage: str | None = None) -> None:
             mask_dir=self.mask_dir,
             extensions=self.extensions,
         )
+
+    @property
+    def name(self) -> str:
+        """Name of the datamodule.
+
+        Folder3D datamodule overrides the name property to provide a custom name.
+        """
+        return self._name