Merge pull request #476 from zapta/develop

Clean up of integration tests

Makefile

@@ -22,13 +22,15 @@ lint:
 	python -m tox -e lint
 
 
-# Tests only, no lint, single python version.
+# Offline tests only, no lint, single python version, skipping online tests.
+# This is a partial but fast test.
 test:	
-	python -m tox --skip-missing-interpreters false -e py312
+	python -m tox --skip-missing-interpreters false -e py312 -- --offline
 
 
-# Tests and lint, single python version.
-# Run this before submitting code.
+# Tests and lint, single python version, all tests including online..
+# This is a thorough but slow test and sufficient for testign before 
+# commiting changes run this before submitting code.
  check:	
 	python -m tox --skip-missing-interpreters false -e lint,py312
 

apio/apio_context.py

@@ -8,6 +8,7 @@
 
 import sys
 import json
+import re
 import platform
 from collections import OrderedDict
 import shutil
@@ -91,14 +92,28 @@ def __init__(
                 fg="yellow",
             )
 
+        # -- A flag to indicate if the system env was already set in this
+        # -- apio session. Used to avoid multiple repeated settings that
+        # -- make the path longer and longer.
+        self.env_was_already_set = False
+
         # -- Save the load project status.
         self._load_project = load_project
 
         # -- Maps the optional project_dir option to a path.
         self.project_dir: Path = util.get_project_dir(project_dir)
+        ApioContext._check_no_spaces_in_dir(self.project_dir, "project")
+
+        # -- Determine apio home dir
+        self.home_dir: Path = ApioContext._get_home_dir()
+        ApioContext._check_no_spaces_in_dir(self.home_dir, "home")
+
+        # -- Determine apio home dir
+        self.packages_dir: Path = ApioContext._get_packages_dir(self.home_dir)
+        ApioContext._check_no_spaces_in_dir(self.packages_dir, "packages")
 
         # -- Profile information, from ~/.apio/profile.json
-        self.profile = Profile()
+        self.profile = Profile(self.home_dir)
 
         # -- Read the platforms information.
         self.platforms = self._load_resource(PLATFORMS_JSON)
@@ -110,7 +125,7 @@ def __init__(
         self.all_packages = self._load_resource(PACKAGES_JSON)
 
         # -- Expand in place the env templates in all_packages.
-        ApioContext._resolve_package_envs(self.all_packages)
+        ApioContext._resolve_package_envs(self.all_packages, self.packages_dir)
 
         # The subset of packages that are applicable to this platform.
         self.platform_packages = self._select_packages_for_platform(
@@ -150,6 +165,23 @@ def __init__(
             self._project = Project(self.project_dir)
             self._project.read()
 
+    @staticmethod
+    def _check_no_spaces_in_dir(dir_path: Path, subject: str):
+        """Give the user an early error message if their apio setup dir
+        contains white space. See https://github.com/FPGAwars/apio/issues/474
+        """
+        # -- Match a single white space in the dir path.
+        if re.search("\\s", str(dir_path)):
+            # -- Here space found. This is a fatal error since we don't hand
+            # -- it well later in the process.
+            click.secho(
+                f"Error: The apio {subject} directory path contains white "
+                "space.",
+                fg="red",
+            )
+            click.secho(f"'{str(dir_path)}'", fg="red")
+            sys.exit(1)
+
     def check_project_loaded(self):
         """Assert that context was created with project loading.."""
         assert (
@@ -279,12 +311,13 @@ def _expand_env_template(template: str, package_path: Path) -> str:
         raise RuntimeError(f"Invalid env template: [{template}]")
 
     @staticmethod
-    def _resolve_package_envs(packages: Dict[str, Dict]) -> None:
+    def _resolve_package_envs(
+        packages: Dict[str, Dict], packages_dir: Path
+    ) -> None:
         """Resolve in place the path and var value templates in the
         given packages dictionary. For example, %p is replaced with
         the package's absolute path."""
 
-        packages_dir = util.get_packages_dir()
         for _, package_config in packages.items():
 
             # -- Get the package root dir.
@@ -456,7 +489,7 @@ def get_package_dir(self, package_name: str) -> Path:
         """Returns the root path of a package with given name."""
 
         package_folder = self.get_package_folder_name(package_name)
-        package_dir = util.get_packages_dir() / package_folder
+        package_dir = self.packages_dir / package_folder
 
         return package_dir
 
@@ -575,3 +608,100 @@ def is_darwin(self) -> bool:
     def is_windows(self) -> bool:
         """Returns True iff platform_id indicates windows."""
         return "windows" in self.platform_id
+
+    @staticmethod
+    def _get_home_dir() -> Path:
+        """Get the absolute apio home dir. This is the apio folder where the
+        profle is located and the packages are installed (unless
+        APIO_PACKAGES_DIR is used).
+        The apio home dir can be overridden using the APIO_HOME_DIR environment
+        varible or in the /etc/apio.json file (in
+        Debian). If not set, the user_HOME/.apio folder is used by default:
+        Ej. Linux:  /home/obijuan/.apio
+        If the folders does not exist, they are created
+        """
+
+        # -- Get the APIO_HOME_DIR env variable
+        # -- It returns None if it was not defined
+        apio_home_dir_env = env_options.get(
+            env_options.APIO_HOME_DIR, default=None
+        )
+
+        # -- Get the home dir. It is what the APIO_HOME_DIR env variable
+        # -- says, or the default folder if None
+        if apio_home_dir_env:
+            home_dir = Path(apio_home_dir_env)
+        else:
+            home_dir = Path.home() / ".apio"
+
+        # -- Make it absolute
+        home_dir = home_dir.absolute()
+
+        # -- Create the folder if it does not exist
+        try:
+            home_dir.mkdir(parents=True, exist_ok=True)
+        except PermissionError:
+            click.secho(
+                f"Error: no usable home directory {home_dir}", fg="red"
+            )
+            sys.exit(1)
+
+        # Return the home_dir as a Path
+        return home_dir
+
+    @staticmethod
+    def _get_packages_dir(home_dir: Path) -> Path:
+        """Return the base directory of apio packages.
+        Packages are installed in the following folder:
+        * Default: $APIO_HOME_DIR/packages
+        * $APIO_PACKAGES_DIR: if the APIO_PACKAGES_DIR env variable is set
+        * INPUT:
+            - pkg_name: Package name (Ex. 'examples')
+        * OUTPUT:
+            - The package absolute folder (PosixPath)
+            (Ex. '/home/obijuan/.apio/packages)
+            The absolute path of the returned directory is guaranteed to have
+            the word packages in it.
+        """
+
+        # -- Get the APIO_PACKAGES_DIR env variable
+        # -- It returns None if it was not defined
+        packaged_dir_override = env_options.get(env_options.APIO_PACKAGES_DIR)
+
+        # -- Handle override.
+        if packaged_dir_override:
+            # -- Verify that the override dir contains the word packages in its
+            # -- absolute path. This is a safety mechanism to prevent
+            # -- uninentional bulk deletions in unintended directories. We
+            # -- check it each time before we perform a package deletion.
+            path = Path(packaged_dir_override).absolute()
+            if "packages" not in str(path).lower():
+                click.secho(
+                    "Error: packages directory path does not contain the word "
+                    f"packages: {str(path)}",
+                    fg="red",
+                )
+                click.secho(
+                    "For safety reasons, if you use the environment variable "
+                    "APIO_PACKAGE_DIR to override\n"
+                    "the packages dir, the new directory must have the word "
+                    "'packages' (case insensitive)\n"
+                    "in its absolute path.",
+                    fg="yellow",
+                )
+                sys.exit(1)
+
+            # -- Override is OK. Use it as the packages dir.
+            packages_dir = Path(packaged_dir_override)
+
+        # -- Else, use the default value.
+        else:
+            # -- Ex '/home/obijuan/.apio/packages/tools-oss-cad-suite'
+            # -- Guaranteed to be absolute.
+            packages_dir = home_dir / "packages"
+
+        # -- Sanity check. If this fails, this is a programming error.
+        assert "packages" in str(packages_dir).lower(), packages_dir
+
+        # -- All done.
+        return packages_dir

apio/cmd_util.py

@@ -156,13 +156,21 @@ def check_exactly_one_param(
     """
     # The the subset of ids of params that where used in the command.
     specified_param_ids = _specified_params(cmd_ctx, param_ids)
-    # If more 2 or more print an error and exit.
-    if len(specified_param_ids) != 1:
+    # If exactly one than we are good.
+    if len(specified_param_ids) == 1:
+        return
+    if len(specified_param_ids) < 1:
+        # -- User specified Less flags than required.
         canonical_aliases = _params_ids_to_aliases(cmd_ctx, param_ids)
         aliases_str = ", ".join(canonical_aliases)
-        fatal_usage_error(
-            cmd_ctx, f"One of [{aliases_str}] must be specified."
+        fatal_usage_error(cmd_ctx, f"Specify one of [{aliases_str}].")
+    else:
+        # -- User specified more flags than allowed.
+        canonical_aliases = _params_ids_to_aliases(
+            cmd_ctx, specified_param_ids
         )
+        aliases_str = ", ".join(canonical_aliases)
+        fatal_usage_error(cmd_ctx, f"Specify only one of [{aliases_str}].")
 
 
 def check_at_least_one_param(

apio/commands/drivers.py

@@ -90,8 +90,8 @@ def cli(
 ):
     """Implements the drivers command."""
 
-    # Make sure these params are exclusive.
-    cmd_util.check_at_most_one_param(
+    # User should select exactly on of these operations.
+    cmd_util.check_exactly_one_param(
         cmd_ctx,
         nameof(ftdi_install, ftdi_uninstall, serial_install, serial_uninstall),
     )

apio/commands/examples.py

@@ -90,8 +90,6 @@ def cli(
     """Manage verilog examples.\n
     Install with `apio packages --install examples`"""
 
-    cmd_ctx.get_help()
-
     # Make sure these params are exclusive.
     cmd_util.check_exactly_one_param(
         cmd_ctx, nameof(list_, fetch_dir, fetch_files)

apio/commands/packages.py

@@ -138,6 +138,7 @@ def _list(apio_ctx: ApioContext, verbose: bool) -> int:
   apio packages --install oss-cad-suite     # Install a specific package.
   apio packages --install [email protected]   # Install a specific version.
   apio packages --uninstall                 # Uninstall all packages.
+  apio packages --uninstall --sayyes        # Same but does not ask yes/no.
   apio packages --uninstall oss-cad-suite   # Uninstall only given package(s).
   apio packages --fix                       # Fix package errors.
 
@@ -223,6 +224,10 @@ def cli(
     cmd_util.check_at_most_one_param(cmd_ctx, nameof(list_, packages))
     cmd_util.check_at_most_one_param(cmd_ctx, nameof(fix, packages))
 
+    cmd_util.check_at_most_one_param(cmd_ctx, nameof(sayyes, list_))
+    cmd_util.check_at_most_one_param(cmd_ctx, nameof(sayyes, install))
+    cmd_util.check_at_most_one_param(cmd_ctx, nameof(sayyes, fix))
+
     # -- Create the apio context.
 
     apio_ctx = ApioContext(project_dir=project_dir, load_project=False)

apio/commands/system.py

@@ -156,11 +156,11 @@ def cli(
 
         # -- Print apio home directory.
         click.secho("Apio home       ", nl=False)
-        click.secho(util.get_home_dir(), fg="cyan")
+        click.secho(apio_ctx.home_dir, fg="cyan")
 
         # -- Print apio home directory.
         click.secho("Apio packages   ", nl=False)
-        click.secho(util.get_packages_dir(), fg="cyan")
+        click.secho(apio_ctx.packages_dir, fg="cyan")
 
         cmd_ctx.exit(0)
 

apio/managers/installer.py

@@ -296,14 +296,13 @@ def install_package(
         print(f"Download URL: {download_url}")
 
     # -- Prepare the packages directory.
-    packages_dir = util.get_packages_dir()
-    packages_dir.mkdir(exist_ok=True)
+    apio_ctx.packages_dir.mkdir(exist_ok=True)
 
     # -- Prepare the package directory.
     package_dir = apio_ctx.get_package_dir(package_name)
 
     # -- Downlod the package file from the remote server.
-    local_file = _download_package_file(download_url, packages_dir)
+    local_file = _download_package_file(download_url, apio_ctx.packages_dir)
     if verbose:
         print(f"Local file: {local_file}")
 
@@ -319,14 +318,14 @@ def install_package(
         # -- Case 1: The package include a top level wrapper directory.
         #
         # -- Unpack the package one level up, in the packages directory.
-        _unpack_package_file(local_file, packages_dir)
+        _unpack_package_file(local_file, apio_ctx.packages_dir)
 
         # -- The uncompressed name may contain a %V placeholder, Replace it
         # -- with target version.
         uncompressed_name = uncompressed_name.replace("%V", target_version)
 
         # -- Construct the local path of the wrapper dir.
-        wrapper_dir = packages_dir / uncompressed_name
+        wrapper_dir = apio_ctx.packages_dir / uncompressed_name
 
         # -- Rename the wrapper dir to the package dir.
         if verbose:
@@ -427,19 +426,20 @@ def fix_packages(
     for dir_name in scan.orphan_dir_names:
         if verbose:
             print(f"Deleting unknown dir '{dir_name}'")
-        # -- Sanity check. Since get_packages_dir() guarranted to include
+        # -- Sanity check. Since apio_ctx.packages_dir is guarranted to include
         # -- the word packages, this can fail only due to programming error.
-        dir_path = util.get_packages_dir() / dir_name
+        dir_path = apio_ctx.packages_dir / dir_name
         assert "packages" in str(dir_path).lower(), dir_path
         # -- Delete.
         shutil.rmtree(dir_path)
 
     for file_name in scan.orphan_file_names:
         if verbose:
             print(f"Deleting unknown file '{file_name}'")
-        # -- Sanity check. Since get_packages_dir() guarranted to include
-        # -- the word packages, this can fail only due to programming error.
-        file_path = util.get_packages_dir() / file_name
+        # -- Sanity check. Since apio_ctx.packages_dir is guarranted to
+        # -- include the word packages, this can fail only due to programming
+        # -- error.
+        file_path = apio_ctx.packages_dir / file_name
         assert "packages" in str(file_path).lower(), dir_path
         # -- Delete.
         file_path.unlink()

apio/managers/old_installer.py

@@ -14,6 +14,7 @@
 import requests
 
 from apio import util
+from apio.apio_context import ApioContext
 from apio.managers.downloader import FileDownloader
 from apio.managers.unpacker import FileUnpacker
 
@@ -37,7 +38,7 @@ class Modifiers:
     def __init__(
         self,
         package: str,
-        apio_ctx=None,
+        apio_ctx: ApioContext = None,
         modifiers=Modifiers(force=False, checkversion=True, verbose=False),
     ):
         """Class initialization. Parameters:
@@ -94,7 +95,7 @@ def __init__(
         # --(It is defined in the resources/packages.json file)
         if self.package in self.apio_ctx.platform_packages:
             # -- Store the package dir
-            self.packages_dir = util.get_home_dir() / dirname
+            self.packages_dir = apio_ctx.packages_dir / dirname
 
             # Get the metadata of the given package
             package_info = self.apio_ctx.platform_packages[self.package]
@@ -141,7 +142,7 @@ def __init__(
                 self.package in self.apio_ctx.profile.packages
                 and modifiers.checkversion is False
             ):
-                self.packages_dir = util.get_home_dir() / dirname
+                self.packages_dir = apio_ctx.home_dir / dirname
 
                 self.package_folder_name = "toolchain-" + package