From c07878450a5ccdd1922705adc73869a1db9222d2 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 14:30:36 +0200 Subject: [PATCH 01/18] Added function to make it independent --- env_prototype/api.py | 4 ++ env_prototype/core.py | 121 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/env_prototype/api.py b/env_prototype/api.py index c035e52..1a30f33 100644 --- a/env_prototype/api.py +++ b/env_prototype/api.py @@ -4,6 +4,8 @@ compute, merge, get_tools, + which, + launch, CycleError, DynamicKeyClashError @@ -15,6 +17,8 @@ "compute", "merge", "get_tools", + "which", + "launch", "CycleError", "DynamicKeyClashError" diff --git a/env_prototype/core.py b/env_prototype/core.py index 0ad5c7f..87f55e3 100644 --- a/env_prototype/core.py +++ b/env_prototype/core.py @@ -1,8 +1,11 @@ import logging import json +import pprint import re import os import platform +import subprocess +import sys from . import lib @@ -154,7 +157,11 @@ def parse(env, platform_name=None): def append(env, env_b): - """Append paths of environment b into environment""" + """Append paths of environment b into environment + + Returns: + env (dict) + """ # todo: should this be refactored to "join" or "extend" # todo: this function name might also be confusing with "merge" env = env.copy() @@ -177,8 +184,8 @@ def get_tools(tools, platform_name=None): tool X can rely on variables of tool Y). Examples: - get_environment(["maya2018", "yeti2.01", "mtoa2018"]) - get_environment(["global", "fusion9", "ofxplugins"]) + get_tools(["maya2018", "yeti2.01", "mtoa2018"]) + get_tools(["global", "fusion9", "ofxplugins"]) Args: tools (list): List of tool names. @@ -209,7 +216,7 @@ def get_tools(tools, platform_name=None): environment = dict() for tool_path in tool_paths: - # Load tool + # Load tool environment try: with open(tool_path, "r") as f: tool_env = json.load(f) @@ -257,3 +264,109 @@ def merge(env, current_env): return result + +def which(program, paths=None): + """Locate `program` in PATH + + Arguments: + program (str): Name of program, e.g. "python" + paths (list): a list of paths + + """ + + def is_exe(fpath): + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + return True + return False + + if paths is None: + paths = os.environ["PATH"].split(os.pathsep) + + for path in paths: + for ext in os.getenv("PATHEXT", "").split(os.pathsep): + fname = program + ext.lower() + abspath = os.path.join(path.strip('"'), fname) + + if is_exe(abspath): + return abspath + + return None + + +def execute(executable, args=None, environment=None, cwd=None): + """Launch a new subprocess of `args` + + Arguments: + executable (str): Relative or absolute path to executable + args (list): Command passed to `subprocess.Popen` + environment (dict, optional): Custom environment passed + to Popen instance. + + Returns: + Popen instance of newly spawned process + + Exceptions: + OSError on internal error + ValueError on `executable` not found + + """ + + CREATE_NO_WINDOW = 0x08000000 + CREATE_NEW_CONSOLE = 0x00000010 + IS_WIN32 = sys.platform == "win32" + PY2 = sys.version_info[0] == 2 + + abspath = executable + + env = (environment or os.environ) + + if PY2: + # Protect against unicode, and other unsupported + # types amongst environment variables + enc = sys.getfilesystemencoding() + env = {k.encode(enc): v.encode(enc) for k, v in env.items()} + + kwargs = dict( + args=[abspath] + args or list(), + env=env, + cwd=cwd, + + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + + # Output `str` through stdout on Python 2 and 3 + universal_newlines=True, + ) + + if env.get("CREATE_NEW_CONSOLE"): + kwargs["creationflags"] = CREATE_NEW_CONSOLE + kwargs.pop("stdout") + kwargs.pop("stderr") + else: + + if IS_WIN32: + kwargs["creationflags"] = CREATE_NO_WINDOW + + popen = subprocess.Popen(**kwargs) + + return popen + + +def launch(tools, executable, args): + + tools_env = get_tools(tools.split(";")) + env = compute(tools_env) + + env = merge(env, current_env=dict(os.environ)) + print("Environment:\n%s" % pprint.pformat(env, indent=4)) + + # Search for the executable within the tool's environment + # by temporarily taking on its `PATH` settings + paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) + exe = which(executable, paths=paths) + + if not exe: + raise ValueError("Unable to find executable: %s" % executable) + + print("Launching: %s" % exe) + execute(exe, environment=env, args=args) From c641d7548c9991e212aef8cf9a83f06228c99b67 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 16:39:19 +0200 Subject: [PATCH 02/18] Added launcher module --- env_prototype/launcher.py | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 env_prototype/launcher.py diff --git a/env_prototype/launcher.py b/env_prototype/launcher.py new file mode 100644 index 0000000..c8296e8 --- /dev/null +++ b/env_prototype/launcher.py @@ -0,0 +1,41 @@ +import os +import pprint + +from . import api + + +def launch(tools, executable, args): + + tools_env = api.get_tools(tools.split(";")) + env = api.compute(tools_env) + + env = api.merge(env, current_env=dict(os.environ)) + print("Environment:\n%s" % pprint.pformat(env, indent=4)) + + # Search for the executable within the tool's environment + # by temporarily taking on its `PATH` settings + paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) + exe = api.which(executable, paths=paths) + + if not exe: + raise ValueError("Unable to find executable: %s" % executable) + + print("Launching: %s" % exe) + api.execute(exe, environment=env, args=args) + + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--tools", + help="The tool environments to include. " + "These should be separated by `;`", + required=True) + parser.add_argument("--executable", + help="The executable to run. ", + required=True) + + kwargs, args = parser.parse_known_args() + + launch(tools=kwargs.tools, executable=kwargs.executable, args=args) From e3b789c5145d4f5c90f396b5263df773a926516a Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 16:39:42 +0200 Subject: [PATCH 03/18] added execute to api --- env_prototype/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/env_prototype/api.py b/env_prototype/api.py index 1a30f33..7a45f6c 100644 --- a/env_prototype/api.py +++ b/env_prototype/api.py @@ -5,7 +5,7 @@ merge, get_tools, which, - launch, + execute, CycleError, DynamicKeyClashError @@ -18,7 +18,7 @@ "merge", "get_tools", "which", - "launch", + "execute", "CycleError", "DynamicKeyClashError" From d27d3885707064a9953b648e95d9fa2526390568 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 16:40:15 +0200 Subject: [PATCH 04/18] changed log.debug to log.error, removed launch function --- env_prototype/core.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/env_prototype/core.py b/env_prototype/core.py index 87f55e3..916fc7d 100644 --- a/env_prototype/core.py +++ b/env_prototype/core.py @@ -1,6 +1,5 @@ import logging import json -import pprint import re import os import platform @@ -222,12 +221,12 @@ def get_tools(tools, platform_name=None): tool_env = json.load(f) log.debug('Read tool successfully: {}'.format(tool_path)) except IOError: - log.debug( + log.error( 'Unable to find the environment file: "{}"'.format(tool_path) ) continue except ValueError as e: - log.debug( + log.error( 'Unable to read the environment file: "{0}", due to:' '\n{1}'.format(tool_path, e) ) @@ -286,7 +285,6 @@ def is_exe(fpath): for ext in os.getenv("PATHEXT", "").split(os.pathsep): fname = program + ext.lower() abspath = os.path.join(path.strip('"'), fname) - if is_exe(abspath): return abspath @@ -350,23 +348,3 @@ def execute(executable, args=None, environment=None, cwd=None): popen = subprocess.Popen(**kwargs) return popen - - -def launch(tools, executable, args): - - tools_env = get_tools(tools.split(";")) - env = compute(tools_env) - - env = merge(env, current_env=dict(os.environ)) - print("Environment:\n%s" % pprint.pformat(env, indent=4)) - - # Search for the executable within the tool's environment - # by temporarily taking on its `PATH` settings - paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) - exe = which(executable, paths=paths) - - if not exe: - raise ValueError("Unable to find executable: %s" % executable) - - print("Launching: %s" % exe) - execute(exe, environment=env, args=args) From bcefa99ef89f796dbebd8b2e41453ba22d686660 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 15:00:11 +0200 Subject: [PATCH 05/18] removed empty line --- env_prototype/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/env_prototype/core.py b/env_prototype/core.py index 916fc7d..d97f905 100644 --- a/env_prototype/core.py +++ b/env_prototype/core.py @@ -341,7 +341,6 @@ def execute(executable, args=None, environment=None, cwd=None): kwargs.pop("stdout") kwargs.pop("stderr") else: - if IS_WIN32: kwargs["creationflags"] = CREATE_NO_WINDOW From 062e590dae064c47245d7d5b2ed04ddfaeb75b82 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 16:25:38 +0200 Subject: [PATCH 06/18] Refactored to use PATH and PATHEXT from env, extended docstrings --- env_prototype/core.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/env_prototype/core.py b/env_prototype/core.py index d97f905..9a59473 100644 --- a/env_prototype/core.py +++ b/env_prototype/core.py @@ -264,12 +264,17 @@ def merge(env, current_env): return result -def which(program, paths=None): +def which(program, env): """Locate `program` in PATH + Ensure `PATHEXT` is declared in the environment if you want to alter the + priority of the system extensions: + + Example : ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC" + Arguments: program (str): Name of program, e.g. "python" - paths (list): a list of paths + env (dict): an environment dictionary """ @@ -278,11 +283,11 @@ def is_exe(fpath): return True return False - if paths is None: - paths = os.environ["PATH"].split(os.pathsep) + paths = env["PATH"].split(os.pathsep) + extensions = env.get("PATHEXT", os.getenv("PATHEXT", "")) for path in paths: - for ext in os.getenv("PATHEXT", "").split(os.pathsep): + for ext in extensions.split(os.pathsep): fname = program + ext.lower() abspath = os.path.join(path.strip('"'), fname) if is_exe(abspath): @@ -299,6 +304,7 @@ def execute(executable, args=None, environment=None, cwd=None): args (list): Command passed to `subprocess.Popen` environment (dict, optional): Custom environment passed to Popen instance. + cwd (str): the current working directory Returns: Popen instance of newly spawned process From 39350fb9329e5dfeaa4ff3121e8f99421482bf83 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 16:26:03 +0200 Subject: [PATCH 07/18] refactored to match signature --- env_prototype/launcher.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/env_prototype/launcher.py b/env_prototype/launcher.py index c8296e8..532232a 100644 --- a/env_prototype/launcher.py +++ b/env_prototype/launcher.py @@ -14,9 +14,7 @@ def launch(tools, executable, args): # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings - paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) - exe = api.which(executable, paths=paths) - + exe = api.which(executable, env) if not exe: raise ValueError("Unable to find executable: %s" % executable) From f5409dff31be2187cafc964cb77e339add19cd35 Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 10 Apr 2018 15:21:00 +0200 Subject: [PATCH 08/18] added launch module to make acre standalone --- acre/launch.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 acre/launch.py diff --git a/acre/launch.py b/acre/launch.py new file mode 100644 index 0000000..a652534 --- /dev/null +++ b/acre/launch.py @@ -0,0 +1,39 @@ +import os +import pprint + +from . import get_tools, compute, merge, which, execute + + +def launch(tools, executable, args): + + tools_env = get_tools(tools.split(";")) + env = compute(tools_env) + + env = merge(env, current_env=dict(os.environ)) + print("Environment:\n%s" % pprint.pformat(env, indent=4)) + + # Search for the executable within the tool's environment + # by temporarily taking on its `PATH` settings + exe = which(executable, env) + if not exe: + raise ValueError("Unable to find executable: %s" % executable) + + print("Launching: %s" % exe) + execute(exe, environment=env, args=args) + + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--tools", + help="The tool environments to include. " + "These should be separated by `;`", + required=True) + parser.add_argument("--executable", + help="The executable to run. ", + required=True) + + kwargs, args = parser.parse_known_args() + + launch(tools=kwargs.tools, executable=kwargs.executable, args=args) From ca765bcf5f0818d8706916c5602ed6d4c4fc5bea Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 10 Apr 2018 16:20:33 +0200 Subject: [PATCH 09/18] Refactored execute to --- acre/__init__.py | 4 ++-- acre/core.py | 2 +- acre/launch.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acre/__init__.py b/acre/__init__.py index 7a45f6c..1a30f33 100644 --- a/acre/__init__.py +++ b/acre/__init__.py @@ -5,7 +5,7 @@ merge, get_tools, which, - execute, + launch, CycleError, DynamicKeyClashError @@ -18,7 +18,7 @@ "merge", "get_tools", "which", - "execute", + "launch", "CycleError", "DynamicKeyClashError" diff --git a/acre/core.py b/acre/core.py index 83a1b12..844033b 100644 --- a/acre/core.py +++ b/acre/core.py @@ -288,7 +288,7 @@ def is_exe(fpath): return None -def execute(executable, args=None, environment=None, cwd=None): +def launch(executable, args=None, environment=None, cwd=None): """Launch a new subprocess of `args` Arguments: diff --git a/acre/launch.py b/acre/launch.py index a652534..e146cc5 100644 --- a/acre/launch.py +++ b/acre/launch.py @@ -1,7 +1,7 @@ import os import pprint -from . import get_tools, compute, merge, which, execute +from . import get_tools, compute, merge, which, launch def launch(tools, executable, args): @@ -19,7 +19,7 @@ def launch(tools, executable, args): raise ValueError("Unable to find executable: %s" % executable) print("Launching: %s" % exe) - execute(exe, environment=env, args=args) + launch(exe, environment=env, args=args) if __name__ == '__main__': From 920c6657f6bdd8873712a2ae8595b31e23bfbb3f Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 11 Apr 2018 11:38:14 +0200 Subject: [PATCH 10/18] refactored conflicting names --- acre/launch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acre/launch.py b/acre/launch.py index e146cc5..c6050a0 100644 --- a/acre/launch.py +++ b/acre/launch.py @@ -4,7 +4,7 @@ from . import get_tools, compute, merge, which, launch -def launch(tools, executable, args): +def launcher(tools, executable, args): tools_env = get_tools(tools.split(";")) env = compute(tools_env) @@ -36,4 +36,4 @@ def launch(tools, executable, args): kwargs, args = parser.parse_known_args() - launch(tools=kwargs.tools, executable=kwargs.executable, args=args) + launcher(tools=kwargs.tools, executable=kwargs.executable, args=args) From 733f31d8524b72edbe114506be17151f33a991cb Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 13 Apr 2018 18:23:40 +0200 Subject: [PATCH 11/18] Delete launch files --- env_prototype/launcher.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 env_prototype/launcher.py diff --git a/env_prototype/launcher.py b/env_prototype/launcher.py deleted file mode 100644 index 532232a..0000000 --- a/env_prototype/launcher.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import pprint - -from . import api - - -def launch(tools, executable, args): - - tools_env = api.get_tools(tools.split(";")) - env = api.compute(tools_env) - - env = api.merge(env, current_env=dict(os.environ)) - print("Environment:\n%s" % pprint.pformat(env, indent=4)) - - # Search for the executable within the tool's environment - # by temporarily taking on its `PATH` settings - exe = api.which(executable, env) - if not exe: - raise ValueError("Unable to find executable: %s" % executable) - - print("Launching: %s" % exe) - api.execute(exe, environment=env, args=args) - - -if __name__ == '__main__': - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--tools", - help="The tool environments to include. " - "These should be separated by `;`", - required=True) - parser.add_argument("--executable", - help="The executable to run. ", - required=True) - - kwargs, args = parser.parse_known_args() - - launch(tools=kwargs.tools, executable=kwargs.executable, args=args) From 0d38e759fabb498f2d1ddee44be1b2ff5ecb83c3 Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 4 Jul 2018 16:01:01 +0200 Subject: [PATCH 12/18] refactored function names, extended docstrings --- acre/__init__.py | 16 ++++++++-------- acre/core.py | 22 +++++++++++----------- acre/launch.py | 8 ++++---- acre/lib.py | 29 +++++++++++++++++++++++++++-- tests/test_dynamic_environments.py | 24 ++++++++++++------------ tests/test_tools.py | 10 +++++----- 6 files changed, 67 insertions(+), 42 deletions(-) diff --git a/acre/__init__.py b/acre/__init__.py index 1a30f33..4bbcd80 100644 --- a/acre/__init__.py +++ b/acre/__init__.py @@ -1,10 +1,10 @@ from .core import ( - parse, + prepare, append, - compute, + build, merge, - get_tools, - which, + discover, + locate, launch, CycleError, @@ -12,12 +12,12 @@ ) __all__ = [ - "parse", + "prepare", "append", - "compute", + "build", "merge", - "get_tools", - "which", + "discover", + "locate", "launch", "CycleError", diff --git a/acre/core.py b/acre/core.py index 844033b..f54c30c 100644 --- a/acre/core.py +++ b/acre/core.py @@ -25,11 +25,11 @@ class DynamicKeyClashError(ValueError): pass -def compute(env, - dynamic_keys=True, - allow_cycle=False, - allow_key_clash=False, - cleanup=True): +def build(env, + dynamic_keys=True, + allow_cycle=False, + allow_key_clash=False, + cleanup=True): """Compute the result from recursive dynamic environment. Note: Keys that are not present in the data will remain unformatted as the @@ -112,7 +112,7 @@ def compute(env, return env -def parse(env, platform_name=None): +def prepare(env, platform_name=None): """Parse environment for platform-specific values Args: @@ -166,11 +166,11 @@ def append(env, env_b): return env -def get_tools(tools, platform_name=None): +def discover(tools, platform_name=None): """Return combined environment for the given set of tools. - This will find merge all the required environment variables of the input - tools into a single dictionary. Then it will do a recursive format to + This will find and merge all the required environment variables of the + input tools into a single dictionary. Then it will do a recursive format to format all dynamic keys and values using the same dictionary. (So that tool X can rely on variables of tool Y). @@ -224,7 +224,7 @@ def get_tools(tools, platform_name=None): ) continue - tool_env = parse(tool_env, platform_name=platform_name) + tool_env = prepare(tool_env, platform_name=platform_name) environment = append(environment, tool_env) return environment @@ -256,7 +256,7 @@ def merge(env, current_env): return result -def which(program, env): +def locate(program, env): """Locate `program` in PATH Ensure `PATHEXT` is declared in the environment if you want to alter the diff --git a/acre/launch.py b/acre/launch.py index c6050a0..38aea2b 100644 --- a/acre/launch.py +++ b/acre/launch.py @@ -1,20 +1,20 @@ import os import pprint -from . import get_tools, compute, merge, which, launch +from . import discover, build, merge, locate, launch def launcher(tools, executable, args): - tools_env = get_tools(tools.split(";")) - env = compute(tools_env) + tools_env = discover(tools.split(";")) + env = build(tools_env) env = merge(env, current_env=dict(os.environ)) print("Environment:\n%s" % pprint.pformat(env, indent=4)) # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings - exe = which(executable, env) + exe = locate(executable, env) if not exe: raise ValueError("Unable to find executable: %s" % executable) diff --git a/acre/lib.py b/acre/lib.py index 09b3062..ecd9172 100644 --- a/acre/lib.py +++ b/acre/lib.py @@ -13,6 +13,12 @@ def uniqify_ordered(seq): See: https://stackoverflow.com/questions/480214/how-do-you-remove- duplicates-from-a-list-in-whilst-preserving-order + Args: + seq(list): list of values + + Returns: + list + """ seen = set() seen_add = seen.add @@ -46,7 +52,14 @@ def __missing__(self, key): def topological_sort(dependency_pairs): - """Sort values subject to dependency constraints""" + """Sort values subject to dependency constraints + + Args: + dependency_pairs(list): list of pairs, [a, b] + + Returns: + namedtuple + """ num_heads = defaultdict(int) # num arrows pointing in tails = defaultdict(list) # list of arrows going out heads = [] # unique list of heads in order first seen @@ -64,12 +77,24 @@ def topological_sort(dependency_pairs): num_heads[t] -= 1 if not num_heads[t]: ordered.append(t) + cyclic = [n for n, heads in num_heads.items() if heads] + return Results(ordered, cyclic) def append_path(self, key, path): - """Append *path* to *key* in *self*.""" + """Append *path* to *key* in *self*. + + Args: + self (dict): environment dictionary + key (str): environment variable name + path (str): path + + Returns: + None + + """ try: if path not in self[key]: self[key] = os.pathsep.join([self[key], str(path)]) diff --git a/tests/test_dynamic_environments.py b/tests/test_dynamic_environments.py index d039d11..df8d5cf 100644 --- a/tests/test_dynamic_environments.py +++ b/tests/test_dynamic_environments.py @@ -17,15 +17,15 @@ def test_parse_platform(self): "B": "universal" } - result = acre.parse(data, platform_name="darwin") + result = acre.prepare(data, platform_name="darwin") self.assertEqual(result["A"], data["A"]["darwin"]) self.assertEqual(result["B"], data["B"]) - result = acre.parse(data, platform_name="windows") + result = acre.prepare(data, platform_name="windows") self.assertEqual(result["A"], data["A"]["windows"]) self.assertEqual(result["B"], data["B"]) - result = acre.parse(data, platform_name="linux") + result = acre.prepare(data, platform_name="linux") self.assertEqual(result["A"], data["A"]["linux"]) self.assertEqual(result["B"], data["B"]) @@ -44,7 +44,7 @@ def test_nesting_deep(self): "I": "deep_{H}" } - result = acre.compute(data) + result = acre.build(data) self.assertEqual(result, { "A": "bla", @@ -66,11 +66,11 @@ def test_cycle(self): } with self.assertRaises(acre.CycleError): - acre.compute(data, allow_cycle=False) + acre.build(data, allow_cycle=False) # If we compute the cycle the result is unknown, it can be either {Y} # or {X} for both values so we just check whether are equal - result = acre.compute(data, allow_cycle=True) + result = acre.build(data, allow_cycle=True) self.assertEqual(result["X"], result["Y"]) def test_dynamic_keys(self): @@ -82,7 +82,7 @@ def test_dynamic_keys(self): "{B}": "this is C" } - env = acre.compute(data) + env = acre.build(data) self.assertEqual(env, { "A": "D", @@ -99,10 +99,10 @@ def test_dynamic_keys_clash_cycle(self): } with self.assertRaises(acre.DynamicKeyClashError): - acre.compute(data, allow_key_clash=False) + acre.build(data, allow_key_clash=False) # Allow to pass (even if unpredictable result) - acre.compute(data, allow_key_clash=True) + acre.build(data, allow_key_clash=True) def test_dynamic_keys_clash(self): """Dynamic key clash captured correctly""" @@ -113,10 +113,10 @@ def test_dynamic_keys_clash(self): } with self.assertRaises(acre.DynamicKeyClashError): - acre.compute(data, allow_key_clash=False) + acre.build(data, allow_key_clash=False) # Allow to pass (even if unpredictable result) - acre.compute(data, allow_key_clash=True) + acre.build(data, allow_key_clash=True) def test_compute_preserve_reference_to_self(self): """acre.compute() does not format key references to itself""" @@ -125,7 +125,7 @@ def test_compute_preserve_reference_to_self(self): "PATH": "{PATH}", "PYTHONPATH": "x;y/{PYTHONPATH}" } - data = acre.compute(data) + data = acre.build(data) self.assertEqual(data, { "PATH": "{PATH}", "PYTHONPATH": "x;y/{PYTHONPATH}" diff --git a/tests/test_tools.py b/tests/test_tools.py index d8a4f93..6d755a7 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -16,7 +16,7 @@ def test_get_tools(self): """Get tools works""" self.setup_sample("sample1") - env = acre.get_tools(["global"]) + env = acre.discover(["global"]) self.assertEqual(env, { "PIPELINE": "P:/pipeline/dev2_1" @@ -27,14 +27,14 @@ def test_get_with_platform(self): self.setup_sample("sample1") - env = acre.get_tools(["maya_2018"], platform_name="darwin") + env = acre.discover(["maya_2018"], platform_name="darwin") self.assertEqual(env["PATH"], "{MAYA_LOCATION}/bin") # Test Mac only path self.assertTrue("DYLD_LIBRARY_PATH" in env) self.assertEqual(env["DYLD_LIBRARY_PATH"], "{MAYA_LOCATION}/MacOS") - env = acre.get_tools(["maya_2018"], platform_name="windows") + env = acre.discover(["maya_2018"], platform_name="windows") self.assertEqual(env["PATH"], ";".join([ "{MAYA_LOCATION}/bin", "C:/Program Files/Common Files/Autodesk Shared/", @@ -47,9 +47,9 @@ def test_get_with_platform(self): def test_compute_tools(self): self.setup_sample("sample1") - env = acre.get_tools(["maya_2018"], platform_name="windows") + env = acre.discover(["maya_2018"], platform_name="windows") - env = acre.compute(env) + env = acre.build(env) self.assertEqual(env["MAYA_LOCATION"], "C:/Program Files/Autodesk/Maya2018") From b63a261fbb22ad0a95f3748f34958e07e7979ba7 Mon Sep 17 00:00:00 2001 From: wikoreman Date: Mon, 10 Sep 2018 11:29:01 +0200 Subject: [PATCH 13/18] Refactored function name, append --> join --- acre/__init__.py | 4 ++-- acre/core.py | 6 ++---- tests/test_dynamic_environments.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/acre/__init__.py b/acre/__init__.py index 4bbcd80..ec62368 100644 --- a/acre/__init__.py +++ b/acre/__init__.py @@ -1,6 +1,6 @@ from .core import ( prepare, - append, + join, build, merge, discover, @@ -13,7 +13,7 @@ __all__ = [ "prepare", - "append", + "join", "build", "merge", "discover", diff --git a/acre/core.py b/acre/core.py index f54c30c..6fe255a 100644 --- a/acre/core.py +++ b/acre/core.py @@ -147,14 +147,12 @@ def prepare(env, platform_name=None): return result -def append(env, env_b): +def join(env, env_b): """Append paths of environment b into environment Returns: env (dict) """ - # todo: should this be refactored to "join" or "extend" - # todo: this function name might also be confusing with "merge" env = env.copy() for variable, value in env_b.items(): for path in value.split(";"): @@ -225,7 +223,7 @@ def discover(tools, platform_name=None): continue tool_env = prepare(tool_env, platform_name=platform_name) - environment = append(environment, tool_env) + environment = join(environment, tool_env) return environment diff --git a/tests/test_dynamic_environments.py b/tests/test_dynamic_environments.py index df8d5cf..7c0724d 100644 --- a/tests/test_dynamic_environments.py +++ b/tests/test_dynamic_environments.py @@ -148,7 +148,7 @@ def test_append(self): _data_a = data_a.copy() _data_b = data_b.copy() - data = acre.append(data_a, data_b) + data = acre.join(data_a, data_b) self.assertEqual(data, { "A": "A;A2", From 7d75d8f65242e0bb9bc40d9670948bdd19a53e98 Mon Sep 17 00:00:00 2001 From: wikoreman Date: Mon, 10 Sep 2018 11:30:14 +0200 Subject: [PATCH 14/18] Added platform separator replacement --- acre/core.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acre/core.py b/acre/core.py index 6fe255a..b0f3898 100644 --- a/acre/core.py +++ b/acre/core.py @@ -128,6 +128,14 @@ def prepare(env, platform_name=None): platform_name = platform_name or PLATFORM + lookup = {"windows": ["/", "\\"], + "linux": ["\\", "/"], + "darwin": ["\\", "/"]} + + translate = lookup.get(platform_name, None) + if translate is None: + raise KeyError("Given platform name `%s` is not supported" % platform) + result = {} for variable, value in env.items(): From ad409c91f6589bab292bdac0f4a7ff70b36ffd58 Mon Sep 17 00:00:00 2001 From: wikoreman Date: Mon, 10 Sep 2018 11:30:47 +0200 Subject: [PATCH 15/18] Added check for url values to skip those --- acre/core.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/acre/core.py b/acre/core.py index b0f3898..a0fc33e 100644 --- a/acre/core.py +++ b/acre/core.py @@ -14,6 +14,8 @@ logging.basicConfig() log = logging.getLogger() +re_url = re.compile(r"^\w+://") + class CycleError(ValueError): """A cyclic dependency in dynamic environment""" @@ -150,6 +152,11 @@ def prepare(env, platform_name=None): if isinstance(value, (list, tuple)): value = ";".join(value) + # Replace the separator to match the given platform's separator + # Skip any value which is a url; ://
+ if not re_url.match(value): + value = value.replace(translate[0], translate[1]) + result[variable] = value return result From 8557b2af022134c8572ffad22c38cec2fe9b8847 Mon Sep 17 00:00:00 2001 From: wikoreman Date: Mon, 10 Sep 2018 11:32:02 +0200 Subject: [PATCH 16/18] refactored launch module, added time.sleep to counter rapid closing --- acre/launch.py | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/acre/launch.py b/acre/launch.py index 38aea2b..8ae809c 100644 --- a/acre/launch.py +++ b/acre/launch.py @@ -1,27 +1,10 @@ import os -import pprint +import sys +import time from . import discover, build, merge, locate, launch -def launcher(tools, executable, args): - - tools_env = discover(tools.split(";")) - env = build(tools_env) - - env = merge(env, current_env=dict(os.environ)) - print("Environment:\n%s" % pprint.pformat(env, indent=4)) - - # Search for the executable within the tool's environment - # by temporarily taking on its `PATH` settings - exe = locate(executable, env) - if not exe: - raise ValueError("Unable to find executable: %s" % executable) - - print("Launching: %s" % exe) - launch(exe, environment=env, args=args) - - if __name__ == '__main__': import argparse @@ -36,4 +19,24 @@ def launcher(tools, executable, args): kwargs, args = parser.parse_known_args() - launcher(tools=kwargs.tools, executable=kwargs.executable, args=args) + # Build environment based on tools + tools = kwargs.tools.split(";") + tools_env = discover(kwargs.tools.split(";")) + env = build(tools_env) + env = merge(env, current_env=dict(os.environ)) + + # Search for the executable within the tool's environment + # by temporarily taking on its `PATH` settings + exe = locate(kwargs.executable, env) + if not exe: + raise ValueError("Unable to find executable: %s" % kwargs.executable) + + print("\n\nLaunching: %s" % exe) + try: + launch(exe, environment=env, args=args) + except Exception as exc: + # Ensure we can capture any exception and give the user (and us) time + # to read it + print(exc) + time.sleep(10) + sys.exit(1) From d58213d4350d5666b708bc0d5d5f8619e600da37 Mon Sep 17 00:00:00 2001 From: wikoreman Date: Tue, 2 Oct 2018 14:47:36 +0200 Subject: [PATCH 17/18] added launch module --- acre/launch.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/acre/launch.py b/acre/launch.py index 8ae809c..b841a91 100644 --- a/acre/launch.py +++ b/acre/launch.py @@ -28,8 +28,13 @@ # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings exe = locate(kwargs.executable, env) - if not exe: - raise ValueError("Unable to find executable: %s" % kwargs.executable) + try: + if not exe: + raise ValueError("Unable to find executable: %s" % kwargs.executable) + except Exception as exc: + print(exc) + time.sleep(10) + sys.exit(1) print("\n\nLaunching: %s" % exe) try: From ba24285007a53529683be4492d0e78313b712855 Mon Sep 17 00:00:00 2001 From: wikoreman Date: Tue, 2 Oct 2018 16:50:01 +0200 Subject: [PATCH 18/18] changed launch to __main__ --- acre/__main__.py | 41 +++++++++++++++++++++++++++++++++++++++++ acre/launch.py | 47 ----------------------------------------------- 2 files changed, 41 insertions(+), 47 deletions(-) create mode 100644 acre/__main__.py delete mode 100644 acre/launch.py diff --git a/acre/__main__.py b/acre/__main__.py new file mode 100644 index 0000000..58d5057 --- /dev/null +++ b/acre/__main__.py @@ -0,0 +1,41 @@ +import os +import sys +import time +import argparse + +from . import discover, build, merge, locate, launch + +parser = argparse.ArgumentParser() +parser.add_argument("--tools", + help="The tool environments to include. " + "These should be separated by `;`", + required=True) +parser.add_argument("--executable", + help="The executable to run. ", + required=True) + +kwargs, args = parser.parse_known_args() + +# Build environment based on tools +tools = kwargs.tools.split(";") +tools_env = discover(kwargs.tools.split(";")) +env = build(tools_env) +env = merge(env, current_env=dict(os.environ)) + +# Search for the executable within the tool's environment +# by temporarily taking on its `PATH` settings +exe = locate(kwargs.executable, env) +try: + if not exe: + raise ValueError("Unable to find executable: %s" % kwargs.executable) +except Exception as exc: + time.sleep(10) + sys.exit(1) + +try: + launch(exe, environment=env, args=args) +except Exception as exc: + # Ensure we can capture any exception and give the user (and us) time + # to read it + time.sleep(10) + sys.exit(1) diff --git a/acre/launch.py b/acre/launch.py deleted file mode 100644 index b841a91..0000000 --- a/acre/launch.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import sys -import time - -from . import discover, build, merge, locate, launch - - -if __name__ == '__main__': - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--tools", - help="The tool environments to include. " - "These should be separated by `;`", - required=True) - parser.add_argument("--executable", - help="The executable to run. ", - required=True) - - kwargs, args = parser.parse_known_args() - - # Build environment based on tools - tools = kwargs.tools.split(";") - tools_env = discover(kwargs.tools.split(";")) - env = build(tools_env) - env = merge(env, current_env=dict(os.environ)) - - # Search for the executable within the tool's environment - # by temporarily taking on its `PATH` settings - exe = locate(kwargs.executable, env) - try: - if not exe: - raise ValueError("Unable to find executable: %s" % kwargs.executable) - except Exception as exc: - print(exc) - time.sleep(10) - sys.exit(1) - - print("\n\nLaunching: %s" % exe) - try: - launch(exe, environment=env, args=args) - except Exception as exc: - # Ensure we can capture any exception and give the user (and us) time - # to read it - print(exc) - time.sleep(10) - sys.exit(1)