Generate pylintrc with a newer version of pylint and fix problems there.

spine_engine/execution_managers/conda_kernel_spec_manager.py

@@ -80,7 +80,7 @@ def _validate_kernelspec_path(self, proposal):
 
     def __init__(self, **kwargs):
         self._conda_executable = kwargs.pop("conda_exe")
-        super(CondaKernelSpecManager, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.log = logging.getLogger(__name__)
         self.log.setLevel(logging.WARNING)
         self._conda_info_cache = None
@@ -140,7 +140,7 @@ def _conda_info(self):
                 ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
                 result = ansi_escape.sub("", p)  # Remove ANSI Escape Sequences, such as ESC[0m
                 conda_info = json.loads(result)
-            except Exception as err:
+            except Exception:
                 conda_info = None
                 self.log.error("Obtaining 'conda info --json' failed")
             self._conda_info_cache = conda_info
@@ -183,12 +183,12 @@ def _all_envs(self):
                 # as created by, say, conda or anaconda-project. The name
                 # of the parent directory, then, provides useful context.
                 if basename(env_base) == "envs" and (env_base != envs_prefix or env_name in all_envs):
-                    env_name = "{}-{}".format(basename(dirname(env_base)), env_name)
+                    env_name = f"{basename(dirname(env_base))}-{env_name}"
             # Further disambiguate, if necessary, with a counter.
             if env_name in all_envs:
                 base_name = env_name
                 for count in range(len(all_envs)):
-                    env_name = "{}-{}".format(base_name, count + 2)
+                    env_name = f"{base_name}-{count + 2}"
                     if env_name not in all_envs:
                         break
             all_envs[env_name] = env_path
@@ -235,7 +235,7 @@ def _all_specs(self):
                 elif kernel_name == "ir":
                     kernel_name = "r"
                 kernel_prefix = "" if env_name == "root" else "env-"
-                kernel_name = "conda-{}{}-{}".format(kernel_prefix, env_name, kernel_name)
+                kernel_name = f"conda-{kernel_prefix}{env_name}-{kernel_name}"
                 # Replace invalid characters with dashes
                 kernel_name = self.clean_kernel_name(kernel_name)
 
@@ -326,7 +326,7 @@ def find_kernel_specs(self):
         if self.conda_only:
             kspecs = {}
         else:
-            kspecs = super(CondaKernelSpecManager, self).find_kernel_specs()
+            kspecs = super().find_kernel_specs()
 
         # add conda envs kernelspecs
         if self.whitelist:
@@ -350,7 +350,7 @@ def get_kernel_spec(self, kernel_name):
             return None
         self.log.info(f"res.argv:{res.argv}")
         if res is None and not self.conda_only:
-            res = super(CondaKernelSpecManager, self).get_kernel_spec(kernel_name)
+            res = super().get_kernel_spec(kernel_name)
         return res
 
     def get_all_specs(self):
@@ -377,7 +377,7 @@ def remove_kernel_spec(self, name):
             self.ensure_native_kernel = False
             # Conda environment kernelspec are only virtual, so remove can only be applied
             # on non-virtual kernels.
-            specs = super(CondaKernelSpecManager, self).find_kernel_specs()
+            specs = super().find_kernel_specs()
         finally:
             self.ensure_native_kernel = save_native
         spec_dir = specs[name]

spine_engine/execution_managers/conda_kernel_spec_runner.py

@@ -47,9 +47,7 @@ def exec_in_env(conda_prefix, env_path, *command):
             os.execvp(quoted_command[0], quoted_command)
         else:
             activate = os.path.join(conda_prefix, "bin", "activate")
-            ecomm = ". '{}' '{}' && echo CONDA_PREFIX=$CONDA_PREFIX && exec {}".format(
-                activate, env_path, " ".join(quoted_command)
-            )
+            ecomm = f". '{activate}' '{env_path}' && echo CONDA_PREFIX=$CONDA_PREFIX && exec {' '.join(quoted_command)}"
             ecomm = ["sh" if "bsd" in sys.platform else "bash", "-c", ecomm]
             os.execvp(ecomm[0], ecomm)
 

spine_engine/execution_managers/kernel_execution_manager.py

@@ -69,9 +69,8 @@ def _make_kernel_manager(self, kernel_name, group_id, server_ip, filter_id):
         """
         if not filter_id == "":
             group_id = filter_id  # Ignore group ID in case filter ID exists
-        for k in self._kernel_managers:
+        for km in self._kernel_managers.values():
             # Reuse kernel manager if using same group id and kernel and it's idle
-            km = self._kernel_managers[k]
             if km.group_id() == group_id and km.kernel_name == kernel_name:
                 if not km.is_busy():
                     return km
@@ -101,11 +100,11 @@ def new_kernel_manager(self, kernel_name, group_id, logger, extra_switches=None,
         conda_exe = kwargs.pop("conda_exe", "")
         if environment == "conda":
             if not os.path.exists(conda_exe):
-                logger.msg_kernel_execution.emit(msg=dict(type="conda_not_found"))
+                logger.msg_kernel_execution.emit({"type": "conda_not_found"})
                 self._kernel_managers.pop(self.get_kernel_manager_key(km))
                 return None
             km.kernel_spec_manager = CondaKernelSpecManager(conda_exe=conda_exe)
-        msg = dict(kernel_name=kernel_name)
+        msg = {"kernel_name": kernel_name}
         if not km.is_alive():
             try:
                 if not km.kernel_spec:
@@ -218,7 +217,7 @@ def kill_kernel_managers(self):
         """Shuts down all kernel managers stored in the factory."""
         while True:
             try:
-                key, km = self._kernel_managers.popitem()
+                _, km = self._kernel_managers.popitem()
                 if km.is_alive():
                     km.shutdown_kernel(now=True)
             except KeyError:
@@ -283,7 +282,7 @@ def __init__(
             **kwargs (optional): Keyword arguments passed to ``KernelManager.start_kernel()``
         """
         super().__init__(logger)
-        self._msg_head = dict(kernel_name=kernel_name)
+        self._msg_head = {"kernel_name": kernel_name}
         self._commands = commands
         self._cmd_failed = False
         self.std_out = kwargs["stdout"] = open(os.devnull, "w")
@@ -306,7 +305,7 @@ def run_until_complete(self):
         if self._kill_completed:
             conn_file = self._kernel_manager.connection_file
             shutdown_kernel_manager(conn_file)
-            self._logger.msg_kernel_execution.emit(dict(type="kernel_shutdown", **self._msg_head))
+            self._logger.msg_kernel_execution.emit({"type": "kernel_shutdown", **self._msg_head})
         if self._cmd_failed or not run_succeeded:
             return -1
         return 0
@@ -315,12 +314,12 @@ def _do_run(self):
         try:
             self._kernel_client.wait_for_ready(timeout=self._startup_timeout)
         except RuntimeError as e:
-            msg = dict(type="execution_failed_to_start", error=str(e), **self._msg_head)
+            msg = {"type": "execution_failed_to_start", "error": str(e), **self._msg_head}
             self._logger.msg_kernel_execution.emit(msg)
             self._kernel_client.stop_channels()
             self._kernel_manager.shutdown_kernel(now=True)
             return False
-        msg = dict(type="execution_started", **self._msg_head)
+        msg = {"type": "execution_started", **self._msg_head}
         self._logger.msg_kernel_execution.emit(msg)
         for cmd in self._commands:
             self._cmd_failed = False
@@ -341,7 +340,7 @@ def _output_hook(self, msg):
         elif msg["header"]["msg_type"] == "status":
             # Set kernel manager busy if execution is starting or in progress
             exec_state = msg["content"]["execution_state"]
-            if exec_state == "busy" or exec_state == "starting":
+            if exec_state in {"busy", "starting"}:
                 self._kernel_manager.set_busy(True)
             else:  # exec_state == 'idle'
                 self._kernel_manager.set_busy(False)
@@ -357,4 +356,4 @@ def stop_execution(self):
             if self._kill_completed:
                 conn_file = self._kernel_manager.connection_file
                 shutdown_kernel_manager(conn_file)
-                self._logger.msg_kernel_execution.emit(dict(type="kernel_shutdown", **self._msg_head))
+                self._logger.msg_kernel_execution.emit({"type": "kernel_shutdown", **self._msg_head})

spine_engine/execution_managers/persistent_execution_manager.py

@@ -54,7 +54,7 @@ def __init__(self, args, group_id):
         self._is_running_lock = Lock()
         self._is_running = True
         self._persistent_resources_release_lock = Lock()
-        self._kwargs = dict(stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        self._kwargs = {"stdin": PIPE, "stdout": PIPE, "stderr": PIPE}
         if sys.platform == "win32":
             self._kwargs["creationflags"] = CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW
             # Setup Popen to not show console in frozen app. Another option is to use
@@ -149,7 +149,7 @@ def _log_stdout(self):
         try:
             for line in iter(self._persistent.stdout.readline, b""):
                 data = line.decode("UTF8", "replace").rstrip()
-                self._msg_queue.put(dict(type="stdout", data=data))
+                self._msg_queue.put({"type": "stdout", "data": data})
         except ValueError:
             pass
 
@@ -158,7 +158,7 @@ def _log_stderr(self):
         try:
             for line in iter(self._persistent.stderr.readline, b""):
                 data = line.decode("UTF8", "replace").rstrip()
-                self._msg_queue.put(dict(type="stderr", data=data))
+                self._msg_queue.put({"type": "stderr", "data": data})
         except ValueError:
             pass
 
@@ -464,6 +464,7 @@ def _release_persistent_resources(self):
 
 
 def _send_ctrl_c(pid):
+    # pylint: disable=possibly-used-before-assignment
     kernel = ctypes.windll.kernel32
     kernel.FreeConsole()
     kernel.AttachConsole(pid)
@@ -549,7 +550,7 @@ def __bool__(self):
 
     @staticmethod
     def _emit_persistent_started(logger, key, language):
-        msg = dict(type="persistent_started", key=key, language=language)
+        msg = {"type": "persistent_started", "key": key, "language": language}
         logger.msg_persistent_execution.emit(msg)
 
     def _get_idle_persistent_managers(self):
@@ -602,7 +603,7 @@ def new_persistent_manager(self, constructor, logger, args, group_id):
         try:
             pm = self.persistent_managers[key] = constructor(args, group_id)
         except OSError as err:
-            msg = dict(type="persistent_failed_to_start", args=" ".join(args), error=str(err))
+            msg = {"type": "persistent_failed_to_start", "args": " ".join(args), "error": str(err)}
             logger.msg_persistent_execution.emit(msg)
             return None
         self._emit_persistent_started(logger, key, pm.language)
@@ -658,8 +659,7 @@ def issue_persistent_command(self, key, cmd):
         pm = self.persistent_managers.get(key)
         if pm is None:
             return ()
-        for msg in pm.issue_command(cmd, add_history=True, catch_exception=False):
-            yield msg
+        yield from pm.issue_command(cmd, add_history=True, catch_exception=False)
 
     def is_persistent_command_complete(self, key, cmd):
         """Checks whether a command is complete.
@@ -688,7 +688,7 @@ def get_persistent_completions(self, key, text):
         """
         pm = self.persistent_managers.get(key)
         if pm is None:
-            return
+            return []
         return pm.get_completions(text)
 
     def get_persistent_history_item(self, key, text, prefix, backwards):
@@ -702,7 +702,7 @@ def get_persistent_history_item(self, key, text, prefix, backwards):
         """
         pm = self.persistent_managers.get(key)
         if pm is None:
-            return
+            return ""
         return pm.get_history_item(text, prefix, backwards)
 
     def kill_manager_processes(self):
@@ -824,10 +824,10 @@ def run_until_complete(self):
             return -1
         self._persistent_manager.set_running_until_completion(True)
         try:
-            msg = dict(type="execution_started", args=" ".join(self._args))
+            msg = {"type": "execution_started", "args": " ".join(self._args)}
             self._logger.msg_persistent_execution.emit(msg)
             fmt_alias = "# Running " + self._alias.rstrip()
-            self._logger.msg_persistent_execution.emit(dict(type="stdin", data=fmt_alias))
+            self._logger.msg_persistent_execution.emit({"type": "stdin", "data": fmt_alias})
             for cmd in self._commands:
                 for msg in self._persistent_manager.issue_command(cmd):
                     if msg["type"] != "stdin":

spine_engine/execution_managers/process_execution_manager.py

@@ -54,10 +54,10 @@ def run_until_complete(self):
                     creationflags=cf,
                 )
             except OSError as e:
-                msg = dict(type="execution_failed_to_start", error=str(e), program=self._program)
+                msg = {"type": "execution_failed_to_start", "error": str(e), "program": self._program}
                 self._logger.msg_standard_execution.emit(msg)
                 return 1
-            msg = dict(type="execution_started", program=self._program, args=" ".join(self._args))
+            msg = {"type": "execution_started", "program": self._program, "args": " ".join(self._args)}
             self._logger.msg_standard_execution.emit(msg)
             running = "# Running" + " ".join([self._program] + self._args)
             self._logger.msg_standard_execution.emit({"type": "stdin", "data": running})

spine_engine/jumpster.py

@@ -153,7 +153,7 @@ def execute(self):
         while not self._is_complete() or active_iters:
             # start iterators
             while len(active_iters) < self._max_concurrent:
-                candidate_steps = self._get_steps_to_execute(limit=(self._max_concurrent - len(active_iters)))
+                candidate_steps = self._get_steps_to_execute(limit=self._max_concurrent - len(active_iters))
                 step_by_key.update({step.key: step for step in candidate_steps})
                 # Add all waiting steps
                 candidate_steps += list(waiting.values())
@@ -257,12 +257,9 @@ def execute(self):
                 del active_iters[key]
         errs = {tid: err for tid, err in errors.items() if err}
         if errs:
+            error_list = "\n".join([f"In thread {tid}: {err.to_string()}" for tid, err in errs.items()])
             raise JumpsterThreadError(
-                "During multithread execution errors occurred in threads:\n{error_list}".format(
-                    error_list="\n".join(
-                        ["In thread {tid}: {err}".format(tid=tid, err=err.to_string()) for tid, err in errs.items()]
-                    )
-                ),
+                f"During multithread execution errors occurred in threads:\n{error_list}",
                 thread_error_infos=list(errs.values()),
             )
 

spine_engine/project_item/connection.py

@@ -157,7 +157,7 @@ def make_logger(self, queue):
         self._logger = QueueLogger(queue, self.name, None, {})
 
     def emit_flash(self):
-        self._logger.flash.emit()
+        self._logger.flash.emit("")
 
 
 DEFAULT_ENABLED_FILTER_TYPES = {ALTERNATIVE_FILTER_TYPE: False, SCENARIO_FILTER_TYPE: True}
@@ -590,7 +590,7 @@ class Jump(ConnectionBase):
     """Represents a conditional jump between two project items."""
 
     def __init__(
-        self, source_name, source_position, destination_name, destination_position, condition={}, cmd_line_args=()
+        self, source_name, source_position, destination_name, destination_position, condition=None, cmd_line_args=()
     ):
         """
         Args:
@@ -664,7 +664,7 @@ def _is_python_script_condition_true(self, jump_counter):
                 script_file.seek(0)
                 python = resolve_current_python_interpreter()
                 result = subprocess.run(
-                    [python, "-", *expanded_args], encoding="utf-8", stdin=script_file, capture_output=True
+                    [python, "-", *expanded_args], encoding="utf-8", stdin=script_file, capture_output=True, check=False
                 )
                 if result.stdout:
                     self._logger.msg_proc.emit(result.stdout)

spine_engine/project_item/executable_item_base.py

@@ -179,7 +179,6 @@ def stop_execution(self):
         """Stops executing this item."""
         self._logger.msg.emit(f"Stopping {self._name}")
 
-    # pylint: disable=no-self-use
     def _output_resources_forward(self):
         """Returns output resources for forward execution.
 
@@ -190,7 +189,6 @@ def _output_resources_forward(self):
         """
         return []
 
-    # pylint: disable=no-self-use
     def _output_resources_backward(self):
         """Returns output resources for backward execution.
 

spine_engine/project_item/project_item_resource.py

@@ -360,7 +360,7 @@ def get_source(resource):
     """
     if resource.type_ in _DATABASE_RESOURCE_TYPES:
         return resource.url
-    elif resource.hasfilepath:
+    if resource.hasfilepath:
         return resource.path
     return None
 

spine_engine/server/certificate_creator.py

@@ -42,8 +42,8 @@ def generate_certificates(base_dir):
             os.mkdir(d)
 
         # create new keys in certificates dir
-        server_public_file, server_secret_file = zmq.auth.create_certificates(keys_dir, "server")
-        client_public_file, client_secret_file = zmq.auth.create_certificates(keys_dir, "client")
+        zmq.auth.create_certificates(keys_dir, "server")
+        zmq.auth.create_certificates(keys_dir, "client")
         # move public keys to appropriate directory
         for key_file in os.listdir(keys_dir):
             if key_file.endswith(".key"):
@@ -58,7 +58,6 @@ def main(args):
     if len(args) < 2:
         script_dir = os.path.dirname(os.path.abspath(__file__))
         base_dir = os.path.join(script_dir, "certs")
-        cert_dir = os.path.join(base_dir, "certificates")
         if os.path.exists(base_dir):
             print(f"Directory {base_dir} already exists. Please remove it to recreate certificates.")
             return 0