Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove setting of metadata in Runner #2107

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions metaflow/client/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,13 +269,19 @@ def __init__(
_parent: Optional["MetaflowObject"] = None,
_namespace_check: bool = True,
_current_namespace: Optional[str] = None,
_current_metadata: Optional[str] = None,
):
self._metaflow = Metaflow()
self._parent = _parent
self._path_components = None
self._attempt = attempt
self._current_namespace = _current_namespace or get_namespace()
self._current_metadata = _current_metadata or get_metadata()
self._namespace_check = _namespace_check

# Set the metadata being passed..
metadata(self._current_metadata)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't want to do this at all. Basically, we create a Metaflow object that has self.metadata in it. We want to initialize self.metadata to be the one from self._current_metadata.


# If the current namespace is False, we disable checking for namespace for this
# and all children objects. Not setting namespace_check to False has the consequence
# of preventing access to children objects after the namespace changes
Expand Down Expand Up @@ -394,6 +400,7 @@ def __iter__(self) -> Iterator["MetaflowObject"]:
_current_namespace=(
self._current_namespace if self._namespace_check else None
),
_current_metadata=self._current_metadata,
)
for obj in unfiltered_children
),
Expand Down Expand Up @@ -509,6 +516,7 @@ def __getitem__(self, id: str) -> "MetaflowObject":
_current_namespace=(
self._current_namespace if self._namespace_check else None
),
_current_metadata=self._current_metadata,
)
else:
raise KeyError(id)
Expand Down Expand Up @@ -540,16 +548,17 @@ def _unpickle_284(self, data):
)

def _unpickle_2124(self, data):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need to add another version of course. So we can unpickle all versions of objects.

if len(data) != 4:
if len(data) != 5:
raise MetaflowInternalError(
"Unexpected size of array: {}".format(len(data))
)
pathspec, attempt, ns, namespace_check = data
pathspec, attempt, ns, namespace_check, metadata = data
self.__init__(
pathspec=pathspec,
attempt=attempt,
_namespace_check=namespace_check,
_current_namespace=ns,
_current_metadata=metadata,
)

_UNPICKLE_FUNC = {"2.8.4": _unpickle_284, "2.12.4": _unpickle_2124}
Expand Down Expand Up @@ -579,6 +588,7 @@ def __setstate__(self, state):
attempt=state.get("_attempt", None),
_namespace_check=state.get("_namespace_check", False),
_current_namespace=None,
_current_metadata=None,
)

def __getstate__(self):
Expand All @@ -601,6 +611,7 @@ def __getstate__(self):
self._attempt,
self._current_namespace,
self._namespace_check,
self._current_metadata,
],
}

Expand Down
10 changes: 4 additions & 6 deletions metaflow/runner/metaflow_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from typing import Dict, Iterator, Optional, Tuple

from metaflow import Run, metadata
from metaflow import Run

from .utils import handle_timeout, clear_and_set_os_environ
from .subprocess_manager import CommandManager, SubprocessManager
Expand Down Expand Up @@ -284,11 +284,9 @@ def __get_executing_run(self, tfp_runner_attribute, command_obj):
# Set the environment variables to what they were before the run executed.
clear_and_set_os_environ(self.old_env)

# Set the correct metadata from the runner_attribute file corresponding to this run.
metadata_for_flow = content.get("metadata")
metadata(metadata_for_flow)

run_object = Run(pathspec, _namespace_check=False)
run_object = Run(
pathspec, _namespace_check=False, _current_metadata=content.get("metadata")
)
return ExecutingRun(self, command_obj, run_object)

def run(self, **kwargs) -> ExecutingRun:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is something in deployer which is where I ran into it.

Expand Down
Loading