From 9bc1c348c2f10daac2fb00e1b7007c6f5e589154 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Wed, 18 Sep 2024 03:26:04 +0530 Subject: [PATCH] Add parsing of profiles --- src/esdl4tulipa/db.py | 17 ++++++- src/esdl4tulipa/mapping.py | 11 +++-- src/esdl4tulipa/parser.py | 90 ++++++++++++++++++++++++++++--------- src/esdl4tulipa/profiles.py | 84 ++++++++++++++++++++++++++++++++++ tests/test_db.py | 19 ++++++++ tests/test_mapping.py | 8 ++-- tests/test_parser.py | 16 ++++--- 7 files changed, 210 insertions(+), 35 deletions(-) create mode 100644 src/esdl4tulipa/profiles.py diff --git a/src/esdl4tulipa/db.py b/src/esdl4tulipa/db.py index 7e4b4c4..1acf4b0 100644 --- a/src/esdl4tulipa/db.py +++ b/src/esdl4tulipa/db.py @@ -4,11 +4,26 @@ import duckdb import pandas as pd from .mapping import TAssets +from .profiles import influx_to_tulipa def insert_into(con: duckdb.DuckDBPyConnection, tbl: str, records: tuple[TAssets, ...]): """Insert Tulipa records into DuckDB.""" df = pd.DataFrame(asdict(i) for i in records) # noqa: F841 + return _insert_into(con, tbl, df) + + +def insert_profile_into(con: duckdb.DuckDBPyConnection, tbl: str, df: pd.DataFrame): + """Insert Tulipa records into DuckDB.""" + df = influx_to_tulipa(df) + return _insert_into(con, tbl, df) + + +def _insert_into(con: duckdb.DuckDBPyConnection, tbl: str, df: pd.DataFrame): # FIXME: use prepared statements - con.execute(f"CREATE TABLE {tbl} AS SELECT * FROM df") + tbl_test = con.sql(f"SELECT name FROM (SHOW TABLES) WHERE name = {tbl!r}").df() + if tbl_test.empty: + con.execute(f"CREATE TABLE {tbl} AS SELECT * FROM df") + else: + con.execute(f"INSERT INTO {tbl} SELECT * FROM df") return tbl diff --git a/src/esdl4tulipa/mapping.py b/src/esdl4tulipa/mapping.py index a51b667..5944c04 100644 --- a/src/esdl4tulipa/mapping.py +++ b/src/esdl4tulipa/mapping.py @@ -19,6 +19,10 @@ def unguarded_is_dataclass(_type: Type[T], /) -> bool: return is_dataclass(_type) +# keys for which there is no ESDL equivalent +_keys_not_in_esdl = ("profile", "from_asset", "to_asset") + + @dataclass(unsafe_hash=True) class AssetData: """Base dataclass to represent :ref:`esdl.esdl.EnergyAsset`.""" @@ -26,6 +30,7 @@ class AssetData: name: str = "" id: str = "" active: bool = False + profile: str = "" @classmethod def esdl_key(cls, key: str) -> str: @@ -65,7 +70,7 @@ def esdl_key(cls, key: str) -> str: # noqa: D102 @dataclass(unsafe_hash=True) -class _producer_t(AssetData): # noqa: D101 +class _cost_n_lifetime_t(AssetData): # noqa: D101 investment_cost: float | None = None variable_cost: float | None = None lifetime: float | None = None @@ -84,7 +89,7 @@ def esdl_key(cls, key: str) -> str: # noqa: D102 @dataclass(unsafe_hash=True) -class producer_t(_producer_t): # noqa: D101 +class producer_t(_cost_n_lifetime_t): # noqa: D101 initial_capacity: float | None = None @classmethod @@ -133,7 +138,7 @@ def esdl_key(cls, key: str) -> str: # noqa: D102 @dataclass(unsafe_hash=True) -class flow_t(_producer_t): # noqa: D101 +class flow_t(_cost_n_lifetime_t): # noqa: D101 from_asset: str = "" to_asset: str = "" capacity: float | None = None diff --git a/src/esdl4tulipa/parser.py b/src/esdl4tulipa/parser.py index ea5a159..e7d450e 100644 --- a/src/esdl4tulipa/parser.py +++ b/src/esdl4tulipa/parser.py @@ -1,19 +1,25 @@ """Load and parse an ESDL file.""" import contextlib +from collections.abc import Callable from dataclasses import fields from functools import reduce from io import StringIO from itertools import chain -from typing import Callable +from typing import Any from typing import Generator +from typing import Protocol from typing import TypeAlias from typing import TypeVar from typing import cast +import pandas as pd from esdl import esdl from esdl.esdl_handler import EnergySystemHandler from pyecore.ecore import EOrderedSet from tabulate import tabulate +from .profiles import gen_profile_name +from .profiles import get_influx_profile +from .profiles import get_profiles from .mapping import TAssets from .mapping import asset_types from .mapping import flow_t @@ -80,8 +86,8 @@ def batched( technical term: `cdr` of an edge). Example, the following edges: - - (from, link, to) -> (link, to) - - (from, to₂) -> (to) + - (from, link, to₁) -> (link, to₁) + - (from, to₂) -> (to₂) - (from, ...) are represented as the sequence: (link, to₁, to₂, ...) @@ -222,6 +228,44 @@ def merge_assets(asset1: TAssets, asset2: TAssets, **overrides) -> flow_t: return flow_t(**merged) +class Maker(Protocol): # noqa: D101 + def __call__( # noqa: D102 + self, + from_: esdl.EnergyAsset, + to_: esdl.EnergyAsset, + link: esdl.EnergyAsset | None = None, + ) -> tuple[Any, ...]: ... + + +def make_flow_quartet( + from_: esdl.EnergyAsset, to_: esdl.EnergyAsset, link: esdl.EnergyAsset | None = None +) -> tuple[flow_t, TAssets, TAssets, list[esdl.InfluxDBProfile]]: + """Make a (flow, from_asset, to_asset, [profile]) quartet.""" + from_asset = fill_asset(from_) + to_asset = fill_asset(to_) + if link is None: + if profiles := get_profiles(from_, to_): + name = gen_profile_name(profiles[0]) # assume one + else: + name = "" + flow = merge_assets(from_asset, to_asset, profile=name) + else: + if profiles := get_profiles(from_, link, to_): + name = gen_profile_name(profiles[0]) # assume one + else: + name = "" + # reset 'name' & 'id' for flow + overrides = { + "from_asset": from_.name, + "to_asset": to_.name, + "name": "", + "id": "", + "profile": name, + } + flow = cast(flow_t, fill_asset(link, **overrides)) + return (flow, from_asset, to_asset, profiles if profiles else []) + + def edge_is_allowed(*assets: esdl.EnergyAsset) -> bool: """Check if the asset combination defines a valid Tulipa edge/flow. @@ -251,7 +295,9 @@ def edge_is_allowed(*assets: esdl.EnergyAsset) -> bool: return {"energynetwork"} == _kinds if len(_kinds) < 2 else True -def edge(*assets: esdl.EnergyAsset) -> tuple[flow_t, TAssets, TAssets]: +def edge( + *assets: esdl.EnergyAsset, maker: Maker = make_flow_quartet +) -> tuple[Any, ...]: """Create a Tulipa flow, and assets from ESDL assets. Parameters @@ -259,10 +305,16 @@ def edge(*assets: esdl.EnergyAsset) -> tuple[flow_t, TAssets, TAssets]: *assets: esdl.EnergyAsset Set of ESDL assets to link and convert. + maker: Maker + Function to create the edge. Normally the function should + return a flow quartet: + + tuple[flow_t, TAssets, TAssets, list[esdl.InfluxDBProfile]] + Returns ------- - tuple[TAssets, ...] - Tuple of (flow, from_asset, to_asset) + tuple[Any, ...] + Typically a tuple of (flow, from_asset, to_asset, [profile]) Raises ------ @@ -293,8 +345,7 @@ def edge(*assets: esdl.EnergyAsset) -> tuple[flow_t, TAssets, TAssets]: | esdl.Storage() | esdl.EnergyNetwork() as a2, ) if edge_is_allowed(a1, a2): - from_asset, to_asset = map(fill_asset, assets) - flow = merge_assets(from_asset, to_asset) + return maker(a1, a2) case ( esdl.Producer() | esdl.Conversion() @@ -307,19 +358,11 @@ def edge(*assets: esdl.EnergyAsset) -> tuple[flow_t, TAssets, TAssets]: | esdl.EnergyNetwork() as a2, ) if edge_is_allowed(a1, link, a2): # NOTE: len(kinds(...)) <= 2 to support EnergyNetwork -> EnergyNetwork - from_asset = fill_asset(a1) - to_asset = fill_asset(a2) - flow = cast( - flow_t, - # reset 'name' & 'id' for flow - fill_asset(link, from_asset=a1.name, to_asset=a2.name, name="", id=""), - ) + return maker(a1, a2, link) case _: # NOTE: unhandled case: asset, transport, ..., asset raise ValueError(f"{assets=}: uncharted territory!") - return (flow, from_asset, to_asset) - def itr_nodes( asset: esdl.EnergyAsset, edges: list[esdl.EnergyAsset], depth: int @@ -366,7 +409,7 @@ def itr_nodes( def hop_nodes( asset: esdl.EnergyAsset, edges: list[esdl.EnergyAsset], depth: int = 1 ) -> list[esdl.EnergyAsset]: - """Find all the assets that have an incoming flow from initially provided asset. + """For out flows from the provided asset, find all the destination assets. Parameters ---------- @@ -505,14 +548,19 @@ def parse_graph( return res -def load(path: str) -> tuple[tuple[flow_t, ...], tuple[TAssets, ...]]: +def load( + path: str, +) -> tuple[tuple[flow_t, ...], tuple[TAssets, ...], tuple[pd.DataFrame, ...]]: """Load ESDL file and parse nodes.""" with contextlib.redirect_stdout(StringIO()): ensys = _HANDLER.load_file(path) edges = parse_graph(ensys, find_edges, []) flows: tuple[flow_t] = tuple(edge[0] for edge in edges) - assets: tuple[TAssets, ...] = tuple(set(chain(*(edge[1:] for edge in edges)))) - return flows, assets + assets: tuple[TAssets, ...] = tuple(set(chain(*(edge[1:3] for edge in edges)))) + profiles = tuple( + get_influx_profile(p) for p in set(edge[-1][0] for edge in edges if edge[-1]) + ) # assume only one + return flows, assets, profiles def debug(path: str) -> esdl.EnergySystem: diff --git a/src/esdl4tulipa/profiles.py b/src/esdl4tulipa/profiles.py new file mode 100644 index 0000000..87f7e6e --- /dev/null +++ b/src/esdl4tulipa/profiles.py @@ -0,0 +1,84 @@ +"""Retrieve profiles associated to energy assets.""" + +from itertools import pairwise +import numpy.testing as np_test +import pandas as pd +from esdl import esdl +from esdl.profiles.influxdbprofilemanager import ConnectionSettings +from esdl.profiles.influxdbprofilemanager import InfluxDBProfileManager + + +def influx_to_tulipa(df: pd.DataFrame) -> pd.DataFrame: + """Convert InfluxDB profile to match Tulipa schema.""" + name = df.attrs["profile"] + *_, influx_field = name.rsplit(":", maxsplit=1) + np_test.assert_array_equal(df.columns, ["datetime", influx_field]) + year = df["datetime"].dt.year + name_dt = pd.DataFrame.from_records( + ( + (name, y, n) + for y, count in year.groupby(year).count().items() + for n in range(1, count + 1) + ), + columns=["name", "year", "period"], + ) + return pd.concat([name_dt, df.iloc[:, -1].rename("value")], axis=1) + + +def get_influx_profile(profile: esdl.InfluxDBProfile): + """Read profiles from TNO's EDR as a `pandas.DataFrame`.""" + settings = ConnectionSettings( + host=profile.host, + port=profile.port, + username="", + password="", + database=profile.database, + ssl=True, + verify_ssl=True, + ) + manager = InfluxDBProfileManager(settings=settings) + manager.load_influxdb( + profile.measurement, + [profile.field], + from_datetime=profile.startDate, + to_datetime=profile.endDate, + filters=profile.filters, + ) + df = pd.DataFrame(manager.profile_data_list, columns=manager.profile_header) + df.attrs = {"profile": gen_profile_name(profile)} + return df + + +def _get_profile(port: esdl.InPort | esdl.OutPort) -> esdl.InfluxDBProfile | None: + if len(port.profile) > 0: + profile = port.profile[0] + if "edr" in profile.host.casefold(): + return profile + + +def gen_profile_name(profile: esdl.InfluxDBProfile): + """Generate a profile name as 'measurement:field'.""" + return f"{profile.measurement}:{profile.field}" + + +def get_profiles(*assets: esdl.EnergyAsset) -> list[esdl.InfluxDBProfile]: + """Read profile associated to the edge.""" + if len(assets) == 1: + profiles = [ + _port1.profile[0] for _port1 in assets[0].port if len(_port1.profile) > 0 + ] + else: + nassets = len(assets) + profiles = [] + for i, (a1, a2) in enumerate(pairwise(assets)): + for _port1 in filter(lambda p: isinstance(p, esdl.OutPort), a1.port): + for _port2 in _port1.connectedTo: + if ( + isinstance(_port2, esdl.InPort) + and a2 == _port2.energyasset + and (_prof := _get_profile(_port1)) + ): + profiles.append(_prof) + if (nassets - 2 == i) and (_prof := _get_profile(_port2)): + profiles.append(_prof) + return profiles diff --git a/tests/test_db.py b/tests/test_db.py index 0bdf2c9..01f535d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -3,6 +3,7 @@ import duckdb from esdl4tulipa.db import insert_into +from esdl4tulipa.db import insert_profile_into from esdl4tulipa.parser import load import pytest @@ -13,6 +14,11 @@ def flows_assets(): return load("tests/data/esdl/norse-mythology-good.esdl") +@pytest.fixture +def flows_assets_profiles(): + return load("tests/data/esdl/Tiny-edr-profiles.esdl") + + def test_insert(flows_assets): tblnames = ("flow_table", "asset_table") records = {k: v for k, v in zip(tblnames, flows_assets)} @@ -33,3 +39,16 @@ def test_insert(flows_assets): assert _type == df.dtypes[f.name] else: assert f.type == df.dtypes[f.name] + + +def test_insert_profile(flows_assets_profiles): + *_, profiles = flows_assets_profiles + profiles = profiles[:3] # only use 3, to shorten time + tbl = "profile_table" + con = duckdb.connect() # can have db file as argument + + for profile in profiles: + assert tbl == insert_profile_into(con, tbl, profile) + + res = con.sql(f"SELECT COUNT(*) FROM {tbl}").df() + assert sum(p.shape[0] for p in profiles) == res.iloc[0, 0] diff --git a/tests/test_mapping.py b/tests/test_mapping.py index 20976cd..b335b0b 100644 --- a/tests/test_mapping.py +++ b/tests/test_mapping.py @@ -3,7 +3,8 @@ from dataclasses import dataclass from esdl4tulipa.mapping import AssetData from esdl4tulipa.mapping import asset_types -from esdl4tulipa.mapping import _producer_t +from esdl4tulipa.mapping import _keys_not_in_esdl +from esdl4tulipa.mapping import _cost_n_lifetime_t import pytest import re @@ -49,11 +50,10 @@ def test_esdl_key(kind): try: assert asset.esdl_key(fld) except AssertionError: - if fld in ("from_asset", "to_asset"): - # from & to nodes have no ESDL equivalent + if fld in _keys_not_in_esdl: pass else: raise - if isinstance(asset, _producer_t) and key_re.match(fld): + if isinstance(asset, _cost_n_lifetime_t) and key_re.match(fld): assert esdl_re.match(asset.esdl_key(fld)) diff --git a/tests/test_parser.py b/tests/test_parser.py index 623cb16..6a354fa 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -193,18 +193,22 @@ def test_parse_graph(): @pytest.mark.parametrize( - "fname, nflows, nassets", + "fname, nflows, nassets, nprofiles", [ - ("Tiny.esdl", 4, 5), - ("norse-mythology-good.esdl", 35, 29), - ("vehicle_charging_etc.esdl", 13, 12), + ("Tiny.esdl", 4, 5, 0), + ("norse-mythology-good.esdl", 35, 29, 0), + ("vehicle_charging_etc.esdl", 13, 12, 0), + ("Tiny-edr-profiles.esdl", 4, 5, 1), + ("norse-mythology-edr-profiles.esdl", 35, 29, 5), + ("vehicle_charging_etc_edr_profiles.esdl", 13, 12, 4), ], ) -def test_load(fname, nflows, nassets): +def test_load(fname, nflows, nassets, nprofiles): # TODO: test more meaningful things - flows, assets = load(f"tests/data/esdl/{fname}") + flows, assets, profiles = load(f"tests/data/esdl/{fname}") assert len(flows) == nflows assert len(assets) == nassets + assert len(profiles) == nprofiles @pytest.mark.parametrize(