Add parsing of profiles

src/esdl4tulipa/db.py

@@ -4,11 +4,26 @@
 import duckdb
 import pandas as pd
 from .mapping import TAssets
+from .profiles import influx_to_tulipa
 
 
 def insert_into(con: duckdb.DuckDBPyConnection, tbl: str, records: tuple[TAssets, ...]):
     """Insert Tulipa records into DuckDB."""
     df = pd.DataFrame(asdict(i) for i in records)  # noqa: F841
+    return _insert_into(con, tbl, df)
+
+
+def insert_profile_into(con: duckdb.DuckDBPyConnection, tbl: str, df: pd.DataFrame):
+    """Insert Tulipa records into DuckDB."""
+    df = influx_to_tulipa(df)
+    return _insert_into(con, tbl, df)
+
+
+def _insert_into(con: duckdb.DuckDBPyConnection, tbl: str, df: pd.DataFrame):
     # FIXME: use prepared statements
-    con.execute(f"CREATE TABLE {tbl} AS SELECT * FROM df")
+    tbl_test = con.sql(f"SELECT name FROM (SHOW TABLES) WHERE name = {tbl!r}").df()
+    if tbl_test.empty:
+        con.execute(f"CREATE TABLE {tbl} AS SELECT * FROM df")
+    else:
+        con.execute(f"INSERT INTO {tbl} SELECT * FROM df")
     return tbl

src/esdl4tulipa/mapping.py

@@ -19,13 +19,18 @@ def unguarded_is_dataclass(_type: Type[T], /) -> bool:
     return is_dataclass(_type)
 
 
+# keys for which there is no ESDL equivalent
+_keys_not_in_esdl = ("profile", "from_asset", "to_asset")
+
+
 @dataclass(unsafe_hash=True)
 class AssetData:
     """Base dataclass to represent :ref:`esdl.esdl.EnergyAsset`."""
 
     name: str = ""
     id: str = ""
     active: bool = False
+    profile: str = ""
 
     @classmethod
     def esdl_key(cls, key: str) -> str:
@@ -65,7 +70,7 @@ def esdl_key(cls, key: str) -> str:  # noqa: D102
 
 
 @dataclass(unsafe_hash=True)
-class _producer_t(AssetData):  # noqa: D101
+class _cost_n_lifetime_t(AssetData):  # noqa: D101
     investment_cost: float | None = None
     variable_cost: float | None = None
     lifetime: float | None = None
@@ -84,7 +89,7 @@ def esdl_key(cls, key: str) -> str:  # noqa: D102
 
 
 @dataclass(unsafe_hash=True)
-class producer_t(_producer_t):  # noqa: D101
+class producer_t(_cost_n_lifetime_t):  # noqa: D101
     initial_capacity: float | None = None
 
     @classmethod
@@ -133,7 +138,7 @@ def esdl_key(cls, key: str) -> str:  # noqa: D102
 
 
 @dataclass(unsafe_hash=True)
-class flow_t(_producer_t):  # noqa: D101
+class flow_t(_cost_n_lifetime_t):  # noqa: D101
     from_asset: str = ""
     to_asset: str = ""
     capacity: float | None = None

src/esdl4tulipa/parser.py

@@ -1,19 +1,25 @@
 """Load and parse an ESDL file."""
 
 import contextlib
+from collections.abc import Callable
 from dataclasses import fields
 from functools import reduce
 from io import StringIO
 from itertools import chain
-from typing import Callable
+from typing import Any
 from typing import Generator
+from typing import Protocol
 from typing import TypeAlias
 from typing import TypeVar
 from typing import cast
+import pandas as pd
 from esdl import esdl
 from esdl.esdl_handler import EnergySystemHandler
 from pyecore.ecore import EOrderedSet
 from tabulate import tabulate
+from .profiles import gen_profile_name
+from .profiles import get_influx_profile
+from .profiles import get_profiles
 from .mapping import TAssets
 from .mapping import asset_types
 from .mapping import flow_t
@@ -80,8 +86,8 @@ def batched(
         technical term: `cdr` of an edge).  Example, the following
         edges:
 
-        - (from, link, to) -> (link, to)
-        - (from, to₂) -> (to)
+        - (from, link, to₁) -> (link, to₁)
+        - (from, to₂) -> (to₂)
         - (from, ...)
 
         are represented as the sequence: (link, to₁, to₂, ...)
@@ -222,6 +228,44 @@ def merge_assets(asset1: TAssets, asset2: TAssets, **overrides) -> flow_t:
     return flow_t(**merged)
 
 
+class Maker(Protocol):  # noqa: D101
+    def __call__(  # noqa: D102
+        self,
+        from_: esdl.EnergyAsset,
+        to_: esdl.EnergyAsset,
+        link: esdl.EnergyAsset | None = None,
+    ) -> tuple[Any, ...]: ...
+
+
+def make_flow_quartet(
+    from_: esdl.EnergyAsset, to_: esdl.EnergyAsset, link: esdl.EnergyAsset | None = None
+) -> tuple[flow_t, TAssets, TAssets, list[esdl.InfluxDBProfile]]:
+    """Make a (flow, from_asset, to_asset, [profile]) quartet."""
+    from_asset = fill_asset(from_)
+    to_asset = fill_asset(to_)
+    if link is None:
+        if profiles := get_profiles(from_, to_):
+            name = gen_profile_name(profiles[0])  # assume one
+        else:
+            name = ""
+        flow = merge_assets(from_asset, to_asset, profile=name)
+    else:
+        if profiles := get_profiles(from_, link, to_):
+            name = gen_profile_name(profiles[0])  # assume one
+        else:
+            name = ""
+        # reset 'name' & 'id' for flow
+        overrides = {
+            "from_asset": from_.name,
+            "to_asset": to_.name,
+            "name": "",
+            "id": "",
+            "profile": name,
+        }
+        flow = cast(flow_t, fill_asset(link, **overrides))
+    return (flow, from_asset, to_asset, profiles if profiles else [])
+
+
 def edge_is_allowed(*assets: esdl.EnergyAsset) -> bool:
     """Check if the asset combination defines a valid Tulipa edge/flow.
 
@@ -251,18 +295,26 @@ def edge_is_allowed(*assets: esdl.EnergyAsset) -> bool:
     return {"energynetwork"} == _kinds if len(_kinds) < 2 else True
 
 
-def edge(*assets: esdl.EnergyAsset) -> tuple[flow_t, TAssets, TAssets]:
+def edge(
+    *assets: esdl.EnergyAsset, maker: Maker = make_flow_quartet
+) -> tuple[Any, ...]:
     """Create a Tulipa flow, and assets from ESDL assets.
 
     Parameters
     ----------
     *assets: esdl.EnergyAsset
         Set of ESDL assets to link and convert.
 
+    maker: Maker
+        Function to create the edge.  Normally the function should
+        return a flow quartet:
+
+          tuple[flow_t, TAssets, TAssets, list[esdl.InfluxDBProfile]]
+
     Returns
     -------
-    tuple[TAssets, ...]
-        Tuple of (flow, from_asset, to_asset)
+    tuple[Any, ...]
+        Typically a tuple of (flow, from_asset, to_asset, [profile])
 
     Raises
     ------
@@ -293,8 +345,7 @@ def edge(*assets: esdl.EnergyAsset) -> tuple[flow_t, TAssets, TAssets]:
             | esdl.Storage()
             | esdl.EnergyNetwork() as a2,
         ) if edge_is_allowed(a1, a2):
-            from_asset, to_asset = map(fill_asset, assets)
-            flow = merge_assets(from_asset, to_asset)
+            return maker(a1, a2)
         case (
             esdl.Producer()
             | esdl.Conversion()
@@ -307,19 +358,11 @@ def edge(*assets: esdl.EnergyAsset) -> tuple[flow_t, TAssets, TAssets]:
             | esdl.EnergyNetwork() as a2,
         ) if edge_is_allowed(a1, link, a2):
             # NOTE: len(kinds(...)) <= 2 to support EnergyNetwork -> EnergyNetwork
-            from_asset = fill_asset(a1)
-            to_asset = fill_asset(a2)
-            flow = cast(
-                flow_t,
-                # reset 'name' & 'id' for flow
-                fill_asset(link, from_asset=a1.name, to_asset=a2.name, name="", id=""),
-            )
+            return maker(a1, a2, link)
         case _:
             # NOTE: unhandled case: asset, transport, ..., asset
             raise ValueError(f"{assets=}: uncharted territory!")
 
-    return (flow, from_asset, to_asset)
-
 
 def itr_nodes(
     asset: esdl.EnergyAsset, edges: list[esdl.EnergyAsset], depth: int
@@ -366,7 +409,7 @@ def itr_nodes(
 def hop_nodes(
     asset: esdl.EnergyAsset, edges: list[esdl.EnergyAsset], depth: int = 1
 ) -> list[esdl.EnergyAsset]:
-    """Find all the assets that have an incoming flow from initially provided asset.
+    """For out flows from the provided asset, find all the destination assets.
 
     Parameters
     ----------
@@ -505,14 +548,19 @@ def parse_graph(
     return res
 
 
-def load(path: str) -> tuple[tuple[flow_t, ...], tuple[TAssets, ...]]:
+def load(
+    path: str,
+) -> tuple[tuple[flow_t, ...], tuple[TAssets, ...], tuple[pd.DataFrame, ...]]:
     """Load ESDL file and parse nodes."""
     with contextlib.redirect_stdout(StringIO()):
         ensys = _HANDLER.load_file(path)
     edges = parse_graph(ensys, find_edges, [])
     flows: tuple[flow_t] = tuple(edge[0] for edge in edges)
-    assets: tuple[TAssets, ...] = tuple(set(chain(*(edge[1:] for edge in edges))))
-    return flows, assets
+    assets: tuple[TAssets, ...] = tuple(set(chain(*(edge[1:3] for edge in edges))))
+    profiles = tuple(
+        get_influx_profile(p) for p in set(edge[-1][0] for edge in edges if edge[-1])
+    )  # assume only one
+    return flows, assets, profiles
 
 
 def debug(path: str) -> esdl.EnergySystem:

src/esdl4tulipa/profiles.py

@@ -0,0 +1,84 @@
+"""Retrieve profiles associated to energy assets."""
+
+from itertools import pairwise
+import numpy.testing as np_test
+import pandas as pd
+from esdl import esdl
+from esdl.profiles.influxdbprofilemanager import ConnectionSettings
+from esdl.profiles.influxdbprofilemanager import InfluxDBProfileManager
+
+
+def influx_to_tulipa(df: pd.DataFrame) -> pd.DataFrame:
+    """Convert InfluxDB profile to match Tulipa schema."""
+    name = df.attrs["profile"]
+    *_, influx_field = name.rsplit(":", maxsplit=1)
+    np_test.assert_array_equal(df.columns, ["datetime", influx_field])
+    year = df["datetime"].dt.year
+    name_dt = pd.DataFrame.from_records(
+        (
+            (name, y, n)
+            for y, count in year.groupby(year).count().items()
+            for n in range(1, count + 1)
+        ),
+        columns=["name", "year", "period"],
+    )
+    return pd.concat([name_dt, df.iloc[:, -1].rename("value")], axis=1)
+
+
+def get_influx_profile(profile: esdl.InfluxDBProfile):
+    """Read profiles from TNO's EDR as a `pandas.DataFrame`."""
+    settings = ConnectionSettings(
+        host=profile.host,
+        port=profile.port,
+        username="",
+        password="",
+        database=profile.database,
+        ssl=True,
+        verify_ssl=True,
+    )
+    manager = InfluxDBProfileManager(settings=settings)
+    manager.load_influxdb(
+        profile.measurement,
+        [profile.field],
+        from_datetime=profile.startDate,
+        to_datetime=profile.endDate,
+        filters=profile.filters,
+    )
+    df = pd.DataFrame(manager.profile_data_list, columns=manager.profile_header)
+    df.attrs = {"profile": gen_profile_name(profile)}
+    return df
+
+
+def _get_profile(port: esdl.InPort | esdl.OutPort) -> esdl.InfluxDBProfile | None:
+    if len(port.profile) > 0:
+        profile = port.profile[0]
+        if "edr" in profile.host.casefold():
+            return profile
+
+
+def gen_profile_name(profile: esdl.InfluxDBProfile):
+    """Generate a profile name as 'measurement:field'."""
+    return f"{profile.measurement}:{profile.field}"
+
+
+def get_profiles(*assets: esdl.EnergyAsset) -> list[esdl.InfluxDBProfile]:
+    """Read profile associated to the edge."""
+    if len(assets) == 1:
+        profiles = [
+            _port1.profile[0] for _port1 in assets[0].port if len(_port1.profile) > 0
+        ]
+    else:
+        nassets = len(assets)
+        profiles = []
+        for i, (a1, a2) in enumerate(pairwise(assets)):
+            for _port1 in filter(lambda p: isinstance(p, esdl.OutPort), a1.port):
+                for _port2 in _port1.connectedTo:
+                    if (
+                        isinstance(_port2, esdl.InPort)
+                        and a2 == _port2.energyasset
+                        and (_prof := _get_profile(_port1))
+                    ):
+                        profiles.append(_prof)
+                    if (nassets - 2 == i) and (_prof := _get_profile(_port2)):
+                        profiles.append(_prof)
+    return profiles

tests/test_db.py

@@ -3,6 +3,7 @@
 import duckdb
 
 from esdl4tulipa.db import insert_into
+from esdl4tulipa.db import insert_profile_into
 from esdl4tulipa.parser import load
 
 import pytest
@@ -13,6 +14,11 @@ def flows_assets():
     return load("tests/data/esdl/norse-mythology-good.esdl")
 
 
+@pytest.fixture
+def flows_assets_profiles():
+    return load("tests/data/esdl/Tiny-edr-profiles.esdl")
+
+
 def test_insert(flows_assets):
     tblnames = ("flow_table", "asset_table")
     records = {k: v for k, v in zip(tblnames, flows_assets)}
@@ -33,3 +39,16 @@ def test_insert(flows_assets):
                 assert _type == df.dtypes[f.name]
             else:
                 assert f.type == df.dtypes[f.name]
+
+
+def test_insert_profile(flows_assets_profiles):
+    *_, profiles = flows_assets_profiles
+    profiles = profiles[:3]  # only use 3, to shorten time
+    tbl = "profile_table"
+    con = duckdb.connect()  # can have db file as argument
+
+    for profile in profiles:
+        assert tbl == insert_profile_into(con, tbl, profile)
+
+    res = con.sql(f"SELECT COUNT(*) FROM {tbl}").df()
+    assert sum(p.shape[0] for p in profiles) == res.iloc[0, 0]

tests/test_mapping.py

@@ -3,7 +3,8 @@
 from dataclasses import dataclass
 from esdl4tulipa.mapping import AssetData
 from esdl4tulipa.mapping import asset_types
-from esdl4tulipa.mapping import _producer_t
+from esdl4tulipa.mapping import _keys_not_in_esdl
+from esdl4tulipa.mapping import _cost_n_lifetime_t
 import pytest
 import re
 
@@ -49,11 +50,10 @@ def test_esdl_key(kind):
         try:
             assert asset.esdl_key(fld)
         except AssertionError:
-            if fld in ("from_asset", "to_asset"):
-                # from & to nodes have no ESDL equivalent
+            if fld in _keys_not_in_esdl:
                 pass
             else:
                 raise
 
-        if isinstance(asset, _producer_t) and key_re.match(fld):
+        if isinstance(asset, _cost_n_lifetime_t) and key_re.match(fld):
             assert esdl_re.match(asset.esdl_key(fld))