diff --git a/examples/PoCTutorial/src/run_grow_tutorial.py b/examples/PoCTutorial/src/run_grow_tutorial.py index 6b02e8f42..14d46b56b 100644 --- a/examples/PoCTutorial/src/run_grow_tutorial.py +++ b/examples/PoCTutorial/src/run_grow_tutorial.py @@ -1,6 +1,6 @@ from pathlib import Path -from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType from mesido.esdl.esdl_parser import ESDLFileParser from mesido.workflows import EndScenarioSizingStaged, run_end_scenario_sizing @@ -13,16 +13,17 @@ class EndScenarioSizingStagedHighs(EndScenarioSizingStaged): import time kwargs = { - "write_result_db_profiles": True, + # '"esdl_profiles_output_type"=None' (default) means no profiles to DB + "esdl_profiles_output_type": ESDLOutputProfilesType.INFLUXDB, "database_connections": [ { "access_type": DBAccessType.WRITE, # DBAccessType.READ or DBAccessType.READ_WRITE - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/examples/municipality/src/run_municipality.py b/examples/municipality/src/run_municipality.py index dc529d66d..bc1d35784 100644 --- a/examples/municipality/src/run_municipality.py +++ b/examples/municipality/src/run_municipality.py @@ -16,12 +16,12 @@ "database_connections": [ { "access_type": DBAccessType.READ, # DBAccessType.WRITE or DBAccessType.READ_WRITE - "influxdb_host": "required_user_input", - "influxdb_port": 1234, - "influxdb_username": "required_user_input", - "influxdb_password": "required_user_input", - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "required_user_input", + "port": 1234, + "username": "required_user_input", + "password": "required_user_input", + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/manual_run/testing_bugs/model/Delft_T.esdl b/manual_run/testing_bugs/model/Delft_T.esdl new file mode 100644 index 000000000..6cfc4d91e --- /dev/null +++ b/manual_run/testing_bugs/model/Delft_T.esdl @@ -0,0 +1,697 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/manual_run/testing_bugs/src/run_grow.py b/manual_run/testing_bugs/src/run_grow.py index 06668149a..e89190c25 100644 --- a/manual_run/testing_bugs/src/run_grow.py +++ b/manual_run/testing_bugs/src/run_grow.py @@ -2,7 +2,7 @@ from mesido.esdl.esdl_parser import ESDLFileParser from mesido.workflows import EndScenarioSizingStaged, run_end_scenario_sizing -from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType if __name__ == "__main__": import time @@ -12,35 +12,55 @@ kwargs = { "use_esdl_ranged_constraint": True, # default value in the code is set to False + "esdl_profiles_output_type": ESDLOutputProfilesType.POSTGRESQL, # 'None' (default) means no profiles to DB "database_connections": [ { "access_type": DBAccessType.READ, # or DBAccessType.WRITE or DBAccessType.READ_WRITE - "influxdb_host": "required_user_input", - "influxdb_port": 1234, - "influxdb_username": "required_user_input", - "influxdb_password": "required_user_input", - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "required_user_input", + "port": 1234, + "username": "required_user_input", + "password": "required_user_input", + "ssl": False, + "verify_ssl": False, }, { - "access_type": DBAccessType.WRITE, # or DBAccessType.WRITE or DBAccessType.READ_WRITE - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "access_type": DBAccessType.READ, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, + }, + { + "access_type": DBAccessType.READ, + "host": "omotes_influxdb", + "port": 8096, + "username": "root", + "password": "9012", + "ssl": False, + "verify_ssl": False, + }, + { + "access_type": DBAccessType.READ_WRITE, + "host": "postgres", + "port": 6432, + "username": "root", + "password": "1234", }, - ], } + esdl_input_file = "Delft_T.esdl" solution = run_end_scenario_sizing( EndScenarioSizingStaged, base_folder=base_folder, - esdl_file_name="Base Netwerk Delft.esdl", + esdl_file_name=esdl_input_file, esdl_parser=ESDLFileParser, - # **kwargs, # Example of usage if needed/used + **kwargs, # Example of usage if needed/used ) + with open(base_folder / "model" / esdl_input_file.replace(".esdl", "_GrowOptimized.esdl"), "w", encoding="utf-8") as f: + f.write(solution.optimized_esdl_string) + print("Execution time: " + time.strftime("%M:%S", time.gmtime(time.time() - start_time))) diff --git a/setup.py b/setup.py index 6c879ddfd..f675a8857 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ # < 81.0.0 needed for pandapipes (still to be removed) # < 82.0.0 needed for pkg_resources (used in rtctools) "setuptools <= 80.9.0", - "pyesdl == 26.3", + "pyesdl[profiles] == 26.5", "pandas >= 1.3.1, < 2.0", "casadi-gil-comp == 3.6.7", "StrEnum == 0.4.15", diff --git a/src/mesido/asset_sizing_mixin.py b/src/mesido/asset_sizing_mixin.py index d42d6f632..1ba7f646e 100644 --- a/src/mesido/asset_sizing_mixin.py +++ b/src/mesido/asset_sizing_mixin.py @@ -12,7 +12,7 @@ from mesido.esdl.asset_to_component_base import AssetStateEnum from mesido.esdl.common import Asset from mesido.esdl.esdl_additional_vars_mixin import get_asset_contraints -from mesido.esdl.profile_parser import InfluxDBProfileReader +from mesido.esdl.profile_parser import ESDLProfileReader from mesido.head_loss_class import HeadLossOption from mesido.network_common import NetworkSettings from mesido.pipe_class import CableClass, GasPipeClass, PipeClass @@ -812,7 +812,7 @@ def _make_max_size_var(name, lb, ub, nominal): if ( qty_asset_profile_constraints > 0 and hasattr(asset_profile_constraints[0], "maximum") - and InfluxDBProfileReader._get_profile_quantity_and_unit( + and ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.WATT @@ -1907,7 +1907,7 @@ def __producer_constraints( if ( qty_asset_profile_constraints > 0 and hasattr(asset_profile_constraints[0], "maximum") - and InfluxDBProfileReader._get_profile_quantity_and_unit( + and ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.WATT @@ -1950,16 +1950,16 @@ def __producer_constraints( # profile is specified without units (xlm/csv) qty_asset_profile_constraints == 0 or ( - InfluxDBProfileReader._get_profile_quantity_and_unit( + ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).physicalQuantity == esdl.PhysicalQuantityEnum.COEFFICIENT and ( - InfluxDBProfileReader._get_profile_quantity_and_unit( + ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.PERCENT - or InfluxDBProfileReader._get_profile_quantity_and_unit( + or ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.NONE diff --git a/src/mesido/esdl/esdl_mixin.py b/src/mesido/esdl/esdl_mixin.py index 2a94f780a..791887618 100644 --- a/src/mesido/esdl/esdl_mixin.py +++ b/src/mesido/esdl/esdl_mixin.py @@ -21,7 +21,7 @@ from mesido.esdl.esdl_model_base import _ESDLModelBase from mesido.esdl.esdl_parser import ESDLStringParser from mesido.esdl.esdl_qth_model import ESDLQTHModel -from mesido.esdl.profile_parser import BaseProfileReader, InfluxDBProfileReader +from mesido.esdl.profile_parser import BaseProfileReader, ESDLProfileReader from mesido.physics_mixin import PhysicsMixin from mesido.pipe_class import GasPipeClass, PipeClass from mesido.pycml.pycml_mixin import PyCMLMixin @@ -57,6 +57,17 @@ class DBAccessType(StrEnum): READ_WRITE = "read_write" +class ESDLOutputProfilesType(StrEnum): + """ + Enumeration for ESDL output profiles type + """ + + INFLUXDB = "influxdb" + POSTGRESQL = "postgresql" + TIME_SERIES_PROFILE = "time_series_profile" + DATE_TIME_PROFILE = "date_time_profile" + + class _ESDLInputException(Exception): pass @@ -131,7 +142,7 @@ def __init__(self, *args, **kwargs) -> None: DBAccessType.WRITE: [], } - profile_reader_class = kwargs.get("profile_reader", InfluxDBProfileReader) + profile_reader_class = kwargs.get("profile_reader", ESDLProfileReader) input_file_name = kwargs.get("input_timeseries_file", None) input_folder = kwargs.get("input_folder") input_file_path = None @@ -142,22 +153,24 @@ def __init__(self, *args, **kwargs) -> None: for dbconnection in database_connection_info: if dbconnection["access_type"] != DBAccessType.WRITE: database_host_port = "{}:{}".format( - dbconnection["influxdb_host"], - dbconnection["influxdb_port"], + dbconnection["host"], + dbconnection["port"], ) read_only_dbase_credentials[database_host_port] = ( - dbconnection["influxdb_username"], - dbconnection["influxdb_password"], + dbconnection["username"], + dbconnection["password"], ) if dbconnection["access_type"] != DBAccessType.READ_WRITE: self._database_credentials[dbconnection["access_type"]].append( { - "influxdb_host": dbconnection["influxdb_host"], - "influxdb_port": dbconnection["influxdb_port"], - "influxdb_username": dbconnection["influxdb_username"], - "influxdb_password": dbconnection["influxdb_password"], - "influxdb_ssl": dbconnection["influxdb_ssl"], - "influxdb_verify_ssl": dbconnection["influxdb_verify_ssl"], + "host": dbconnection["host"], + "port": dbconnection["port"], + "username": dbconnection["username"], + "password": dbconnection["password"], + "ssl": dbconnection["ssl"] if "ssl" in dbconnection else False, + "verify_ssl": ( + dbconnection["verify_ssl"] if "verify_ssl" in dbconnection else False + ), } ) elif dbconnection["access_type"] == DBAccessType.READ_WRITE: @@ -165,12 +178,16 @@ def __init__(self, *args, **kwargs) -> None: for rw in both_read_and_write: self._database_credentials[rw].append( { - "influxdb_host": dbconnection["influxdb_host"], - "influxdb_port": dbconnection["influxdb_port"], - "influxdb_username": dbconnection["influxdb_username"], - "influxdb_password": dbconnection["influxdb_password"], - "influxdb_ssl": dbconnection["influxdb_ssl"], - "influxdb_verify_ssl": dbconnection["influxdb_verify_ssl"], + "host": dbconnection["host"], + "port": dbconnection["port"], + "username": dbconnection["username"], + "password": dbconnection["password"], + "ssl": dbconnection["ssl"] if "ssl" in dbconnection else False, + "verify_ssl": ( + dbconnection["verify_ssl"] + if "verify_ssl" in dbconnection + else False + ), } ) else: diff --git a/src/mesido/esdl/profile_parser.py b/src/mesido/esdl/profile_parser.py index 69fd2e37e..5600fd875 100644 --- a/src/mesido/esdl/profile_parser.py +++ b/src/mesido/esdl/profile_parser.py @@ -3,11 +3,11 @@ import sys from collections import defaultdict from pathlib import Path -from typing import Dict, Optional, Set, Tuple +from typing import Dict, Optional, Set, Tuple, Union import esdl -from esdl.profiles.influxdbprofilemanager import ConnectionSettings -from esdl.profiles.influxdbprofilemanager import InfluxDBProfileManager +from esdl.profiles.credentials import Credentials +from esdl.profiles.profile_utils import close_db_connections, load_profile_data_and_header from esdl.units.conversion import ENERGY_IN_J, POWER_IN_W, convert_to_unit from mesido.esdl.common import Asset @@ -241,7 +241,7 @@ def _load_profiles_from_source( raise NotImplementedError -class InfluxDBProfileReader(BaseProfileReader): +class ESDLProfileReader(BaseProfileReader): asset_type_to_variable_name_conversion = { esdl.esdl.HeatingDemand: ".target_heat_demand", esdl.esdl.GenericConsumer: ".target_heat_demand", @@ -255,6 +255,18 @@ class InfluxDBProfileReader(BaseProfileReader): esdl.esdl.PVInstallation: ".maximum_electricity_source", } + supported_profiles = ( + esdl.DataTableProfile, + esdl.InfluxDBProfile, + esdl.TimeSeriesProfile, + esdl.DateTimeProfile, + esdl.ProfileReference, + ) + + # Python 3.10 compatible + SupportedProfilesType = Union.__getitem__(tuple(supported_profiles)) + # SupportedProfilesType = Union[*supported_profiles] + def __init__( self, energy_system: esdl.EnergySystem, @@ -267,10 +279,9 @@ def __init__( file_path=file_path, use_esdl_ranged_contraint=use_esdl_ranged_contraint, ) - self._df = pd.DataFrame() - self._database_credentials = ( - database_credentials if database_credentials is not None else {"": ("", "")} - ) + if database_credentials is not None: + for cred_id, (username, password) in database_credentials.items(): + Credentials.add_credential(cred_id, username, password) def _load_profiles_from_source( self, @@ -289,7 +300,7 @@ def _load_profiles_from_source( unique_profiles_attributes = [] # a list containning lists of attributes unique_series = [] for profile in [ - x for x in self._energy_system.eAllContents() if isinstance(x, esdl.InfluxDBProfile) + x for x in self._energy_system.eAllContents() if isinstance(x, self.supported_profiles) ]: if profile.profileType == esdl.ProfileTypeEnum.OUTPUT: continue @@ -334,11 +345,13 @@ def _load_profiles_from_source( f"specified to be loaded for each asset covers exactly the " f"same timeseries. " ) + close_db_connections() + # Loop trough all the requried profiles in the energy system and assign the profile data: # - series: use the unique series data, without reading from the database again # - other profile info: get it from the specific profile for profile in [ - x for x in self._energy_system.eAllContents() if isinstance(x, esdl.InfluxDBProfile) + x for x in self._energy_system.eAllContents() if isinstance(x, self.supported_profiles) ]: if profile.profileType == esdl.ProfileTypeEnum.OUTPUT: continue @@ -412,7 +425,7 @@ def _load_profiles_from_source( self._profiles[idx] = profiles.copy() # @staticmethod - def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) -> pd.Series: + def _load_profile_timeseries_from_database(self, profile: SupportedProfilesType) -> pd.Series: """ Function to load the profiles from an InfluxDB. Returns a timeseries with the data for the asset. @@ -429,54 +442,25 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) # to avoid circular import issue from mesido.workflows.utils.error_types import NetworkErrors, potential_error_to_error - if profile.id in self._df: - return self._df[profile.id] - - profile_host = profile.host - - ssl_setting = False - if "https" in profile_host: - profile_host = profile_host[8:] - ssl_setting = True - elif "http" in profile_host: - profile_host = profile_host[7:] - if profile.port == 443: - ssl_setting = True - influx_host = "{}:{}".format(profile_host, profile.port) - - username, password = self._database_credentials.get(influx_host, (None, None)) - - conn_settings = ConnectionSettings( - host=profile.host, - port=profile.port, - username=username, - password=password, - database=profile.database, - ssl=ssl_setting, - verify_ssl=ssl_setting, - ) - try: - time_series_data = InfluxDBProfileManager(conn_settings) - except Exception: + profile_raw_data, _ = load_profile_data_and_header( + profile, + enable_cache=True, + apply_multiplier=False, + close_connection_after_load=False, + ) + except Exception as e: container = profile.eContainer() asset = container.energyasset get_potential_errors().add_potential_issue( MesidoAssetIssueType.ASSET_PROFILE_AVAILABILITY, asset.id, - f"Asset named {asset.name}: Database {profile.database}" - f" is not available in the host.", + f"Error retrieving profile for asset '{asset.name}' from host '{profile.host}'" + f" and database '{profile.database}': {str(e)}.", ) potential_error_to_error(NetworkErrors.HEAT_NETWORK_ERRORS) - time_series_data.load_influxdb( - profile.measurement, - [profile.field], - profile.startDate, - profile.endDate, - ) - - if not time_series_data.profile_data_list: # if time_series_data.profile_data_list == []: + if not profile_raw_data: # if time_series_data.profile_data_list == []: container = profile.eContainer() asset = container.energyasset get_potential_errors().add_potential_issue( @@ -488,15 +472,15 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) potential_error_to_error(NetworkErrors.HEAT_NETWORK_ERRORS) - for x in time_series_data.profile_data_list: + for x in profile_raw_data: if len(x) != 2: raise RuntimeError( "InfluxDB profile currently only supports parsing exactly one " "profile for each asset" ) - x_0, x_1 = map(list, zip(*time_series_data.profile_data_list)) - if not time_series_data.profile_data_list[0][0].tzinfo: + x_0, x_1 = map(list, zip(*profile_raw_data)) + if not profile_raw_data[0][0].tzinfo: index = pd.DatetimeIndex( data=x_0, tz=datetime.timezone.utc, @@ -507,7 +491,6 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) data = x_1 series = pd.Series(data=data, index=index) - self._df[profile.id] = series return series diff --git a/src/mesido/util.py b/src/mesido/util.py index 6e7c2547f..5028bc99a 100644 --- a/src/mesido/util.py +++ b/src/mesido/util.py @@ -1,5 +1,5 @@ from mesido.esdl.esdl_parser import BaseESDLParser -from mesido.esdl.profile_parser import BaseProfileReader, InfluxDBProfileReader +from mesido.esdl.profile_parser import BaseProfileReader, ESDLProfileReader from rtctools.optimization.collocated_integrated_optimization_problem import ( CollocatedIntegratedOptimizationProblem, @@ -13,7 +13,7 @@ def run_esdl_mesido_optimization( esdl_file_name: str = "", base_folder: str = "", esdl_string: str = "", - profile_reader: BaseProfileReader = InfluxDBProfileReader, + profile_reader: BaseProfileReader = ESDLProfileReader, input_timeseries_file: str = "", *args, **kwargs, diff --git a/src/mesido/workflows/gas_elect_workflow.py b/src/mesido/workflows/gas_elect_workflow.py index 946aa1caf..2c10fc729 100644 --- a/src/mesido/workflows/gas_elect_workflow.py +++ b/src/mesido/workflows/gas_elect_workflow.py @@ -196,16 +196,16 @@ def main(runinfo_path, log_level): logger.info("Gas and electricity workflow") kwargs = { - "write_result_db_profiles": False, + "esdl_profiles_output_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/src/mesido/workflows/grow_workflow.py b/src/mesido/workflows/grow_workflow.py index 6582eac12..231000aaf 100644 --- a/src/mesido/workflows/grow_workflow.py +++ b/src/mesido/workflows/grow_workflow.py @@ -875,16 +875,16 @@ def main(runinfo_path, log_level): logger.info("Run Scenario Sizing") kwargs = { - "write_result_db_profiles": False, + "esdl_profiles_output_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/src/mesido/workflows/io/write_output.py b/src/mesido/workflows/io/write_output.py index 4fd1ae730..96e280b8f 100644 --- a/src/mesido/workflows/io/write_output.py +++ b/src/mesido/workflows/io/write_output.py @@ -11,14 +11,19 @@ from typing import Dict, Union import esdl -from esdl.profiles.influxdbprofilemanager import ConnectionSettings -from esdl.profiles.influxdbprofilemanager import InfluxDBProfileManager -from esdl.profiles.profilemanager import ProfileManager +from esdl.profiles.datatableprofilemanager import DataTableProfileManager +from esdl.profiles.profile_utils import ( + append_values_to_date_time_profile, + create_data_table_profile, + create_date_time_profile, + create_time_series_profile, + save_data_table_profiles_to_database, +) import mesido.esdl.esdl_parser from mesido.constants import GRAVITATIONAL_CONSTANT from mesido.esdl.edr_pipe_class import EDRPipeClass -from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType from mesido.financial_mixin import calculate_annuity_factor from mesido.network_common import NetworkSettings from mesido.post_processing.post_processing_utils import pipe_pressure, pipe_velocity @@ -33,6 +38,8 @@ logger = logging.getLogger("mesido") +TIME_STEP_SECONDS_TIMESERIESPROFILE = 3600 + class ScenarioOutput: __optimized_energy_system_handler = None @@ -42,24 +49,29 @@ def __init__(self, **kwargs): self.model_folder = kwargs.get("model_folder") self.output_folder = kwargs.get("output_folder") self.esdl_file_name = kwargs.get("esdl_file_name", "ESDL_file.esdl") - # Settings for influxdb when writing out result profile data to it + # Settings for DataTableProfile when writing out result profile data to it # Default settings - self.write_result_db_profiles = False - self.influxdb_username = None - self.influxdb_password = None + self.username = None + self.password = None + self.esdl_profiles_output_type = kwargs.get("esdl_profiles_output_type", None) + self.write_esdl_profiles_to_db = self.esdl_profiles_output_type in [ + ESDLOutputProfilesType.POSTGRESQL, + ESDLOutputProfilesType.INFLUXDB, + ] base_error_string = "Missing influxdb setting for writing result profile data:" try: - self.write_result_db_profiles = kwargs["write_result_db_profiles"] - - if self.write_result_db_profiles not in [True, False]: + if self.esdl_profiles_output_type is not None and not isinstance( + self.esdl_profiles_output_type, ESDLOutputProfilesType + ): logger.error( - "Current setting of write_result_db_profiles is: " - f"{self.write_result_db_profiles} and it should be set to True or False" + "Current setting of esdl_profiles_output_type is: " + f"{self.esdl_profiles_output_type} and it should be None " + "or a value of ESDLOutputProfilesType" ) sys.exit(1) - if self.write_result_db_profiles: + if self.esdl_profiles_output_type is not None: database_connection_write = self._database_credentials.get(DBAccessType.WRITE, []) if len(database_connection_write) == 0: @@ -74,10 +86,10 @@ def __init__(self, **kwargs): else: database_connection_input = database_connection_write[0] try: - self.influxdb_host = database_connection_input["influxdb_host"] - if len(self.influxdb_host) == 0: + self.host = database_connection_input["host"] + if len(self.host) == 0: logger.error( - "Current setting of influxdb_host is an empty string and it should" + "Current setting of host is an empty string and it should" " be the name of the host" ) sys.exit(1) @@ -85,43 +97,43 @@ def __init__(self, **kwargs): logger.error(f"{base_error_string} host") sys.exit(1) try: - self.influxdb_port = database_connection_input["influxdb_port"] - if not isinstance(self.influxdb_port, int): + self.port = database_connection_input["port"] + if not isinstance(self.port, int): logger.error( - "Current setting of influxdb_port is: " - f"{self.influxdb_port} and it should be set to int value (port number)" + "Current setting of port is: " + f"{self.port} and it should be set to int value (port number)" ) sys.exit(1) except KeyError: logger.error(f"{base_error_string} port") sys.exit(1) try: - self.influxdb_username = database_connection_input["influxdb_username"] + self.username = database_connection_input["username"] except KeyError: logger.error(f"{base_error_string} username") sys.exit(1) try: - self.influxdb_password = database_connection_input["influxdb_password"] + self.password = database_connection_input["password"] except KeyError: logger.error(f"{base_error_string} password") sys.exit(1) try: - self.influxdb_ssl = database_connection_input["influxdb_ssl"] - if self.influxdb_ssl not in [True, False]: + self.ssl = database_connection_input["ssl"] + if self.ssl not in [True, False]: logger.error( - "Current setting of influxdb_ssl is: " - f"{self.influxdb_ssl} and it should be set to True or False" + "Current setting of ssl is: " + f"{self.ssl} and it should be set to True or False" ) sys.exit(1) except KeyError: logger.error(f"{base_error_string} ssl") sys.exit(1) try: - self.influxdb_verify_ssl = database_connection_input["influxdb_verify_ssl"] - if self.influxdb_verify_ssl not in [True, False]: + self.verify_ssl = database_connection_input["verify_ssl"] + if self.verify_ssl not in [True, False]: logger.error( - "Current setting of influxdb_verify_ssl is: " - f"{self.influxdb_verify_ssl} and it should be set to True or False" + "Current setting of verify_ssl is: " + f"{self.verify_ssl} and it should be set to True or False" ) sys.exit(1) except KeyError: @@ -134,6 +146,35 @@ def __init__(self, **kwargs): def get_optimized_esh(self): return self.__optimized_energy_system_handler + def _resample_profile_data_to_fixed_timestep(self, profile_data): + if not profile_data: + return [] + + sorted_profile_data = sorted(profile_data, key=lambda row: row[0]) + step = datetime.timedelta(seconds=TIME_STEP_SECONDS_TIMESERIESPROFILE) + + start_datetime = sorted_profile_data[0][0] + end_datetime = sorted_profile_data[-1][0] + duration_seconds = (end_datetime - start_datetime).total_seconds() + n_steps = int(np.ceil(duration_seconds / TIME_STEP_SECONDS_TIMESERIESPROFILE)) + 1 + + values = [] + row_index = 0 + latest_value = sorted_profile_data[0][1] + + for step_index in range(n_steps): + target_datetime = start_datetime + step_index * step + while ( + row_index + 1 < len(sorted_profile_data) + and sorted_profile_data[row_index + 1][0] <= target_datetime + ): + row_index += 1 + latest_value = sorted_profile_data[row_index][1] + + values.append(latest_value) + + return values + def _write_html_output(self, template_name="mpc_buffer_sizing_output"): from jinja2 import Environment, FileSystemLoader @@ -505,7 +546,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -532,7 +574,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -558,7 +601,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -571,10 +615,12 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) distribution=esdl.StringLabelDistribution( stringItem=[ esdl.StringItem( - label="Installation", value=tot_timehorizon_install_cost_euro + label="Installation", + value=tot_timehorizon_install_cost_euro, ), esdl.StringItem( - label="Investment", value=tot_timehorizon_invest_cost_euro + label="Investment", + value=tot_timehorizon_invest_cost_euro, ), esdl.StringItem( label="Variable OPEX", @@ -587,7 +633,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -603,7 +650,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -619,7 +667,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -636,7 +685,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -651,7 +701,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.ENERGY, unit=esdl.UnitEnum.WATTHOUR + physicalQuantity=esdl.PhysicalQuantityEnum.ENERGY, + unit=esdl.UnitEnum.WATTHOUR, ), ) ) @@ -788,7 +839,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) 100.0, ) estimated_energy_from_regional_source_perc[subarea.name] = min( - (100.0 - estimated_energy_from_local_source_perc[subarea.name]), 100.0 + (100.0 - estimated_energy_from_local_source_perc[subarea.name]), + 100.0, ) else: estimated_energy_from_local_source_perc[subarea.name] = 0.0 @@ -870,7 +922,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) kpis.kpi.append( esdl.DoubleKPI( value=round( - estimated_energy_from_regional_source_perc[subarea.name], 1 + estimated_energy_from_regional_source_perc[subarea.name], + 1, ), name="Estimated energy from regional source(s) [%]", quantityAndUnit=esdl.esdl.QuantityAndUnitType( @@ -881,7 +934,10 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ) kpis.kpi.append( esdl.DoubleKPI( - value=round(total_energy_consumed_locally_wh[subarea.name] / 1.0e9, 1), + value=round( + total_energy_consumed_locally_wh[subarea.name] / 1.0e9, + 1, + ), name="Total energy consumed [GWh]", quantityAndUnit=esdl.esdl.QuantityAndUnitType( physicalQuantity=esdl.PhysicalQuantityEnum.ENERGY, @@ -929,7 +985,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) stringItem=polygon_area_string_item ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -1070,7 +1127,12 @@ def _write_updated_esdl( logger.warning(f"ESDL update: asset {asset_id} has not been updated") # Pipes: - edr_pipe_properties_to_copy = ["innerDiameter", "outerDiameter", "diameter", "material"] + edr_pipe_properties_to_copy = [ + "innerDiameter", + "outerDiameter", + "diameter", + "material", + ] esh_edr = EnergySystemHandler() @@ -1135,20 +1197,10 @@ def _write_updated_esdl( # - Fields: profile value for the specific variable # - Tags used as filters: simulationRun, assetClass, assetName, assetId, capability - if self.write_result_db_profiles: - logger.info("Writing asset result profile data to influxDB") + if self.esdl_profiles_output_type is not None: + logger.info(f"Writing asset results to profile type '{self.esdl_profiles_output_type}'") results = self.extract_results() - influxdb_conn_settings = ConnectionSettings( - host=self.influxdb_host, - port=self.influxdb_port, - username=self.influxdb_username, - password=self.influxdb_password, - database=output_energy_system_id, - ssl=self.influxdb_ssl, - verify_ssl=self.influxdb_verify_ssl, - ) - capabilities = [ esdl.Transport, esdl.Conversion, @@ -1319,10 +1371,6 @@ def _write_updated_esdl( for v in var_pops: variables_two_hydraulic_system.remove(v) - profiles = ProfileManager() - profiles.profile_type = "DATETIME_LIST" - profiles.profile_header = ["datetime"] # + general_headers - # Get index of outport which will be used to assign the profile data to index_outport = -1 for ip in range(len(asset.port)): @@ -1343,14 +1391,9 @@ def _write_updated_esdl( ) sys.exit(1) + asset_esdl_output_profiles = {} + asset_esdl_output_profiles_data_dict = {} for ii in range(len(self.times())): - if not self.io.datetimes[ii].tzinfo: - data_row = [ - self.io.datetimes[ii].replace(tzinfo=datetime.timezone.utc) - ] - else: - data_row = [self.io.datetimes[ii]] - try: # For all components dealing with one hydraulic system if isinstance( @@ -1374,10 +1417,16 @@ def _write_updated_esdl( traceback.print_exc() sys.exit(1) - for variable in variables_names: + for variable_name in variables_names: + if not self.io.datetimes[ii].tzinfo: + data_row = [ + self.io.datetimes[ii].replace(tzinfo=datetime.timezone.utc) + ] + else: + data_row = [self.io.datetimes[ii]] + if ii == 0: - # Set header for each column - profiles.profile_header.append(variable) + asset_esdl_output_profiles_data_dict[variable_name] = [] # Set profile database attributes for the esdl asset if not self.io.datetimes[0].tzinfo: start_date_time = self.io.datetimes[0].replace( @@ -1386,7 +1435,7 @@ def _write_updated_esdl( logger.warning( f"No timezone specified for the output profile: " f"default UTC has been used for asset {asset_name} " - f"variable {variable}" + f"variable {variable_name}" ) else: start_date_time = self.io.datetimes[0] @@ -1397,81 +1446,115 @@ def _write_updated_esdl( else: end_date_time = self.io.datetimes[-1] - profile_attributes = esdl.InfluxDBProfile( - database=output_energy_system_id, - measurement=carrier_id, - field=profiles.profile_header[-1], - port=self.influxdb_port, - host=self.influxdb_host, - startDate=start_date_time, - endDate=end_date_time, - id=str(uuid.uuid4()), - filters='"assetId"=' + f"'{str(asset_id)}'", - profileType=esdl.ProfileTypeEnum.OUTPUT, - ) # Assign quantity and units variable - if variable in ["Heat_flow", "Pump_power"]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.POWER, - unit=esdl.UnitEnum.WATT, - multiplier=esdl.MultiplierEnum.NONE, - ) + if variable_name in ["Heat_flow", "Pump_power"]: + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.POWER, + unit=esdl.UnitEnum.WATT, + multiplier=esdl.MultiplierEnum.NONE, ) - elif variable in [ + + elif variable_name in [ f"{commodity}In.H", f"Primary.{commodity}In.H", f"Secondary.{commodity}In.H", ]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.PRESSURE, - unit=esdl.UnitEnum.PASCAL, - multiplier=esdl.MultiplierEnum.NONE, - ) + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.PRESSURE, + unit=esdl.UnitEnum.PASCAL, + multiplier=esdl.MultiplierEnum.NONE, ) - elif variable in [ + + elif variable_name in [ f"{commodity}In.Q", f"Primary.{commodity}In.Q", f"Secondary.{commodity}In.Q", ]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.FLOW, - unit=esdl.UnitEnum.CUBIC_METRE, - perTimeUnit=esdl.TimeUnitEnum.SECOND, - multiplier=esdl.MultiplierEnum.NONE, - ) + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.FLOW, + unit=esdl.UnitEnum.CUBIC_METRE, + perTimeUnit=esdl.TimeUnitEnum.SECOND, + multiplier=esdl.MultiplierEnum.NONE, ) - elif variable in ["PostProc.Velocity"]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.SPEED, - unit=esdl.UnitEnum.METRE, - perTimeUnit=esdl.TimeUnitEnum.SECOND, - multiplier=esdl.MultiplierEnum.NONE, - ) + + elif variable_name in ["PostProc.Velocity"]: + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.SPEED, + unit=esdl.UnitEnum.METRE, + perTimeUnit=esdl.TimeUnitEnum.SECOND, + multiplier=esdl.MultiplierEnum.NONE, ) + else: logger.warning( f"No profile units will be written to the ESDL for: " - f"{asset_id}. + {variable}" + f"{asset_id}. + {variable_name}" ) # Write the source of profiles (Optimizer) - profile_attributes.dataSource = esdl.esdl.DataSource( + data_source = esdl.DataSource( id=str(uuid.uuid4()), name="Optimizer", description="This was created in the optimizer", type=esdl.DataSourceTypeEnum.MODEL, ) - - # Write result OUTPUT profiles on the optimized esdl - asset.port[index_outport].profile.append(profile_attributes) + if self.write_esdl_profiles_to_db: + if ( + self.esdl_profiles_output_type + == ESDLOutputProfilesType.POSTGRESQL + ): + db_type = esdl.DatabaseTypeEnum.POSTGRESQL + else: + db_type = esdl.DatabaseTypeEnum.INFLUXDB + esdl_profile = create_data_table_profile( + es=energy_system, + database_name=output_energy_system_id, + table_name=carrier_id, + column_name=variable_name, + start_date=start_date_time, + end_date=end_date_time, + db_host=self.host, + db_port=self.port, + filter='"assetId"=' + f"'{str(asset_id)}'", + db_type=db_type, + profile_type=esdl.ProfileTypeEnum.OUTPUT, + quantity_and_unit_type=quantity_and_unit, + data_source=data_source, + ) + asset_esdl_output_profiles[variable_name] = ( + DataTableProfileManager(esdl_profile) + ) + elif ( + self.esdl_profiles_output_type + == ESDLOutputProfilesType.TIME_SERIES_PROFILE + ): + esdl_profile = create_time_series_profile( + es=energy_system, + name=variable_name, + start_date=start_date_time, + timestep_in_seconds=TIME_STEP_SECONDS_TIMESERIESPROFILE, + values=[], # fill later + profile_type=esdl.ProfileTypeEnum.OUTPUT, + quantity_and_unit_type=quantity_and_unit, + data_source=data_source, + ) + asset_esdl_output_profiles[variable_name] = esdl_profile + else: # ESDLOutputProfilesType.DATE_TIME_PROFILE + esdl_profile = create_date_time_profile( + es=energy_system, + name=variable_name, + datetime_and_values=[], # fill later + profile_type=esdl.ProfileTypeEnum.OUTPUT, + quantity_and_unit_type=quantity_and_unit, + data_source=data_source, + ) + asset_esdl_output_profiles[variable_name] = esdl_profile + # Write result OUTPUT profiles on the optimized esdl + asset.port[index_outport].profile.append(esdl_profile) # Add variable values in new column conversion_factor = 0.0 - if variable in [ + if variable_name in [ f"{commodity}In.H", f"Primary.{commodity}In.H", f"Secondary.{commodity}In.H", @@ -1479,24 +1562,29 @@ def _write_updated_esdl( conversion_factor = GRAVITATIONAL_CONSTANT * 988.0 else: conversion_factor = 1.0 - if variable not in ["PostProc.Velocity", "PostProc.Pressure"]: + if variable_name not in [ + "PostProc.Velocity", + "PostProc.Pressure", + ]: data_row.append( - results[f"{asset_id}." + variable][ii] * conversion_factor + results[f"{asset_id}." + variable_name][ii] + * conversion_factor ) + # The variable evaluation below seems unnecessary, but it would be # used we expand the list of post process type variables - elif variable in ["PostProc.Velocity", "PostProc.Pressure"]: - data_row.append(post_processed[variable][ii]) + elif variable_name in [ + "PostProc.Velocity", + "PostProc.Pressure", + ]: + data_row.append(post_processed[variable_name][ii]) - profiles.profile_data_list.append(data_row) - # end time steps - profiles.num_profile_items = len(profiles.profile_data_list) - profiles.start_datetime = profiles.profile_data_list[0][0] - profiles.end_datetime = profiles.profile_data_list[-1][0] + if self.write_esdl_profiles_to_db: + asset_esdl_output_profiles[ + variable_name + ].profile_data_list.append(data_row) - influxdb_profile_manager = InfluxDBProfileManager( - influxdb_conn_settings, profiles - ) + asset_esdl_output_profiles_data_dict[variable_name].append(data_row) optim_simulation_tag = { "simulationRun": simulation_id, @@ -1506,11 +1594,31 @@ def _write_updated_esdl( "assetClass": asset_class, "capability": capability, } - _ = influxdb_profile_manager.save_influxdb( - measurement=carrier_id, - field_names=influxdb_profile_manager.profile_header[1:], - tags=optim_simulation_tag, - ) + for ( + variable_name, + profile_data, + ) in asset_esdl_output_profiles_data_dict.items(): + if self.write_esdl_profiles_to_db: + asset_esdl_output_profiles[variable_name].profile_data_list = ( + profile_data + ) + elif ( + self.esdl_profiles_output_type + == ESDLOutputProfilesType.TIME_SERIES_PROFILE + ): + asset_esdl_output_profiles[variable_name].values.extend( + self._resample_profile_data_to_fixed_timestep(profile_data) + ) + else: # ESDLOutputProfilesType.DATE_TIME_PROFILE + append_values_to_date_time_profile( + asset_esdl_output_profiles[variable_name], + profile_data, + ) + if self.write_esdl_profiles_to_db: + save_data_table_profiles_to_database( + list(asset_esdl_output_profiles.values()), + optim_simulation_tag, + ) # -- Test tags -- # do not delete - to be used in test case # prof_loaded_from_influxdb = InfluxDBProfileManager(influxdb_conn_settings) diff --git a/src/mesido/workflows/multicommodity_simulator_workflow.py b/src/mesido/workflows/multicommodity_simulator_workflow.py index 58228bbb1..2bf777d0a 100644 --- a/src/mesido/workflows/multicommodity_simulator_workflow.py +++ b/src/mesido/workflows/multicommodity_simulator_workflow.py @@ -824,16 +824,16 @@ def main(runinfo_path, log_level): logger.info("Run Network Simulator") kwargs = { - "write_result_db_profiles": False, + "esdl_profiles_output_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/src/mesido/workflows/simulator_workflow.py b/src/mesido/workflows/simulator_workflow.py index 7e3d2f8e7..dd6149636 100644 --- a/src/mesido/workflows/simulator_workflow.py +++ b/src/mesido/workflows/simulator_workflow.py @@ -354,16 +354,16 @@ def main(runinfo_path, log_level): logger.info("Run Network Simulator") kwargs = { - "write_result_db_profiles": False, + "esdl_profiles_output_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/tests/test_head_loss.py b/tests/test_head_loss.py index 79f06af40..458c78c12 100644 --- a/tests/test_head_loss.py +++ b/tests/test_head_loss.py @@ -81,16 +81,16 @@ def energy_system_options(self): # Do not delete kwargs: this is used to manualy check writing out of profile data kwargs = { - "write_result_db_profiles": False, + "esdl_profiles_output_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/tests/test_multiple_in_and_out_port_components.py b/tests/test_multiple_in_and_out_port_components.py index fba151fea..4ebcbc6aa 100644 --- a/tests/test_multiple_in_and_out_port_components.py +++ b/tests/test_multiple_in_and_out_port_components.py @@ -52,16 +52,16 @@ def energy_system_options(self): # Do not delete kwargs: this is used to manualy check writing out of profile data kwargs = { - "write_result_db_profiles": False, + "esdl_profiles_output_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } @@ -355,16 +355,16 @@ def energy_system_options(self): # Do not delete kwargs: this is used to manualy check writing out of profile data kwargs = { - "write_result_db_profiles": False, + "esdl_profiles_output_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/tests/test_potential_errors.py b/tests/test_potential_errors.py index 7dcf3bb70..dc2e93993 100644 --- a/tests/test_potential_errors.py +++ b/tests/test_potential_errors.py @@ -7,7 +7,7 @@ import esdl from mesido.esdl.esdl_parser import ESDLFileParser -from mesido.esdl.profile_parser import InfluxDBProfileReader +from mesido.esdl.profile_parser import ESDLProfileReader from mesido.exceptions import MesidoAssetIssueError from mesido.potential_errors import MesidoAssetIssueType, PotentialErrors from mesido.workflows import EndScenarioSizingStaged @@ -20,7 +20,7 @@ from utils_test_scaling import create_log_list_scaling -class MockInfluxDBProfileReader(InfluxDBProfileReader): +class MockESDLProfileReader(ESDLProfileReader): def __init__( self, energy_system: esdl.EnergySystem, @@ -74,7 +74,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_1.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -113,7 +113,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_2.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -141,7 +141,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_3.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -172,7 +172,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_4.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -214,8 +214,10 @@ def test_asset_potential_errors(self): ) np.testing.assert_equal( cm.exception.message_per_asset_id["2ab92324-f86e-4976-9a6e-f7454b77ba3c"], - "Asset named HeatingDemand_2ab9: Input profile " - "demand1_MW_wrong_name in WarmingUp default profiles is not available in the database.", + "Error retrieving profile for asset 'HeatingDemand_2ab9' from host " + "'profiles.warmingup.info' and database 'energy_profiles': Failed to load from " + "InfluxDB: Failed to load profile data from InfluxDB for profile with " + "id:c3c18c06-1500-4e55-ae1d-56a82e806387 from host 'profiles.warmingup.info:443'.", ) # Check that the ResidualHeatSource multiplier's error is picked up @@ -229,7 +231,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_5.esdl", - profile_reader=InfluxDBProfileReader, + profile_reader=ESDLProfileReader, ) problem.pre() np.testing.assert_equal( diff --git a/tests/test_profile_parsing.py b/tests/test_profile_parsing.py index 0daecf1fd..695b57abf 100644 --- a/tests/test_profile_parsing.py +++ b/tests/test_profile_parsing.py @@ -9,7 +9,7 @@ import esdl from mesido.esdl.esdl_parser import ESDLFileParser -from mesido.esdl.profile_parser import InfluxDBProfileReader, ProfileReaderFromFile +from mesido.esdl.profile_parser import ESDLProfileReader, ProfileReaderFromFile from mesido.workflows import EndScenarioSizingStaged from mesido.workflows.utils.adapt_profiles import ( adapt_hourly_profile_averages_timestep_size, @@ -21,7 +21,7 @@ import pandas as pd -class MockInfluxDBProfileReader(InfluxDBProfileReader): +class MockESDLProfileReader(ESDLProfileReader): def __init__( self, energy_system: esdl.EnergySystem, @@ -163,7 +163,7 @@ def test_loading_from_influx(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre()