Add FlexMeasures plugins, USEF protocol, and Cariflex simulator

- flexmeasures-entsoe: ENTSO-E data plugin
- flexmeasures-weather: Weather data plugin
- USEF Flex Trading Protocol PDF (2.4MB)
- Cariflex simulator (publishes to Redis)
- Dashboard Grafana updated with correct InfluxDB queries
- All tools extracted in /tools/

tools/flexmeasures-weather/flexmeasures_weather/cli/commands.py

@@ -0,0 +1,188 @@
+from flask import current_app
+
+from flask.cli import with_appcontext
+import click
+from flexmeasures.data.models.time_series import Sensor
+
+from flexmeasures.data.transactional import task_with_status_report
+from flexmeasures.data.config import db
+
+from .. import flexmeasures_weather_bp
+from .schemas.weather_sensor import WeatherSensorSchema
+from ..utils.modeling import (
+    get_or_create_weather_station,
+    get_weather_station_by_asset_id,
+)
+from ..utils.locating import get_locations, get_location_by_asset_id
+from ..utils.filing import make_file_path
+from ..utils.weather import (
+    save_forecasts_in_db,
+    save_forecasts_as_json,
+    get_supported_sensor_spec,
+)
+from ..sensor_specs import mapping
+
+"""
+TODO: allow to also pass an asset ID or name for the weather station (instead of location) to both commands?
+      See https://github.com/FlexMeasures/flexmeasures-weather
+"""
+
+supported_sensors_list = ", ".join(
+    [str(sensor_specs["fm_sensor_name"]) for sensor_specs in mapping]
+)
+
+
+@flexmeasures_weather_bp.cli.command("register-weather-sensor")
+@with_appcontext
+@click.option(
+    "--name",
+    required=True,
+    help=f"Name of the sensor. Has to be from the supported list ({supported_sensors_list})",
+)
+@click.option(
+    "--asset-id",
+    required=False,
+    type=int,
+    help="The asset id of the weather station (you can also give its location).",
+)
+@click.option(
+    "--latitude",
+    required=False,
+    type=float,
+    help="Latitude of where you want to measure.",
+)
+@click.option(
+    "--longitude",
+    required=False,
+    type=float,
+    help="Longitude of where you want to measure.",
+)
+@click.option(
+    "--timezone",
+    default="UTC",
+    help="The timezone of the sensor data as string, e.g. 'UTC' (default) or 'Europe/Amsterdam'",
+)
+def add_weather_sensor(**args):
+    """
+    Add a weather sensor.
+    This will first create a weather station asset if none exists at the location yet.
+
+    """
+    errors = WeatherSensorSchema().validate(args)
+    if errors:
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] Please correct the following errors:\n{errors}.\n Use the --help flag to learn more."
+        )
+        raise click.Abort
+    if args["asset_id"] is not None:
+        weather_station = get_weather_station_by_asset_id(args["asset_id"])
+    elif args["latitude"] is not None and args["longitude"] is not None:
+        weather_station = get_or_create_weather_station(
+            args["latitude"], args["longitude"]
+        )
+    else:
+        raise Exception(
+            "Arguments are missing to register a weather sensor. Provide either '--asset-id' or ('--latitude' and '--longitude')."
+        )
+
+    sensor = Sensor.query.filter(
+        Sensor.name == args["name"].lower(),
+        Sensor.generic_asset == weather_station,
+    ).one_or_none()
+    if sensor:
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] A '{args['name']}' weather sensor already exists at this weather station (the station's ID is {weather_station.id})."
+        )
+        return
+    fm_sensor_specs = get_supported_sensor_spec(args["name"])
+    fm_sensor_specs["generic_asset"] = weather_station
+    fm_sensor_specs["timezone"] = args["timezone"]
+    fm_sensor_specs["name"] = fm_sensor_specs.pop("fm_sensor_name")
+    fm_sensor_specs.pop("OWM_sensor_name")
+    fm_sensor_specs.pop("WAPI_sensor_name")
+    sensor = Sensor(**fm_sensor_specs)
+    sensor.attributes = fm_sensor_specs["attributes"]
+
+    db.session.add(sensor)
+    db.session.commit()
+    click.echo(
+        f"[FLEXMEASURES-WEATHER] Successfully created weather sensor with ID {sensor.id}, at weather station with ID {weather_station.id}"
+    )
+    click.echo(
+        f"[FLEXMEASURES-WEATHER] You can access this sensor at its entity address {sensor.entity_address}"
+    )
+
+
+@flexmeasures_weather_bp.cli.command("get-weather-forecasts")
+@with_appcontext
+@click.option(
+    "--location",
+    type=str,
+    required=False,
+    help='Measurement location(s). "latitude,longitude" or "top-left-latitude,top-left-longitude:'
+    'bottom-right-latitude,bottom-right-longitude." The first format defines one location to measure.'
+    " The second format defines a region of interest with several (>=4) locations"
+    ' (see also the "method" and "num_cells" parameters for details on how to use this feature).',
+)
+@click.option(
+    "--asset-id",
+    type=int,
+    required=False,
+    help="ID of a weather station asset - forecasts will be gotten for its location. If present, --location will be ignored.",
+)
+@click.option(
+    "--store-in-db/--store-as-json-files",
+    default=True,
+    help="Store forecasts in the database, or simply save as json files (defaults to database).",
+)
+@click.option(
+    "--num_cells",
+    type=int,
+    default=1,
+    help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter. Defaults to 1.",
+)
+@click.option(
+    "--method",
+    default="hex",
+    type=click.Choice(["hex", "square"]),
+    help="Grid creation method. Only used if a region of interest has been mapped in the location parameter.",
+)
+@click.option(
+    "--region",
+    type=str,
+    default="",
+    help="Name of the region (will create sub-folder if you store json files).",
+)
+@task_with_status_report("get-weather-forecasts")
+def collect_weather_data(location, asset_id, store_in_db, num_cells, method, region):
+    """
+    Collect weather forecasts from the Weather Provider API.
+    This will be done for one or more locations, for which we first identify relevant weather stations.
+
+    This function can get weather data for one location or for several locations within
+    a geometrical grid (See the --location parameter).
+    """
+
+    api_key = str(
+        current_app.config.get(
+            "WEATHERAPI_KEY", current_app.config.get("OPENWEATHERMAP_API_KEY", "")
+        )
+    )
+    if api_key == "":
+        raise Exception("[FLEXMEASURES-WEATHER] Setting WEATHERAPI_KEY not available.")
+    if asset_id is not None:
+        locations = [get_location_by_asset_id(asset_id)]
+    elif location is not None:
+        locations = get_locations(location, num_cells, method)
+    else:
+        raise Warning(
+            "[FLEXMEASURES-WEATHER] Pass either location or asset-id to get weather forecasts."
+        )
+
+    # Save the results
+    if store_in_db:
+        save_forecasts_in_db(api_key, locations)
+    else:
+        save_forecasts_as_json(
+            api_key, locations, data_path=make_file_path(current_app, region)
+        )

tools/flexmeasures-weather/flexmeasures_weather/cli/schemas/weather_sensor.py

@@ -0,0 +1,43 @@
+from marshmallow import (
+    Schema,
+    validates,
+    ValidationError,
+    fields,
+    validate,
+)
+
+import pytz
+
+from ...utils.weather import get_supported_sensor_spec, get_supported_sensors_str
+
+
+class WeatherSensorSchema(Schema):
+    """
+    Schema for the weather sensor registration.
+    Based on flexmeasures.Sensor, plus some attributes for the weather station asset.
+    """
+
+    name = fields.Str(required=True)
+    timezone = fields.Str()
+    asset_id = fields.Int(required=False, allow_none=True)
+    latitude = fields.Float(
+        required=False, validate=validate.Range(min=-90, max=90), allow_none=True
+    )
+    longitude = fields.Float(
+        required=False, validate=validate.Range(min=-180, max=180), allow_none=True
+    )
+
+    @validates("name")
+    def validate_name_is_supported(self, name: str, **kwargs):
+        if get_supported_sensor_spec(name):
+            return
+        raise ValidationError(
+            f"Weather sensors with name '{name}' are not supported by flexmeasures-weather. For now, the following is supported: [{get_supported_sensors_str()}]"
+        )
+
+    @validates("timezone")
+    def validate_timezone(self, timezone: str, **kwargs):
+        try:
+            pytz.timezone(timezone)
+        except pytz.UnknownTimeZoneError:
+            raise ValidationError(f"Timezone {timezone} is unknown!")

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/conftest.py

@@ -0,0 +1 @@
+from flexmeasures.conftest import run_as_cli  # noqa: F401

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_get_forecasts.py

@@ -0,0 +1,105 @@
+import logging
+
+import pytest
+from flexmeasures.data.models.time_series import TimedBelief
+
+from ..commands import collect_weather_data
+from ...utils import weather
+from .utils import mock_api_response
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+
+def test_get_weather_forecasts_to_db(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test if we can process forecast and save them to the database.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    print(result.output)
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    for wind_speed in (100, 90):
+        assert wind_speed in [belief.event_value for belief in beliefs]
+
+
+def test_get_weather_forecasts_wapi_mapping(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test that WeatherAPI provider-specific field names are mapped independently.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "WAPI")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    expected_values = [pytest.approx(100 / 3.6), pytest.approx(90 / 3.6)]
+    assert [belief.event_value for belief in beliefs] == expected_values
+
+
+def test_get_weather_forecasts_no_close_sensors(
+    app, db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db, caplog
+):
+    """
+    Looking for a location too far away from existing weather station.
+    Check we get a warning.
+    """
+    weather_station = add_weather_sensors_fresh_db["wind"].generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    with caplog.at_level(logging.WARNING):
+        result = runner.invoke(
+            collect_weather_data,
+            [
+                "--location",
+                f"{weather_station.latitude - 5},{weather_station.longitude}",
+            ],
+        )
+        print(result.output)
+        assert "Reported task get-weather-forecasts status as True" in result.output
+        assert "no sufficiently close weather sensor found" in caplog.text

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_register.py

@@ -0,0 +1,47 @@
+import pytest
+from flexmeasures import Sensor
+
+from ..commands import add_weather_sensor
+from .utils import cli_params_from_dict
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+sensor_params = {"name": "wind speed", "latitude": 30, "longitude": 40}
+
+
+@pytest.mark.parametrize(
+    "invalid_param, invalid_value, expected_msg",
+    [
+        ("name", "windd-speed", "not supported by flexmeasures-weather"),
+        ("latitude", 93, "less than or equal to 90"),
+        ("timezone", "Erope/Amsterdam", "is unknown"),
+    ],
+)
+def test_register_weather_sensor_invalid_data(
+    app, db, invalid_param, invalid_value, expected_msg
+):
+    test_sensor_params = sensor_params.copy()
+    test_sensor_params[invalid_param] = invalid_value
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(test_sensor_params))
+    assert "Aborted" in result.output
+    assert expected_msg in result.output
+
+
+def test_register_weather_sensor(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    sensor = Sensor.query.filter(Sensor.name == sensor_params["name"]).one_or_none()
+    assert sensor is not None
+
+
+def test_register_weather_sensor_twice(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "already exists" in result.output

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/utils.py

@@ -0,0 +1,37 @@
+from typing import List
+from datetime import datetime, timedelta
+from flask import current_app
+from flexmeasures.utils.time_utils import as_server_time, get_timezone
+
+
+def cli_params_from_dict(d) -> List[str]:
+    cli_params = []
+    for k, v in d.items():
+        cli_params.append(f"--{k}")
+        cli_params.append(v)
+    return cli_params
+
+
+def mock_api_response(api_key, location):
+    mock_date = datetime.now()
+    mock_date_tz_aware = as_server_time(
+        datetime.fromtimestamp(mock_date.timestamp(), tz=get_timezone())
+    ).replace(second=0, microsecond=0)
+
+    provider = str(current_app.config.get("WEATHER_PROVIDER", ""))
+    date_key = "dt"
+    temp_key = "temp"
+    wind_speed_key = "wind_speed"
+    if provider == "WAPI":
+        date_key = "time_epoch"
+        temp_key = "temp_c"
+        wind_speed_key = "wind_kph"
+
+    return mock_date_tz_aware, [
+        {date_key: mock_date.timestamp(), temp_key: 40, wind_speed_key: 100},
+        {
+            date_key: (mock_date + timedelta(hours=1)).timestamp(),
+            temp_key: 42,
+            wind_speed_key: 90,
+        },
+    ]