Add FlexMeasures plugins, USEF protocol, and Cariflex simulator

- flexmeasures-entsoe: ENTSO-E data plugin
- flexmeasures-weather: Weather data plugin
- USEF Flex Trading Protocol PDF (2.4MB)
- Cariflex simulator (publishes to Redis)
- Dashboard Grafana updated with correct InfluxDB queries
- All tools extracted in /tools/

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/conftest.py

@@ -0,0 +1 @@
+from flexmeasures.conftest import run_as_cli  # noqa: F401

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_get_forecasts.py

@@ -0,0 +1,105 @@
+import logging
+
+import pytest
+from flexmeasures.data.models.time_series import TimedBelief
+
+from ..commands import collect_weather_data
+from ...utils import weather
+from .utils import mock_api_response
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+
+def test_get_weather_forecasts_to_db(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test if we can process forecast and save them to the database.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    print(result.output)
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    for wind_speed in (100, 90):
+        assert wind_speed in [belief.event_value for belief in beliefs]
+
+
+def test_get_weather_forecasts_wapi_mapping(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test that WeatherAPI provider-specific field names are mapped independently.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "WAPI")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    expected_values = [pytest.approx(100 / 3.6), pytest.approx(90 / 3.6)]
+    assert [belief.event_value for belief in beliefs] == expected_values
+
+
+def test_get_weather_forecasts_no_close_sensors(
+    app, db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db, caplog
+):
+    """
+    Looking for a location too far away from existing weather station.
+    Check we get a warning.
+    """
+    weather_station = add_weather_sensors_fresh_db["wind"].generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    with caplog.at_level(logging.WARNING):
+        result = runner.invoke(
+            collect_weather_data,
+            [
+                "--location",
+                f"{weather_station.latitude - 5},{weather_station.longitude}",
+            ],
+        )
+        print(result.output)
+        assert "Reported task get-weather-forecasts status as True" in result.output
+        assert "no sufficiently close weather sensor found" in caplog.text

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_register.py

@@ -0,0 +1,47 @@
+import pytest
+from flexmeasures import Sensor
+
+from ..commands import add_weather_sensor
+from .utils import cli_params_from_dict
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+sensor_params = {"name": "wind speed", "latitude": 30, "longitude": 40}
+
+
+@pytest.mark.parametrize(
+    "invalid_param, invalid_value, expected_msg",
+    [
+        ("name", "windd-speed", "not supported by flexmeasures-weather"),
+        ("latitude", 93, "less than or equal to 90"),
+        ("timezone", "Erope/Amsterdam", "is unknown"),
+    ],
+)
+def test_register_weather_sensor_invalid_data(
+    app, db, invalid_param, invalid_value, expected_msg
+):
+    test_sensor_params = sensor_params.copy()
+    test_sensor_params[invalid_param] = invalid_value
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(test_sensor_params))
+    assert "Aborted" in result.output
+    assert expected_msg in result.output
+
+
+def test_register_weather_sensor(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    sensor = Sensor.query.filter(Sensor.name == sensor_params["name"]).one_or_none()
+    assert sensor is not None
+
+
+def test_register_weather_sensor_twice(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "already exists" in result.output

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/utils.py

@@ -0,0 +1,37 @@
+from typing import List
+from datetime import datetime, timedelta
+from flask import current_app
+from flexmeasures.utils.time_utils import as_server_time, get_timezone
+
+
+def cli_params_from_dict(d) -> List[str]:
+    cli_params = []
+    for k, v in d.items():
+        cli_params.append(f"--{k}")
+        cli_params.append(v)
+    return cli_params
+
+
+def mock_api_response(api_key, location):
+    mock_date = datetime.now()
+    mock_date_tz_aware = as_server_time(
+        datetime.fromtimestamp(mock_date.timestamp(), tz=get_timezone())
+    ).replace(second=0, microsecond=0)
+
+    provider = str(current_app.config.get("WEATHER_PROVIDER", ""))
+    date_key = "dt"
+    temp_key = "temp"
+    wind_speed_key = "wind_speed"
+    if provider == "WAPI":
+        date_key = "time_epoch"
+        temp_key = "temp_c"
+        wind_speed_key = "wind_kph"
+
+    return mock_date_tz_aware, [
+        {date_key: mock_date.timestamp(), temp_key: 40, wind_speed_key: 100},
+        {
+            date_key: (mock_date + timedelta(hours=1)).timestamp(),
+            temp_key: 42,
+            wind_speed_key: 90,
+        },
+    ]