Add FlexMeasures plugins, USEF protocol, and Cariflex simulator

- flexmeasures-entsoe: ENTSO-E data plugin
- flexmeasures-weather: Weather data plugin
- USEF Flex Trading Protocol PDF (2.4MB)
- Cariflex simulator (publishes to Redis)
- Dashboard Grafana updated with correct InfluxDB queries
- All tools extracted in /tools/

tools/flexmeasures-weather/flexmeasures_weather/__init__.py

@@ -0,0 +1,68 @@
+__version__ = "Unknown version"
+
+
+"""
+The __init__ for the flexmeasures-weather FlexMeasures plugin.
+
+FlexMeasures registers the BluePrint objects it finds in here.
+"""
+
+
+from importlib.metadata import version, PackageNotFoundError
+
+from flask import Blueprint
+
+from .utils.blueprinting import ensure_bp_routes_are_loaded_fresh
+
+# Overwriting version (if possible) from the package metadata
+# ― if this plugin has been installed as a package.
+# This uses importlib.metadata behaviour added in Python 3.8.
+# Note that we rely on git tags (via setuptools_scm) to define that version.
+try:
+    __version__ = version("flexmeasures_weather")
+except PackageNotFoundError:
+    # package is not installed
+    pass
+
+
+DEFAULT_FILE_PATH_LOCATION = "weather-forecasts"
+DEFAULT_DATA_SOURCE_NAME = "Weather"
+DEFAULT_WEATHER_STATION_NAME = "weather station (created by FM-Weather)"
+WEATHER_STATION_TYPE_NAME = "weather station"
+DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE = 1
+
+__version__ = "0.1"
+__settings__ = {
+    "WEATHER_FILE_PATH_LOCATION": dict(
+        description="Location of JSON files (if you store weather data in this form). Absolute path.",
+        level="debug",
+    ),
+    "WEATHER_DATA_SOURCE_NAME": dict(
+        description=f"Name of the data source for Weather data, defaults to '{DEFAULT_DATA_SOURCE_NAME}'",
+        level="debug",
+    ),
+    "WEATHER_STATION_NAME": dict(
+        description=f"Name of the weather station asset, defaults to '{DEFAULT_WEATHER_STATION_NAME}'",
+        level="debug",
+    ),
+    "WEATHER_MAXIMAL_DEGREE_LOCATION_DISTANCE": dict(
+        descripion=f"Maximum distance (in degrees latitude & longitude) for weather stations from forecast location, defaults to {DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE}",
+        level="debug",
+    ),
+    "WEATHER_PROVIDER": dict(
+        description="Provider for weather data. Permissible options are 'OWM' (OpenWeatherMap) or 'WAPI' (WeatherAPI).",
+        level="error",
+    ),
+    "WEATHERAPI_KEY": dict(
+        description="API key for OWM or WAPI, whatever you have chosen.",
+        level="error",
+    ),
+}
+
+# CLI
+flexmeasures_weather_bp: Blueprint = Blueprint(
+    "flexmeasures-weather CLI", __name__, cli_group="weather"
+)
+flexmeasures_weather_bp.cli.help = "flexmeasures-weather CLI commands"
+ensure_bp_routes_are_loaded_fresh("cli.commands")
+from flexmeasures_weather.cli import commands  # noqa: E402,F401

tools/flexmeasures-weather/flexmeasures_weather/cli/commands.py

@@ -0,0 +1,188 @@
+from flask import current_app
+
+from flask.cli import with_appcontext
+import click
+from flexmeasures.data.models.time_series import Sensor
+
+from flexmeasures.data.transactional import task_with_status_report
+from flexmeasures.data.config import db
+
+from .. import flexmeasures_weather_bp
+from .schemas.weather_sensor import WeatherSensorSchema
+from ..utils.modeling import (
+    get_or_create_weather_station,
+    get_weather_station_by_asset_id,
+)
+from ..utils.locating import get_locations, get_location_by_asset_id
+from ..utils.filing import make_file_path
+from ..utils.weather import (
+    save_forecasts_in_db,
+    save_forecasts_as_json,
+    get_supported_sensor_spec,
+)
+from ..sensor_specs import mapping
+
+"""
+TODO: allow to also pass an asset ID or name for the weather station (instead of location) to both commands?
+      See https://github.com/FlexMeasures/flexmeasures-weather
+"""
+
+supported_sensors_list = ", ".join(
+    [str(sensor_specs["fm_sensor_name"]) for sensor_specs in mapping]
+)
+
+
+@flexmeasures_weather_bp.cli.command("register-weather-sensor")
+@with_appcontext
+@click.option(
+    "--name",
+    required=True,
+    help=f"Name of the sensor. Has to be from the supported list ({supported_sensors_list})",
+)
+@click.option(
+    "--asset-id",
+    required=False,
+    type=int,
+    help="The asset id of the weather station (you can also give its location).",
+)
+@click.option(
+    "--latitude",
+    required=False,
+    type=float,
+    help="Latitude of where you want to measure.",
+)
+@click.option(
+    "--longitude",
+    required=False,
+    type=float,
+    help="Longitude of where you want to measure.",
+)
+@click.option(
+    "--timezone",
+    default="UTC",
+    help="The timezone of the sensor data as string, e.g. 'UTC' (default) or 'Europe/Amsterdam'",
+)
+def add_weather_sensor(**args):
+    """
+    Add a weather sensor.
+    This will first create a weather station asset if none exists at the location yet.
+
+    """
+    errors = WeatherSensorSchema().validate(args)
+    if errors:
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] Please correct the following errors:\n{errors}.\n Use the --help flag to learn more."
+        )
+        raise click.Abort
+    if args["asset_id"] is not None:
+        weather_station = get_weather_station_by_asset_id(args["asset_id"])
+    elif args["latitude"] is not None and args["longitude"] is not None:
+        weather_station = get_or_create_weather_station(
+            args["latitude"], args["longitude"]
+        )
+    else:
+        raise Exception(
+            "Arguments are missing to register a weather sensor. Provide either '--asset-id' or ('--latitude' and '--longitude')."
+        )
+
+    sensor = Sensor.query.filter(
+        Sensor.name == args["name"].lower(),
+        Sensor.generic_asset == weather_station,
+    ).one_or_none()
+    if sensor:
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] A '{args['name']}' weather sensor already exists at this weather station (the station's ID is {weather_station.id})."
+        )
+        return
+    fm_sensor_specs = get_supported_sensor_spec(args["name"])
+    fm_sensor_specs["generic_asset"] = weather_station
+    fm_sensor_specs["timezone"] = args["timezone"]
+    fm_sensor_specs["name"] = fm_sensor_specs.pop("fm_sensor_name")
+    fm_sensor_specs.pop("OWM_sensor_name")
+    fm_sensor_specs.pop("WAPI_sensor_name")
+    sensor = Sensor(**fm_sensor_specs)
+    sensor.attributes = fm_sensor_specs["attributes"]
+
+    db.session.add(sensor)
+    db.session.commit()
+    click.echo(
+        f"[FLEXMEASURES-WEATHER] Successfully created weather sensor with ID {sensor.id}, at weather station with ID {weather_station.id}"
+    )
+    click.echo(
+        f"[FLEXMEASURES-WEATHER] You can access this sensor at its entity address {sensor.entity_address}"
+    )
+
+
+@flexmeasures_weather_bp.cli.command("get-weather-forecasts")
+@with_appcontext
+@click.option(
+    "--location",
+    type=str,
+    required=False,
+    help='Measurement location(s). "latitude,longitude" or "top-left-latitude,top-left-longitude:'
+    'bottom-right-latitude,bottom-right-longitude." The first format defines one location to measure.'
+    " The second format defines a region of interest with several (>=4) locations"
+    ' (see also the "method" and "num_cells" parameters for details on how to use this feature).',
+)
+@click.option(
+    "--asset-id",
+    type=int,
+    required=False,
+    help="ID of a weather station asset - forecasts will be gotten for its location. If present, --location will be ignored.",
+)
+@click.option(
+    "--store-in-db/--store-as-json-files",
+    default=True,
+    help="Store forecasts in the database, or simply save as json files (defaults to database).",
+)
+@click.option(
+    "--num_cells",
+    type=int,
+    default=1,
+    help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter. Defaults to 1.",
+)
+@click.option(
+    "--method",
+    default="hex",
+    type=click.Choice(["hex", "square"]),
+    help="Grid creation method. Only used if a region of interest has been mapped in the location parameter.",
+)
+@click.option(
+    "--region",
+    type=str,
+    default="",
+    help="Name of the region (will create sub-folder if you store json files).",
+)
+@task_with_status_report("get-weather-forecasts")
+def collect_weather_data(location, asset_id, store_in_db, num_cells, method, region):
+    """
+    Collect weather forecasts from the Weather Provider API.
+    This will be done for one or more locations, for which we first identify relevant weather stations.
+
+    This function can get weather data for one location or for several locations within
+    a geometrical grid (See the --location parameter).
+    """
+
+    api_key = str(
+        current_app.config.get(
+            "WEATHERAPI_KEY", current_app.config.get("OPENWEATHERMAP_API_KEY", "")
+        )
+    )
+    if api_key == "":
+        raise Exception("[FLEXMEASURES-WEATHER] Setting WEATHERAPI_KEY not available.")
+    if asset_id is not None:
+        locations = [get_location_by_asset_id(asset_id)]
+    elif location is not None:
+        locations = get_locations(location, num_cells, method)
+    else:
+        raise Warning(
+            "[FLEXMEASURES-WEATHER] Pass either location or asset-id to get weather forecasts."
+        )
+
+    # Save the results
+    if store_in_db:
+        save_forecasts_in_db(api_key, locations)
+    else:
+        save_forecasts_as_json(
+            api_key, locations, data_path=make_file_path(current_app, region)
+        )

tools/flexmeasures-weather/flexmeasures_weather/cli/schemas/weather_sensor.py

@@ -0,0 +1,43 @@
+from marshmallow import (
+    Schema,
+    validates,
+    ValidationError,
+    fields,
+    validate,
+)
+
+import pytz
+
+from ...utils.weather import get_supported_sensor_spec, get_supported_sensors_str
+
+
+class WeatherSensorSchema(Schema):
+    """
+    Schema for the weather sensor registration.
+    Based on flexmeasures.Sensor, plus some attributes for the weather station asset.
+    """
+
+    name = fields.Str(required=True)
+    timezone = fields.Str()
+    asset_id = fields.Int(required=False, allow_none=True)
+    latitude = fields.Float(
+        required=False, validate=validate.Range(min=-90, max=90), allow_none=True
+    )
+    longitude = fields.Float(
+        required=False, validate=validate.Range(min=-180, max=180), allow_none=True
+    )
+
+    @validates("name")
+    def validate_name_is_supported(self, name: str, **kwargs):
+        if get_supported_sensor_spec(name):
+            return
+        raise ValidationError(
+            f"Weather sensors with name '{name}' are not supported by flexmeasures-weather. For now, the following is supported: [{get_supported_sensors_str()}]"
+        )
+
+    @validates("timezone")
+    def validate_timezone(self, timezone: str, **kwargs):
+        try:
+            pytz.timezone(timezone)
+        except pytz.UnknownTimeZoneError:
+            raise ValidationError(f"Timezone {timezone} is unknown!")

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/conftest.py

@@ -0,0 +1 @@
+from flexmeasures.conftest import run_as_cli  # noqa: F401

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_get_forecasts.py

@@ -0,0 +1,105 @@
+import logging
+
+import pytest
+from flexmeasures.data.models.time_series import TimedBelief
+
+from ..commands import collect_weather_data
+from ...utils import weather
+from .utils import mock_api_response
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+
+def test_get_weather_forecasts_to_db(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test if we can process forecast and save them to the database.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    print(result.output)
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    for wind_speed in (100, 90):
+        assert wind_speed in [belief.event_value for belief in beliefs]
+
+
+def test_get_weather_forecasts_wapi_mapping(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test that WeatherAPI provider-specific field names are mapped independently.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "WAPI")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    expected_values = [pytest.approx(100 / 3.6), pytest.approx(90 / 3.6)]
+    assert [belief.event_value for belief in beliefs] == expected_values
+
+
+def test_get_weather_forecasts_no_close_sensors(
+    app, db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db, caplog
+):
+    """
+    Looking for a location too far away from existing weather station.
+    Check we get a warning.
+    """
+    weather_station = add_weather_sensors_fresh_db["wind"].generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    with caplog.at_level(logging.WARNING):
+        result = runner.invoke(
+            collect_weather_data,
+            [
+                "--location",
+                f"{weather_station.latitude - 5},{weather_station.longitude}",
+            ],
+        )
+        print(result.output)
+        assert "Reported task get-weather-forecasts status as True" in result.output
+        assert "no sufficiently close weather sensor found" in caplog.text

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_register.py

@@ -0,0 +1,47 @@
+import pytest
+from flexmeasures import Sensor
+
+from ..commands import add_weather_sensor
+from .utils import cli_params_from_dict
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+sensor_params = {"name": "wind speed", "latitude": 30, "longitude": 40}
+
+
+@pytest.mark.parametrize(
+    "invalid_param, invalid_value, expected_msg",
+    [
+        ("name", "windd-speed", "not supported by flexmeasures-weather"),
+        ("latitude", 93, "less than or equal to 90"),
+        ("timezone", "Erope/Amsterdam", "is unknown"),
+    ],
+)
+def test_register_weather_sensor_invalid_data(
+    app, db, invalid_param, invalid_value, expected_msg
+):
+    test_sensor_params = sensor_params.copy()
+    test_sensor_params[invalid_param] = invalid_value
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(test_sensor_params))
+    assert "Aborted" in result.output
+    assert expected_msg in result.output
+
+
+def test_register_weather_sensor(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    sensor = Sensor.query.filter(Sensor.name == sensor_params["name"]).one_or_none()
+    assert sensor is not None
+
+
+def test_register_weather_sensor_twice(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "already exists" in result.output

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/utils.py

@@ -0,0 +1,37 @@
+from typing import List
+from datetime import datetime, timedelta
+from flask import current_app
+from flexmeasures.utils.time_utils import as_server_time, get_timezone
+
+
+def cli_params_from_dict(d) -> List[str]:
+    cli_params = []
+    for k, v in d.items():
+        cli_params.append(f"--{k}")
+        cli_params.append(v)
+    return cli_params
+
+
+def mock_api_response(api_key, location):
+    mock_date = datetime.now()
+    mock_date_tz_aware = as_server_time(
+        datetime.fromtimestamp(mock_date.timestamp(), tz=get_timezone())
+    ).replace(second=0, microsecond=0)
+
+    provider = str(current_app.config.get("WEATHER_PROVIDER", ""))
+    date_key = "dt"
+    temp_key = "temp"
+    wind_speed_key = "wind_speed"
+    if provider == "WAPI":
+        date_key = "time_epoch"
+        temp_key = "temp_c"
+        wind_speed_key = "wind_kph"
+
+    return mock_date_tz_aware, [
+        {date_key: mock_date.timestamp(), temp_key: 40, wind_speed_key: 100},
+        {
+            date_key: (mock_date + timedelta(hours=1)).timestamp(),
+            temp_key: 42,
+            wind_speed_key: 90,
+        },
+    ]

tools/flexmeasures-weather/flexmeasures_weather/conftest.py

@@ -0,0 +1,72 @@
+from typing import Dict
+from datetime import timedelta
+
+import pytest
+from flask_sqlalchemy import SQLAlchemy
+from flexmeasures.app import create as create_flexmeasures_app
+from flexmeasures.conftest import db, fresh_db  # noqa: F401
+from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
+from flexmeasures.data.models.time_series import Sensor
+
+from flexmeasures_weather import WEATHER_STATION_TYPE_NAME
+
+
+@pytest.fixture(scope="session")
+def app():
+    print("APP FIXTURE")
+
+    # Adding this plugin, making sure the name is known (as last part of plugin path)
+    test_app = create_flexmeasures_app(
+        env="testing", plugins=["../flexmeasures_weather"]
+    )
+
+    # Establish an application context before running the tests.
+    ctx = test_app.app_context()
+    ctx.push()
+
+    yield test_app
+
+    ctx.pop()
+
+    print("DONE WITH APP FIXTURE")
+
+
+@pytest.fixture(scope="module")
+def add_weather_sensors(db) -> Dict[str, Sensor]:  # noqa: F811
+    return create_weather_sensors(db)
+
+
+@pytest.fixture(scope="function")
+def add_weather_sensors_fresh_db(fresh_db) -> Dict[str, Sensor]:  # noqa: F811
+    return create_weather_sensors(fresh_db)
+
+
+def create_weather_sensors(db: SQLAlchemy):  # noqa: F811
+    """Add a weather station asset with two weather sensors."""
+    weather_station_type = GenericAssetType(name=WEATHER_STATION_TYPE_NAME)
+    db.session.add(weather_station_type)
+
+    weather_station = GenericAsset(
+        name="Test weather station",
+        generic_asset_type=weather_station_type,
+        latitude=33.4843866,
+        longitude=126,
+    )
+    db.session.add(weather_station)
+
+    wind_sensor = Sensor(
+        name="wind speed",
+        generic_asset=weather_station,
+        event_resolution=timedelta(minutes=60),
+        unit="m/s",
+    )
+    db.session.add(wind_sensor)
+
+    temp_sensor = Sensor(
+        name="temperature",
+        generic_asset=weather_station,
+        event_resolution=timedelta(minutes=60),
+        unit="°C",
+    )
+    db.session.add(temp_sensor)
+    return {"wind": wind_sensor, "temperature": temp_sensor}

tools/flexmeasures-weather/flexmeasures_weather/sensor_specs.py

@@ -0,0 +1,51 @@
+from datetime import timedelta
+
+
+"""
+This maps sensor specs which we can use in FlexMeasures to Weather labels.
+Note: Sensor names we use in FM need to be unique per weather station.
+At the moment, we only extract from Weather hourly data.
+"""
+
+
+weather_attributes = {
+    "daily_seasonality": True,
+    "weekly_seasonality": False,
+    "yearly_seasonality": True,
+}
+
+
+mapping = [
+    dict(
+        fm_sensor_name="temperature",
+        OWM_sensor_name="temp",
+        WAPI_sensor_name="temp_c",
+        unit="°C",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+    dict(
+        fm_sensor_name="wind speed",
+        OWM_sensor_name="wind_speed",
+        WAPI_sensor_name="wind_kph",
+        unit="m/s",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+    dict(
+        fm_sensor_name="cloud cover",
+        OWM_sensor_name="clouds",
+        WAPI_sensor_name="cloud",
+        unit="%",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+    dict(
+        fm_sensor_name="irradiance",  # in save_forecasts_to_db, we catch this name and do the actual computation to get to the irradiance
+        OWM_sensor_name="clouds",
+        WAPI_sensor_name="cloud",
+        unit="W/m²",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+]

tools/flexmeasures-weather/flexmeasures_weather/utils/blueprinting.py

@@ -0,0 +1,22 @@
+import sys
+import importlib
+
+
+def ensure_bp_routes_are_loaded_fresh(module_name):
+    """
+    Reload a module if it has been loaded before.
+    It's useful for situations in which some other process has read
+    the module before, but you need some action to happen which only
+    happens during module import ― decorators are a good example.
+
+    One use case is pytest, which reads all python code when it collects tests.
+    In our case, that happens before FlexMeasures' import mechanism
+    has had a chance to know which blueprints a plugin has.
+    Seemingly, the importing code (plugin's __init__) can be imported later
+    than the imported module (containing @route decorators).
+    Re-importing helps to get this order right when FlexMeasures reads the
+    plugin's __init__.
+    """
+    m_name = "flexmeasures_weather." + module_name
+    if m_name in sys.modules:
+        importlib.reload(sys.modules[m_name])

tools/flexmeasures-weather/flexmeasures_weather/utils/filing.py

@@ -0,0 +1,25 @@
+import os
+
+import click
+from flask import Flask, current_app
+
+from flexmeasures_weather import DEFAULT_FILE_PATH_LOCATION
+
+
+def make_file_path(app: Flask, region: str) -> str:
+    """Ensure and return path for weather data"""
+    file_path = current_app.config.get(
+        "WEATHER_FILE_PATH_LOCATION", DEFAULT_FILE_PATH_LOCATION
+    )
+    data_path = os.path.join(app.root_path, file_path)
+    if not os.path.exists(data_path):
+        click.echo("[FLEXMEASURES-WEATHER] Creating %s ..." % data_path)
+        os.mkdir(data_path)
+    # optional: extend with subpath for region
+    if region is not None and region != "":
+        region_data_path = "%s/%s" % (data_path, region)
+        if not os.path.exists(region_data_path):
+            click.echo("[FLEXMEASURES-WEATHER] Creating %s ..." % region_data_path)
+            os.mkdir(region_data_path)
+        data_path = region_data_path
+    return data_path

tools/flexmeasures-weather/flexmeasures_weather/utils/locating.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from typing import Tuple, List, Optional
+
+import click
+from flask import current_app
+
+from flexmeasures.utils.grid_cells import LatLngGrid, get_cell_nums
+from flexmeasures import Sensor
+from flexmeasures.data.models.generic_assets import GenericAsset
+from flexmeasures.utils import flexmeasures_inflection
+
+from .. import WEATHER_STATION_TYPE_NAME
+
+
+def get_locations(
+    location: str,
+    num_cells: int,
+    method: str,
+) -> List[Tuple[float, float]]:
+    """
+    Get locations for getting forecasts for, by parsing the location string, which possibly opens a latitude/longitude grid with several neatly ordered locations.
+    """
+    if (
+        location.count(",") == 0
+        or location.count(",") != location.count(":") + 1
+        or location.count(":") == 1
+        and (
+            location.find(",") > location.find(":")
+            or location.find(",", location.find(",") + 1) < location.find(":")
+        )
+    ):
+        raise Exception(
+            '[FLEXMEASURES-WEATHER] location parameter "%s" seems malformed. Please use "latitude,longitude" or '
+            ' "top-left-latitude,top-left-longitude:bottom-right-latitude,bottom-right-longitude"'
+            % location
+        )
+
+    location_identifiers = tuple(location.split(":"))
+
+    if len(location_identifiers) == 1:
+        ll = location_identifiers[0].split(",")
+        locations = [(float(ll[0]), float(ll[1]))]
+        click.echo("[FLEXMEASURES-WEATHER] Only one location: %s,%s." % locations[0])
+    elif len(location_identifiers) == 2:
+        click.echo(
+            "[FLEXMEASURES-WEATHER] Making a grid of locations between top/left %s and bottom/right %s ..."
+            % location_identifiers
+        )
+        top_left = tuple(float(s) for s in location_identifiers[0].split(","))
+        if len(top_left) != 2:
+            raise Exception(
+                "[FLEXMEASURES-WEATHER] top-left parameter '%s' is invalid."
+                % location_identifiers[0]
+            )
+        bottom_right = tuple(float(s) for s in location_identifiers[1].split(","))
+        if len(bottom_right) != 2:
+            raise Exception(
+                "[FLEXMEASURES-WEATHER] bottom-right parameter '%s' is invalid."
+                % location_identifiers[1]
+            )
+
+        num_lat, num_lng = get_cell_nums(top_left, bottom_right, num_cells)
+
+        locations = LatLngGrid(
+            top_left=top_left,
+            bottom_right=bottom_right,
+            num_cells_lat=num_lat,
+            num_cells_lng=num_lng,
+        ).get_locations(method)
+    else:
+        raise Exception(
+            "[FLEXMEASURES-WEATHER] location parameter '%s' has too many locations."
+            % location
+        )
+    return locations
+
+
+def find_weather_sensor_by_location(
+    location: Tuple[float, float],
+    max_degree_difference_for_nearest_weather_sensor: int,
+    sensor_name: str,
+) -> Sensor | None:
+    """
+    Try to find a weather sensor of fitting type close by.
+    Return None if the nearest weather sensor is further away than some minimum degrees or if no sensor was found at all.
+    """
+    weather_sensor: Optional[Sensor] = Sensor.find_closest(
+        generic_asset_type_name=WEATHER_STATION_TYPE_NAME,
+        sensor_name=sensor_name,
+        lat=location[0],
+        lng=location[1],
+        n=1,
+    )
+    if weather_sensor is None:
+        current_app.logger.warning(
+            "[FLEXMEASURES-WEATHER] No weather sensor set up yet for measuring %s. Try the register-weather-sensor CLI task."
+            % sensor_name
+        )
+        return None
+    weather_station: GenericAsset = weather_sensor.generic_asset
+    if abs(
+        location[0] - weather_station.location[0]
+    ) > max_degree_difference_for_nearest_weather_sensor or abs(
+        location[1] - weather_station.location[1]
+        > max_degree_difference_for_nearest_weather_sensor
+    ):
+        current_app.logger.warning(
+            f"[FLEXMEASURES-WEATHER] We found a weather station, but no sufficiently close weather sensor found (within {max_degree_difference_for_nearest_weather_sensor} {flexmeasures_inflection.pluralize('degree', max_degree_difference_for_nearest_weather_sensor)} distance) for measuring {sensor_name}! We're looking for: {location}, closest available: ({weather_station.location})"
+        )
+        return None
+    return weather_sensor
+
+
+def get_location_by_asset_id(asset_id: int) -> Tuple[float, float]:
+    """Get location for forecasting by passing an asset id"""
+    asset = GenericAsset.query.filter(GenericAsset.id == asset_id).one_or_none()
+    if asset.generic_asset_type.name != WEATHER_STATION_TYPE_NAME:
+        raise Exception(
+            f"Asset {asset} does not seem to be a weather station we should use ― we expect an asset with type '{WEATHER_STATION_TYPE_NAME}'."
+        )
+    if asset is None:
+        raise Exception(
+            "[FLEXMEASURES-WEATHER] No asset found for the given asset id %s."
+            % asset_id
+        )
+    return (asset.latitude, asset.longitude)

tools/flexmeasures-weather/flexmeasures_weather/utils/modeling.py

@@ -0,0 +1,124 @@
+from packaging import version
+
+from flask import current_app
+from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
+from flexmeasures import Source, __version__ as flexmeasures_version
+from flexmeasures.data import db
+from flexmeasures.data.services.data_sources import get_or_create_source
+
+from flexmeasures_weather import DEFAULT_DATA_SOURCE_NAME
+from flexmeasures_weather import WEATHER_STATION_TYPE_NAME
+from flexmeasures_weather import DEFAULT_WEATHER_STATION_NAME
+
+
+if version.parse(flexmeasures_version) < version.parse("0.13"):
+    SOURCE_TYPE = "forecasting script"
+else:
+    SOURCE_TYPE = "forecaster"
+
+FM_SUPPORTS_ACCOUNT_LINKED_SOURCES = version.parse(
+    flexmeasures_version
+) >= version.parse("0.32")
+
+if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+    from flexmeasures import Account
+else:
+    Account = None
+
+
+def get_or_create_weather_account():
+    """Make sure we have an account for the weather provider service."""
+    if Account is None:
+        raise RuntimeError(
+            "FlexMeasures Account model is unavailable before FlexMeasures 0.32."
+        )
+    account_name = current_app.config.get(
+        "WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+    )
+    weather_account = Account.query.filter(
+        Account.name == account_name,
+    ).one_or_none()
+    if weather_account is None:
+        weather_account = Account(name=account_name)
+        db.session.add(weather_account)
+        db.session.flush()
+    return weather_account
+
+
+def get_or_create_owm_data_source() -> Source:
+    """Make sure we have a weather provider data source of the configured type."""
+    source_kwargs = dict(
+        source=current_app.config.get(
+            "WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+        ),
+        source_type=SOURCE_TYPE,
+        flush=False,
+    )
+    if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+        source_kwargs["account"] = get_or_create_weather_account()
+    return get_or_create_source(**source_kwargs)
+
+
+def get_or_create_owm_data_source_for_derived_data() -> Source:
+    owm_source_name = current_app.config.get(
+        "WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+    )
+    source_kwargs = dict(
+        source=f"FlexMeasures {owm_source_name}",
+        source_type=SOURCE_TYPE,
+        flush=False,
+    )
+    if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+        source_kwargs["account"] = get_or_create_weather_account()
+    return get_or_create_source(**source_kwargs)
+
+
+def get_or_create_weather_station_type() -> GenericAssetType:
+    """Make sure a weather station type exists"""
+    weather_station_type = GenericAssetType.query.filter(
+        GenericAssetType.name == WEATHER_STATION_TYPE_NAME,
+    ).one_or_none()
+    if weather_station_type is None:
+        weather_station_type = GenericAssetType(
+            name=WEATHER_STATION_TYPE_NAME,
+            description="A weather station with various sensors.",
+        )
+        db.session.add(weather_station_type)
+    return weather_station_type
+
+
+def get_or_create_weather_station(latitude: float, longitude: float) -> GenericAsset:
+    """Make sure a weather station exists at this location."""
+    station_name = current_app.config.get(
+        "WEATHER_STATION_NAME", DEFAULT_WEATHER_STATION_NAME
+    )
+    weather_station = GenericAsset.query.filter(
+        GenericAsset.latitude == latitude, GenericAsset.longitude == longitude
+    ).one_or_none()
+    if weather_station is None:
+        weather_station_type = get_or_create_weather_station_type()
+        weather_station = GenericAsset(
+            name=station_name,
+            generic_asset_type=weather_station_type,
+            latitude=latitude,
+            longitude=longitude,
+        )
+        db.session.add(weather_station)
+    return weather_station
+
+
+def get_weather_station_by_asset_id(asset_id: int) -> GenericAsset:
+    weather_station = GenericAsset.query.filter(
+        GenericAsset.generic_asset_type_id == asset_id
+    ).one_or_none()
+    if weather_station is None:
+        raise Exception(
+            f"[FLEXMEASURES-WEATHER] Weather station is not present for the given asset id '{asset_id}'."
+        )
+
+    if weather_station.latitude is None or weather_station.longitude is None:
+        raise Exception(
+            f"[FLEXMEASURES-WEATHER] Weather station {weather_station} is missing location information [Latitude, Longitude]."
+        )
+
+    return weather_station

tools/flexmeasures-weather/flexmeasures_weather/utils/radiating.py

@@ -0,0 +1,37 @@
+from datetime import datetime
+
+import pandas as pd
+from pvlib.location import Location
+
+
+def compute_irradiance(
+    latitude: float, longitude: float, dt: datetime, cloud_coverage: float
+) -> float:
+    """Compute the irradiance received on a location at a specific time.
+    This uses pvlib to
+    1)  compute clear-sky irradiance as Global Horizontal Irradiance (GHI),
+        which includes both Direct Normal Irradiance (DNI)
+        and Diffuse Horizontal Irradiance (DHI).
+    2)  adjust the GHI for cloud coverage
+    """
+    site = Location(latitude, longitude, tz=dt.tzinfo)
+    solpos = site.get_solarposition(pd.DatetimeIndex([dt]))
+    ghi_clear = site.get_clearsky(pd.DatetimeIndex([dt]), solar_position=solpos).loc[
+        dt
+    ]["ghi"]
+    return ghi_clear_to_ghi(ghi_clear, cloud_coverage)
+
+
+def ghi_clear_to_ghi(ghi_clear: float, cloud_coverage: float) -> float:
+    """Compute global horizontal irradiance (GHI) from clear-sky GHI, given a cloud coverage between 0 and 1.
+
+    References
+    ----------
+    Perez, R., Moore, K., Wilcox, S., Renne, D., Zelenka, A., 2007.
+    Forecasting solar radiation – preliminary evaluation of an
+    approach based upon the national forecast database. Solar Energy
+    81, 809–812.
+    """
+    if cloud_coverage < 0 or cloud_coverage > 1:
+        raise ValueError("cloud_coverage should lie in the interval [0, 1]")
+    return (1 - 0.87 * cloud_coverage**1.9) * ghi_clear

tools/flexmeasures-weather/flexmeasures_weather/utils/tests/test_modeling.py

@@ -0,0 +1,194 @@
+from types import SimpleNamespace
+
+import pytest
+
+from flexmeasures import Asset
+
+import flexmeasures_weather.utils.modeling as modeling
+from flexmeasures_weather import DEFAULT_DATA_SOURCE_NAME, DEFAULT_WEATHER_STATION_NAME
+from flexmeasures_weather.utils.modeling import (
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    SOURCE_TYPE,
+    get_or_create_owm_data_source,
+    get_or_create_owm_data_source_for_derived_data,
+    get_or_create_weather_account,
+    get_or_create_weather_station,
+)
+
+
+def test_creating_two_weather_stations(fresh_db):
+    get_or_create_weather_station(50, 40)
+    get_or_create_weather_station(40, 50)
+    assert Asset.query.filter(Asset.name == DEFAULT_WEATHER_STATION_NAME).count() == 2
+
+
+# The version-branch tests below still use monkeypatching to isolate source
+# creation side effects without requiring multiple FlexMeasures installs.
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Weather source accounts are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_weather_account(fresh_db):
+    weather_account = get_or_create_weather_account()
+
+    assert weather_account.name == DEFAULT_DATA_SOURCE_NAME
+    assert (
+        modeling.Account.query.filter(
+            modeling.Account.name == weather_account.name
+        ).count()
+        == 1
+    )
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_data_source_registers_weather_source_on_weather_account(
+    fresh_db,
+):
+    data_source = get_or_create_owm_data_source()
+
+    assert data_source.type == SOURCE_TYPE
+    assert data_source.account is not None
+    assert data_source.account.name == data_source.name
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_data_source_for_derived_data_uses_weather_account(fresh_db):
+    derived_data_source = get_or_create_owm_data_source_for_derived_data()
+
+    assert derived_data_source.type == SOURCE_TYPE
+    assert derived_data_source.account is not None
+    assert derived_data_source.account.name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_data_source_passes_weather_account_when_supported(
+    fresh_db, monkeypatch
+):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, account, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                account=account,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, account=account)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_derived_data_source_passes_weather_account_when_supported(
+    fresh_db, monkeypatch
+):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, account, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                account=account,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, account=account)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source_for_derived_data()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Legacy source creation without accounts is only used on FlexMeasures < 0.32.",
+)
+def test_get_or_create_owm_data_source_omits_account_when_not_supported(monkeypatch):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, name=source)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs == {
+        "source": DEFAULT_DATA_SOURCE_NAME,
+        "source_type": SOURCE_TYPE,
+        "flush": False,
+    }
+
+
+@pytest.mark.skipif(
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Legacy source creation without accounts is only used on FlexMeasures < 0.32.",
+)
+def test_get_or_create_owm_derived_data_source_omits_account_when_not_supported(
+    monkeypatch,
+):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, name=source)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source_for_derived_data()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs == {
+        "source": f"FlexMeasures {DEFAULT_DATA_SOURCE_NAME}",
+        "source_type": SOURCE_TYPE,
+        "flush": False,
+    }

tools/flexmeasures-weather/flexmeasures_weather/utils/weather.py

@@ -0,0 +1,357 @@
+from __future__ import annotations
+
+from typing import Tuple, List, Dict, Optional, Any
+import os
+from datetime import datetime, timedelta
+import json
+
+import click
+from flask import current_app
+import requests
+from humanize import naturaldelta
+from timely_beliefs import BeliefsDataFrame
+from flexmeasures.utils.time_utils import as_server_time, get_timezone, server_now
+from flexmeasures.data.models.time_series import Sensor, TimedBelief
+from flexmeasures.data.utils import save_to_db
+
+from flexmeasures_weather import DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE
+from .locating import find_weather_sensor_by_location
+from ..sensor_specs import mapping
+from .modeling import (
+    get_or_create_owm_data_source,
+    get_or_create_owm_data_source_for_derived_data,
+)
+from .radiating import compute_irradiance
+from zoneinfo import ZoneInfo
+
+API_VERSION = "3.0"
+
+
+def get_supported_sensor_spec(name: str) -> Optional[dict]:
+    """
+    Find the specs from a sensor by name.
+    """
+    for supported_sensor_spec in mapping:
+        if supported_sensor_spec["fm_sensor_name"] == name:
+            return supported_sensor_spec.copy()
+    return None
+
+
+def get_supported_sensors_str() -> str:
+    """A string - list of supported sensors, also revealing their unit"""
+    return ", ".join(
+        [
+            f"{sensor_specs['fm_sensor_name']} ({sensor_specs['unit']})"
+            for sensor_specs in mapping
+        ]
+    )
+
+
+def process_weatherapi_data(
+    data: List[Dict[str, Any]], hour_no: int
+) -> List[Dict[str, Any]]:
+    """
+    Processes raw WeatherAPI forecast data into a format similar to OpenWeatherMap's format.
+
+    Args:
+        data (List[Dict[str, Any]]): A list of forecast day dictionaries from WeatherAPI,
+            each containing an 'hour' key with 24 hourly entries.
+        hour_no (int): The index of the current hour to start from.
+
+    Returns:
+        List[Dict[str, Any]]: A list of 48 hourly forecast entries, each mapped to the
+        expected structure with fields like temperature, humidity, wind, and condition.
+    """
+    first_day = data[0]["hour"]
+    second_day = data[1]["hour"]
+    third_day = data[2]["hour"]
+    combined = first_day + second_day + third_day
+
+    relevant = combined[hour_no : hour_no + 48]
+    return relevant
+
+
+def call_openweatherapi(
+    api_key: str, location: Tuple[float, float]
+) -> Tuple[datetime, List[Dict]]:
+    """
+    Make a single "one-call" to the Open Weather API and return the API timestamp as well as the 48 hourly forecasts.
+    See https://openweathermap.org/api/one-call-3 for docs.
+    Note that the first forecast is about the current hour.
+    """
+    check_openweathermap_version(API_VERSION)
+    query_str = f"lat={location[0]}&lon={location[1]}&units=metric&exclude=minutely,daily,alerts&appid={api_key}"
+    res = requests.get(
+        f"http://api.openweathermap.org/data/{API_VERSION}/onecall?{query_str}"
+    )
+    assert (
+        res.status_code == 200
+    ), f"OpenWeatherMap returned status code {res.status_code}: {res.text}"
+    data = res.json()
+    time_of_api_call = as_server_time(
+        datetime.fromtimestamp(data["current"]["dt"], tz=get_timezone())
+    ).replace(second=0, microsecond=0)
+    return time_of_api_call, data["hourly"]
+
+
+def call_weatherapi(
+    api_key: str, location: Tuple[float, float], days: int = 3
+) -> Tuple[datetime, List[Dict]]:
+    """
+    Makes a request to the WeatherAPI to retrieve hourly weather forecast data.
+
+    Args:
+        api_key (str): API key for authenticating with the Weather API.
+        location (Tuple[float, float]): A tuple containing the latitude and longitude.
+        days (int, optional): Number of days to request the forecast for (default is 3, including current day).
+
+    Returns:
+        Tuple[datetime, List[Dict]]:
+            - The timestamp of the API call.
+            - A list of hourly forecast data as dictionaries. Note that the first forecast is about the current hour.
+
+    Raises:
+        AssertionError: If the response from the Weather API is not successful (HTTP status 200).
+    """
+
+    latitude, longitude = location[0], location[1]
+
+    query_str = f"http://api.weatherapi.com/v1/forecast.json?key={api_key}&q={latitude},{longitude}&days={days}&aqi=yes&alerts=yes"
+    res = requests.get(query_str)
+
+    assert (
+        res.status_code == 200
+    ), f"Weather API returned status code {res.status_code}: {res.text}"
+
+    data = res.json()
+
+    # get the time of the api call
+    time_of_call = int(data["location"]["localtime_epoch"])
+    local_timezone = ZoneInfo(data["location"]["tz_id"])
+    local_time = datetime.fromtimestamp(time_of_call, local_timezone)
+    time_of_api_call = as_server_time(local_time)
+    time_of_api_call = time_of_api_call.replace(second=0, microsecond=0)
+
+    print(f"Time of API call in WAPI is {time_of_api_call}")
+
+    relevant = data["forecast"]["forecastday"]
+    hour_no = local_time.hour
+
+    hourly = process_weatherapi_data(relevant, hour_no)
+    return time_of_api_call, hourly
+
+
+def call_api(
+    api_key: str, location: Tuple[float, float]
+) -> Tuple[datetime, List[Dict]]:
+    """
+    Dispatches the weather API call based on the configured provider.
+
+    Args:
+        api_key (str): API key for the selected weather service provider.
+        location (Tuple[float, float]): Latitude and longitude tuple.
+
+    Returns:
+        Tuple[datetime, List[Dict]]:
+            - Timestamp of the API call.
+            - List of hourly forecast data.
+
+    Raises:
+        Exception: If an invalid weather provider is configured.
+    """
+
+    provider = str(current_app.config.get("WEATHER_PROVIDER", "OWM"))
+    if provider not in ["OWM", "WAPI"]:
+        raise Exception(
+            "Invalid provider name. Please set WEATHER_PROVIDER setting in config file to either OWM or WAPI, the two permissible options."
+        )
+
+    if provider == "OWM":
+        click.secho("Calling Open Weather Map")
+        return call_openweatherapi(api_key, location)
+    else:
+        click.secho("Calling Weather API")
+        return call_weatherapi(api_key, location)
+
+
+def save_forecasts_in_db(  # noqa: C901
+    api_key: str,
+    locations: List[Tuple[float, float]],
+):
+    """Process the response from Weather Provider API into timed beliefs.
+    Collects all forecasts for all locations and all sensors at all locations, then bulk-saves them.
+    """
+    click.echo("[FLEXMEASURES-WEATHER] Getting weather forecasts:")
+    click.echo("[FLEXMEASURES-WEATHER] Latitude, Longitude")
+    click.echo("[FLEXMEASURES-WEATHER] -----------------------")
+    max_degree_difference_for_nearest_weather_sensor = current_app.config.get(
+        "WEATHER_MAXIMAL_DEGREE_LOCATION_DISTANCE",
+        DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE,
+    )
+    provider = str(current_app.config.get("WEATHER_PROVIDER", ""))
+    if provider not in ["OWM", "WAPI"]:
+        raise Exception(
+            "Invalid provider name. Please set WEATHER_PROVIDER setting in config file to either OWM or WAPI, the two permissible options."
+        )
+    for location in locations:
+        click.echo("[FLEXMEASURES] %s, %s" % location)
+        weather_sensors: Dict[str, Sensor] = (
+            {}
+        )  # keep track of the sensors to save lookups
+        db_forecasts: Dict[Sensor, List[TimedBelief]] = {}  # collect beliefs per sensor
+
+        now = server_now()
+        time_of_api_call, forecasts = call_api(api_key, location)
+        diff_fm_owm = now - time_of_api_call
+        if abs(diff_fm_owm) > timedelta(minutes=10):
+            click.echo(
+                f"[FLEXMEASURES-WEATHER] Warning: difference between this server and Weather Provider is {naturaldelta(diff_fm_owm)}"
+            )
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] Called weather provider {provider} API successfully at {now}."
+        )
+
+        # loop through forecasts, including the one of current hour (horizon 0)
+        for fc in forecasts:
+            time_key = fc["dt"] if provider == "OWM" else fc["time_epoch"]
+            fc_datetime = as_server_time(
+                datetime.fromtimestamp(time_key, get_timezone())
+            )
+            click.echo(
+                f"[FLEXMEASURES-WEATHER] Processing forecast for {fc_datetime} ..."
+            )
+            data_source = get_or_create_owm_data_source()
+            for sensor_specs in mapping:
+                sensor_name = str(sensor_specs["fm_sensor_name"])
+                provider_response_label = sensor_specs[f"{provider}_sensor_name"]
+                if provider_response_label in fc:
+                    weather_sensor = get_weather_sensor(
+                        sensor_specs,
+                        location,
+                        weather_sensors,
+                        max_degree_difference_for_nearest_weather_sensor,
+                    )
+                    if weather_sensor is not None:
+                        click.echo(
+                            f"Found pre-configured weather sensor {weather_sensor.name} ..."
+                        )
+                        if weather_sensor not in db_forecasts.keys():
+                            db_forecasts[weather_sensor] = []
+
+                        fc_value = fc[provider_response_label]
+
+                        if provider_response_label == "wind_kph":
+                            # convert wind speed from kph to m/s
+                            fc_value = fc[provider_response_label] / 3.6
+
+                        # the irradiance is not available in Provider -> we compute it ourselves
+                        if sensor_name == "irradiance":
+                            fc_value = compute_irradiance(
+                                location[0],
+                                location[1],
+                                fc_datetime,
+                                # Provider sends cloud cover in percent, we need a ratio
+                                fc_value / 100.0,
+                            )
+                            data_source = (
+                                get_or_create_owm_data_source_for_derived_data()
+                            )
+
+                        db_forecasts[weather_sensor].append(
+                            TimedBelief(
+                                event_start=fc_datetime,
+                                belief_time=now,
+                                event_value=fc_value,
+                                sensor=weather_sensor,
+                                source=data_source,
+                            )
+                        )
+                else:
+                    # we will not fail here, but issue a warning
+                    msg = "No label '%s' in response data for time %s" % (
+                        provider_response_label,
+                        fc_datetime,
+                    )
+                    click.echo("[FLEXMEASURES-WEATHER] %s" % msg)
+                    current_app.logger.warning(msg)
+    for sensor in db_forecasts.keys():
+        click.echo(f"[FLEXMEASURES-WEATHER] Saving {sensor.name} forecasts ...")
+        if len(db_forecasts[sensor]) == 0:
+            # This is probably a serious problem
+            raise Exception(
+                "Nothing to put in the database was produced. That does not seem right..."
+            )
+        status = save_to_db(BeliefsDataFrame(db_forecasts[sensor]))
+        if status == "success_but_nothing_new":
+            current_app.logger.info(
+                "[FLEXMEASURES-WEATHER] Done. These beliefs had already been saved before."
+            )
+        elif status == "success_with_unchanged_beliefs_skipped":
+            current_app.logger.info(
+                "[FLEXMEASURES-WEATHER] Done. Some beliefs had already been saved before."
+            )
+
+
+def get_weather_sensor(
+    sensor_specs: dict,
+    location: Tuple[float, float],
+    weather_sensors: Dict[str, Sensor],
+    max_degree_difference_for_nearest_weather_sensor: int,
+) -> Sensor | None:
+    """Get the weather sensor for this own response label and location, if we haven't retrieved it already."""
+    sensor_name = str(sensor_specs["fm_sensor_name"])
+    if sensor_name in weather_sensors:
+        weather_sensor = weather_sensors[sensor_name]
+    else:
+        weather_sensor = find_weather_sensor_by_location(
+            location,
+            max_degree_difference_for_nearest_weather_sensor,
+            sensor_name=sensor_name,
+        )
+        weather_sensors[sensor_name] = weather_sensor
+    if (
+        weather_sensor is not None
+        and weather_sensor.event_resolution != sensor_specs["event_resolution"]
+    ):
+        raise Exception(
+            f"[FLEXMEASURES-WEATHER] The weather sensor found for {sensor_name} has an unfitting event resolution (should be {sensor_specs['event_resolution']}, but is {weather_sensor.event_resolution}."
+        )
+    return weather_sensor
+
+
+def save_forecasts_as_json(
+    api_key: str, locations: List[Tuple[float, float]], data_path: str
+):
+    """Get forecasts, then store each as a raw JSON file, for later processing."""
+    click.echo("[FLEXMEASURES-WEATHER] Getting weather forecasts:")
+    click.echo("[FLEXMEASURES-WEATHER] Latitude, Longitude")
+    click.echo("[FLEXMEASURES-WEATHER] ----------------------")
+    for location in locations:
+        click.echo("[FLEXMEASURES-WEATHER] %s, %s" % location)
+        now = server_now()
+        time_of_api_call, forecasts = call_api(api_key, location)
+        diff_fm_owm = now - time_of_api_call
+        if abs(diff_fm_owm) > timedelta(minutes=10):
+            click.echo(
+                f"[FLEXMEASURES-WEATHER] Warning: difference between this server and Weather Provider is {naturaldelta(diff_fm_owm)}"
+            )
+        now_str = now.strftime("%Y-%m-%dT%H-%M-%S")
+        path_to_files = os.path.join(data_path, now_str)
+        if not os.path.exists(path_to_files):
+            click.echo(f"[FLEXMEASURES-WEATHER] Making directory: {path_to_files} ...")
+            os.mkdir(path_to_files)
+        forecasts_file = "%s/forecast_lat_%s_lng_%s.json" % (
+            path_to_files,
+            str(location[0]),
+            str(location[1]),
+        )
+        with open(forecasts_file, "w") as outfile:
+            json.dump(forecasts, outfile)
+
+
+def check_openweathermap_version(api_version: str):
+    supported_versions = ["2.5", "3.0"]
+    if api_version not in supported_versions:
+        current_app.logger.warning(
+            f"This plugin may not be fully compatible with OpenWeatherMap API version {api_version}. We tested with versions {supported_versions}"
+        )