Add FlexMeasures plugins, USEF protocol, and Cariflex simulator
- flexmeasures-entsoe: ENTSO-E data plugin - flexmeasures-weather: Weather data plugin - USEF Flex Trading Protocol PDF (2.4MB) - Cariflex simulator (publishes to Redis) - Dashboard Grafana updated with correct InfluxDB queries - All tools extracted in /tools/
This commit is contained in:
Eric F
BIN
tools/flexmeasures-weather/flexmeasures_weather/utils/.DS_Store
vendored
Normal file
BIN
tools/flexmeasures-weather/flexmeasures_weather/utils/.DS_Store
vendored
Normal file
Binary file not shown.
@@ -0,0 +1,22 @@
|
||||
import sys
|
||||
import importlib
|
||||
|
||||
|
||||
def ensure_bp_routes_are_loaded_fresh(module_name):
|
||||
"""
|
||||
Reload a module if it has been loaded before.
|
||||
It's useful for situations in which some other process has read
|
||||
the module before, but you need some action to happen which only
|
||||
happens during module import ― decorators are a good example.
|
||||
|
||||
One use case is pytest, which reads all python code when it collects tests.
|
||||
In our case, that happens before FlexMeasures' import mechanism
|
||||
has had a chance to know which blueprints a plugin has.
|
||||
Seemingly, the importing code (plugin's __init__) can be imported later
|
||||
than the imported module (containing @route decorators).
|
||||
Re-importing helps to get this order right when FlexMeasures reads the
|
||||
plugin's __init__.
|
||||
"""
|
||||
m_name = "flexmeasures_weather." + module_name
|
||||
if m_name in sys.modules:
|
||||
importlib.reload(sys.modules[m_name])
|
||||
@@ -0,0 +1,25 @@
|
||||
import os
|
||||
|
||||
import click
|
||||
from flask import Flask, current_app
|
||||
|
||||
from flexmeasures_weather import DEFAULT_FILE_PATH_LOCATION
|
||||
|
||||
|
||||
def make_file_path(app: Flask, region: str) -> str:
|
||||
"""Ensure and return path for weather data"""
|
||||
file_path = current_app.config.get(
|
||||
"WEATHER_FILE_PATH_LOCATION", DEFAULT_FILE_PATH_LOCATION
|
||||
)
|
||||
data_path = os.path.join(app.root_path, file_path)
|
||||
if not os.path.exists(data_path):
|
||||
click.echo("[FLEXMEASURES-WEATHER] Creating %s ..." % data_path)
|
||||
os.mkdir(data_path)
|
||||
# optional: extend with subpath for region
|
||||
if region is not None and region != "":
|
||||
region_data_path = "%s/%s" % (data_path, region)
|
||||
if not os.path.exists(region_data_path):
|
||||
click.echo("[FLEXMEASURES-WEATHER] Creating %s ..." % region_data_path)
|
||||
os.mkdir(region_data_path)
|
||||
data_path = region_data_path
|
||||
return data_path
|
||||
@@ -0,0 +1,127 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple, List, Optional
|
||||
|
||||
import click
|
||||
from flask import current_app
|
||||
|
||||
from flexmeasures.utils.grid_cells import LatLngGrid, get_cell_nums
|
||||
from flexmeasures import Sensor
|
||||
from flexmeasures.data.models.generic_assets import GenericAsset
|
||||
from flexmeasures.utils import flexmeasures_inflection
|
||||
|
||||
from .. import WEATHER_STATION_TYPE_NAME
|
||||
|
||||
|
||||
def get_locations(
|
||||
location: str,
|
||||
num_cells: int,
|
||||
method: str,
|
||||
) -> List[Tuple[float, float]]:
|
||||
"""
|
||||
Get locations for getting forecasts for, by parsing the location string, which possibly opens a latitude/longitude grid with several neatly ordered locations.
|
||||
"""
|
||||
if (
|
||||
location.count(",") == 0
|
||||
or location.count(",") != location.count(":") + 1
|
||||
or location.count(":") == 1
|
||||
and (
|
||||
location.find(",") > location.find(":")
|
||||
or location.find(",", location.find(",") + 1) < location.find(":")
|
||||
)
|
||||
):
|
||||
raise Exception(
|
||||
'[FLEXMEASURES-WEATHER] location parameter "%s" seems malformed. Please use "latitude,longitude" or '
|
||||
' "top-left-latitude,top-left-longitude:bottom-right-latitude,bottom-right-longitude"'
|
||||
% location
|
||||
)
|
||||
|
||||
location_identifiers = tuple(location.split(":"))
|
||||
|
||||
if len(location_identifiers) == 1:
|
||||
ll = location_identifiers[0].split(",")
|
||||
locations = [(float(ll[0]), float(ll[1]))]
|
||||
click.echo("[FLEXMEASURES-WEATHER] Only one location: %s,%s." % locations[0])
|
||||
elif len(location_identifiers) == 2:
|
||||
click.echo(
|
||||
"[FLEXMEASURES-WEATHER] Making a grid of locations between top/left %s and bottom/right %s ..."
|
||||
% location_identifiers
|
||||
)
|
||||
top_left = tuple(float(s) for s in location_identifiers[0].split(","))
|
||||
if len(top_left) != 2:
|
||||
raise Exception(
|
||||
"[FLEXMEASURES-WEATHER] top-left parameter '%s' is invalid."
|
||||
% location_identifiers[0]
|
||||
)
|
||||
bottom_right = tuple(float(s) for s in location_identifiers[1].split(","))
|
||||
if len(bottom_right) != 2:
|
||||
raise Exception(
|
||||
"[FLEXMEASURES-WEATHER] bottom-right parameter '%s' is invalid."
|
||||
% location_identifiers[1]
|
||||
)
|
||||
|
||||
num_lat, num_lng = get_cell_nums(top_left, bottom_right, num_cells)
|
||||
|
||||
locations = LatLngGrid(
|
||||
top_left=top_left,
|
||||
bottom_right=bottom_right,
|
||||
num_cells_lat=num_lat,
|
||||
num_cells_lng=num_lng,
|
||||
).get_locations(method)
|
||||
else:
|
||||
raise Exception(
|
||||
"[FLEXMEASURES-WEATHER] location parameter '%s' has too many locations."
|
||||
% location
|
||||
)
|
||||
return locations
|
||||
|
||||
|
||||
def find_weather_sensor_by_location(
|
||||
location: Tuple[float, float],
|
||||
max_degree_difference_for_nearest_weather_sensor: int,
|
||||
sensor_name: str,
|
||||
) -> Sensor | None:
|
||||
"""
|
||||
Try to find a weather sensor of fitting type close by.
|
||||
Return None if the nearest weather sensor is further away than some minimum degrees or if no sensor was found at all.
|
||||
"""
|
||||
weather_sensor: Optional[Sensor] = Sensor.find_closest(
|
||||
generic_asset_type_name=WEATHER_STATION_TYPE_NAME,
|
||||
sensor_name=sensor_name,
|
||||
lat=location[0],
|
||||
lng=location[1],
|
||||
n=1,
|
||||
)
|
||||
if weather_sensor is None:
|
||||
current_app.logger.warning(
|
||||
"[FLEXMEASURES-WEATHER] No weather sensor set up yet for measuring %s. Try the register-weather-sensor CLI task."
|
||||
% sensor_name
|
||||
)
|
||||
return None
|
||||
weather_station: GenericAsset = weather_sensor.generic_asset
|
||||
if abs(
|
||||
location[0] - weather_station.location[0]
|
||||
) > max_degree_difference_for_nearest_weather_sensor or abs(
|
||||
location[1] - weather_station.location[1]
|
||||
> max_degree_difference_for_nearest_weather_sensor
|
||||
):
|
||||
current_app.logger.warning(
|
||||
f"[FLEXMEASURES-WEATHER] We found a weather station, but no sufficiently close weather sensor found (within {max_degree_difference_for_nearest_weather_sensor} {flexmeasures_inflection.pluralize('degree', max_degree_difference_for_nearest_weather_sensor)} distance) for measuring {sensor_name}! We're looking for: {location}, closest available: ({weather_station.location})"
|
||||
)
|
||||
return None
|
||||
return weather_sensor
|
||||
|
||||
|
||||
def get_location_by_asset_id(asset_id: int) -> Tuple[float, float]:
|
||||
"""Get location for forecasting by passing an asset id"""
|
||||
asset = GenericAsset.query.filter(GenericAsset.id == asset_id).one_or_none()
|
||||
if asset.generic_asset_type.name != WEATHER_STATION_TYPE_NAME:
|
||||
raise Exception(
|
||||
f"Asset {asset} does not seem to be a weather station we should use ― we expect an asset with type '{WEATHER_STATION_TYPE_NAME}'."
|
||||
)
|
||||
if asset is None:
|
||||
raise Exception(
|
||||
"[FLEXMEASURES-WEATHER] No asset found for the given asset id %s."
|
||||
% asset_id
|
||||
)
|
||||
return (asset.latitude, asset.longitude)
|
||||
@@ -0,0 +1,124 @@
|
||||
from packaging import version
|
||||
|
||||
from flask import current_app
|
||||
from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
|
||||
from flexmeasures import Source, __version__ as flexmeasures_version
|
||||
from flexmeasures.data import db
|
||||
from flexmeasures.data.services.data_sources import get_or_create_source
|
||||
|
||||
from flexmeasures_weather import DEFAULT_DATA_SOURCE_NAME
|
||||
from flexmeasures_weather import WEATHER_STATION_TYPE_NAME
|
||||
from flexmeasures_weather import DEFAULT_WEATHER_STATION_NAME
|
||||
|
||||
|
||||
if version.parse(flexmeasures_version) < version.parse("0.13"):
|
||||
SOURCE_TYPE = "forecasting script"
|
||||
else:
|
||||
SOURCE_TYPE = "forecaster"
|
||||
|
||||
FM_SUPPORTS_ACCOUNT_LINKED_SOURCES = version.parse(
|
||||
flexmeasures_version
|
||||
) >= version.parse("0.32")
|
||||
|
||||
if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
|
||||
from flexmeasures import Account
|
||||
else:
|
||||
Account = None
|
||||
|
||||
|
||||
def get_or_create_weather_account():
|
||||
"""Make sure we have an account for the weather provider service."""
|
||||
if Account is None:
|
||||
raise RuntimeError(
|
||||
"FlexMeasures Account model is unavailable before FlexMeasures 0.32."
|
||||
)
|
||||
account_name = current_app.config.get(
|
||||
"WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
|
||||
)
|
||||
weather_account = Account.query.filter(
|
||||
Account.name == account_name,
|
||||
).one_or_none()
|
||||
if weather_account is None:
|
||||
weather_account = Account(name=account_name)
|
||||
db.session.add(weather_account)
|
||||
db.session.flush()
|
||||
return weather_account
|
||||
|
||||
|
||||
def get_or_create_owm_data_source() -> Source:
|
||||
"""Make sure we have a weather provider data source of the configured type."""
|
||||
source_kwargs = dict(
|
||||
source=current_app.config.get(
|
||||
"WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
|
||||
),
|
||||
source_type=SOURCE_TYPE,
|
||||
flush=False,
|
||||
)
|
||||
if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
|
||||
source_kwargs["account"] = get_or_create_weather_account()
|
||||
return get_or_create_source(**source_kwargs)
|
||||
|
||||
|
||||
def get_or_create_owm_data_source_for_derived_data() -> Source:
|
||||
owm_source_name = current_app.config.get(
|
||||
"WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
|
||||
)
|
||||
source_kwargs = dict(
|
||||
source=f"FlexMeasures {owm_source_name}",
|
||||
source_type=SOURCE_TYPE,
|
||||
flush=False,
|
||||
)
|
||||
if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
|
||||
source_kwargs["account"] = get_or_create_weather_account()
|
||||
return get_or_create_source(**source_kwargs)
|
||||
|
||||
|
||||
def get_or_create_weather_station_type() -> GenericAssetType:
|
||||
"""Make sure a weather station type exists"""
|
||||
weather_station_type = GenericAssetType.query.filter(
|
||||
GenericAssetType.name == WEATHER_STATION_TYPE_NAME,
|
||||
).one_or_none()
|
||||
if weather_station_type is None:
|
||||
weather_station_type = GenericAssetType(
|
||||
name=WEATHER_STATION_TYPE_NAME,
|
||||
description="A weather station with various sensors.",
|
||||
)
|
||||
db.session.add(weather_station_type)
|
||||
return weather_station_type
|
||||
|
||||
|
||||
def get_or_create_weather_station(latitude: float, longitude: float) -> GenericAsset:
|
||||
"""Make sure a weather station exists at this location."""
|
||||
station_name = current_app.config.get(
|
||||
"WEATHER_STATION_NAME", DEFAULT_WEATHER_STATION_NAME
|
||||
)
|
||||
weather_station = GenericAsset.query.filter(
|
||||
GenericAsset.latitude == latitude, GenericAsset.longitude == longitude
|
||||
).one_or_none()
|
||||
if weather_station is None:
|
||||
weather_station_type = get_or_create_weather_station_type()
|
||||
weather_station = GenericAsset(
|
||||
name=station_name,
|
||||
generic_asset_type=weather_station_type,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
)
|
||||
db.session.add(weather_station)
|
||||
return weather_station
|
||||
|
||||
|
||||
def get_weather_station_by_asset_id(asset_id: int) -> GenericAsset:
|
||||
weather_station = GenericAsset.query.filter(
|
||||
GenericAsset.generic_asset_type_id == asset_id
|
||||
).one_or_none()
|
||||
if weather_station is None:
|
||||
raise Exception(
|
||||
f"[FLEXMEASURES-WEATHER] Weather station is not present for the given asset id '{asset_id}'."
|
||||
)
|
||||
|
||||
if weather_station.latitude is None or weather_station.longitude is None:
|
||||
raise Exception(
|
||||
f"[FLEXMEASURES-WEATHER] Weather station {weather_station} is missing location information [Latitude, Longitude]."
|
||||
)
|
||||
|
||||
return weather_station
|
||||
@@ -0,0 +1,37 @@
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
from pvlib.location import Location
|
||||
|
||||
|
||||
def compute_irradiance(
|
||||
latitude: float, longitude: float, dt: datetime, cloud_coverage: float
|
||||
) -> float:
|
||||
"""Compute the irradiance received on a location at a specific time.
|
||||
This uses pvlib to
|
||||
1) compute clear-sky irradiance as Global Horizontal Irradiance (GHI),
|
||||
which includes both Direct Normal Irradiance (DNI)
|
||||
and Diffuse Horizontal Irradiance (DHI).
|
||||
2) adjust the GHI for cloud coverage
|
||||
"""
|
||||
site = Location(latitude, longitude, tz=dt.tzinfo)
|
||||
solpos = site.get_solarposition(pd.DatetimeIndex([dt]))
|
||||
ghi_clear = site.get_clearsky(pd.DatetimeIndex([dt]), solar_position=solpos).loc[
|
||||
dt
|
||||
]["ghi"]
|
||||
return ghi_clear_to_ghi(ghi_clear, cloud_coverage)
|
||||
|
||||
|
||||
def ghi_clear_to_ghi(ghi_clear: float, cloud_coverage: float) -> float:
|
||||
"""Compute global horizontal irradiance (GHI) from clear-sky GHI, given a cloud coverage between 0 and 1.
|
||||
|
||||
References
|
||||
----------
|
||||
Perez, R., Moore, K., Wilcox, S., Renne, D., Zelenka, A., 2007.
|
||||
Forecasting solar radiation – preliminary evaluation of an
|
||||
approach based upon the national forecast database. Solar Energy
|
||||
81, 809–812.
|
||||
"""
|
||||
if cloud_coverage < 0 or cloud_coverage > 1:
|
||||
raise ValueError("cloud_coverage should lie in the interval [0, 1]")
|
||||
return (1 - 0.87 * cloud_coverage**1.9) * ghi_clear
|
||||
@@ -0,0 +1,194 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from flexmeasures import Asset
|
||||
|
||||
import flexmeasures_weather.utils.modeling as modeling
|
||||
from flexmeasures_weather import DEFAULT_DATA_SOURCE_NAME, DEFAULT_WEATHER_STATION_NAME
|
||||
from flexmeasures_weather.utils.modeling import (
|
||||
FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
SOURCE_TYPE,
|
||||
get_or_create_owm_data_source,
|
||||
get_or_create_owm_data_source_for_derived_data,
|
||||
get_or_create_weather_account,
|
||||
get_or_create_weather_station,
|
||||
)
|
||||
|
||||
|
||||
def test_creating_two_weather_stations(fresh_db):
|
||||
get_or_create_weather_station(50, 40)
|
||||
get_or_create_weather_station(40, 50)
|
||||
assert Asset.query.filter(Asset.name == DEFAULT_WEATHER_STATION_NAME).count() == 2
|
||||
|
||||
|
||||
# The version-branch tests below still use monkeypatching to isolate source
|
||||
# creation side effects without requiring multiple FlexMeasures installs.
|
||||
@pytest.mark.skipif(
|
||||
not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
reason="Weather source accounts are only supported on FlexMeasures >= 0.32.",
|
||||
)
|
||||
def test_get_or_create_weather_account(fresh_db):
|
||||
weather_account = get_or_create_weather_account()
|
||||
|
||||
assert weather_account.name == DEFAULT_DATA_SOURCE_NAME
|
||||
assert (
|
||||
modeling.Account.query.filter(
|
||||
modeling.Account.name == weather_account.name
|
||||
).count()
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
|
||||
)
|
||||
def test_get_or_create_owm_data_source_registers_weather_source_on_weather_account(
|
||||
fresh_db,
|
||||
):
|
||||
data_source = get_or_create_owm_data_source()
|
||||
|
||||
assert data_source.type == SOURCE_TYPE
|
||||
assert data_source.account is not None
|
||||
assert data_source.account.name == data_source.name
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
|
||||
)
|
||||
def test_get_or_create_owm_data_source_for_derived_data_uses_weather_account(fresh_db):
|
||||
derived_data_source = get_or_create_owm_data_source_for_derived_data()
|
||||
|
||||
assert derived_data_source.type == SOURCE_TYPE
|
||||
assert derived_data_source.account is not None
|
||||
assert derived_data_source.account.name == DEFAULT_DATA_SOURCE_NAME
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
|
||||
)
|
||||
def test_get_or_create_owm_data_source_passes_weather_account_when_supported(
|
||||
fresh_db, monkeypatch
|
||||
):
|
||||
captured_kwargs = {}
|
||||
|
||||
def fake_get_or_create_source(source, source_type, account, flush):
|
||||
captured_kwargs.update(
|
||||
dict(
|
||||
source=source,
|
||||
source_type=source_type,
|
||||
account=account,
|
||||
flush=flush,
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(type=source_type, account=account)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"flexmeasures_weather.utils.modeling.get_or_create_source",
|
||||
fake_get_or_create_source,
|
||||
)
|
||||
|
||||
data_source = get_or_create_owm_data_source()
|
||||
|
||||
assert data_source.type == SOURCE_TYPE
|
||||
assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
|
||||
)
|
||||
def test_get_or_create_owm_derived_data_source_passes_weather_account_when_supported(
|
||||
fresh_db, monkeypatch
|
||||
):
|
||||
captured_kwargs = {}
|
||||
|
||||
def fake_get_or_create_source(source, source_type, account, flush):
|
||||
captured_kwargs.update(
|
||||
dict(
|
||||
source=source,
|
||||
source_type=source_type,
|
||||
account=account,
|
||||
flush=flush,
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(type=source_type, account=account)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"flexmeasures_weather.utils.modeling.get_or_create_source",
|
||||
fake_get_or_create_source,
|
||||
)
|
||||
|
||||
data_source = get_or_create_owm_data_source_for_derived_data()
|
||||
|
||||
assert data_source.type == SOURCE_TYPE
|
||||
assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
reason="Legacy source creation without accounts is only used on FlexMeasures < 0.32.",
|
||||
)
|
||||
def test_get_or_create_owm_data_source_omits_account_when_not_supported(monkeypatch):
|
||||
captured_kwargs = {}
|
||||
|
||||
def fake_get_or_create_source(source, source_type, flush):
|
||||
captured_kwargs.update(
|
||||
dict(
|
||||
source=source,
|
||||
source_type=source_type,
|
||||
flush=flush,
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(type=source_type, name=source)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"flexmeasures_weather.utils.modeling.get_or_create_source",
|
||||
fake_get_or_create_source,
|
||||
)
|
||||
|
||||
data_source = get_or_create_owm_data_source()
|
||||
|
||||
assert data_source.type == SOURCE_TYPE
|
||||
assert captured_kwargs == {
|
||||
"source": DEFAULT_DATA_SOURCE_NAME,
|
||||
"source_type": SOURCE_TYPE,
|
||||
"flush": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
|
||||
reason="Legacy source creation without accounts is only used on FlexMeasures < 0.32.",
|
||||
)
|
||||
def test_get_or_create_owm_derived_data_source_omits_account_when_not_supported(
|
||||
monkeypatch,
|
||||
):
|
||||
captured_kwargs = {}
|
||||
|
||||
def fake_get_or_create_source(source, source_type, flush):
|
||||
captured_kwargs.update(
|
||||
dict(
|
||||
source=source,
|
||||
source_type=source_type,
|
||||
flush=flush,
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(type=source_type, name=source)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"flexmeasures_weather.utils.modeling.get_or_create_source",
|
||||
fake_get_or_create_source,
|
||||
)
|
||||
|
||||
data_source = get_or_create_owm_data_source_for_derived_data()
|
||||
|
||||
assert data_source.type == SOURCE_TYPE
|
||||
assert captured_kwargs == {
|
||||
"source": f"FlexMeasures {DEFAULT_DATA_SOURCE_NAME}",
|
||||
"source_type": SOURCE_TYPE,
|
||||
"flush": False,
|
||||
}
|
||||
357
tools/flexmeasures-weather/flexmeasures_weather/utils/weather.py
Normal file
357
tools/flexmeasures-weather/flexmeasures_weather/utils/weather.py
Normal file
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple, List, Dict, Optional, Any
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
import json
|
||||
|
||||
import click
|
||||
from flask import current_app
|
||||
import requests
|
||||
from humanize import naturaldelta
|
||||
from timely_beliefs import BeliefsDataFrame
|
||||
from flexmeasures.utils.time_utils import as_server_time, get_timezone, server_now
|
||||
from flexmeasures.data.models.time_series import Sensor, TimedBelief
|
||||
from flexmeasures.data.utils import save_to_db
|
||||
|
||||
from flexmeasures_weather import DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE
|
||||
from .locating import find_weather_sensor_by_location
|
||||
from ..sensor_specs import mapping
|
||||
from .modeling import (
|
||||
get_or_create_owm_data_source,
|
||||
get_or_create_owm_data_source_for_derived_data,
|
||||
)
|
||||
from .radiating import compute_irradiance
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
API_VERSION = "3.0"
|
||||
|
||||
|
||||
def get_supported_sensor_spec(name: str) -> Optional[dict]:
|
||||
"""
|
||||
Find the specs from a sensor by name.
|
||||
"""
|
||||
for supported_sensor_spec in mapping:
|
||||
if supported_sensor_spec["fm_sensor_name"] == name:
|
||||
return supported_sensor_spec.copy()
|
||||
return None
|
||||
|
||||
|
||||
def get_supported_sensors_str() -> str:
|
||||
"""A string - list of supported sensors, also revealing their unit"""
|
||||
return ", ".join(
|
||||
[
|
||||
f"{sensor_specs['fm_sensor_name']} ({sensor_specs['unit']})"
|
||||
for sensor_specs in mapping
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def process_weatherapi_data(
|
||||
data: List[Dict[str, Any]], hour_no: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Processes raw WeatherAPI forecast data into a format similar to OpenWeatherMap's format.
|
||||
|
||||
Args:
|
||||
data (List[Dict[str, Any]]): A list of forecast day dictionaries from WeatherAPI,
|
||||
each containing an 'hour' key with 24 hourly entries.
|
||||
hour_no (int): The index of the current hour to start from.
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: A list of 48 hourly forecast entries, each mapped to the
|
||||
expected structure with fields like temperature, humidity, wind, and condition.
|
||||
"""
|
||||
first_day = data[0]["hour"]
|
||||
second_day = data[1]["hour"]
|
||||
third_day = data[2]["hour"]
|
||||
combined = first_day + second_day + third_day
|
||||
|
||||
relevant = combined[hour_no : hour_no + 48]
|
||||
return relevant
|
||||
|
||||
|
||||
def call_openweatherapi(
|
||||
api_key: str, location: Tuple[float, float]
|
||||
) -> Tuple[datetime, List[Dict]]:
|
||||
"""
|
||||
Make a single "one-call" to the Open Weather API and return the API timestamp as well as the 48 hourly forecasts.
|
||||
See https://openweathermap.org/api/one-call-3 for docs.
|
||||
Note that the first forecast is about the current hour.
|
||||
"""
|
||||
check_openweathermap_version(API_VERSION)
|
||||
query_str = f"lat={location[0]}&lon={location[1]}&units=metric&exclude=minutely,daily,alerts&appid={api_key}"
|
||||
res = requests.get(
|
||||
f"http://api.openweathermap.org/data/{API_VERSION}/onecall?{query_str}"
|
||||
)
|
||||
assert (
|
||||
res.status_code == 200
|
||||
), f"OpenWeatherMap returned status code {res.status_code}: {res.text}"
|
||||
data = res.json()
|
||||
time_of_api_call = as_server_time(
|
||||
datetime.fromtimestamp(data["current"]["dt"], tz=get_timezone())
|
||||
).replace(second=0, microsecond=0)
|
||||
return time_of_api_call, data["hourly"]
|
||||
|
||||
|
||||
def call_weatherapi(
|
||||
api_key: str, location: Tuple[float, float], days: int = 3
|
||||
) -> Tuple[datetime, List[Dict]]:
|
||||
"""
|
||||
Makes a request to the WeatherAPI to retrieve hourly weather forecast data.
|
||||
|
||||
Args:
|
||||
api_key (str): API key for authenticating with the Weather API.
|
||||
location (Tuple[float, float]): A tuple containing the latitude and longitude.
|
||||
days (int, optional): Number of days to request the forecast for (default is 3, including current day).
|
||||
|
||||
Returns:
|
||||
Tuple[datetime, List[Dict]]:
|
||||
- The timestamp of the API call.
|
||||
- A list of hourly forecast data as dictionaries. Note that the first forecast is about the current hour.
|
||||
|
||||
Raises:
|
||||
AssertionError: If the response from the Weather API is not successful (HTTP status 200).
|
||||
"""
|
||||
|
||||
latitude, longitude = location[0], location[1]
|
||||
|
||||
query_str = f"http://api.weatherapi.com/v1/forecast.json?key={api_key}&q={latitude},{longitude}&days={days}&aqi=yes&alerts=yes"
|
||||
res = requests.get(query_str)
|
||||
|
||||
assert (
|
||||
res.status_code == 200
|
||||
), f"Weather API returned status code {res.status_code}: {res.text}"
|
||||
|
||||
data = res.json()
|
||||
|
||||
# get the time of the api call
|
||||
time_of_call = int(data["location"]["localtime_epoch"])
|
||||
local_timezone = ZoneInfo(data["location"]["tz_id"])
|
||||
local_time = datetime.fromtimestamp(time_of_call, local_timezone)
|
||||
time_of_api_call = as_server_time(local_time)
|
||||
time_of_api_call = time_of_api_call.replace(second=0, microsecond=0)
|
||||
|
||||
print(f"Time of API call in WAPI is {time_of_api_call}")
|
||||
|
||||
relevant = data["forecast"]["forecastday"]
|
||||
hour_no = local_time.hour
|
||||
|
||||
hourly = process_weatherapi_data(relevant, hour_no)
|
||||
return time_of_api_call, hourly
|
||||
|
||||
|
||||
def call_api(
|
||||
api_key: str, location: Tuple[float, float]
|
||||
) -> Tuple[datetime, List[Dict]]:
|
||||
"""
|
||||
Dispatches the weather API call based on the configured provider.
|
||||
|
||||
Args:
|
||||
api_key (str): API key for the selected weather service provider.
|
||||
location (Tuple[float, float]): Latitude and longitude tuple.
|
||||
|
||||
Returns:
|
||||
Tuple[datetime, List[Dict]]:
|
||||
- Timestamp of the API call.
|
||||
- List of hourly forecast data.
|
||||
|
||||
Raises:
|
||||
Exception: If an invalid weather provider is configured.
|
||||
"""
|
||||
|
||||
provider = str(current_app.config.get("WEATHER_PROVIDER", "OWM"))
|
||||
if provider not in ["OWM", "WAPI"]:
|
||||
raise Exception(
|
||||
"Invalid provider name. Please set WEATHER_PROVIDER setting in config file to either OWM or WAPI, the two permissible options."
|
||||
)
|
||||
|
||||
if provider == "OWM":
|
||||
click.secho("Calling Open Weather Map")
|
||||
return call_openweatherapi(api_key, location)
|
||||
else:
|
||||
click.secho("Calling Weather API")
|
||||
return call_weatherapi(api_key, location)
|
||||
|
||||
|
||||
def save_forecasts_in_db( # noqa: C901
|
||||
api_key: str,
|
||||
locations: List[Tuple[float, float]],
|
||||
):
|
||||
"""Process the response from Weather Provider API into timed beliefs.
|
||||
Collects all forecasts for all locations and all sensors at all locations, then bulk-saves them.
|
||||
"""
|
||||
click.echo("[FLEXMEASURES-WEATHER] Getting weather forecasts:")
|
||||
click.echo("[FLEXMEASURES-WEATHER] Latitude, Longitude")
|
||||
click.echo("[FLEXMEASURES-WEATHER] -----------------------")
|
||||
max_degree_difference_for_nearest_weather_sensor = current_app.config.get(
|
||||
"WEATHER_MAXIMAL_DEGREE_LOCATION_DISTANCE",
|
||||
DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE,
|
||||
)
|
||||
provider = str(current_app.config.get("WEATHER_PROVIDER", ""))
|
||||
if provider not in ["OWM", "WAPI"]:
|
||||
raise Exception(
|
||||
"Invalid provider name. Please set WEATHER_PROVIDER setting in config file to either OWM or WAPI, the two permissible options."
|
||||
)
|
||||
for location in locations:
|
||||
click.echo("[FLEXMEASURES] %s, %s" % location)
|
||||
weather_sensors: Dict[str, Sensor] = (
|
||||
{}
|
||||
) # keep track of the sensors to save lookups
|
||||
db_forecasts: Dict[Sensor, List[TimedBelief]] = {} # collect beliefs per sensor
|
||||
|
||||
now = server_now()
|
||||
time_of_api_call, forecasts = call_api(api_key, location)
|
||||
diff_fm_owm = now - time_of_api_call
|
||||
if abs(diff_fm_owm) > timedelta(minutes=10):
|
||||
click.echo(
|
||||
f"[FLEXMEASURES-WEATHER] Warning: difference between this server and Weather Provider is {naturaldelta(diff_fm_owm)}"
|
||||
)
|
||||
click.echo(
|
||||
f"[FLEXMEASURES-WEATHER] Called weather provider {provider} API successfully at {now}."
|
||||
)
|
||||
|
||||
# loop through forecasts, including the one of current hour (horizon 0)
|
||||
for fc in forecasts:
|
||||
time_key = fc["dt"] if provider == "OWM" else fc["time_epoch"]
|
||||
fc_datetime = as_server_time(
|
||||
datetime.fromtimestamp(time_key, get_timezone())
|
||||
)
|
||||
click.echo(
|
||||
f"[FLEXMEASURES-WEATHER] Processing forecast for {fc_datetime} ..."
|
||||
)
|
||||
data_source = get_or_create_owm_data_source()
|
||||
for sensor_specs in mapping:
|
||||
sensor_name = str(sensor_specs["fm_sensor_name"])
|
||||
provider_response_label = sensor_specs[f"{provider}_sensor_name"]
|
||||
if provider_response_label in fc:
|
||||
weather_sensor = get_weather_sensor(
|
||||
sensor_specs,
|
||||
location,
|
||||
weather_sensors,
|
||||
max_degree_difference_for_nearest_weather_sensor,
|
||||
)
|
||||
if weather_sensor is not None:
|
||||
click.echo(
|
||||
f"Found pre-configured weather sensor {weather_sensor.name} ..."
|
||||
)
|
||||
if weather_sensor not in db_forecasts.keys():
|
||||
db_forecasts[weather_sensor] = []
|
||||
|
||||
fc_value = fc[provider_response_label]
|
||||
|
||||
if provider_response_label == "wind_kph":
|
||||
# convert wind speed from kph to m/s
|
||||
fc_value = fc[provider_response_label] / 3.6
|
||||
|
||||
# the irradiance is not available in Provider -> we compute it ourselves
|
||||
if sensor_name == "irradiance":
|
||||
fc_value = compute_irradiance(
|
||||
location[0],
|
||||
location[1],
|
||||
fc_datetime,
|
||||
# Provider sends cloud cover in percent, we need a ratio
|
||||
fc_value / 100.0,
|
||||
)
|
||||
data_source = (
|
||||
get_or_create_owm_data_source_for_derived_data()
|
||||
)
|
||||
|
||||
db_forecasts[weather_sensor].append(
|
||||
TimedBelief(
|
||||
event_start=fc_datetime,
|
||||
belief_time=now,
|
||||
event_value=fc_value,
|
||||
sensor=weather_sensor,
|
||||
source=data_source,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# we will not fail here, but issue a warning
|
||||
msg = "No label '%s' in response data for time %s" % (
|
||||
provider_response_label,
|
||||
fc_datetime,
|
||||
)
|
||||
click.echo("[FLEXMEASURES-WEATHER] %s" % msg)
|
||||
current_app.logger.warning(msg)
|
||||
for sensor in db_forecasts.keys():
|
||||
click.echo(f"[FLEXMEASURES-WEATHER] Saving {sensor.name} forecasts ...")
|
||||
if len(db_forecasts[sensor]) == 0:
|
||||
# This is probably a serious problem
|
||||
raise Exception(
|
||||
"Nothing to put in the database was produced. That does not seem right..."
|
||||
)
|
||||
status = save_to_db(BeliefsDataFrame(db_forecasts[sensor]))
|
||||
if status == "success_but_nothing_new":
|
||||
current_app.logger.info(
|
||||
"[FLEXMEASURES-WEATHER] Done. These beliefs had already been saved before."
|
||||
)
|
||||
elif status == "success_with_unchanged_beliefs_skipped":
|
||||
current_app.logger.info(
|
||||
"[FLEXMEASURES-WEATHER] Done. Some beliefs had already been saved before."
|
||||
)
|
||||
|
||||
|
||||
def get_weather_sensor(
|
||||
sensor_specs: dict,
|
||||
location: Tuple[float, float],
|
||||
weather_sensors: Dict[str, Sensor],
|
||||
max_degree_difference_for_nearest_weather_sensor: int,
|
||||
) -> Sensor | None:
|
||||
"""Get the weather sensor for this own response label and location, if we haven't retrieved it already."""
|
||||
sensor_name = str(sensor_specs["fm_sensor_name"])
|
||||
if sensor_name in weather_sensors:
|
||||
weather_sensor = weather_sensors[sensor_name]
|
||||
else:
|
||||
weather_sensor = find_weather_sensor_by_location(
|
||||
location,
|
||||
max_degree_difference_for_nearest_weather_sensor,
|
||||
sensor_name=sensor_name,
|
||||
)
|
||||
weather_sensors[sensor_name] = weather_sensor
|
||||
if (
|
||||
weather_sensor is not None
|
||||
and weather_sensor.event_resolution != sensor_specs["event_resolution"]
|
||||
):
|
||||
raise Exception(
|
||||
f"[FLEXMEASURES-WEATHER] The weather sensor found for {sensor_name} has an unfitting event resolution (should be {sensor_specs['event_resolution']}, but is {weather_sensor.event_resolution}."
|
||||
)
|
||||
return weather_sensor
|
||||
|
||||
|
||||
def save_forecasts_as_json(
|
||||
api_key: str, locations: List[Tuple[float, float]], data_path: str
|
||||
):
|
||||
"""Get forecasts, then store each as a raw JSON file, for later processing."""
|
||||
click.echo("[FLEXMEASURES-WEATHER] Getting weather forecasts:")
|
||||
click.echo("[FLEXMEASURES-WEATHER] Latitude, Longitude")
|
||||
click.echo("[FLEXMEASURES-WEATHER] ----------------------")
|
||||
for location in locations:
|
||||
click.echo("[FLEXMEASURES-WEATHER] %s, %s" % location)
|
||||
now = server_now()
|
||||
time_of_api_call, forecasts = call_api(api_key, location)
|
||||
diff_fm_owm = now - time_of_api_call
|
||||
if abs(diff_fm_owm) > timedelta(minutes=10):
|
||||
click.echo(
|
||||
f"[FLEXMEASURES-WEATHER] Warning: difference between this server and Weather Provider is {naturaldelta(diff_fm_owm)}"
|
||||
)
|
||||
now_str = now.strftime("%Y-%m-%dT%H-%M-%S")
|
||||
path_to_files = os.path.join(data_path, now_str)
|
||||
if not os.path.exists(path_to_files):
|
||||
click.echo(f"[FLEXMEASURES-WEATHER] Making directory: {path_to_files} ...")
|
||||
os.mkdir(path_to_files)
|
||||
forecasts_file = "%s/forecast_lat_%s_lng_%s.json" % (
|
||||
path_to_files,
|
||||
str(location[0]),
|
||||
str(location[1]),
|
||||
)
|
||||
with open(forecasts_file, "w") as outfile:
|
||||
json.dump(forecasts, outfile)
|
||||
|
||||
|
||||
def check_openweathermap_version(api_version: str):
|
||||
supported_versions = ["2.5", "3.0"]
|
||||
if api_version not in supported_versions:
|
||||
current_app.logger.warning(
|
||||
f"This plugin may not be fully compatible with OpenWeatherMap API version {api_version}. We tested with versions {supported_versions}"
|
||||
)
|
||||
Reference in New Issue
Block a user