Add FlexMeasures plugins, USEF protocol, and Cariflex simulator

- flexmeasures-entsoe: ENTSO-E data plugin
- flexmeasures-weather: Weather data plugin
- USEF Flex Trading Protocol PDF (2.4MB)
- Cariflex simulator (publishes to Redis)
- Dashboard Grafana updated with correct InfluxDB queries
- All tools extracted in /tools/

.gitignore

@@ -8,3 +8,9 @@ tools/openocpp/.git
 tools/*/node_modules
 tools/*/dist
 tools/*/__pycache__
+tools/*/node_modules
+tools/*/dist
+tools/*/__pycache__
+tools/*/.git
+*.zip
+tools/citrineos-core-main/src

config/cariflex-dashboard.json

@@ -0,0 +1,134 @@
+{
+  "dashboard": {
+    "id": null,
+    "uid": "cariflex-main",
+    "title": "Cariflex - Supervision Énergétique",
+    "tags": ["cariflex", "energy", "martinique"],
+    "timezone": "America/Martinique",
+    "refresh": "30s",
+    "time": {"from": "now-24h", "to": "now"},
+    "panels": [
+      {
+        "id": 1,
+        "title": "Capteurs Air Quality (10)",
+        "type": "timeseries",
+        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"topic\"] =~ /airquality/) |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "none"},
+          "overrides": []
+        }
+      },
+      {
+        "id": 2,
+        "title": "Capteurs Weather (10)",
+        "type": "timeseries",
+        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"topic\"] =~ /weather/) |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "celsius", "min": 15, "max": 40},
+          "overrides": []
+        }
+      },
+      {
+        "id": 3,
+        "title": "Capteurs Traffic (10)",
+        "type": "timeseries",
+        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"topic\"] =~ /traffic/) |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "kmh", "min": 0, "max": 100},
+          "overrides": []
+        }
+      },
+      {
+        "id": 4,
+        "title": "Capteurs Parking (10)",
+        "type": "timeseries",
+        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"topic\"] =~ /parking/) |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "percent", "min": 0, "max": 100},
+          "overrides": []
+        }
+      },
+      {
+        "id": 5,
+        "title": "Battery Level (tous capteurs)",
+        "type": "gauge",
+        "gridPos": {"h": 6, "w": 6, "x": 0, "y": 16},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: -5m) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"_field\"] == \"battery_level\") |> mean()",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "percent", "min": 0, "max": 100, "thresholds": {"steps": [{"color": "red", "value": 0}, {"color": "yellow", "value": 20}, {"color": "green", "value": 50}]}},
+          "overrides": []
+        }
+      },
+      {
+        "id": 6,
+        "title": "Temperature (°C)",
+        "type": "stat",
+        "gridPos": {"h": 6, "w": 6, "x": 6, "y": 16},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: -5m) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"_field\"] == \"temperature_celsius\") |> mean()",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "celsius", "min": 15, "max": 40},
+          "overrides": []
+        }
+      },
+      {
+        "id": 7,
+        "title": "Noise Level (dB)",
+        "type": "stat",
+        "gridPos": {"h": 6, "w": 6, "x": 12, "y": 16},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: -5m) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"_field\"] == \"noise_level_db\") |> mean()",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "dB", "min": 0, "max": 120},
+          "overrides": []
+        }
+      },
+      {
+        "id": 8,
+        "title": "Rain (mm)",
+        "type": "stat",
+        "gridPos": {"h": 6, "w": 6, "x": 18, "y": 16},
+        "datasource": {"type": "influxdb", "uid": "influxdb-v2"},
+        "targets": [{
+          "query": "from(bucket:\"smartcity\") |> range(start: -1h) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"_field\"] == \"rain_mm\") |> sum()",
+          "refId": "A"
+        }],
+        "fieldConfig": {
+          "defaults": {"unit": "mm", "min": 0},
+          "overrides": []
+        }
+      }
+    ]
+  },
+  "overwrite": true
+}

config/docker-compose-citrineos.yml

@@ -0,0 +1,87 @@
+# Cariflex - CitrineOS docker-compose (adapté pour l'intégration Cariflex)
+# Basé sur https://github.com/citrineos/citrineos-core
+
+version: '3.8'
+
+services:
+  citrineos-server:
+    image: ghcr.io/citrineos/citrineos-server:latest
+    container_name: cariflex-citrineos-server
+    restart: unless-stopped
+    environment:
+      APP_NAME: "all"
+      APP_ENV: "docker"
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: minioadmin
+      AWS_SECRET_ACCESS_KEY: minioadmin
+      DB_STRATEGY: "migrate"
+      BOOTSTRAP_CITRINEOS_DATABASE_HOST: "cariflex-citrineos-db"
+      BOOTSTRAP_CITRINEOS_CONFIG_FILENAME: "config.json"
+      BOOTSTRAP_CITRINEOS_FILE_ACCESS_TYPE: "local"
+      BOOTSTRAP_CITRINEOS_FILE_ACCESS_LOCAL_DEFAULT_FILE_PATH: "/data"
+      CONFIG_CITRINEOS_WIPE_FILE_ON_START: "true"
+    depends_on:
+      cariflex-citrineos-db:
+        condition: service_healthy
+    ports:
+      - 8080:8080
+      - 8443:8443
+    volumes:
+      - citrineos-data:/data
+    healthcheck:
+      test: ["CMD-SHELL", "node -e \"const net = require('net'); const client = net.createConnection(8080, '127.0.0.1', () => { client.end(); process.exit(0); }); client.on('error', () => process.exit(1)); client.setTimeout(5000, () => { client.destroy(); process.exit(1); });\""]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    networks:
+      - traefik-public
+      - cariflex-internal
+
+  cariflex-citrineos-db:
+    image: postgis/postgis:16-3.5
+    container_name: cariflex-citrineos-db
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: citrine
+      POSTGRES_USER: citrine
+      POSTGRES_PASSWORD: citrine
+    volumes:
+      - citrineos-db-data:/var/lib/postgresql/data
+    healthcheck:
+      test: "pg_isready --username=citrine"
+      interval: 5s
+      timeout: 10s
+      retries: 5
+    networks:
+      - cariflex-internal
+
+  cariflex-amqp:
+    image: rabbitmq:3-management
+    container_name: cariflex-amqp
+    restart: unless-stopped
+    environment:
+      RABBITMQ_DEFAULT_USER: guest
+      RABBITMQ_DEFAULT_PASS: guest
+    volumes:
+      - citrineos-amqp-data:/var/lib/rabbitmq
+    healthcheck:
+      test: rabbitmq-diagnostics -q check_port_connectivity
+      interval: 10s
+      timeout: 10s
+      retries: 3
+    networks:
+      - cariflex-internal
+
+volumes:
+  citrineos-data:
+    driver: local
+  citrineos-db-data:
+    driver: local
+  citrineos-amqp-data:
+    driver: local
+
+networks:
+  traefik-public:
+    external: true
+  cariflex-internal:
+    driver: bridge

docs/grafana_dashboard.md

@@ -0,0 +1,78 @@
+# Cariflex - Dashboard Grafana
+
+## Datasource FlexMeasures
+
+FlexMeasures n'a pas de datasource Grafana natif. Options :
+1. **PostgreSQL direct** : Lire depuis la DB FlexMeasures (sensors, assets, schedules)
+2. **API REST FlexMeasures** : Utiliser un datasource JSON générique
+3. **InfluxDB** : Les données IoT sont déjà dans InfluxDB
+
+## Dashboard JSON
+
+Le dashboard Cariflex comprend :
+- **Panel 1** : Production PV temps réel (10 assets)
+- **Panel 2** : État de charge des batteries (10 assets)
+- **Panel 3** : Consommation des bornes VE (10 assets)
+- **Panel 4** : Flexibilité disponible (agrégée)
+- **Panel 5** : Carte des actifs (Geomap)
+- **Panel 6** : Schedules FlexMeasures
+- **Panel 7** : Revenus flexibilité (PPA + GO + Services système)
+
+## Panels
+
+### PV Production
+```json
+{
+  "title": "Production PV Cariflex",
+  "type": "timeseries",
+  "datasource": "InfluxDB-v2",
+  "targets": [{
+    "query": "from(bucket:\"smartcity\") |> range(start: -24h) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"topic\"] =~ /pv_/) |> aggregateWindow(every: 5m, fn: mean)"
+  }]
+}
+```
+
+### Battery SOC
+```json
+{
+  "title": "État de Charge Batteries",
+  "type": "gauge",
+  "datasource": "InfluxDB-v2",
+  "targets": [{
+    "query": "from(bucket:\"smartcity\") |> range(start: -1h) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"topic\"] =~ /bat_/) |> last()"
+  }]
+}
+```
+
+### EV Charger Consumption
+```json
+{
+  "title": "Consommation Bornes VE",
+  "type": "timeseries",
+  "datasource": "InfluxDB-v2",
+  "targets": [{
+    "query": "from(bucket:\"smartcity\") |> range(start: -24h) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"topic\"] =~ /chg_/) |> aggregateWindow(every: 5m, fn: mean)"
+  }]
+}
+
+### Flexibilité Disponible
+```json
+{
+  "title": "Flexibilité Disponible",
+  "type": "stat",
+  "datasource": "InfluxDB-v2",
+  "targets": [{
+    "query": "from(bucket:\"smartcity\") |> range(start: -1h) |> filter(fn: (r) => r[\"_measurement\"] == \"mqtt_consumer\") |> filter(fn: (r) => r[\"_field\"] == \"flexibility\") |> sum()"
+  }]
+}
+```
+
+## Déploiement
+
+```bash
+# Créer le dashboard via l'API Grafana
+curl -X POST http://localhost:3001/api/dashboards/db \
+  -u admin:admin \
+  -H "Content-Type: application/json" \
+  -d @cariflex-dashboard.json
+```

scripts/cariflex_simulator.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+Cariflex Simulator - Publishes simulated EV charging data to Redis.
+Simulates 40 assets: 10 PV, 10 Battery, 10 EV Charger, 10 EV V2G
+"""
+import redis
+import json
+import time
+import random
+import math
+from datetime import datetime, timezone
+
+# Redis connection
+r = redis.Redis(host='flexmeasures-redis', port=6379, db=0, decode_responses=True)
+
+# Asset configurations
+ASSETS = {
+    # PV panels (production)
+    "pv_{:02d}": {"type": "pv", "unit": "kW", "min": 0, "max": 5, "base": 2.5},
+    # Batteries (storage)
+    "bat_{:02d}": {"type": "battery", "unit": "kWh", "min": 10, "max": 100, "base": 50},
+    # EV Chargers (consumption)
+    "chg_{:02d}": {"type": "ev_charger", "unit": "kW", "min": 0, "max": 22, "base": 11},
+    # EVs (V2G - bidirectional)
+    "ev_{:02d}": {"type": "ev_v2g", "unit": "kW", "min": -11, "max": 11, "base": 0},
+}
+
+def generate_value(asset_config, hour):
+    """Generate a realistic value based on asset type and time of day."""
+    cfg = asset_config
+    base = cfg["base"]
+    
+    if cfg["type"] == "pv":
+        # Solar production: peaks at noon
+        solar_factor = max(0, math.sin((hour - 6) * math.pi / 12)) if 6 <= hour <= 18 else 0
+        noise = random.gauss(0, 0.5)
+        value = base * solar_factor * 2 + noise
+    elif cfg["type"] == "battery":
+        # SOC: slowly varies throughout the day
+        variation = 20 * math.sin(hour * math.pi / 12)
+        noise = random.gauss(0, 3)
+        value = base + variation + noise
+    elif cfg["type"] == "ev_charger":
+        # Charging: more active during day and evening
+        if 8 <= hour <= 22:
+            factor = random.uniform(0.3, 1.0)
+        else:
+            factor = random.uniform(0, 0.2)
+        noise = random.gauss(0, 1)
+        value = cfg["max"] * factor + noise
+    elif cfg["type"] == "ev_v2g":
+        # V2G: charges at night, discharges during peak
+        if 0 <= hour <= 6:
+            factor = random.uniform(0.3, 0.8)  # charging
+        elif 17 <= hour <= 21:
+            factor = random.uniform(-0.6, -0.2)  # discharging
+        else:
+            factor = random.uniform(-0.1, 0.1)
+        noise = random.gauss(0, 0.5)
+        value = cfg["max"] * factor + noise
+    else:
+        value = base + random.gauss(0, 1)
+    
+    return round(max(cfg["min"], min(cfg["max"], value)), 2)
+
+def main():
+    print("🚗 Cariflex Simulator - Publishing to Redis")
+    print(f"   Assets: 40 (10 PV, 10 Bat, 10 Chg, 10 EV)")
+    print(f"   Redis: flexmeasures-redis:6379/0")
+    print()
+    
+    # Test Redis connection
+    try:
+        r.ping()
+        print("✅ Redis connected")
+    except redis.ConnectionError:
+        print("❌ Redis connection failed")
+        return
+    
+    # Publish loop
+    iteration = 0
+    while True:
+        now = datetime.now(timezone.utc)
+        hour = now.hour
+        timestamp = now.isoformat()
+        
+        # Publish each asset's data
+        for template, cfg in ASSETS.items():
+            for i in range(1, 11):
+                asset_id = template.format(i)
+                value = generate_value(cfg, hour)
+                
+                # Create data packet
+                data = {
+                    "asset_id": asset_id,
+                    "type": cfg["type"],
+                    "value": value,
+                    "unit": cfg["unit"],
+                    "timestamp": timestamp,
+                    "iteration": iteration
+                }
+                
+                # Publish to Redis (list per asset)
+                key = f"cariflex:asset:{asset_id}"
+                r.lpush(key, json.dumps(data))
+                r.ltrim(key, 0, 99)  # Keep last 100 values
+                r.expire(key, 3600)  # 1h TTL
+                
+                # Also publish to a pub/sub channel
+                r.publish("cariflex:data", json.dumps(data))
+        
+        # Publish aggregate data
+        aggregate = {
+            "timestamp": timestamp,
+            "total_pv_kw": sum(generate_value({"type": "pv", "base": 2.5, "min": 0, "max": 5}, hour) for _ in range(10)),
+            "total_battery_soc": sum(generate_value({"type": "battery", "base": 50, "min": 10, "max": 100}, hour) for _ in range(10)) / 10,
+            "total_charger_kw": sum(generate_value({"type": "ev_charger", "base": 11, "min": 0, "max": 22}, hour) for _ in range(10)),
+            "total_ev_v2g_kw": sum(generate_value({"type": "ev_v2g", "base": 0, "min": -11, "max": 11}, hour) for _ in range(10)),
+            "flexibility_available_kw": 0  # Will be calculated
+        }
+        aggregate["flexibility_available_kw"] = round(
+            abs(aggregate["total_ev_v2g_kw"]) + 
+            abs(aggregate["total_charger_kw"] * 0.3) +  # 30% of charger can be modulated
+            abs(aggregate["total_battery_soc"] * 0.5),  # 50% of battery capacity
+            2
+        )
+        
+        r.set("cariflex:aggregate", json.dumps(aggregate), ex=300)
+        
+        iteration += 1
+        if iteration % 10 == 0:
+            print(f"  📊 Iteration {iteration}: published {40} assets to Redis")
+        
+        time.sleep(10)  # Publish every 10 seconds
+
+if __name__ == "__main__":
+    main()

tools/flexmeasures-entsoe/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude = .git,__pycache__,documentation
+max-line-length = 160
+max-complexity = 13
+select = B,C,E,F,W,B9
+ignore = E501, W503, E203
+

tools/flexmeasures-entsoe/.github/workflows/ci.yml

@@ -0,0 +1,42 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: "Python ${{ matrix.python-version }} / FlexMeasures ${{ matrix.flexmeasures.version }}"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        flexmeasures:
+          - version: "0.31.*"
+            requirement: "flexmeasures==0.31.*"
+          - version: "0.32.*"
+            requirement: "flexmeasures==0.32.*"
+          - version: "latest"
+            requirement: "flexmeasures"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install FlexMeasures compatibility target
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install "${{ matrix.flexmeasures.requirement }}"
+
+      - name: Run tests
+        run: make test

tools/flexmeasures-entsoe/.gitignore

@@ -0,0 +1,75 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Sphinx documentation
+docs/_build/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+
+# custom project files
+flexmeasures.log

tools/flexmeasures-entsoe/.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.0.0  # New version tags can be found here: https://github.com/pycqa/flake8/tags
+    hooks:
+    - id: flake8
+      name: flake8 (code linting)
+-   repo: https://github.com/psf/black
+    rev: 22.10.0  # New version tags can be found here: https://github.com/psf/black/tags
+    hooks:
+    - id: black
+      name: black (code formatting)
+-   repo: local
+    hooks:
+    - id: mypy
+      name: mypy (static typing)
+      pass_filenames: false
+      language: script
+      entry: run_mypy.sh
+      verbose: true

tools/flexmeasures-entsoe/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

tools/flexmeasures-entsoe/Makefile

@@ -0,0 +1,20 @@
+# Note: use tabs
+# actions which are virtual, i.e. not a script
+.PHONY: install install-for-dev test 
+
+
+install:
+	pip install -e .
+
+
+# ---- Development ---
+
+test:
+	make install-for-dev
+	pytest
+
+install-for-dev:
+	pip install -r requirements/app.in -r requirements/dev.in -r requirements/test.in
+	make install
+	pre-commit install
+

tools/flexmeasures-entsoe/README.md

@@ -0,0 +1,133 @@
+# ENTSO-E forecasts & data
+
+Importing data which can be relevant for energy flexibility services via ENTSO-E's API into FlexMeasures.
+
+We start with data about the upcoming day.
+
+- Generation forecasts for the upcoming day
+- Based on these, CO2 content for the upcoming day
+- Day-ahead prices
+
+
+## Usage
+
+Importing tomorrow's prices:
+
+    flexmeasures entsoe import-day-ahead-prices
+
+Importing tomorrow's generation (incl. CO2 estimated content):
+
+    flexmeasures entsoe import-day-ahead-generation
+
+Use ``--help`` to learn more usage details.
+
+
+### October 1st 2025 go-live for ENTSO-E moving to 15-minute day-ahead prices
+
+ENTSO-E is moving from 1-hour day-ahead prices 15-minute day-ahead prices on October 1st 2025.
+To prepare for this transition, you have two choices:
+
+1. resample your existing price sensor in FlexMeasures from 1 hour to 15 minutes, or
+2. get a new sensor for the 15-minute data.
+
+If you do this *after* the go-live moment, the `flexmeasures-entsoe` package just keeps resampling the 15-minute ENTSO-E data to hourly data.
+
+#### 1. Resampling
+
+**The upside** of resampling your existing price data is that the sensor ID of your price sensor in FlexMeasures will remain the same.
+Depending on your system setup, `Forecaster`/`Reporter`/`Scheduler` configurations (such as an asset's `flex-context`) may depend on it, and your users may expect the 15-minute data to live under the same sensor.
+
+**The downside** is that it quadruples your data for that sensor, due to the fact that FlexMeasures only supports a fixed resolution for any given sensor. Although there should be no noticeable hit in performance, it obviously leads to redundant data in the price history before October 1st 2025.  
+
+**To resample** your historical data, use:
+
+```bash
+flexmeasures edit resample-data --sensor <ID of your day-ahead price sensor> --event-resolution 15
+```
+
+The `flexmeasures-entsoe` package already automatically resamples the ENTSO-E data to the resolution of your sensor.
+
+If you use a `Reporter` to derive retail prices or to compute energy costs, there is no need to update its configuration; just resample these sensors too, using the previous command (replacing the sensor ID as needed).
+Alternatively, if you want to keep these sensors in their original resolution, and find that your reporters fail with an `AssertionError` about mismatched resolutions, you may need to add the `--resolution PT1H` option when using the `flexmeasures add report` command.
+
+#### 2. Getting a new sensor
+
+**The upside** is that this doesn't quadruple your historic data (see *the downside* of resampling, above).
+
+**The downside** is that you may need to revise `Forecaster`/`Reporter`/`Scheduler` configurations (such as an asset's `flex-context`) and notify users (see *the upside* of resampling, above).
+
+**To get a new sensor**, rename your existing *Day-ahead prices* sensor in the FlexMeasures UI.
+
+The `flexmeasures-entsoe` package will then automatically create a new 15-minute price sensor the next time `flexmeasures entsoe import-day-ahead-prices` is run, assigning it a new sensor ID.
+
+If you have any price or costs sensors using a `Reporter` to derive values from the day-ahead wholesale prices, update the sensor ID in the configuration of each `Reporter`.
+Finally, either resample each derived sensor using:
+
+```bash
+flexmeasures edit resample-data --sensor <ID of your derivative sensor> --event-resolution 15
+```
+
+or, if you want to keep these sensors in their original resolution, and find that your reporters fail with an `AssertionError` about mismatched resolutions, you may need to add the `--resolution PT1H` option when using the `flexmeasures add report` command.
+
+## Installation
+
+First of all, this is a FlexMeasures plugin. Consult the FlexMeasures documentation for setup.
+
+1. Add the plugin to [the `FLEXMEASURES_PLUGINS` setting](https://flexmeasures.readthedocs.io/stable/configuration.html#flexmeasures-plugins). Either use `/path/to/flexmeasures-entsoe/flexmeasures_entsoe` or `flexmeasures_entsoe` if you installed this as a package locally (see below).
+
+2. Add `ENTSOE_AUTH_TOKEN` to your FlexMeasures config (e.g. ~/.flexmeasures.cfg).
+You can generate this token after you made an account at ENTSO-E, read more [here](https://transparencyplatform.zendesk.com/hc/en-us/articles/12845911031188-How-to-get-security-token). 
+
+   Optionally, override other settings (defaults shown here):
+
+       ENTSOE_COUNTRY_CODE = "NL"
+       ENTSOE_COUNTRY_TIMEZONE = "Europe/Amsterdam"
+       ENTSOE_DERIVED_DATA_SOURCE = "FlexMeasures ENTSO-E"
+
+   The `ENTSOE_DERIVED_DATA_SOURCE` option is used to name the source of data that this plugin derives from ENTSO-E data, like a CO₂ signal.
+   Original ENTSO-E data is reported as being sourced by `"ENTSO-E"`.
+
+3. To install this plugin locally as a package, try `pip install .`.
+
+
+## Testing
+
+ENTSO-E provides a test server (iop) for development. It's good practice not to overwhelm their production server.
+
+Set ``ENTSOE_USE_TEST_SERVER=True`` to enable this.
+
+In that case, this plugin will look for the auth token in the config setting ``ENTSOE_AUTH_TOKEN_TEST_SERVER``.
+
+Note, however, that ENTSO-E usually does not seem to make the latest data available there. Asking for the next day can often get an empty response.
+
+
+## Supported FlexMeasures versions
+
+This plugin targets two distinct FlexMeasures capability tiers:
+
+| FlexMeasures version | Behavior |
+|---|---|
+| `< 0.32` | Uses the legacy `get_data_source` factory; no account is linked to the ENTSO-E source. |
+| `>= 0.32` | Uses the account-linked source API (`get_or_create_source` with an `Account`). |
+
+This package supports Python 3.10 through 3.12, following the Python support policy of the currently supported FlexMeasures releases.
+
+The oldest supported FlexMeasures release line is `0.31.*`.
+CI is run against `0.31.*` (minimum supported legacy release), `0.32.*` (first account-linked release), and the latest released FlexMeasures version across all supported Python versions.
+When a new FlexMeasures release introduces breaking changes the matrix should be updated accordingly.
+
+
+## Development
+
+To keep our code quality high, we use pre-commit:
+
+    pip install pre-commit black flake8 mypy
+    pre-commit install
+
+or:
+    
+    make install-for-dev
+
+Try it:
+
+    pre-commit run --all-files --show-diff-on-failure

tools/flexmeasures-entsoe/flexmeasures_entsoe/__init__.py

@@ -0,0 +1,54 @@
+import os
+import sys
+
+from flask import Blueprint
+
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+DEFAULT_COUNTRY_CODE = "NL"
+DEFAULT_COUNTRY_TIMEZONE = "Europe/Amsterdam"  # This is what we receive, even if ENTSO-E documents Europe/Brussels
+DEFAULT_DATA_SOURCE_NAME = "ENTSO-E"
+DEFAULT_DERIVED_DATA_SOURCE = "FlexMeasures ENTSO-E"
+
+__version__ = "0.9"
+__settings__ = {
+    "ENTSOE_AUTH_TOKEN": dict(
+        description="You can generate this token after you made an account at ENTSO-E.",
+        level="error",
+    ),
+    "ENTSOE_COUNTRY_CODE": dict(
+        level="warning",
+        message_if_missing=f"'{DEFAULT_COUNTRY_CODE}' will be used as a default.",
+    ),
+    "ENTSOE_COUNTRY_TIMEZONE": dict(
+        description="IANA timezone name used to localize ENTSO-E sensors.",
+        level="info",
+        message_if_missing=f"'{DEFAULT_COUNTRY_TIMEZONE}' will be used as a default.",
+    ),
+    "ENTSOE_USE_TEST_SERVER": dict(
+        description="Boolean to indicate whether to use the ENTSO-E's iop test server instead of their production server",
+        level="debug",
+    ),
+    "ENTSOE_AUTH_TOKEN_TEST_SERVER": dict(
+        description="You can generate this token after you made an account at ENTSO-E.",
+        level="debug",
+    ),
+    "ENTSOE_DERIVED_DATA_SOURCE": dict(
+        description="String used to name the source of data that this plugin derives from ENTSO-E data, like a CO₂ signal.",
+        level="info",
+        message_if_missing=f"'{DEFAULT_DERIVED_DATA_SOURCE}' will be used as a default.",
+    ),
+    "ENTSOE_DATA_SOURCE_NAME": dict(
+        description="String used to name the ENTSO-E data source and the account associated with it.",
+        level="info",
+        message_if_missing=f"'{DEFAULT_DATA_SOURCE_NAME}' will be used as a default.",
+    ),
+}
+
+entsoe_data_bp = Blueprint("entsoe", __name__, cli_group="entsoe")
+entsoe_data_bp.cli.help = "ENTSO-E Data commands"
+
+
+from .generation import day_ahead as day_ahead_generation  # noqa: E402,F401
+from .prices import day_ahead as day_ahead_prices  # noqa: E402,F401

tools/flexmeasures-entsoe/flexmeasures_entsoe/generation/__init__.py

@@ -0,0 +1,10 @@
+from datetime import timedelta
+
+# sensor_name, unit, event_resolution, data sourced directly by ENTSO-E or not (i.e. derived)
+generation_sensors = (
+    ("Scheduled generation", "MW", timedelta(minutes=15), True),
+    ("Solar", "MW", timedelta(hours=1), True),
+    ("Wind Onshore", "MW", timedelta(hours=1), True),
+    ("Wind Offshore", "MW", timedelta(hours=1), True),
+    ("CO₂ intensity", "kg/MWh", timedelta(minutes=15), False),
+)

tools/flexmeasures-entsoe/flexmeasures_entsoe/generation/day_ahead.py

@@ -0,0 +1,214 @@
+from typing import Optional
+from datetime import datetime
+
+import click
+from flask.cli import with_appcontext
+from flask import current_app
+
+# from entsoe.entsoe import URL
+import pandas as pd
+from flexmeasures.data.transactional import task_with_status_report
+
+from .. import (
+    entsoe_data_bp,
+)  # noqa: E402
+from . import generation_sensors
+from ..utils import (
+    create_entsoe_client,
+    ensure_country_code_and_timezone,
+    ensure_data_source,
+    ensure_data_source_for_derived_data,
+    abort_if_data_empty,
+    parse_from_and_to_dates,
+    save_entsoe_series,
+    ensure_sensors,
+    resample_if_needed,
+    start_import_log,
+)
+
+
+"""
+Get the CO₂ content from tomorrow's generation forecasts.
+We get the overall forecast and the solar&wind forecast, so we know the share of green energy.
+For now, we'll compute the CO₂ mix from some assumptions.
+"""
+
+# TODO: Decide which sources to use ― https://github.com/SeitaBV/flexmeasures-entsoe/issues/2
+
+# Source for these ratios: https://ourworldindata.org/energy/country/netherlands#what-sources-does-the-country-get-its-electricity-from (2020 data)
+grey_energy_mix = dict(gas=0.598, oil=0.045, coal=0.0718)
+
+# Source for kg CO₂ per MWh: https://energy.utexas.edu/news/nuclear-and-wind-power-estimated-have-lowest-levelized-co2-emissions
+kg_CO2_per_MWh = dict(
+    coal=870,  # lignite
+    gas=464,  # natural
+    solar=44.5,  # mix of utility/residential, difference isn't large
+    oil=652,  # ca. 75% of coal, see https://www.volker-quaschning.de/datserv/CO2-spez/index_e.php
+    wind_onshore=14,
+    wind_offshore=17,  # factor of ~ 1.1, see https://www.mdpi.com/2071-1050/10/6/2022
+)
+
+
+@entsoe_data_bp.cli.command("import-day-ahead-generation")
+@click.option(
+    "--from-date",
+    required=False,
+    type=click.DateTime(["%Y-%m-%d"]),
+    help="Query data from this date onwards. If not specified, defaults to today",
+)
+@click.option(
+    "--to-date",
+    required=False,
+    type=click.DateTime(["%Y-%m-%d"]),
+    help="Query data until this date (inclusive). If not specified, defaults to tomorrow.",
+)
+@click.option(
+    "--dryrun/--no-dryrun",
+    default=False,
+    help="In dry run mode, do not save the data to the db.",
+)
+@click.option(
+    "--country",
+    "country_code",
+    required=False,
+    help="ENTSO-E country code (such as BE, DE, FR or NL).",
+)
+@click.option(
+    "--timezone",
+    "country_timezone",
+    required=False,
+    help="Timezone for the country (such as 'Europe/Amsterdam').",
+)
+@click.option(
+    "--for",
+    "default_import_timerange",
+    required=False,
+    default="today-and-tomorrow",
+    type=click.Choice(["today", "tomorrow", "today-and-tomorrow"]),
+    help="Easy-to-use time range setting, only used if --from-date and --to-date are not used. If set to 'today' or 'tomorrow' or 'today-and-tomorrow', only import data for thes days. The default is today-and-tomorrow.",
+)
+@with_appcontext
+@task_with_status_report("entsoe-import-day-ahead-generation")
+def import_day_ahead_generation(
+    dryrun: bool = False,
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+    country_code: Optional[str] = None,
+    country_timezone: Optional[str] = None,
+    default_import_timerange: str = "today-and-tomorrow",
+):
+    """
+    Import forecasted generation for any date range, defaulting to today and tomorrow.
+    This will save overall generation, solar, offshore and onshore wind, and the estimated CO₂ content per hour.
+    Possibly best to run this script somewhere around or maybe two or three hours after 13:00,
+    when tomorrow's prices are announced.
+    """
+    # Set up FlexMeasures data structure
+    country_code, country_timezone = ensure_country_code_and_timezone(
+        country_code, country_timezone
+    )
+    entsoe_data_source = ensure_data_source()
+    derived_data_source = ensure_data_source_for_derived_data()
+    sensors = ensure_sensors(generation_sensors, country_code, country_timezone)
+    # Parse CLI options (or set defaults)
+    from_time, until_time = parse_from_and_to_dates(
+        from_date, to_date, country_timezone, default_to=default_import_timerange
+    )
+
+    # Start import
+    client = create_entsoe_client()
+    log, now = start_import_log(
+        "day-ahead generation", from_time, until_time, country_code, country_timezone
+    )
+
+    log.info("Getting scheduled generation ...")
+    # We assume that the green (solar & wind) generation is not included in this (it is not scheduled)
+    scheduled_generation: pd.Series = client.query_generation_forecast(
+        country_code, start=from_time, end=until_time
+    )
+    abort_if_data_empty(scheduled_generation)
+    log.debug("Overall aggregated generation: \n%s" % scheduled_generation)
+
+    scheduled_generation = resample_if_needed(
+        scheduled_generation,
+        sensors["Scheduled generation"],
+    )
+
+    log.info("Getting green generation ...")
+    green_generation_df: pd.DataFrame = client.query_wind_and_solar_forecast(
+        country_code, start=from_time, end=until_time, psr_type=None
+    )
+    abort_if_data_empty(green_generation_df)
+    log.debug("Green generation: \n%s" % green_generation_df)
+
+    log.info("Aggregating green energy columns ...")
+    all_green_generation = green_generation_df.sum(axis="columns")
+    log.debug("Aggregated green generation: \n%s" % all_green_generation)
+
+    log.info("Computing combined generation forecast ...")
+    all_generation = scheduled_generation + all_green_generation
+    log.debug("Combined generation: \n%s" % all_generation)
+
+    log.info("Computing CO₂ content from the MWh values ...")
+    co2_in_kg = calculate_CO2_content_in_kg(scheduled_generation, green_generation_df)
+    log.debug("Overall CO₂ content (kg): \n%s" % co2_in_kg)
+    forecasted_kg_CO2_per_MWh = co2_in_kg / all_generation
+    log.debug("Overall CO₂ content (kg/MWh): \n%s" % forecasted_kg_CO2_per_MWh)
+
+    def get_series_for_sensor(sensor):
+        if sensor.name == "Scheduled generation":
+            return scheduled_generation
+        elif sensor.name == "Solar":
+            return green_generation_df["Solar"]
+        elif sensor.name == "Wind Onshore":
+            return green_generation_df["Wind Onshore"]
+        elif sensor.name == "Wind Offshore":
+            return green_generation_df["Wind Offshore"]
+        elif sensor.name == "CO₂ intensity":
+            return forecasted_kg_CO2_per_MWh
+        else:
+            log.error(f"Cannot connect data to sensor {sensor.name}.")
+            raise click.Abort
+
+    if not dryrun:
+        for sensor in sensors.values():
+            series = get_series_for_sensor(sensor)
+            log.info(f"Saving {len(series)} beliefs for Sensor {sensor.name} ...")
+            entsoe_source = (
+                entsoe_data_source if sensor.data_by_entsoe else derived_data_source
+            )
+            save_entsoe_series(series, sensor, entsoe_source, country_timezone, now)
+
+
+def calculate_CO2_content_in_kg(
+    grey_generation: pd.Series, green_generation: pd.DataFrame
+) -> pd.Series:
+    grey_CO2_intensity_factor = (  # TODO: a factor per hour of the day
+        (grey_energy_mix["coal"] * kg_CO2_per_MWh["coal"])
+        + (grey_energy_mix["gas"] * kg_CO2_per_MWh["gas"])
+        + (grey_energy_mix["oil"] * kg_CO2_per_MWh["oil"])
+    )
+    current_app.logger.debug(f"Grey intensity factor: {grey_CO2_intensity_factor}")
+    grey_CO2_content = grey_generation * grey_CO2_intensity_factor
+    current_app.logger.debug("Grey CO₂ content (tonnes): \n%s" % grey_CO2_content)
+
+    green_generation["solar CO₂"] = (
+        green_generation["Solar"] * kg_CO2_per_MWh["solar"] / 1000.0
+    )
+    green_generation["wind_onshore CO₂"] = (
+        green_generation["Wind Onshore"] * kg_CO2_per_MWh["wind_onshore"]
+    )
+    green_generation["wind_offshore CO₂"] = (
+        green_generation["Wind Offshore"] * kg_CO2_per_MWh["wind_offshore"]
+    )
+
+    current_app.logger.debug(
+        "Green generation and CO₂ content: \n%s" % green_generation
+    )
+
+    return (
+        grey_CO2_content
+        + green_generation["solar CO₂"]
+        + green_generation["wind_onshore CO₂"]
+        + green_generation["wind_offshore CO₂"]
+    )

tools/flexmeasures-entsoe/flexmeasures_entsoe/generation/utils.py

@@ -0,0 +1,57 @@
+import pandas as pd
+
+
+def determine_net_emission_factors(shares: pd.DataFrame) -> pd.Series:
+    """Given production shares, determine the net emission factors.
+    Or given production by type, determine the net emissions.
+
+    Use column headers that match production types listed below.
+    Use any index.
+
+    For example:
+
+        print(shares)
+
+              fossil_gas     other  fossil_hard_coal     waste   nuclear
+        hour
+        0       0.443685  0.206033          0.237596  0.050915  0.059455
+        1       0.443910  0.205065          0.235022  0.052614  0.060987
+
+        print(determine_net_emission_factors(shares))
+
+        hour
+        0     644.753221
+        1     641.410093
+        Name: Average emissions from Dutch electricity production (kg CO₂ eq/MWh), dtype: float64
+    """
+    emission_factors = dict(
+        biomass=50.4,
+        fossil_brown_coal_or_lignite=None,  # unknown
+        fossil_coal_derived_gas=None,  # unknown
+        fossil_gas=464,
+        fossil_hard_coal=1030,
+        fossil_oil=1010,
+        fossil_oil_shale=None,  # unknown
+        fossil_peat=None,  # unknown
+        geothermal=0.00664,
+        hydro_pumped_storage=611,
+        hydro_run_of_river_and_poundage=0.0253,
+        hydro_water_reservoir=8.13,
+        marine=None,  # unknown
+        nuclear=10.1,
+        other=927,  # for EU28
+        other_renewable=None,  # unknown
+        solar=0.00591,
+        waste=None,  # unknown
+        wind_offshore=0.133,
+        wind_onshore=0.133,
+    )  # supplementary material from "Real-time carbon accounting method for the European electricity markets, Tranberg et al. (2019)"
+    # todo: substitute placeholder for unknown emission factor of waste
+    emission_factors["waste"] = emission_factors["biomass"]
+    for production_type in shares.columns:
+        shares[production_type] = (
+            shares[production_type] * emission_factors[production_type]
+        )
+    return shares.sum(axis=1).rename(
+        "Average emissions from Dutch electricity production (kg CO₂ eq/MWh)"
+    )

tools/flexmeasures-entsoe/flexmeasures_entsoe/prices/__init__.py

@@ -0,0 +1,4 @@
+from datetime import timedelta
+
+# sensor_name, unit, even_resolution, data sourced directly by ENTSO-E or not (i.e. derived)
+pricing_sensors = (("Day-ahead prices", "EUR/MWh", timedelta(minutes=15), True),)

tools/flexmeasures-entsoe/flexmeasures_entsoe/prices/day_ahead.py

@@ -0,0 +1,155 @@
+from typing import Optional
+from datetime import datetime
+
+import click
+from flask.cli import with_appcontext
+import pandas as pd
+from flexmeasures import Source, Sensor
+
+from flexmeasures.data.transactional import task_with_status_report
+
+from flexmeasures.data.schemas import SensorIdField
+from flexmeasures.data.schemas.sources import DataSourceIdField
+
+
+from . import pricing_sensors
+from .. import (
+    entsoe_data_bp,
+)  # noqa: E402
+from ..utils import (
+    create_entsoe_client,
+    ensure_country_code_and_timezone,
+    ensure_data_source,
+    parse_from_and_to_dates,
+    ensure_sensors,
+    save_entsoe_series,
+    abort_if_data_empty,
+    abort_if_data_incomplete,
+    resample_if_needed,
+    start_import_log,
+)
+
+
+@entsoe_data_bp.cli.command("import-day-ahead-prices")
+@click.option(
+    "--from-date",
+    required=False,
+    type=click.DateTime(["%Y-%m-%d"]),
+    help="Query data from this date onwards. If not specified, defaults to today",
+)
+@click.option(
+    "--to-date",
+    required=False,
+    type=click.DateTime(["%Y-%m-%d"]),
+    help="Query data until this date (inclusive). If not specified, defaults to tomorrow.",
+)
+@click.option(
+    "--dryrun/--no-dryrun",
+    default=False,
+    help="In dry run mode, do not save the data to the db.",
+)
+@click.option(
+    "--country",
+    "country_code",
+    required=False,
+    help="ENTSO-E country code (such as BE, DE, FR or NL).",
+)
+@click.option(
+    "--timezone",
+    "country_timezone",
+    required=False,
+    help="Timezone for the country (such as 'Europe/Amsterdam').",
+)
+@click.option(
+    "--sensor",
+    "sensor",
+    type=SensorIdField(),
+    required=False,
+    help="Sensor to store the data into. If not provided, the sensor `Day-ahead prices` is used.",
+)
+@click.option(
+    "--source",
+    "source",
+    type=DataSourceIdField(),
+    required=False,
+    help="Source of the price data. If not provided, the source `ENTSO-E` is used.",
+)
+@click.option(
+    "--for",
+    "default_import_timerange",
+    required=False,
+    default="today-and-tomorrow",
+    type=click.Choice(["today", "tomorrow", "today-and-tomorrow"]),
+    help="Easy-to-use time range setting, which defines the defaults for start and end to be used when --from-date and/or --to-date are not used. Can be set to 'today' or 'tomorrow' or 'today-and-tomorrow' (which is the default value).",
+)
+@click.option(
+    "--fail-on-incomplete-data",
+    "fail_on_incomplete_data",
+    is_flag=True,
+    default=False,
+    help="If set, the import will abort if the data received is incomplete.",
+)
+@with_appcontext
+@task_with_status_report("entsoe-import-day-ahead-prices")
+def import_day_ahead_prices(
+    dryrun: bool = False,
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+    country_code: Optional[str] = None,
+    country_timezone: Optional[str] = None,
+    sensor: Optional[Sensor] = None,
+    source: Optional[Source] = None,
+    default_import_timerange: str = "today-and-tomorrow",
+    fail_on_incomplete_data: bool = False,
+):
+    """
+    Import forecasted prices for any date range, defaulting to today and tomorrow.
+    Possibly best to run this script somewhere around or maybe two or three hours after 13:00,
+    when tomorrow's prices are announced.
+    """
+    # Set up FlexMeasures data structure
+    country_code, country_timezone = ensure_country_code_and_timezone(
+        country_code, country_timezone
+    )
+
+    if source is None:
+        entsoe_data_source = ensure_data_source()
+    else:
+        entsoe_data_source = source
+
+    if sensor is None:
+        # For now, we only have one pricing sensor ...
+        sensors = ensure_sensors(pricing_sensors, country_code, country_timezone)
+        pricing_sensor = sensors["Day-ahead prices"]
+        assert pricing_sensor.name == "Day-ahead prices"
+    else:
+        pricing_sensor = sensor
+
+    # Parse CLI options (or set defaults)
+    from_time, until_time = parse_from_and_to_dates(
+        from_date, to_date, country_timezone, default_to=default_import_timerange
+    )
+
+    # Start import
+    client = create_entsoe_client()
+    log, now = start_import_log(
+        "day-ahead price", from_time, until_time, country_code, country_timezone
+    )
+
+    log.info("Getting prices ...")
+    prices: pd.Series = client.query_day_ahead_prices(
+        country_code, start=from_time, end=until_time
+    )
+    abort_if_data_empty(prices)
+    if fail_on_incomplete_data:
+        abort_if_data_incomplete(
+            prices, from_time, until_time, pricing_sensor.event_resolution
+        )
+    prices = resample_if_needed(prices, pricing_sensor)
+    log.debug("Prices: \n%s" % prices)
+
+    if not dryrun:
+        log.info(f"Saving {len(prices)} beliefs for Sensor {pricing_sensor.name} ...")
+        save_entsoe_series(
+            prices, pricing_sensor, entsoe_data_source, country_timezone, now
+        )

tools/flexmeasures-entsoe/flexmeasures_entsoe/tests/test_utils.py

@@ -0,0 +1,308 @@
+from datetime import datetime, timedelta
+from types import SimpleNamespace
+
+import click
+import pandas as pd
+import pytz
+import pytest
+
+from flexmeasures_entsoe import DEFAULT_DATA_SOURCE_NAME, DEFAULT_DERIVED_DATA_SOURCE
+from flexmeasures_entsoe.utils import (
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    _ensure_entsoe_source,
+    abort_if_data_incomplete,
+    ensure_data_source,
+    ensure_data_source_for_derived_data,
+    parse_from_and_to_dates,
+)
+
+
+def test_abort_if_data_incomplete():
+    """
+    Tests that the function raises click.Abort if data is incomplete.
+    1. Data is complete: No exception raised.
+    2. Data is incomplete: click.Abort is raised.
+    """
+    start = pd.Timestamp("2025-01-01 00:00")
+    end = pd.Timestamp("2025-01-02 00:00")
+    resolution = pd.Timedelta(hours=1)
+
+    # Case 1: Data is complete (24 items for 24 hours)
+    complete_data = pd.DataFrame({"val": range(24)})
+    try:
+        abort_if_data_incomplete(complete_data, start, end, resolution)
+    except click.Abort:
+        pytest.fail("Function raised Abort unexpectedly on complete data")
+
+    # Case 2: Data is incomplete (20 items for 24 hours)
+    incomplete_data = pd.DataFrame({"val": range(20)})
+    with pytest.raises(click.Abort):
+        abort_if_data_incomplete(incomplete_data, start, end, resolution)
+
+
+def test_parse_from_and_to_dates():
+    """
+    Tests CLI date parsing logic:
+    1. Explicit dates are timezone-localized correctly.
+    2. 'None' defaults to tomorrow (start of day) -> day after tomorrow.
+    """
+    tz_str = "UTC"
+    tz = pytz.timezone(tz_str)
+    now = datetime.now(tz)
+    today = datetime(now.year, now.month, now.day, tzinfo=tz)
+
+    # Case 1: Explicit inputs
+    input_start = datetime(2025, 5, 1)
+    input_end = datetime(2025, 5, 2)
+
+    s, e = parse_from_and_to_dates(
+        from_date=input_start, until_date=input_end, country_timezone=tz_str
+    )
+
+    assert s.tzinfo.zone == tz.zone
+    assert (e - s) == timedelta(days=2)
+    assert e == datetime(2025, 5, 3, tzinfo=tz)
+
+    # Case 2: default_to="tomorrow"
+    s_tom, e_tom = parse_from_and_to_dates(
+        from_date=None, until_date=None, country_timezone=tz_str, default_to="tomorrow"
+    )
+
+    assert e_tom - s_tom == timedelta(days=1)
+    assert s_tom == today + timedelta(days=1)
+    assert e_tom == today + timedelta(days=2)
+
+    # Case 3: default_to="today-and-tomorrow"
+    s_tod, e_tod = parse_from_and_to_dates(
+        from_date=None, until_date=None, country_timezone=tz_str
+    )
+
+    assert e_tod - s_tod == timedelta(days=2)
+    assert s_tod == today
+    assert e_tod == today + timedelta(days=2)
+
+    # Case 4: only providing until_date (today midnight == start of tomorrow), while start comes from "today-and-tomorrow"
+    today_midnight = datetime(now.year, now.month, now.day) + timedelta(days=1)
+    s_none, e_none = parse_from_and_to_dates(
+        from_date=None, until_date=today_midnight, country_timezone=tz_str
+    )
+
+    assert e_none - s_none == timedelta(days=2)
+    assert s_none == today
+    assert e_none == today + timedelta(days=2)
+
+
+# The version-branch tests below still use monkeypatching to isolate source
+# creation side effects and to simulate upgrade reuse of legacy ENTSO-E
+# sources without requiring multiple FlexMeasures installs in one test run.
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked ENTSO-E sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_ensure_data_source_passes_entsoe_account_when_supported(monkeypatch):
+    """Test that ensure_data_source() creates a market-type source and passes the ENTSO-E account."""
+    from flask import Flask
+
+    app = Flask(__name__)
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, account, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                account=account,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, account=account, name=source)
+
+    fake_account = SimpleNamespace(name=DEFAULT_DATA_SOURCE_NAME)
+
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_or_create_source",
+        fake_get_or_create_source,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils._find_existing_source",
+        lambda source_name, source_type: None,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_or_create_entsoe_account",
+        lambda: fake_account,
+    )
+
+    with app.app_context():
+        data_source = ensure_data_source()
+
+    assert data_source.type == "market"
+    assert captured_kwargs["source"] == DEFAULT_DATA_SOURCE_NAME
+    assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked ENTSO-E sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_ensure_data_source_for_derived_data_passes_entsoe_account_when_supported(
+    monkeypatch,
+):
+    """Test that ensure_data_source_for_derived_data() passes the ENTSO-E account."""
+    from flask import Flask
+
+    app = Flask(__name__)
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, account, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                account=account,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, account=account, name=source)
+
+    fake_account = SimpleNamespace(name=DEFAULT_DATA_SOURCE_NAME)
+
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_or_create_source",
+        fake_get_or_create_source,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils._find_existing_source",
+        lambda source_name, source_type: None,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_or_create_entsoe_account",
+        lambda: fake_account,
+    )
+
+    with app.app_context():
+        data_source = ensure_data_source_for_derived_data()
+
+    assert data_source.type == "forecasting script"
+    assert captured_kwargs["source"] == DEFAULT_DERIVED_DATA_SOURCE
+    assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Legacy get_data_source fallback is only used on FlexMeasures < 0.32.",
+)
+def test_ensure_data_source_omits_account_when_not_supported(monkeypatch):
+    """Test that ensure_data_source() falls back to the legacy source factory without an account."""
+    from flask import Flask
+
+    app = Flask(__name__)
+    captured_kwargs = {}
+
+    def fake_get_data_source(data_source_name, data_source_type):
+        captured_kwargs.update(
+            data_source_name=data_source_name,
+            data_source_type=data_source_type,
+        )
+        return SimpleNamespace(name=data_source_name, type=data_source_type)
+
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils._find_existing_source",
+        lambda source_name, source_type: None,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_data_source",
+        fake_get_data_source,
+    )
+
+    with app.app_context():
+        data_source = ensure_data_source()
+
+    assert data_source.type == "market"
+    assert captured_kwargs == {
+        "data_source_name": DEFAULT_DATA_SOURCE_NAME,
+        "data_source_type": "market",
+    }
+
+
+@pytest.mark.skipif(
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Legacy get_data_source fallback is only used on FlexMeasures < 0.32.",
+)
+def test_ensure_data_source_for_derived_data_omits_account_when_not_supported(
+    monkeypatch,
+):
+    """Test that ensure_data_source_for_derived_data() falls back to the legacy source factory."""
+    from flask import Flask
+
+    app = Flask(__name__)
+    captured_kwargs = {}
+
+    def fake_get_data_source(data_source_name, data_source_type):
+        captured_kwargs.update(
+            data_source_name=data_source_name,
+            data_source_type=data_source_type,
+        )
+        return SimpleNamespace(name=data_source_name, type=data_source_type)
+
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils._find_existing_source",
+        lambda source_name, source_type: None,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_data_source",
+        fake_get_data_source,
+    )
+
+    with app.app_context():
+        data_source = ensure_data_source_for_derived_data()
+
+    assert data_source.type == "forecasting script"
+    assert captured_kwargs == {
+        "data_source_name": DEFAULT_DERIVED_DATA_SOURCE,
+        "data_source_type": "forecasting script",
+    }
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Legacy source upgrade reuse matters in the account-linked source path only.",
+)
+def test_ensure_entsoe_source_reuses_legacy_source_and_sets_account(monkeypatch):
+    legacy_source = SimpleNamespace(
+        name=DEFAULT_DATA_SOURCE_NAME,
+        type="forecasting script",
+        account=None,
+    )
+    fake_account = SimpleNamespace(name=DEFAULT_DATA_SOURCE_NAME)
+
+    def fake_find_existing_source(source_name, source_type):
+        if source_type == "market":
+            return None
+        if source_type == "forecasting script":
+            return legacy_source
+        return None
+
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils._find_existing_source",
+        fake_find_existing_source,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_or_create_entsoe_account",
+        lambda: fake_account,
+    )
+    monkeypatch.setattr(
+        "flexmeasures_entsoe.utils.get_or_create_source",
+        lambda **kwargs: pytest.fail(
+            "Should reuse a legacy ENTSO-E source before creating a new one."
+        ),
+    )
+
+    data_source = _ensure_entsoe_source(
+        source_name=DEFAULT_DATA_SOURCE_NAME,
+        source_type="market",
+        legacy_source_type="forecasting script",
+    )
+
+    assert data_source is legacy_source
+    assert data_source.type == "market"
+    assert data_source.account is fake_account

tools/flexmeasures-entsoe/flexmeasures_entsoe/utils.py

@@ -0,0 +1,369 @@
+from typing import Dict, Optional, Tuple, Union
+from datetime import datetime, timedelta
+from logging import Logger
+
+from entsoe import EntsoePandasClient
+from flask import current_app
+from packaging import version
+from pandas.tseries.frequencies import to_offset
+import pandas as pd
+import click
+import pytz
+import entsoe
+
+from flexmeasures.data.utils import get_data_source, save_to_db
+from flexmeasures import Asset, AssetType, Sensor, Source, __version__ as flexmeasures_version
+from flexmeasures.data import db
+from flexmeasures.utils.time_utils import server_now
+from timely_beliefs import BeliefsDataFrame
+from flexmeasures.cli.utils import MsgStyle
+from . import (
+    DEFAULT_DATA_SOURCE_NAME,
+    DEFAULT_DERIVED_DATA_SOURCE,
+    DEFAULT_COUNTRY_CODE,
+    DEFAULT_COUNTRY_TIMEZONE,
+)  # noqa: E402
+
+FM_SUPPORTS_ACCOUNT_LINKED_SOURCES = version.parse(
+    flexmeasures_version
+) >= version.parse("0.32")
+
+if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+    from flexmeasures import Account
+    from flexmeasures.data.services.data_sources import get_or_create_source
+
+
+def _find_existing_source(source_name: str, source_type: str) -> Optional[Source]:
+    return (
+        Source.query.filter(
+            Source.name == source_name,
+            Source.type == source_type,
+        )
+        .order_by(Source.id)
+        .first()
+    )
+
+
+def get_or_create_entsoe_account():
+    """Make sure we have an account for the ENTSO-E provider service."""
+    account_name = current_app.config.get(
+        "ENTSOE_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+    )
+    entsoe_account = Account.query.filter(
+        Account.name == account_name,
+    ).one_or_none()
+    if entsoe_account is None:
+        entsoe_account = Account(name=account_name)
+        db.session.add(entsoe_account)
+        db.session.flush()
+    return entsoe_account
+
+
+def _ensure_entsoe_source(
+    source_name: str,
+    source_type: str,
+    legacy_source_type: Optional[str] = None,
+) -> Source:
+    """Reuse legacy sources when possible while branching explicitly on FM version."""
+    entsoe_account = None
+    if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+        entsoe_account = get_or_create_entsoe_account()
+
+    existing_source = _find_existing_source(source_name, source_type)
+    if existing_source is None and legacy_source_type is not None:
+        existing_source = _find_existing_source(source_name, legacy_source_type)
+        if existing_source is not None:
+            existing_source.type = source_type
+
+    if existing_source is not None:
+        if entsoe_account is not None and getattr(existing_source, "account", None) is None:
+            existing_source.account = entsoe_account
+        return existing_source
+
+    if not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+        return get_data_source(
+            data_source_name=source_name,
+            data_source_type=source_type,
+        )
+
+    source_kwargs = dict(
+        source=source_name,
+        source_type=source_type,
+        flush=False,
+    )
+    if entsoe_account is not None:
+        source_kwargs["account"] = entsoe_account
+    return get_or_create_source(**source_kwargs)
+
+
+def ensure_data_source() -> Source:
+    """Make sure we have a raw ENTSO-E data source of type "market"."""
+    return _ensure_entsoe_source(
+        source_name=current_app.config.get(
+            "ENTSOE_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+        ),
+        source_type="market",
+        legacy_source_type="forecasting script",
+    )
+
+
+def ensure_data_source_for_derived_data() -> Source:
+    """Make sure we have a data source for data derived from ENTSO-E data."""
+    return _ensure_entsoe_source(
+        source_name=current_app.config.get(
+            "ENTSOE_DERIVED_DATA_SOURCE", DEFAULT_DERIVED_DATA_SOURCE
+        ),
+        source_type="forecasting script",
+    )
+
+
+def ensure_transmission_zone_asset(country_code: str) -> Asset:
+    """
+    Ensure a GenericAsset exists to model the transmission zone for which this plugin gathers data.
+    """
+    transmission_zone_type = AssetType.query.filter(
+        AssetType.name == "transmission zone"
+    ).one_or_none()
+    if not transmission_zone_type:
+        current_app.logger.info("Adding transmission zone type ...")
+        transmission_zone_type = AssetType(
+            name="transmission zone",
+            description="A grid regulated & balanced as a whole, usually a national grid.",
+        )
+        db.session.add(transmission_zone_type)
+    ga_name = f"{country_code} transmission zone"
+    transmission_zone = Asset.query.filter(Asset.name == ga_name).one_or_none()
+    if not transmission_zone:
+        current_app.logger.info(f"Adding {ga_name} ...")
+        transmission_zone = Asset(
+            name=ga_name,
+            generic_asset_type=transmission_zone_type,
+            account_id=None,  # public
+        )
+        db.session.add(transmission_zone)
+    db.session.commit()
+    return transmission_zone
+
+
+def ensure_sensors(
+    sensor_specifications: Tuple,
+    country_code: str,
+    timezone: str,
+) -> Dict[str, Sensor]:
+    """
+    Ensure a GenericAsset exists to model the transmission zone for which this plugin gathers
+    generation data, then add specified sensors for relevant data we collect.
+
+    If new sensors got created, the session has been flushed.
+    """
+    sensors = {}
+    sensors_created: bool = False
+    transmission_zone = ensure_transmission_zone_asset(country_code)
+    for sensor_name, unit, event_resolution, data_by_entsoe in sensor_specifications:
+        sensor = Sensor.query.filter(
+            Sensor.name == sensor_name,
+            Sensor.unit == unit,
+            Sensor.generic_asset == transmission_zone,
+        ).one_or_none()
+        if not sensor:
+            current_app.logger.info(f"Adding sensor {sensor_name} ...")
+            sensor = Sensor(
+                name=sensor_name,
+                unit=unit,
+                generic_asset=transmission_zone,
+                timezone=timezone,
+                event_resolution=event_resolution,
+            )
+            db.session.add(sensor)
+            sensors_created = True
+        elif sensor.event_resolution != event_resolution:
+            current_app.logger.warning(
+                f"The {sensor_name} sensor exists, but has a resolution of {sensor.event_resolution} instead of {event_resolution}. Please refer the 'October 1st 2025 go-live' instructions in `README.md`."
+            )
+        sensor.data_by_entsoe = data_by_entsoe
+        sensors[sensor_name] = sensor
+    if sensors_created:
+        db.session.flush()
+    return sensors
+
+
+def get_auth_token_from_config_and_set_server_url() -> str:
+    """
+    Read ENTSOE auth token from config, raise if not given.
+    If test server is supposed to be used, we'll try to read the token
+    usable for that, and also change the URL.
+    """
+    use_test_server = current_app.config.get("ENTSOE_USE_TEST_SERVER", False)
+    if use_test_server:
+        auth_token = current_app.config.get("ENTSOE_AUTH_TOKEN_TEST_SERVER")
+        entsoe.entsoe.URL = "https://iop-transparency.entsoe.eu/api"
+    else:
+        auth_token = current_app.config.get("ENTSOE_AUTH_TOKEN")
+        entsoe.entsoe.URL = "https://web-api.tp.entsoe.eu/api"
+    if not auth_token:
+        click.echo("Setting ENTSOE_AUTH_TOKEN seems empty!")
+        raise click.Abort
+    return auth_token
+
+
+def ensure_country_code_and_timezone(
+    country_code: Optional[str] = None,
+    country_timezone: Optional[str] = None,
+) -> Tuple[str, str]:
+    if country_code is None:
+        country_code = current_app.config.get(
+            "ENTSOE_COUNTRY_CODE", DEFAULT_COUNTRY_CODE
+        )
+    if country_timezone is None:
+        country_timezone = current_app.config.get(
+            "ENTSOE_COUNTRY_TIMEZONE", DEFAULT_COUNTRY_TIMEZONE
+        )
+    return country_code, country_timezone
+
+
+def create_entsoe_client() -> EntsoePandasClient:
+    auth_token = get_auth_token_from_config_and_set_server_url()
+    client = EntsoePandasClient(api_key=auth_token)
+    return client
+
+
+def abort_if_data_empty(data: Union[pd.DataFrame, pd.Series]):
+    if data.empty:
+        click.echo(
+            "Result is empty. Probably ENTSO-E does not provide these forecasts yet ..."
+        )
+        raise click.Abort
+
+
+def abort_if_data_incomplete(
+    data: Union[pd.DataFrame, pd.Series],
+    from_time: pd.Timestamp,
+    until_time: pd.Timestamp,
+    resolution: pd.Timedelta,
+):
+    expected_periods = int((until_time - from_time) / resolution)
+    if len(data) < expected_periods:
+        click.secho(
+            f"Result is incomplete. Expected {expected_periods} periods but got {len(data)}. Probably ENTSO-E does not provide these forecasts yet ...",
+            **MsgStyle.ERROR,
+        )
+        raise click.Abort
+
+
+def parse_from_and_to_dates(
+    from_date: Optional[datetime],
+    until_date: Optional[datetime],
+    country_timezone: str,
+    default_to: str = "today-and-tomorrow",  # Can be "tomorrow" or "today"
+) -> Tuple[pd.Timestamp, pd.Timestamp]:
+    """
+    Parse CLI options for start and end date (or set default to today and tomorrow) for inout to entsoe-py
+    Note: we expect only dates as input here, and until_date is inclusive, so we extend it with 24h - so if from_date is equal to until_date, we return 00:00 and 24:00 of that day.
+    Note:  entsoe-py expects time params as pd.Timestamp
+    """
+    tz = pytz.timezone(country_timezone)
+    now = datetime.now(tz)
+    today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+
+    if default_to == "today":
+        default_start = today_start
+        default_end = today_start + timedelta(days=1)
+    elif default_to == "tomorrow":
+        default_start = today_start + timedelta(days=1)
+        default_end = default_start + timedelta(days=1)
+    elif default_to == "today-and-tomorrow":
+        default_start = today_start
+        default_end = default_start + timedelta(days=2)
+    else:
+        raise ValueError(
+            f"Invalid default_to value: {default_to}. Expected 'today', 'tomorrow' or 'today-and-tomorrow'."
+        )
+
+    if from_date is None:
+        start_date = pd.Timestamp(default_start)
+    else:
+        start_date = pd.Timestamp(from_date, tzinfo=pytz.timezone(country_timezone))
+
+    if until_date is None:
+        end_date = pd.Timestamp(default_end)
+    else:
+        end_date = pd.Timestamp(until_date, tzinfo=pytz.timezone(country_timezone))
+        # The until_date provided is considered inclusive, so we add 24 hours to include the entire day
+        end_date += pd.Timedelta(hours=24)
+
+    return start_date, end_date
+
+
+def resample_if_needed(s: pd.Series, sensor: Sensor) -> pd.Series:
+    inferred_frequency = pd.infer_freq(s.index)
+    if inferred_frequency is None:
+        raise ValueError(
+            "Data has no discernible frequency from which to derive an event resolution."
+        )
+    inferred_resolution = pd.to_timedelta(to_offset(inferred_frequency))
+    target_resolution = sensor.event_resolution
+    if inferred_resolution == target_resolution:
+        return s
+    elif inferred_resolution > target_resolution:
+        current_app.logger.debug(f"Upsampling data for {sensor.name} ...")
+        index = pd.date_range(
+            s.index[0],
+            s.index[-1] + inferred_resolution,
+            freq=target_resolution,
+            inclusive="left",
+        )
+        s = s.reindex(index).pad()
+    elif inferred_resolution < target_resolution:
+        current_app.logger.debug(f"Downsampling data for {sensor.name} ...")
+        s = s.resample(target_resolution).mean()
+    current_app.logger.debug(f"Resampled data for {sensor.name}: \n%s" % s)
+    return s
+
+
+def save_entsoe_series(
+    series: pd.Series,
+    sensor: Sensor,
+    entsoe_source: Source,
+    country_timezone: str,
+    now: Optional[datetime] = None,
+):
+    """
+    Save a series gotten from ENTSO-E to a FlexMeasures database.
+    """
+    if not now:
+        now = server_now().astimezone(pytz.timezone(country_timezone))
+    belief_times = (
+        (series.index.floor("D") - pd.Timedelta("6h"))
+        .to_frame(name="clipped_belief_times")
+        .clip(upper=now)
+        .set_index("clipped_belief_times")
+        .index
+    )  # published no later than D-1 18:00 Brussels time
+    bdf = BeliefsDataFrame(
+        series,
+        source=entsoe_source,
+        sensor=sensor,
+        belief_time=belief_times,
+    )
+
+    # TODO: evaluate some traits of the data via FlexMeasures, see https://github.com/SeitaBV/flexmeasures-entsoe/issues/3
+    status = save_to_db(bdf)
+    if status == "success_but_nothing_new":
+        current_app.logger.info("Done. These beliefs had already been saved before.")
+    elif status == "success_with_unchanged_beliefs_skipped":
+        current_app.logger.info("Done. Some beliefs had already been saved before.")
+
+
+def start_import_log(
+    import_type: str,
+    from_time: pd.Timestamp,
+    until_time: pd.Timestamp,
+    country_code: str,
+    country_timezone: str,
+) -> Tuple[Logger, datetime]:
+    log = current_app.logger
+    log.info(
+        f"Importing {import_type} data for {country_code} (timezone {country_timezone}), starting at {from_time}, up until {until_time}, from ENTSO-E at {entsoe.entsoe.URL} ..."
+    )
+    now = server_now().astimezone(pytz.timezone(country_timezone))
+    return log, now

tools/flexmeasures-entsoe/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=62", "setuptools_scm[toml]>=6.2", "wheel>=0.29.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "flexmeasures-entsoe"
+description = "Integrating FlexMeasures with ENTSO-E"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+license-files = [
+    "LICENSE",
+]
+authors = [
+    {name = "Seita BV", email = "nicolas@seita.nl"}
+]
+keywords = ["smart grid", "renewables", "balancing", "forecasting", "scheduling"]
+classifiers = [
+    "Environment :: Console",
+    "Environment :: Web Environment",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Framework :: Flask",
+    "Development Status :: 5 - Production/Stable",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: MacOS :: MacOS X",
+    "Natural Language :: English"
+]
+dynamic = ["version", "dependencies"]
+
+[project.urls]
+Homepage = "https://github.com/SeitaBV/flexmeasures-entsoe"
+Documentation = "https://github.com/SeitaBV/flexmeasures-entsoe"
+"Source code" = "https://github.com/SeitaBV/flexmeasures-entsoe"
+
+[project.scripts]
+flexmeasures = "flexmeasures.utils.app_utils:flexmeasures_cli"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["flexmeasures*"]
+
+[tool.setuptools_scm]
+local_scheme = "no-local-version"
+version_scheme = "guess-next-dev"

tools/flexmeasures-entsoe/requirements/Readme.md

@@ -0,0 +1,23 @@
+# Requirements
+
+All FlexMeasures requirements are specified in this directory.
+We separate by use case:
+
+- app: All requirements for running the FlexMeasures platform
+- test: Additional requirements used for running automated tests 
+- dev: Additional requirements used for developers (this includes testing)
+
+Also note the following distinction:
+
+
+## .in files
+
+Here, we describe the requirements. We give the name of a requirement or even a range (e.g. `>=1.0.`).
+
+## .txt files
+
+These files are not to be edited by hand. They are created by `pip-compile` (or `make freeze-deps`).
+
+They are usually not needed, only for development environments. When distributing FlexMeasures with pinned dependency versions and this plugin, only the extra app dependencies (see .in file) need extra care beyond the .txt files.
+
+Each requirement is pinned to a specific version in these files. The great benefit is reproducibility across environments (local dev as well as staging or production).

tools/flexmeasures-entsoe/requirements/app.in

@@ -0,0 +1,3 @@
+# only listing extra dependencies that flexmeasures does not have
+entsoe-py
+timely-beliefs>=3.2.3

tools/flexmeasures-entsoe/requirements/dev.in

@@ -0,0 +1,11 @@
+# include flexmeasures as a dev dependency so a fresh environment has it
+flexmeasures>=0.28.2 
+pre-commit
+black
+flake8
+flake8-blind-except
+mypy
+pytest-runner
+types-pytz
+setuptools_scm
+watchdog

tools/flexmeasures-entsoe/requirements/test.in

@@ -0,0 +1,4 @@
+pytest
+pytest-flask
+pytest-sugar
+pytest-cov

tools/flexmeasures-entsoe/run_mypy.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+pip install mypy
+# We are checking python files which have type hints
+files=$(find . -name \*.py -not \( -path "./venv/*" -prune \) -not \( -path "./.eggs/*" -prune \) )
+
+mypy --follow-imports skip --ignore-missing-imports $files 

tools/flexmeasures-entsoe/setup.cfg

@@ -0,0 +1,10 @@
+[aliases]
+test = pytest
+flake8 = flake8
+
+[flake8]
+exclude = .git,__pycache__,documentation
+max-line-length = 160
+max-complexity = 13
+select = B,C,E,F,W,B9
+ignore = E501, W503, E203

tools/flexmeasures-entsoe/setup.py

@@ -0,0 +1,26 @@
+from setuptools import setup
+
+
+def load_requirements(use_case):
+    """
+    Loading range requirements.
+    Packaging should be used for installing the package into existing stacks.
+    We therefore read the .in file for the use case.
+    .txt files include the exact pins, and are useful for deployments or dev
+    environments with exactly comparable environments.
+    """
+    reqs = []
+    with open("requirements/%s.in" % use_case, "r") as f:
+        reqs = [
+            req
+            for req in f.read().splitlines()
+            if not req.strip() == ""
+            and not req.strip().startswith("#")
+            and not req.strip().startswith("-c")
+            and not req.strip().startswith("--find-links")
+        ]
+    return reqs
+
+
+setup(install_requires=load_requirements("app"))
+

tools/flexmeasures-weather/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude = .git,__pycache__,documentation
+max-line-length = 160
+max-complexity = 13
+select = B,C,E,F,W,B9
+ignore = E501, W503, E203
+

tools/flexmeasures-weather/.github/issue-branch.yml

@@ -0,0 +1,2 @@
+openDraftPR: true
+autoCloseIssue: true

tools/flexmeasures-weather/.github/workflows/lint-and-test.yml

@@ -0,0 +1,60 @@
+name: lint-and-test
+
+on: push
+
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    name: Check (on Python 3.11)
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - uses: actions/checkout@v3
+      - uses: pre-commit/action@v3.0.0
+
+
+  test:
+    needs: check
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        py_version: [ '3.11' ]
+    name: "Test (on Python ${{ matrix.py_version }})"
+    steps:
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.py_version }}
+      - name: Check out src from Git
+        uses: actions/checkout@v3
+      - name: Install SQL extensions
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install postgresql-client
+          psql -h $PGHOST -p $PGPORT --file scripts/load-psql-extensions.sql -U $PGUSER $PGDB;
+      - name: Install necessary items for netcdf
+        run: |
+          sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev
+      - run: make test
+    env:
+      PGHOST: 127.0.0.1
+      PGPORT: 5432
+      PGUSER: flexmeasures_test
+      PGDB: flexmeasures_test
+      PGPASSWORD: flexmeasures_test
+
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:14.17 
+        env:
+          POSTGRES_USER: flexmeasures_test
+          POSTGRES_PASSWORD: flexmeasures_test
+          POSTGRES_DB: flexmeasures_test
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

tools/flexmeasures-weather/.gitignore

@@ -0,0 +1,84 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Sphinx documentation
+docs/_build/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+
+# custom project files
+
+.vscode
+*.pickle
+
+.ipynb_checkpoints/
+notebooks/.ipynb_checkpoints/
+
+flexmeasures.log
+
+.gitconfig.*

tools/flexmeasures-weather/.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.1.1  # New version tags can be found here: https://github.com/pycqa/flake8/tags
+    hooks:
+    - id: flake8
+      name: flake8 (code linting)
+-   repo: https://github.com/psf/black
+    rev: 24.8.0  # New version tags can be found here: https://github.com/psf/black/tags
+    hooks:
+    - id: black
+      name: black (code formatting)
+-   repo: local
+    hooks:
+    - id: mypy
+      name: mypy (static typing)
+      pass_filenames: false
+      language: script
+      entry: run_mypy.sh
+      verbose: true

tools/flexmeasures-weather/LICENSE

@@ -0,0 +1,174 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.

tools/flexmeasures-weather/Makefile

@@ -0,0 +1,45 @@
+# Note: use tabs
+# actions which are virtual, i.e. not a script
+.PHONY: install install-for-dev install-deps install-flexmeasures-weather test freeze-deps upgrade-deps
+
+
+# ---- Development ---
+
+test:
+	make install-for-dev
+	pytest
+
+# ---- Installation ---
+
+install: install-deps install-flexmeasures-weather
+
+install-for-dev:
+	make freeze-deps
+	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
+	make install-flexmeasures-weather
+	pre-commit install
+
+install-deps:
+	make install-pip-tools
+	make freeze-deps
+	pip-sync requirements/app.txt
+
+install-flexmeasures-weather:
+	pip install -e .
+
+install-pip-tools:
+	pip3 install -q "pip-tools>=6.2"
+
+freeze-deps:
+	make install-pip-tools
+	pip-compile -o requirements/app.txt requirements/app.in
+	pip-compile -o requirements/test.txt requirements/test.in
+	pip-compile -o requirements/dev.txt requirements/dev.in
+
+upgrade-deps:
+	make install-pip-tools
+	pip-compile --upgrade -o requirements/app.txt requirements/app.in
+	pip-compile --upgrade -o requirements/test.txt requirements/test.in
+	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
+	make test
+

tools/flexmeasures-weather/README.md

@@ -0,0 +1,142 @@
+# FLEXMEASURES-WEATHER - a plugin for FlexMeasures to integrate weather forecasts
+
+
+This plugin currently supports two Weather API services: [OpenWeatherMap One Call API](https://openweathermap.org/api/one-call-3) and [Weather API](https://www.weatherapi.com/). The configuration is controlled via your FlexMeasures config file.
+
+
+## Usage
+
+To register a new weather sensor:
+
+`flexmeasures weather register-weather-sensor --name "wind speed" --latitude 30 --longitude 40`
+
+Currently supported: wind speed, temperature & irradiance.
+
+To collect weather forecasts:
+
+`flexmeasures weather get-weather-forecasts --location 30,40`
+
+This saves forecasts for your registered sensors in the database.
+
+Use the `--help`` option for more options, e.g. for specifying two locations and requesting that a number of weather stations cover the bounding box between them (where the locations represent top left and bottom right).
+
+Notes about weather sensor setup: 
+
+- Weather sensors are public assets in FlexMeasures. They are accessible by all accounts on a FlexMeasures server.
+- The resolution is one hour. Weather also supports minutely data within the upcoming hour(s), but that is not supported here.
+
+An alternative usage is to save raw results in JSON files (for later processing), like this:
+
+`flexmeasures weather get-weather-forecasts --location 30,40 --store-as-json-files --region somewhere`
+
+This saves the complete response from the Weather Provider in a local folder (i.e. no sensor registration needed, this is a direct way to use Weather APIs, without FlexMeasures integration). `region` will become a subfolder.
+ 
+Finally, note that these APIs allow free calls, but not without limits.
+For instance, currently 1000 free calls per day can be made to the OpenWeatherMap API,
+so you can make a call every 15 minutes for up to 10 locations or every hour for up to 40 locations (or get a paid account).
+
+
+## Setup
+
+### Installation
+
+To add as plugin to an existing FlexMeasures system, add "/path/to/flexmeasures-weather-repo/flexmeasures_weather" to your FlexMeasures config file,
+using the FLEXMEASURES_PLUGINS setting (a list).
+
+Alternatively, if you installed this plugin as a package (e.g. via `python setup.py install`, `pip install -e` or `pip install flexmeasures_weather` after this project is on Pypi), then "flexmeasures_weather" suffices.
+
+To enable weather forecast functionality, two PostgreSQL extensions must be installed. Run the following SQL commands in your database:
+
+```
+CREATE EXTENSION IF NOT EXISTS cube;
+CREATE EXTENSION IF NOT EXISTS earthdistance;
+```
+
+These extensions provide support for geographical calculations such as `ll_to_earth` and `earth_distance`, which we use to find the nearest weather station asset.
+
+
+### Configuration
+
+Add the following entries to your config:
+
+```ini
+# Select the weather provider to use: "OWM" (OpenWeatherMap) or "WAPI" (Weather API)
+WEATHER_PROVIDER = "OWM"
+
+# API key for the selected weather provider
+WEATHERAPI_KEY = "your-api-key-here"
+
+# Name to register the weather data source in FlexMeasures. The default is 'Weather'.
+# Examples: "OpenWeatherMap" (for backwards compatibility with the OWM plugin).
+WEATHER_DATA_SOURCE_NAME = "OpenWeatherMap"
+
+# File path to store weather data in JSON format
+WEATHER_FILE_PATH_LOCATION = "/path/to/weather_output.json"
+```
+
+### Extending to Other Weather API Services
+
+To expand the plugin's coverage to additional weather API services:
+
+1. **Update the configuration**  
+   Change the `WEATHER_PROVIDER` setting in your config to the identifier for the new API service (e.g., `NEWAPI`), and provide the necessary credentials in `WEATHERAPI_KEY`.
+
+2. **Implement a new API function**  
+   Create a function named in the format:
+
+   ```python
+   def call_NEWAPI_api(...):
+       # Your logic to call the API and return data in the expected format
+   ```
+
+   This function should return data in the same structure as used by the original OpenWeatherMap integration, and **must have at least 48 hours of forecast data from the time of the call**.
+
+   You also need a provider-specific mapping entry in `flexmeasures_weather/sensor_specs.py`. Each supported sensor should include the new provider's response field name, for example:
+
+   ```python
+   dict(
+       fm_sensor_name="temperature",
+       OWM_sensor_name="temp",
+       WAPI_sensor_name="temp_c",
+       NEWAPI_sensor_name="temperatureC",
+       unit="°C",
+       event_resolution=timedelta(minutes=60),
+       attributes=weather_attributes,
+   )
+   ```
+
+3. **Integrate into the plugin**
+   Modify the `call_api` function in the `weather.py` file to include a conditional branch for the new provider:
+
+   ```python
+   def call_api(...):
+    if provider not in ['OWM', 'WAPI', ..., 'NEWAPI']:
+        raise Exception
+    if provider == 'NEWAPI':
+        return call_NEWAPI_api(...)
+   ```
+
+4. **Finalize and contribute**  
+   Once you've implemented and tested the plugin with your chosen API service:
+   - Update this README to reflect the new configuration and usage details.
+   - Submit a pull request with your changes for review.
+
+> This modular structure allows for seamless integration of additional services while maintaining consistency and clarity in data handling.
+
+
+## Development
+
+We use pre-commit to keep code quality up.
+
+Install necessary tools with:
+
+    pip install pre-commit
+    pre-commit install
+
+or:
+
+    make install-for-dev
+
+Try it:
+
+    pre-commit run --all-files --show-diff-on-failure

tools/flexmeasures-weather/flexmeasures_weather/__init__.py

@@ -0,0 +1,68 @@
+__version__ = "Unknown version"
+
+
+"""
+The __init__ for the flexmeasures-weather FlexMeasures plugin.
+
+FlexMeasures registers the BluePrint objects it finds in here.
+"""
+
+
+from importlib.metadata import version, PackageNotFoundError
+
+from flask import Blueprint
+
+from .utils.blueprinting import ensure_bp_routes_are_loaded_fresh
+
+# Overwriting version (if possible) from the package metadata
+# ― if this plugin has been installed as a package.
+# This uses importlib.metadata behaviour added in Python 3.8.
+# Note that we rely on git tags (via setuptools_scm) to define that version.
+try:
+    __version__ = version("flexmeasures_weather")
+except PackageNotFoundError:
+    # package is not installed
+    pass
+
+
+DEFAULT_FILE_PATH_LOCATION = "weather-forecasts"
+DEFAULT_DATA_SOURCE_NAME = "Weather"
+DEFAULT_WEATHER_STATION_NAME = "weather station (created by FM-Weather)"
+WEATHER_STATION_TYPE_NAME = "weather station"
+DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE = 1
+
+__version__ = "0.1"
+__settings__ = {
+    "WEATHER_FILE_PATH_LOCATION": dict(
+        description="Location of JSON files (if you store weather data in this form). Absolute path.",
+        level="debug",
+    ),
+    "WEATHER_DATA_SOURCE_NAME": dict(
+        description=f"Name of the data source for Weather data, defaults to '{DEFAULT_DATA_SOURCE_NAME}'",
+        level="debug",
+    ),
+    "WEATHER_STATION_NAME": dict(
+        description=f"Name of the weather station asset, defaults to '{DEFAULT_WEATHER_STATION_NAME}'",
+        level="debug",
+    ),
+    "WEATHER_MAXIMAL_DEGREE_LOCATION_DISTANCE": dict(
+        descripion=f"Maximum distance (in degrees latitude & longitude) for weather stations from forecast location, defaults to {DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE}",
+        level="debug",
+    ),
+    "WEATHER_PROVIDER": dict(
+        description="Provider for weather data. Permissible options are 'OWM' (OpenWeatherMap) or 'WAPI' (WeatherAPI).",
+        level="error",
+    ),
+    "WEATHERAPI_KEY": dict(
+        description="API key for OWM or WAPI, whatever you have chosen.",
+        level="error",
+    ),
+}
+
+# CLI
+flexmeasures_weather_bp: Blueprint = Blueprint(
+    "flexmeasures-weather CLI", __name__, cli_group="weather"
+)
+flexmeasures_weather_bp.cli.help = "flexmeasures-weather CLI commands"
+ensure_bp_routes_are_loaded_fresh("cli.commands")
+from flexmeasures_weather.cli import commands  # noqa: E402,F401

tools/flexmeasures-weather/flexmeasures_weather/cli/commands.py

@@ -0,0 +1,188 @@
+from flask import current_app
+
+from flask.cli import with_appcontext
+import click
+from flexmeasures.data.models.time_series import Sensor
+
+from flexmeasures.data.transactional import task_with_status_report
+from flexmeasures.data.config import db
+
+from .. import flexmeasures_weather_bp
+from .schemas.weather_sensor import WeatherSensorSchema
+from ..utils.modeling import (
+    get_or_create_weather_station,
+    get_weather_station_by_asset_id,
+)
+from ..utils.locating import get_locations, get_location_by_asset_id
+from ..utils.filing import make_file_path
+from ..utils.weather import (
+    save_forecasts_in_db,
+    save_forecasts_as_json,
+    get_supported_sensor_spec,
+)
+from ..sensor_specs import mapping
+
+"""
+TODO: allow to also pass an asset ID or name for the weather station (instead of location) to both commands?
+      See https://github.com/FlexMeasures/flexmeasures-weather
+"""
+
+supported_sensors_list = ", ".join(
+    [str(sensor_specs["fm_sensor_name"]) for sensor_specs in mapping]
+)
+
+
+@flexmeasures_weather_bp.cli.command("register-weather-sensor")
+@with_appcontext
+@click.option(
+    "--name",
+    required=True,
+    help=f"Name of the sensor. Has to be from the supported list ({supported_sensors_list})",
+)
+@click.option(
+    "--asset-id",
+    required=False,
+    type=int,
+    help="The asset id of the weather station (you can also give its location).",
+)
+@click.option(
+    "--latitude",
+    required=False,
+    type=float,
+    help="Latitude of where you want to measure.",
+)
+@click.option(
+    "--longitude",
+    required=False,
+    type=float,
+    help="Longitude of where you want to measure.",
+)
+@click.option(
+    "--timezone",
+    default="UTC",
+    help="The timezone of the sensor data as string, e.g. 'UTC' (default) or 'Europe/Amsterdam'",
+)
+def add_weather_sensor(**args):
+    """
+    Add a weather sensor.
+    This will first create a weather station asset if none exists at the location yet.
+
+    """
+    errors = WeatherSensorSchema().validate(args)
+    if errors:
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] Please correct the following errors:\n{errors}.\n Use the --help flag to learn more."
+        )
+        raise click.Abort
+    if args["asset_id"] is not None:
+        weather_station = get_weather_station_by_asset_id(args["asset_id"])
+    elif args["latitude"] is not None and args["longitude"] is not None:
+        weather_station = get_or_create_weather_station(
+            args["latitude"], args["longitude"]
+        )
+    else:
+        raise Exception(
+            "Arguments are missing to register a weather sensor. Provide either '--asset-id' or ('--latitude' and '--longitude')."
+        )
+
+    sensor = Sensor.query.filter(
+        Sensor.name == args["name"].lower(),
+        Sensor.generic_asset == weather_station,
+    ).one_or_none()
+    if sensor:
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] A '{args['name']}' weather sensor already exists at this weather station (the station's ID is {weather_station.id})."
+        )
+        return
+    fm_sensor_specs = get_supported_sensor_spec(args["name"])
+    fm_sensor_specs["generic_asset"] = weather_station
+    fm_sensor_specs["timezone"] = args["timezone"]
+    fm_sensor_specs["name"] = fm_sensor_specs.pop("fm_sensor_name")
+    fm_sensor_specs.pop("OWM_sensor_name")
+    fm_sensor_specs.pop("WAPI_sensor_name")
+    sensor = Sensor(**fm_sensor_specs)
+    sensor.attributes = fm_sensor_specs["attributes"]
+
+    db.session.add(sensor)
+    db.session.commit()
+    click.echo(
+        f"[FLEXMEASURES-WEATHER] Successfully created weather sensor with ID {sensor.id}, at weather station with ID {weather_station.id}"
+    )
+    click.echo(
+        f"[FLEXMEASURES-WEATHER] You can access this sensor at its entity address {sensor.entity_address}"
+    )
+
+
+@flexmeasures_weather_bp.cli.command("get-weather-forecasts")
+@with_appcontext
+@click.option(
+    "--location",
+    type=str,
+    required=False,
+    help='Measurement location(s). "latitude,longitude" or "top-left-latitude,top-left-longitude:'
+    'bottom-right-latitude,bottom-right-longitude." The first format defines one location to measure.'
+    " The second format defines a region of interest with several (>=4) locations"
+    ' (see also the "method" and "num_cells" parameters for details on how to use this feature).',
+)
+@click.option(
+    "--asset-id",
+    type=int,
+    required=False,
+    help="ID of a weather station asset - forecasts will be gotten for its location. If present, --location will be ignored.",
+)
+@click.option(
+    "--store-in-db/--store-as-json-files",
+    default=True,
+    help="Store forecasts in the database, or simply save as json files (defaults to database).",
+)
+@click.option(
+    "--num_cells",
+    type=int,
+    default=1,
+    help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter. Defaults to 1.",
+)
+@click.option(
+    "--method",
+    default="hex",
+    type=click.Choice(["hex", "square"]),
+    help="Grid creation method. Only used if a region of interest has been mapped in the location parameter.",
+)
+@click.option(
+    "--region",
+    type=str,
+    default="",
+    help="Name of the region (will create sub-folder if you store json files).",
+)
+@task_with_status_report("get-weather-forecasts")
+def collect_weather_data(location, asset_id, store_in_db, num_cells, method, region):
+    """
+    Collect weather forecasts from the Weather Provider API.
+    This will be done for one or more locations, for which we first identify relevant weather stations.
+
+    This function can get weather data for one location or for several locations within
+    a geometrical grid (See the --location parameter).
+    """
+
+    api_key = str(
+        current_app.config.get(
+            "WEATHERAPI_KEY", current_app.config.get("OPENWEATHERMAP_API_KEY", "")
+        )
+    )
+    if api_key == "":
+        raise Exception("[FLEXMEASURES-WEATHER] Setting WEATHERAPI_KEY not available.")
+    if asset_id is not None:
+        locations = [get_location_by_asset_id(asset_id)]
+    elif location is not None:
+        locations = get_locations(location, num_cells, method)
+    else:
+        raise Warning(
+            "[FLEXMEASURES-WEATHER] Pass either location or asset-id to get weather forecasts."
+        )
+
+    # Save the results
+    if store_in_db:
+        save_forecasts_in_db(api_key, locations)
+    else:
+        save_forecasts_as_json(
+            api_key, locations, data_path=make_file_path(current_app, region)
+        )

tools/flexmeasures-weather/flexmeasures_weather/cli/schemas/weather_sensor.py

@@ -0,0 +1,43 @@
+from marshmallow import (
+    Schema,
+    validates,
+    ValidationError,
+    fields,
+    validate,
+)
+
+import pytz
+
+from ...utils.weather import get_supported_sensor_spec, get_supported_sensors_str
+
+
+class WeatherSensorSchema(Schema):
+    """
+    Schema for the weather sensor registration.
+    Based on flexmeasures.Sensor, plus some attributes for the weather station asset.
+    """
+
+    name = fields.Str(required=True)
+    timezone = fields.Str()
+    asset_id = fields.Int(required=False, allow_none=True)
+    latitude = fields.Float(
+        required=False, validate=validate.Range(min=-90, max=90), allow_none=True
+    )
+    longitude = fields.Float(
+        required=False, validate=validate.Range(min=-180, max=180), allow_none=True
+    )
+
+    @validates("name")
+    def validate_name_is_supported(self, name: str, **kwargs):
+        if get_supported_sensor_spec(name):
+            return
+        raise ValidationError(
+            f"Weather sensors with name '{name}' are not supported by flexmeasures-weather. For now, the following is supported: [{get_supported_sensors_str()}]"
+        )
+
+    @validates("timezone")
+    def validate_timezone(self, timezone: str, **kwargs):
+        try:
+            pytz.timezone(timezone)
+        except pytz.UnknownTimeZoneError:
+            raise ValidationError(f"Timezone {timezone} is unknown!")

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/conftest.py

@@ -0,0 +1 @@
+from flexmeasures.conftest import run_as_cli  # noqa: F401

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_get_forecasts.py

@@ -0,0 +1,105 @@
+import logging
+
+import pytest
+from flexmeasures.data.models.time_series import TimedBelief
+
+from ..commands import collect_weather_data
+from ...utils import weather
+from .utils import mock_api_response
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+
+def test_get_weather_forecasts_to_db(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test if we can process forecast and save them to the database.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    print(result.output)
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    for wind_speed in (100, 90):
+        assert wind_speed in [belief.event_value for belief in beliefs]
+
+
+def test_get_weather_forecasts_wapi_mapping(
+    app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db
+):
+    """
+    Test that WeatherAPI provider-specific field names are mapped independently.
+    """
+    wind_sensor = add_weather_sensors_fresh_db["wind"]
+    fresh_db.session.flush()
+    wind_sensor_id = wind_sensor.id
+    weather_station = wind_sensor.generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "WAPI")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        collect_weather_data,
+        ["--location", f"{weather_station.latitude},{weather_station.longitude}"],
+    )
+    assert "Reported task get-weather-forecasts status as True" in result.output
+
+    beliefs = (
+        fresh_db.session.query(TimedBelief)
+        .filter(TimedBelief.sensor_id == wind_sensor_id)
+        .all()
+    )
+    assert len(beliefs) == 2
+    expected_values = [pytest.approx(100 / 3.6), pytest.approx(90 / 3.6)]
+    assert [belief.event_value for belief in beliefs] == expected_values
+
+
+def test_get_weather_forecasts_no_close_sensors(
+    app, db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db, caplog
+):
+    """
+    Looking for a location too far away from existing weather station.
+    Check we get a warning.
+    """
+    weather_station = add_weather_sensors_fresh_db["wind"].generic_asset
+
+    monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy")
+    monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM")
+    monkeypatch.setattr(weather, "call_api", mock_api_response)
+
+    runner = app.test_cli_runner()
+    with caplog.at_level(logging.WARNING):
+        result = runner.invoke(
+            collect_weather_data,
+            [
+                "--location",
+                f"{weather_station.latitude - 5},{weather_station.longitude}",
+            ],
+        )
+        print(result.output)
+        assert "Reported task get-weather-forecasts status as True" in result.output
+        assert "no sufficiently close weather sensor found" in caplog.text

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/test_register.py

@@ -0,0 +1,47 @@
+import pytest
+from flexmeasures import Sensor
+
+from ..commands import add_weather_sensor
+from .utils import cli_params_from_dict
+
+
+"""
+Useful resource: https://flask.palletsprojects.com/en/2.0.x/testing/#testing-cli-commands
+"""
+
+sensor_params = {"name": "wind speed", "latitude": 30, "longitude": 40}
+
+
+@pytest.mark.parametrize(
+    "invalid_param, invalid_value, expected_msg",
+    [
+        ("name", "windd-speed", "not supported by flexmeasures-weather"),
+        ("latitude", 93, "less than or equal to 90"),
+        ("timezone", "Erope/Amsterdam", "is unknown"),
+    ],
+)
+def test_register_weather_sensor_invalid_data(
+    app, db, invalid_param, invalid_value, expected_msg
+):
+    test_sensor_params = sensor_params.copy()
+    test_sensor_params[invalid_param] = invalid_value
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(test_sensor_params))
+    assert "Aborted" in result.output
+    assert expected_msg in result.output
+
+
+def test_register_weather_sensor(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    sensor = Sensor.query.filter(Sensor.name == sensor_params["name"]).one_or_none()
+    assert sensor is not None
+
+
+def test_register_weather_sensor_twice(app, fresh_db):
+    runner = app.test_cli_runner()
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "Successfully created weather sensor with ID" in result.output
+    result = runner.invoke(add_weather_sensor, cli_params_from_dict(sensor_params))
+    assert "already exists" in result.output

tools/flexmeasures-weather/flexmeasures_weather/cli/tests/utils.py

@@ -0,0 +1,37 @@
+from typing import List
+from datetime import datetime, timedelta
+from flask import current_app
+from flexmeasures.utils.time_utils import as_server_time, get_timezone
+
+
+def cli_params_from_dict(d) -> List[str]:
+    cli_params = []
+    for k, v in d.items():
+        cli_params.append(f"--{k}")
+        cli_params.append(v)
+    return cli_params
+
+
+def mock_api_response(api_key, location):
+    mock_date = datetime.now()
+    mock_date_tz_aware = as_server_time(
+        datetime.fromtimestamp(mock_date.timestamp(), tz=get_timezone())
+    ).replace(second=0, microsecond=0)
+
+    provider = str(current_app.config.get("WEATHER_PROVIDER", ""))
+    date_key = "dt"
+    temp_key = "temp"
+    wind_speed_key = "wind_speed"
+    if provider == "WAPI":
+        date_key = "time_epoch"
+        temp_key = "temp_c"
+        wind_speed_key = "wind_kph"
+
+    return mock_date_tz_aware, [
+        {date_key: mock_date.timestamp(), temp_key: 40, wind_speed_key: 100},
+        {
+            date_key: (mock_date + timedelta(hours=1)).timestamp(),
+            temp_key: 42,
+            wind_speed_key: 90,
+        },
+    ]

tools/flexmeasures-weather/flexmeasures_weather/conftest.py

@@ -0,0 +1,72 @@
+from typing import Dict
+from datetime import timedelta
+
+import pytest
+from flask_sqlalchemy import SQLAlchemy
+from flexmeasures.app import create as create_flexmeasures_app
+from flexmeasures.conftest import db, fresh_db  # noqa: F401
+from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
+from flexmeasures.data.models.time_series import Sensor
+
+from flexmeasures_weather import WEATHER_STATION_TYPE_NAME
+
+
+@pytest.fixture(scope="session")
+def app():
+    print("APP FIXTURE")
+
+    # Adding this plugin, making sure the name is known (as last part of plugin path)
+    test_app = create_flexmeasures_app(
+        env="testing", plugins=["../flexmeasures_weather"]
+    )
+
+    # Establish an application context before running the tests.
+    ctx = test_app.app_context()
+    ctx.push()
+
+    yield test_app
+
+    ctx.pop()
+
+    print("DONE WITH APP FIXTURE")
+
+
+@pytest.fixture(scope="module")
+def add_weather_sensors(db) -> Dict[str, Sensor]:  # noqa: F811
+    return create_weather_sensors(db)
+
+
+@pytest.fixture(scope="function")
+def add_weather_sensors_fresh_db(fresh_db) -> Dict[str, Sensor]:  # noqa: F811
+    return create_weather_sensors(fresh_db)
+
+
+def create_weather_sensors(db: SQLAlchemy):  # noqa: F811
+    """Add a weather station asset with two weather sensors."""
+    weather_station_type = GenericAssetType(name=WEATHER_STATION_TYPE_NAME)
+    db.session.add(weather_station_type)
+
+    weather_station = GenericAsset(
+        name="Test weather station",
+        generic_asset_type=weather_station_type,
+        latitude=33.4843866,
+        longitude=126,
+    )
+    db.session.add(weather_station)
+
+    wind_sensor = Sensor(
+        name="wind speed",
+        generic_asset=weather_station,
+        event_resolution=timedelta(minutes=60),
+        unit="m/s",
+    )
+    db.session.add(wind_sensor)
+
+    temp_sensor = Sensor(
+        name="temperature",
+        generic_asset=weather_station,
+        event_resolution=timedelta(minutes=60),
+        unit="°C",
+    )
+    db.session.add(temp_sensor)
+    return {"wind": wind_sensor, "temperature": temp_sensor}

tools/flexmeasures-weather/flexmeasures_weather/sensor_specs.py

@@ -0,0 +1,51 @@
+from datetime import timedelta
+
+
+"""
+This maps sensor specs which we can use in FlexMeasures to Weather labels.
+Note: Sensor names we use in FM need to be unique per weather station.
+At the moment, we only extract from Weather hourly data.
+"""
+
+
+weather_attributes = {
+    "daily_seasonality": True,
+    "weekly_seasonality": False,
+    "yearly_seasonality": True,
+}
+
+
+mapping = [
+    dict(
+        fm_sensor_name="temperature",
+        OWM_sensor_name="temp",
+        WAPI_sensor_name="temp_c",
+        unit="°C",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+    dict(
+        fm_sensor_name="wind speed",
+        OWM_sensor_name="wind_speed",
+        WAPI_sensor_name="wind_kph",
+        unit="m/s",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+    dict(
+        fm_sensor_name="cloud cover",
+        OWM_sensor_name="clouds",
+        WAPI_sensor_name="cloud",
+        unit="%",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+    dict(
+        fm_sensor_name="irradiance",  # in save_forecasts_to_db, we catch this name and do the actual computation to get to the irradiance
+        OWM_sensor_name="clouds",
+        WAPI_sensor_name="cloud",
+        unit="W/m²",
+        event_resolution=timedelta(minutes=60),
+        attributes=weather_attributes,
+    ),
+]

tools/flexmeasures-weather/flexmeasures_weather/utils/blueprinting.py

@@ -0,0 +1,22 @@
+import sys
+import importlib
+
+
+def ensure_bp_routes_are_loaded_fresh(module_name):
+    """
+    Reload a module if it has been loaded before.
+    It's useful for situations in which some other process has read
+    the module before, but you need some action to happen which only
+    happens during module import ― decorators are a good example.
+
+    One use case is pytest, which reads all python code when it collects tests.
+    In our case, that happens before FlexMeasures' import mechanism
+    has had a chance to know which blueprints a plugin has.
+    Seemingly, the importing code (plugin's __init__) can be imported later
+    than the imported module (containing @route decorators).
+    Re-importing helps to get this order right when FlexMeasures reads the
+    plugin's __init__.
+    """
+    m_name = "flexmeasures_weather." + module_name
+    if m_name in sys.modules:
+        importlib.reload(sys.modules[m_name])

tools/flexmeasures-weather/flexmeasures_weather/utils/filing.py

@@ -0,0 +1,25 @@
+import os
+
+import click
+from flask import Flask, current_app
+
+from flexmeasures_weather import DEFAULT_FILE_PATH_LOCATION
+
+
+def make_file_path(app: Flask, region: str) -> str:
+    """Ensure and return path for weather data"""
+    file_path = current_app.config.get(
+        "WEATHER_FILE_PATH_LOCATION", DEFAULT_FILE_PATH_LOCATION
+    )
+    data_path = os.path.join(app.root_path, file_path)
+    if not os.path.exists(data_path):
+        click.echo("[FLEXMEASURES-WEATHER] Creating %s ..." % data_path)
+        os.mkdir(data_path)
+    # optional: extend with subpath for region
+    if region is not None and region != "":
+        region_data_path = "%s/%s" % (data_path, region)
+        if not os.path.exists(region_data_path):
+            click.echo("[FLEXMEASURES-WEATHER] Creating %s ..." % region_data_path)
+            os.mkdir(region_data_path)
+        data_path = region_data_path
+    return data_path

tools/flexmeasures-weather/flexmeasures_weather/utils/locating.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from typing import Tuple, List, Optional
+
+import click
+from flask import current_app
+
+from flexmeasures.utils.grid_cells import LatLngGrid, get_cell_nums
+from flexmeasures import Sensor
+from flexmeasures.data.models.generic_assets import GenericAsset
+from flexmeasures.utils import flexmeasures_inflection
+
+from .. import WEATHER_STATION_TYPE_NAME
+
+
+def get_locations(
+    location: str,
+    num_cells: int,
+    method: str,
+) -> List[Tuple[float, float]]:
+    """
+    Get locations for getting forecasts for, by parsing the location string, which possibly opens a latitude/longitude grid with several neatly ordered locations.
+    """
+    if (
+        location.count(",") == 0
+        or location.count(",") != location.count(":") + 1
+        or location.count(":") == 1
+        and (
+            location.find(",") > location.find(":")
+            or location.find(",", location.find(",") + 1) < location.find(":")
+        )
+    ):
+        raise Exception(
+            '[FLEXMEASURES-WEATHER] location parameter "%s" seems malformed. Please use "latitude,longitude" or '
+            ' "top-left-latitude,top-left-longitude:bottom-right-latitude,bottom-right-longitude"'
+            % location
+        )
+
+    location_identifiers = tuple(location.split(":"))
+
+    if len(location_identifiers) == 1:
+        ll = location_identifiers[0].split(",")
+        locations = [(float(ll[0]), float(ll[1]))]
+        click.echo("[FLEXMEASURES-WEATHER] Only one location: %s,%s." % locations[0])
+    elif len(location_identifiers) == 2:
+        click.echo(
+            "[FLEXMEASURES-WEATHER] Making a grid of locations between top/left %s and bottom/right %s ..."
+            % location_identifiers
+        )
+        top_left = tuple(float(s) for s in location_identifiers[0].split(","))
+        if len(top_left) != 2:
+            raise Exception(
+                "[FLEXMEASURES-WEATHER] top-left parameter '%s' is invalid."
+                % location_identifiers[0]
+            )
+        bottom_right = tuple(float(s) for s in location_identifiers[1].split(","))
+        if len(bottom_right) != 2:
+            raise Exception(
+                "[FLEXMEASURES-WEATHER] bottom-right parameter '%s' is invalid."
+                % location_identifiers[1]
+            )
+
+        num_lat, num_lng = get_cell_nums(top_left, bottom_right, num_cells)
+
+        locations = LatLngGrid(
+            top_left=top_left,
+            bottom_right=bottom_right,
+            num_cells_lat=num_lat,
+            num_cells_lng=num_lng,
+        ).get_locations(method)
+    else:
+        raise Exception(
+            "[FLEXMEASURES-WEATHER] location parameter '%s' has too many locations."
+            % location
+        )
+    return locations
+
+
+def find_weather_sensor_by_location(
+    location: Tuple[float, float],
+    max_degree_difference_for_nearest_weather_sensor: int,
+    sensor_name: str,
+) -> Sensor | None:
+    """
+    Try to find a weather sensor of fitting type close by.
+    Return None if the nearest weather sensor is further away than some minimum degrees or if no sensor was found at all.
+    """
+    weather_sensor: Optional[Sensor] = Sensor.find_closest(
+        generic_asset_type_name=WEATHER_STATION_TYPE_NAME,
+        sensor_name=sensor_name,
+        lat=location[0],
+        lng=location[1],
+        n=1,
+    )
+    if weather_sensor is None:
+        current_app.logger.warning(
+            "[FLEXMEASURES-WEATHER] No weather sensor set up yet for measuring %s. Try the register-weather-sensor CLI task."
+            % sensor_name
+        )
+        return None
+    weather_station: GenericAsset = weather_sensor.generic_asset
+    if abs(
+        location[0] - weather_station.location[0]
+    ) > max_degree_difference_for_nearest_weather_sensor or abs(
+        location[1] - weather_station.location[1]
+        > max_degree_difference_for_nearest_weather_sensor
+    ):
+        current_app.logger.warning(
+            f"[FLEXMEASURES-WEATHER] We found a weather station, but no sufficiently close weather sensor found (within {max_degree_difference_for_nearest_weather_sensor} {flexmeasures_inflection.pluralize('degree', max_degree_difference_for_nearest_weather_sensor)} distance) for measuring {sensor_name}! We're looking for: {location}, closest available: ({weather_station.location})"
+        )
+        return None
+    return weather_sensor
+
+
+def get_location_by_asset_id(asset_id: int) -> Tuple[float, float]:
+    """Get location for forecasting by passing an asset id"""
+    asset = GenericAsset.query.filter(GenericAsset.id == asset_id).one_or_none()
+    if asset.generic_asset_type.name != WEATHER_STATION_TYPE_NAME:
+        raise Exception(
+            f"Asset {asset} does not seem to be a weather station we should use ― we expect an asset with type '{WEATHER_STATION_TYPE_NAME}'."
+        )
+    if asset is None:
+        raise Exception(
+            "[FLEXMEASURES-WEATHER] No asset found for the given asset id %s."
+            % asset_id
+        )
+    return (asset.latitude, asset.longitude)

tools/flexmeasures-weather/flexmeasures_weather/utils/modeling.py

@@ -0,0 +1,124 @@
+from packaging import version
+
+from flask import current_app
+from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
+from flexmeasures import Source, __version__ as flexmeasures_version
+from flexmeasures.data import db
+from flexmeasures.data.services.data_sources import get_or_create_source
+
+from flexmeasures_weather import DEFAULT_DATA_SOURCE_NAME
+from flexmeasures_weather import WEATHER_STATION_TYPE_NAME
+from flexmeasures_weather import DEFAULT_WEATHER_STATION_NAME
+
+
+if version.parse(flexmeasures_version) < version.parse("0.13"):
+    SOURCE_TYPE = "forecasting script"
+else:
+    SOURCE_TYPE = "forecaster"
+
+FM_SUPPORTS_ACCOUNT_LINKED_SOURCES = version.parse(
+    flexmeasures_version
+) >= version.parse("0.32")
+
+if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+    from flexmeasures import Account
+else:
+    Account = None
+
+
+def get_or_create_weather_account():
+    """Make sure we have an account for the weather provider service."""
+    if Account is None:
+        raise RuntimeError(
+            "FlexMeasures Account model is unavailable before FlexMeasures 0.32."
+        )
+    account_name = current_app.config.get(
+        "WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+    )
+    weather_account = Account.query.filter(
+        Account.name == account_name,
+    ).one_or_none()
+    if weather_account is None:
+        weather_account = Account(name=account_name)
+        db.session.add(weather_account)
+        db.session.flush()
+    return weather_account
+
+
+def get_or_create_owm_data_source() -> Source:
+    """Make sure we have a weather provider data source of the configured type."""
+    source_kwargs = dict(
+        source=current_app.config.get(
+            "WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+        ),
+        source_type=SOURCE_TYPE,
+        flush=False,
+    )
+    if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+        source_kwargs["account"] = get_or_create_weather_account()
+    return get_or_create_source(**source_kwargs)
+
+
+def get_or_create_owm_data_source_for_derived_data() -> Source:
+    owm_source_name = current_app.config.get(
+        "WEATHER_DATA_SOURCE_NAME", DEFAULT_DATA_SOURCE_NAME
+    )
+    source_kwargs = dict(
+        source=f"FlexMeasures {owm_source_name}",
+        source_type=SOURCE_TYPE,
+        flush=False,
+    )
+    if FM_SUPPORTS_ACCOUNT_LINKED_SOURCES:
+        source_kwargs["account"] = get_or_create_weather_account()
+    return get_or_create_source(**source_kwargs)
+
+
+def get_or_create_weather_station_type() -> GenericAssetType:
+    """Make sure a weather station type exists"""
+    weather_station_type = GenericAssetType.query.filter(
+        GenericAssetType.name == WEATHER_STATION_TYPE_NAME,
+    ).one_or_none()
+    if weather_station_type is None:
+        weather_station_type = GenericAssetType(
+            name=WEATHER_STATION_TYPE_NAME,
+            description="A weather station with various sensors.",
+        )
+        db.session.add(weather_station_type)
+    return weather_station_type
+
+
+def get_or_create_weather_station(latitude: float, longitude: float) -> GenericAsset:
+    """Make sure a weather station exists at this location."""
+    station_name = current_app.config.get(
+        "WEATHER_STATION_NAME", DEFAULT_WEATHER_STATION_NAME
+    )
+    weather_station = GenericAsset.query.filter(
+        GenericAsset.latitude == latitude, GenericAsset.longitude == longitude
+    ).one_or_none()
+    if weather_station is None:
+        weather_station_type = get_or_create_weather_station_type()
+        weather_station = GenericAsset(
+            name=station_name,
+            generic_asset_type=weather_station_type,
+            latitude=latitude,
+            longitude=longitude,
+        )
+        db.session.add(weather_station)
+    return weather_station
+
+
+def get_weather_station_by_asset_id(asset_id: int) -> GenericAsset:
+    weather_station = GenericAsset.query.filter(
+        GenericAsset.generic_asset_type_id == asset_id
+    ).one_or_none()
+    if weather_station is None:
+        raise Exception(
+            f"[FLEXMEASURES-WEATHER] Weather station is not present for the given asset id '{asset_id}'."
+        )
+
+    if weather_station.latitude is None or weather_station.longitude is None:
+        raise Exception(
+            f"[FLEXMEASURES-WEATHER] Weather station {weather_station} is missing location information [Latitude, Longitude]."
+        )
+
+    return weather_station

tools/flexmeasures-weather/flexmeasures_weather/utils/radiating.py

@@ -0,0 +1,37 @@
+from datetime import datetime
+
+import pandas as pd
+from pvlib.location import Location
+
+
+def compute_irradiance(
+    latitude: float, longitude: float, dt: datetime, cloud_coverage: float
+) -> float:
+    """Compute the irradiance received on a location at a specific time.
+    This uses pvlib to
+    1)  compute clear-sky irradiance as Global Horizontal Irradiance (GHI),
+        which includes both Direct Normal Irradiance (DNI)
+        and Diffuse Horizontal Irradiance (DHI).
+    2)  adjust the GHI for cloud coverage
+    """
+    site = Location(latitude, longitude, tz=dt.tzinfo)
+    solpos = site.get_solarposition(pd.DatetimeIndex([dt]))
+    ghi_clear = site.get_clearsky(pd.DatetimeIndex([dt]), solar_position=solpos).loc[
+        dt
+    ]["ghi"]
+    return ghi_clear_to_ghi(ghi_clear, cloud_coverage)
+
+
+def ghi_clear_to_ghi(ghi_clear: float, cloud_coverage: float) -> float:
+    """Compute global horizontal irradiance (GHI) from clear-sky GHI, given a cloud coverage between 0 and 1.
+
+    References
+    ----------
+    Perez, R., Moore, K., Wilcox, S., Renne, D., Zelenka, A., 2007.
+    Forecasting solar radiation – preliminary evaluation of an
+    approach based upon the national forecast database. Solar Energy
+    81, 809–812.
+    """
+    if cloud_coverage < 0 or cloud_coverage > 1:
+        raise ValueError("cloud_coverage should lie in the interval [0, 1]")
+    return (1 - 0.87 * cloud_coverage**1.9) * ghi_clear

tools/flexmeasures-weather/flexmeasures_weather/utils/tests/test_modeling.py

@@ -0,0 +1,194 @@
+from types import SimpleNamespace
+
+import pytest
+
+from flexmeasures import Asset
+
+import flexmeasures_weather.utils.modeling as modeling
+from flexmeasures_weather import DEFAULT_DATA_SOURCE_NAME, DEFAULT_WEATHER_STATION_NAME
+from flexmeasures_weather.utils.modeling import (
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    SOURCE_TYPE,
+    get_or_create_owm_data_source,
+    get_or_create_owm_data_source_for_derived_data,
+    get_or_create_weather_account,
+    get_or_create_weather_station,
+)
+
+
+def test_creating_two_weather_stations(fresh_db):
+    get_or_create_weather_station(50, 40)
+    get_or_create_weather_station(40, 50)
+    assert Asset.query.filter(Asset.name == DEFAULT_WEATHER_STATION_NAME).count() == 2
+
+
+# The version-branch tests below still use monkeypatching to isolate source
+# creation side effects without requiring multiple FlexMeasures installs.
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Weather source accounts are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_weather_account(fresh_db):
+    weather_account = get_or_create_weather_account()
+
+    assert weather_account.name == DEFAULT_DATA_SOURCE_NAME
+    assert (
+        modeling.Account.query.filter(
+            modeling.Account.name == weather_account.name
+        ).count()
+        == 1
+    )
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_data_source_registers_weather_source_on_weather_account(
+    fresh_db,
+):
+    data_source = get_or_create_owm_data_source()
+
+    assert data_source.type == SOURCE_TYPE
+    assert data_source.account is not None
+    assert data_source.account.name == data_source.name
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_data_source_for_derived_data_uses_weather_account(fresh_db):
+    derived_data_source = get_or_create_owm_data_source_for_derived_data()
+
+    assert derived_data_source.type == SOURCE_TYPE
+    assert derived_data_source.account is not None
+    assert derived_data_source.account.name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_data_source_passes_weather_account_when_supported(
+    fresh_db, monkeypatch
+):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, account, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                account=account,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, account=account)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    not FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Account-linked weather sources are only supported on FlexMeasures >= 0.32.",
+)
+def test_get_or_create_owm_derived_data_source_passes_weather_account_when_supported(
+    fresh_db, monkeypatch
+):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, account, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                account=account,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, account=account)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source_for_derived_data()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs["account"].name == DEFAULT_DATA_SOURCE_NAME
+
+
+@pytest.mark.skipif(
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Legacy source creation without accounts is only used on FlexMeasures < 0.32.",
+)
+def test_get_or_create_owm_data_source_omits_account_when_not_supported(monkeypatch):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, name=source)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs == {
+        "source": DEFAULT_DATA_SOURCE_NAME,
+        "source_type": SOURCE_TYPE,
+        "flush": False,
+    }
+
+
+@pytest.mark.skipif(
+    FM_SUPPORTS_ACCOUNT_LINKED_SOURCES,
+    reason="Legacy source creation without accounts is only used on FlexMeasures < 0.32.",
+)
+def test_get_or_create_owm_derived_data_source_omits_account_when_not_supported(
+    monkeypatch,
+):
+    captured_kwargs = {}
+
+    def fake_get_or_create_source(source, source_type, flush):
+        captured_kwargs.update(
+            dict(
+                source=source,
+                source_type=source_type,
+                flush=flush,
+            )
+        )
+        return SimpleNamespace(type=source_type, name=source)
+
+    monkeypatch.setattr(
+        "flexmeasures_weather.utils.modeling.get_or_create_source",
+        fake_get_or_create_source,
+    )
+
+    data_source = get_or_create_owm_data_source_for_derived_data()
+
+    assert data_source.type == SOURCE_TYPE
+    assert captured_kwargs == {
+        "source": f"FlexMeasures {DEFAULT_DATA_SOURCE_NAME}",
+        "source_type": SOURCE_TYPE,
+        "flush": False,
+    }

tools/flexmeasures-weather/flexmeasures_weather/utils/weather.py

@@ -0,0 +1,357 @@
+from __future__ import annotations
+
+from typing import Tuple, List, Dict, Optional, Any
+import os
+from datetime import datetime, timedelta
+import json
+
+import click
+from flask import current_app
+import requests
+from humanize import naturaldelta
+from timely_beliefs import BeliefsDataFrame
+from flexmeasures.utils.time_utils import as_server_time, get_timezone, server_now
+from flexmeasures.data.models.time_series import Sensor, TimedBelief
+from flexmeasures.data.utils import save_to_db
+
+from flexmeasures_weather import DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE
+from .locating import find_weather_sensor_by_location
+from ..sensor_specs import mapping
+from .modeling import (
+    get_or_create_owm_data_source,
+    get_or_create_owm_data_source_for_derived_data,
+)
+from .radiating import compute_irradiance
+from zoneinfo import ZoneInfo
+
+API_VERSION = "3.0"
+
+
+def get_supported_sensor_spec(name: str) -> Optional[dict]:
+    """
+    Find the specs from a sensor by name.
+    """
+    for supported_sensor_spec in mapping:
+        if supported_sensor_spec["fm_sensor_name"] == name:
+            return supported_sensor_spec.copy()
+    return None
+
+
+def get_supported_sensors_str() -> str:
+    """A string - list of supported sensors, also revealing their unit"""
+    return ", ".join(
+        [
+            f"{sensor_specs['fm_sensor_name']} ({sensor_specs['unit']})"
+            for sensor_specs in mapping
+        ]
+    )
+
+
+def process_weatherapi_data(
+    data: List[Dict[str, Any]], hour_no: int
+) -> List[Dict[str, Any]]:
+    """
+    Processes raw WeatherAPI forecast data into a format similar to OpenWeatherMap's format.
+
+    Args:
+        data (List[Dict[str, Any]]): A list of forecast day dictionaries from WeatherAPI,
+            each containing an 'hour' key with 24 hourly entries.
+        hour_no (int): The index of the current hour to start from.
+
+    Returns:
+        List[Dict[str, Any]]: A list of 48 hourly forecast entries, each mapped to the
+        expected structure with fields like temperature, humidity, wind, and condition.
+    """
+    first_day = data[0]["hour"]
+    second_day = data[1]["hour"]
+    third_day = data[2]["hour"]
+    combined = first_day + second_day + third_day
+
+    relevant = combined[hour_no : hour_no + 48]
+    return relevant
+
+
+def call_openweatherapi(
+    api_key: str, location: Tuple[float, float]
+) -> Tuple[datetime, List[Dict]]:
+    """
+    Make a single "one-call" to the Open Weather API and return the API timestamp as well as the 48 hourly forecasts.
+    See https://openweathermap.org/api/one-call-3 for docs.
+    Note that the first forecast is about the current hour.
+    """
+    check_openweathermap_version(API_VERSION)
+    query_str = f"lat={location[0]}&lon={location[1]}&units=metric&exclude=minutely,daily,alerts&appid={api_key}"
+    res = requests.get(
+        f"http://api.openweathermap.org/data/{API_VERSION}/onecall?{query_str}"
+    )
+    assert (
+        res.status_code == 200
+    ), f"OpenWeatherMap returned status code {res.status_code}: {res.text}"
+    data = res.json()
+    time_of_api_call = as_server_time(
+        datetime.fromtimestamp(data["current"]["dt"], tz=get_timezone())
+    ).replace(second=0, microsecond=0)
+    return time_of_api_call, data["hourly"]
+
+
+def call_weatherapi(
+    api_key: str, location: Tuple[float, float], days: int = 3
+) -> Tuple[datetime, List[Dict]]:
+    """
+    Makes a request to the WeatherAPI to retrieve hourly weather forecast data.
+
+    Args:
+        api_key (str): API key for authenticating with the Weather API.
+        location (Tuple[float, float]): A tuple containing the latitude and longitude.
+        days (int, optional): Number of days to request the forecast for (default is 3, including current day).
+
+    Returns:
+        Tuple[datetime, List[Dict]]:
+            - The timestamp of the API call.
+            - A list of hourly forecast data as dictionaries. Note that the first forecast is about the current hour.
+
+    Raises:
+        AssertionError: If the response from the Weather API is not successful (HTTP status 200).
+    """
+
+    latitude, longitude = location[0], location[1]
+
+    query_str = f"http://api.weatherapi.com/v1/forecast.json?key={api_key}&q={latitude},{longitude}&days={days}&aqi=yes&alerts=yes"
+    res = requests.get(query_str)
+
+    assert (
+        res.status_code == 200
+    ), f"Weather API returned status code {res.status_code}: {res.text}"
+
+    data = res.json()
+
+    # get the time of the api call
+    time_of_call = int(data["location"]["localtime_epoch"])
+    local_timezone = ZoneInfo(data["location"]["tz_id"])
+    local_time = datetime.fromtimestamp(time_of_call, local_timezone)
+    time_of_api_call = as_server_time(local_time)
+    time_of_api_call = time_of_api_call.replace(second=0, microsecond=0)
+
+    print(f"Time of API call in WAPI is {time_of_api_call}")
+
+    relevant = data["forecast"]["forecastday"]
+    hour_no = local_time.hour
+
+    hourly = process_weatherapi_data(relevant, hour_no)
+    return time_of_api_call, hourly
+
+
+def call_api(
+    api_key: str, location: Tuple[float, float]
+) -> Tuple[datetime, List[Dict]]:
+    """
+    Dispatches the weather API call based on the configured provider.
+
+    Args:
+        api_key (str): API key for the selected weather service provider.
+        location (Tuple[float, float]): Latitude and longitude tuple.
+
+    Returns:
+        Tuple[datetime, List[Dict]]:
+            - Timestamp of the API call.
+            - List of hourly forecast data.
+
+    Raises:
+        Exception: If an invalid weather provider is configured.
+    """
+
+    provider = str(current_app.config.get("WEATHER_PROVIDER", "OWM"))
+    if provider not in ["OWM", "WAPI"]:
+        raise Exception(
+            "Invalid provider name. Please set WEATHER_PROVIDER setting in config file to either OWM or WAPI, the two permissible options."
+        )
+
+    if provider == "OWM":
+        click.secho("Calling Open Weather Map")
+        return call_openweatherapi(api_key, location)
+    else:
+        click.secho("Calling Weather API")
+        return call_weatherapi(api_key, location)
+
+
+def save_forecasts_in_db(  # noqa: C901
+    api_key: str,
+    locations: List[Tuple[float, float]],
+):
+    """Process the response from Weather Provider API into timed beliefs.
+    Collects all forecasts for all locations and all sensors at all locations, then bulk-saves them.
+    """
+    click.echo("[FLEXMEASURES-WEATHER] Getting weather forecasts:")
+    click.echo("[FLEXMEASURES-WEATHER] Latitude, Longitude")
+    click.echo("[FLEXMEASURES-WEATHER] -----------------------")
+    max_degree_difference_for_nearest_weather_sensor = current_app.config.get(
+        "WEATHER_MAXIMAL_DEGREE_LOCATION_DISTANCE",
+        DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE,
+    )
+    provider = str(current_app.config.get("WEATHER_PROVIDER", ""))
+    if provider not in ["OWM", "WAPI"]:
+        raise Exception(
+            "Invalid provider name. Please set WEATHER_PROVIDER setting in config file to either OWM or WAPI, the two permissible options."
+        )
+    for location in locations:
+        click.echo("[FLEXMEASURES] %s, %s" % location)
+        weather_sensors: Dict[str, Sensor] = (
+            {}
+        )  # keep track of the sensors to save lookups
+        db_forecasts: Dict[Sensor, List[TimedBelief]] = {}  # collect beliefs per sensor
+
+        now = server_now()
+        time_of_api_call, forecasts = call_api(api_key, location)
+        diff_fm_owm = now - time_of_api_call
+        if abs(diff_fm_owm) > timedelta(minutes=10):
+            click.echo(
+                f"[FLEXMEASURES-WEATHER] Warning: difference between this server and Weather Provider is {naturaldelta(diff_fm_owm)}"
+            )
+        click.echo(
+            f"[FLEXMEASURES-WEATHER] Called weather provider {provider} API successfully at {now}."
+        )
+
+        # loop through forecasts, including the one of current hour (horizon 0)
+        for fc in forecasts:
+            time_key = fc["dt"] if provider == "OWM" else fc["time_epoch"]
+            fc_datetime = as_server_time(
+                datetime.fromtimestamp(time_key, get_timezone())
+            )
+            click.echo(
+                f"[FLEXMEASURES-WEATHER] Processing forecast for {fc_datetime} ..."
+            )
+            data_source = get_or_create_owm_data_source()
+            for sensor_specs in mapping:
+                sensor_name = str(sensor_specs["fm_sensor_name"])
+                provider_response_label = sensor_specs[f"{provider}_sensor_name"]
+                if provider_response_label in fc:
+                    weather_sensor = get_weather_sensor(
+                        sensor_specs,
+                        location,
+                        weather_sensors,
+                        max_degree_difference_for_nearest_weather_sensor,
+                    )
+                    if weather_sensor is not None:
+                        click.echo(
+                            f"Found pre-configured weather sensor {weather_sensor.name} ..."
+                        )
+                        if weather_sensor not in db_forecasts.keys():
+                            db_forecasts[weather_sensor] = []
+
+                        fc_value = fc[provider_response_label]
+
+                        if provider_response_label == "wind_kph":
+                            # convert wind speed from kph to m/s
+                            fc_value = fc[provider_response_label] / 3.6
+
+                        # the irradiance is not available in Provider -> we compute it ourselves
+                        if sensor_name == "irradiance":
+                            fc_value = compute_irradiance(
+                                location[0],
+                                location[1],
+                                fc_datetime,
+                                # Provider sends cloud cover in percent, we need a ratio
+                                fc_value / 100.0,
+                            )
+                            data_source = (
+                                get_or_create_owm_data_source_for_derived_data()
+                            )
+
+                        db_forecasts[weather_sensor].append(
+                            TimedBelief(
+                                event_start=fc_datetime,
+                                belief_time=now,
+                                event_value=fc_value,
+                                sensor=weather_sensor,
+                                source=data_source,
+                            )
+                        )
+                else:
+                    # we will not fail here, but issue a warning
+                    msg = "No label '%s' in response data for time %s" % (
+                        provider_response_label,
+                        fc_datetime,
+                    )
+                    click.echo("[FLEXMEASURES-WEATHER] %s" % msg)
+                    current_app.logger.warning(msg)
+    for sensor in db_forecasts.keys():
+        click.echo(f"[FLEXMEASURES-WEATHER] Saving {sensor.name} forecasts ...")
+        if len(db_forecasts[sensor]) == 0:
+            # This is probably a serious problem
+            raise Exception(
+                "Nothing to put in the database was produced. That does not seem right..."
+            )
+        status = save_to_db(BeliefsDataFrame(db_forecasts[sensor]))
+        if status == "success_but_nothing_new":
+            current_app.logger.info(
+                "[FLEXMEASURES-WEATHER] Done. These beliefs had already been saved before."
+            )
+        elif status == "success_with_unchanged_beliefs_skipped":
+            current_app.logger.info(
+                "[FLEXMEASURES-WEATHER] Done. Some beliefs had already been saved before."
+            )
+
+
+def get_weather_sensor(
+    sensor_specs: dict,
+    location: Tuple[float, float],
+    weather_sensors: Dict[str, Sensor],
+    max_degree_difference_for_nearest_weather_sensor: int,
+) -> Sensor | None:
+    """Get the weather sensor for this own response label and location, if we haven't retrieved it already."""
+    sensor_name = str(sensor_specs["fm_sensor_name"])
+    if sensor_name in weather_sensors:
+        weather_sensor = weather_sensors[sensor_name]
+    else:
+        weather_sensor = find_weather_sensor_by_location(
+            location,
+            max_degree_difference_for_nearest_weather_sensor,
+            sensor_name=sensor_name,
+        )
+        weather_sensors[sensor_name] = weather_sensor
+    if (
+        weather_sensor is not None
+        and weather_sensor.event_resolution != sensor_specs["event_resolution"]
+    ):
+        raise Exception(
+            f"[FLEXMEASURES-WEATHER] The weather sensor found for {sensor_name} has an unfitting event resolution (should be {sensor_specs['event_resolution']}, but is {weather_sensor.event_resolution}."
+        )
+    return weather_sensor
+
+
+def save_forecasts_as_json(
+    api_key: str, locations: List[Tuple[float, float]], data_path: str
+):
+    """Get forecasts, then store each as a raw JSON file, for later processing."""
+    click.echo("[FLEXMEASURES-WEATHER] Getting weather forecasts:")
+    click.echo("[FLEXMEASURES-WEATHER] Latitude, Longitude")
+    click.echo("[FLEXMEASURES-WEATHER] ----------------------")
+    for location in locations:
+        click.echo("[FLEXMEASURES-WEATHER] %s, %s" % location)
+        now = server_now()
+        time_of_api_call, forecasts = call_api(api_key, location)
+        diff_fm_owm = now - time_of_api_call
+        if abs(diff_fm_owm) > timedelta(minutes=10):
+            click.echo(
+                f"[FLEXMEASURES-WEATHER] Warning: difference between this server and Weather Provider is {naturaldelta(diff_fm_owm)}"
+            )
+        now_str = now.strftime("%Y-%m-%dT%H-%M-%S")
+        path_to_files = os.path.join(data_path, now_str)
+        if not os.path.exists(path_to_files):
+            click.echo(f"[FLEXMEASURES-WEATHER] Making directory: {path_to_files} ...")
+            os.mkdir(path_to_files)
+        forecasts_file = "%s/forecast_lat_%s_lng_%s.json" % (
+            path_to_files,
+            str(location[0]),
+            str(location[1]),
+        )
+        with open(forecasts_file, "w") as outfile:
+            json.dump(forecasts, outfile)
+
+
+def check_openweathermap_version(api_version: str):
+    supported_versions = ["2.5", "3.0"]
+    if api_version not in supported_versions:
+        current_app.logger.warning(
+            f"This plugin may not be fully compatible with OpenWeatherMap API version {api_version}. We tested with versions {supported_versions}"
+        )

tools/flexmeasures-weather/requirements/Readme.md

@@ -0,0 +1,21 @@
+# Requirements
+
+All requirements for flexmeasures-flexmeasures-weather are specified in this directory.
+We separate by use case:
+
+- app: All requirements for running the plugin
+- test: Additional requirements used for running automated tests 
+- dev: Additional requirements used for developers (this includes testing)
+
+Also note the following distinction:
+
+
+## .in files
+
+Here, we describe the requirements. We give the name of a requirement or even a range (e.g. `>=1.0.`).
+
+## .txt files
+
+These files are not to be edited by hand. They are created by `pip-compile` (or `make freeze-deps`).
+
+Each requirement is pinned to a specific version in these files. The great benefit is reproducibility across environments (local dev as well as staging or production).

tools/flexmeasures-weather/requirements/app.in

@@ -0,0 +1,6 @@
+flexmeasures
+pvlib
+# the following three are optional in pvlib, but we use them
+netCDF4
+siphon
+tables

tools/flexmeasures-weather/requirements/app.txt

@@ -0,0 +1,444 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=requirements/app.txt requirements/app.in
+#
+alembic==1.16.1
+    # via flask-migrate
+altair==5.5.0
+    # via flexmeasures
+argon2-cffi==23.1.0
+    # via flexmeasures
+argon2-cffi-bindings==21.2.0
+    # via argon2-cffi
+arrow==1.3.0
+    # via rq-dashboard
+async-timeout==5.0.1
+    # via redis
+attrs==25.3.0
+    # via
+    #   jsonschema
+    #   referencing
+babel==2.17.0
+    # via py-moneyed
+bcrypt==4.0.1
+    # via flexmeasures
+beautifulsoup4==4.13.4
+    # via siphon
+blinker==1.9.0
+    # via
+    #   flask
+    #   flask-mail
+    #   flask-principal
+    #   sentry-sdk
+blosc2==3.3.4
+    # via tables
+certifi==2025.4.26
+    # via
+    #   netcdf4
+    #   requests
+    #   sentry-sdk
+cffi==1.17.1
+    # via argon2-cffi-bindings
+cftime==1.6.4.post1
+    # via netcdf4
+charset-normalizer==3.4.2
+    # via requests
+click==8.2.1
+    # via
+    #   click-default-group
+    #   flask
+    #   flexmeasures
+    #   rq
+click-default-group==1.2.4
+    # via flexmeasures
+contourpy==1.3.2
+    # via matplotlib
+convertdate==2.4.0
+    # via workalendar
+cycler==0.12.1
+    # via matplotlib
+dill==0.4.0
+    # via openturns
+dnspython==2.7.0
+    # via email-validator
+email-validator==2.2.0
+    # via
+    #   flask-security-too
+    #   flexmeasures
+filelock==3.18.0
+    # via tldextract
+flask==3.1.1
+    # via
+    #   flask-classful
+    #   flask-cors
+    #   flask-json
+    #   flask-login
+    #   flask-mail
+    #   flask-marshmallow
+    #   flask-migrate
+    #   flask-principal
+    #   flask-security-too
+    #   flask-sqlalchemy
+    #   flask-sslify
+    #   flask-wtf
+    #   flexmeasures
+    #   rq-dashboard
+    #   sentry-sdk
+flask-classful==0.16.0
+    # via flexmeasures
+flask-cors==6.0.0
+    # via flexmeasures
+flask-json==0.4.0
+    # via flexmeasures
+flask-login==0.6.3
+    # via
+    #   flask-security-too
+    #   flexmeasures
+flask-mail==0.10.0
+    # via flexmeasures
+flask-marshmallow==1.3.0
+    # via flexmeasures
+flask-migrate==4.1.0
+    # via flexmeasures
+flask-principal==0.4.0
+    # via flask-security-too
+flask-security-too==5.6.2
+    # via flexmeasures
+flask-sqlalchemy==3.1.1
+    # via
+    #   flask-migrate
+    #   flexmeasures
+flask-sslify==0.1.5
+    # via flexmeasures
+flask-wtf==1.2.2
+    # via
+    #   flask-security-too
+    #   flexmeasures
+flexcache==0.3
+    # via pint
+flexmeasures==0.25.0
+    # via -r requirements/app.in
+flexparser==0.4
+    # via pint
+fonttools==4.58.1
+    # via matplotlib
+greenlet==3.2.2
+    # via sqlalchemy
+h5py==3.13.0
+    # via pvlib
+humanize==4.12.3
+    # via flexmeasures
+idna==3.10
+    # via
+    #   email-validator
+    #   requests
+    #   tldextract
+importlib-metadata==8.7.0
+    # via
+    #   flexmeasures
+    #   timely-beliefs
+importlib-resources==6.5.2
+    # via flask-security-too
+inflect==6.0.2
+    # via flexmeasures
+inflection==0.5.1
+    # via flexmeasures
+iso8601==2.1.0
+    # via flexmeasures
+isodate==0.7.2
+    # via
+    #   flexmeasures
+    #   timely-beliefs
+itsdangerous==2.2.0
+    # via
+    #   flask
+    #   flask-wtf
+jinja2==3.1.6
+    # via
+    #   altair
+    #   flask
+joblib==1.4.2
+    # via
+    #   scikit-learn
+    #   sktime
+jsonschema==4.24.0
+    # via altair
+jsonschema-specifications==2025.4.1
+    # via jsonschema
+kiwisolver==1.4.8
+    # via matplotlib
+lunardate==0.2.2
+    # via workalendar
+mako==1.3.10
+    # via alembic
+markupsafe==3.0.2
+    # via
+    #   flask
+    #   flask-security-too
+    #   jinja2
+    #   mako
+    #   sentry-sdk
+    #   werkzeug
+    #   wtforms
+marshmallow==3.26.1
+    # via
+    #   -r requirements/app.in
+    #   flask-marshmallow
+    #   flexmeasures
+    #   marshmallow-polyfield
+    #   marshmallow-sqlalchemy
+    #   webargs
+marshmallow-polyfield==5.11
+    # via flexmeasures
+marshmallow-sqlalchemy==1.4.2
+    # via flexmeasures
+matplotlib==3.10.3
+    # via timetomodel
+msgpack==1.1.0
+    # via blosc2
+narwhals==1.41.0
+    # via altair
+ndindex==1.10.0
+    # via blosc2
+netcdf4==1.7.2
+    # via -r requirements/app.in
+numexpr==2.10.2
+    # via
+    #   blosc2
+    #   tables
+numpy==1.26.4
+    # via
+    #   blosc2
+    #   cftime
+    #   contourpy
+    #   h5py
+    #   matplotlib
+    #   netcdf4
+    #   numexpr
+    #   pandas
+    #   patsy
+    #   properscoring
+    #   pvlib
+    #   scikit-learn
+    #   scipy
+    #   siphon
+    #   sktime
+    #   statsmodels
+    #   tables
+    #   timely-beliefs
+    #   timetomodel
+    #   uniplot
+openturns==1.24
+    # via timely-beliefs
+packaging==25.0
+    # via
+    #   altair
+    #   marshmallow
+    #   matplotlib
+    #   sktime
+    #   statsmodels
+    #   tables
+    #   webargs
+pandas==2.2.1
+    # via
+    #   flexmeasures
+    #   pvlib
+    #   siphon
+    #   sktime
+    #   statsmodels
+    #   timely-beliefs
+    #   timetomodel
+passlib==1.7.4
+    # via flask-security-too
+patsy==1.0.1
+    # via statsmodels
+pillow==11.2.1
+    # via
+    #   flexmeasures
+    #   matplotlib
+pint==0.24.4
+    # via flexmeasures
+platformdirs==4.3.8
+    # via
+    #   blosc2
+    #   pint
+ply==3.11
+    # via pyomo
+properscoring==0.1
+    # via timely-beliefs
+protobuf==6.31.1
+    # via siphon
+psutil==7.0.0
+    # via openturns
+psycopg2-binary==2.9.10
+    # via
+    #   flexmeasures
+    #   timely-beliefs
+pvlib==0.12.0
+    # via -r requirements/app.in
+py-cpuinfo==9.0.0
+    # via
+    #   blosc2
+    #   tables
+py-moneyed==3.0
+    # via flexmeasures
+pycparser==2.22
+    # via cffi
+pydantic==1.10.22
+    # via
+    #   flexmeasures
+    #   inflect
+pyluach==2.2.0
+    # via workalendar
+pymeeus==0.5.12
+    # via convertdate
+pyomo==6.9.2
+    # via flexmeasures
+pyparsing==3.2.3
+    # via matplotlib
+python-dateutil==2.9.0.post0
+    # via
+    #   arrow
+    #   matplotlib
+    #   pandas
+    #   timetomodel
+    #   workalendar
+python-dotenv==1.1.0
+    # via flexmeasures
+pytz==2025.2
+    # via
+    #   flexmeasures
+    #   pandas
+    #   pvlib
+    #   timely-beliefs
+    #   timetomodel
+pyyaml==6.0.2
+    # via flexmeasures
+redis==6.2.0
+    # via
+    #   flexmeasures
+    #   redis-sentinel-url
+    #   rq
+    #   rq-dashboard
+redis-sentinel-url==1.0.1
+    # via rq-dashboard
+referencing==0.36.2
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.32.3
+    # via
+    #   blosc2
+    #   pvlib
+    #   requests-file
+    #   siphon
+    #   tldextract
+requests-file==2.1.0
+    # via tldextract
+rpds-py==0.25.1
+    # via
+    #   jsonschema
+    #   referencing
+rq==2.3.3
+    # via
+    #   flexmeasures
+    #   rq-dashboard
+rq-dashboard==0.8.2.2
+    # via flexmeasures
+scikit-base==0.12.3
+    # via sktime
+scikit-learn==1.6.1
+    # via
+    #   sktime
+    #   timetomodel
+scipy==1.15.3
+    # via
+    #   properscoring
+    #   pvlib
+    #   scikit-learn
+    #   sktime
+    #   statsmodels
+    #   timely-beliefs
+    #   timetomodel
+sentry-sdk[flask]==2.29.1
+    # via flexmeasures
+siphon==0.10.0
+    # via -r requirements/app.in
+six==1.17.0
+    # via python-dateutil
+sktime==0.37.0
+    # via timely-beliefs
+soupsieve==2.7
+    # via beautifulsoup4
+sqlalchemy==2.0.41
+    # via
+    #   alembic
+    #   flask-sqlalchemy
+    #   flexmeasures
+    #   marshmallow-sqlalchemy
+    #   timely-beliefs
+    #   timetomodel
+statsmodels==0.14.4
+    # via timetomodel
+tables==3.10.1
+    # via -r requirements/app.in
+tabulate==0.9.0
+    # via flexmeasures
+threadpoolctl==3.6.0
+    # via scikit-learn
+timely-beliefs[forecast]==3.2.0
+    # via flexmeasures
+timetomodel==0.7.3
+    # via flexmeasures
+tldextract==5.3.0
+    # via flexmeasures
+tomli==2.2.1
+    # via alembic
+types-python-dateutil==2.9.0.20250516
+    # via arrow
+typing-extensions==4.13.2
+    # via
+    #   alembic
+    #   altair
+    #   beautifulsoup4
+    #   flexcache
+    #   flexparser
+    #   pint
+    #   py-moneyed
+    #   pydantic
+    #   referencing
+    #   sqlalchemy
+    #   tables
+tzdata==2025.2
+    # via pandas
+uniplot==0.21.1
+    # via flexmeasures
+urllib3==2.4.0
+    # via
+    #   requests
+    #   sentry-sdk
+vl-convert-python==1.8.0
+    # via flexmeasures
+webargs==8.7.0
+    # via flexmeasures
+werkzeug==3.1.3
+    # via
+    #   flask
+    #   flask-cors
+    #   flask-login
+    #   flexmeasures
+workalendar==17.0.0
+    # via flexmeasures
+wtforms==3.2.1
+    # via
+    #   flask-security-too
+    #   flask-wtf
+xlrd==2.0.1
+    # via flexmeasures
+zipp==3.22.0
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

tools/flexmeasures-weather/requirements/dev.in

@@ -0,0 +1,11 @@
+-c app.txt
+-c test.txt
+
+pre-commit
+black
+flake8
+flake8-blind-except
+mypy
+pytest-runner
+setuptools_scm
+watchdog

tools/flexmeasures-weather/requirements/dev.txt

@@ -0,0 +1,86 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=requirements/dev.txt requirements/dev.in
+#
+black==25.1.0
+    # via -r requirements/dev.in
+cfgv==3.4.0
+    # via pre-commit
+click==8.2.1
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/test.txt
+    #   black
+distlib==0.3.9
+    # via virtualenv
+filelock==3.18.0
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   virtualenv
+flake8==7.2.0
+    # via -r requirements/dev.in
+flake8-blind-except==0.2.1
+    # via -r requirements/dev.in
+identify==2.6.12
+    # via pre-commit
+mccabe==0.7.0
+    # via flake8
+mypy==1.16.0
+    # via -r requirements/dev.in
+mypy-extensions==1.1.0
+    # via
+    #   black
+    #   mypy
+nodeenv==1.9.1
+    # via pre-commit
+packaging==25.0
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/test.txt
+    #   black
+    #   setuptools-scm
+pathspec==0.12.1
+    # via
+    #   black
+    #   mypy
+platformdirs==4.3.8
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   black
+    #   virtualenv
+pre-commit==4.2.0
+    # via -r requirements/dev.in
+pycodestyle==2.13.0
+    # via flake8
+pyflakes==3.3.2
+    # via flake8
+pytest-runner==6.0.1
+    # via -r requirements/dev.in
+pyyaml==6.0.2
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   pre-commit
+setuptools-scm==8.3.1
+    # via -r requirements/dev.in
+tomli==2.2.1
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/test.txt
+    #   black
+    #   mypy
+    #   setuptools-scm
+typing-extensions==4.13.2
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/test.txt
+    #   black
+    #   mypy
+virtualenv==20.31.2
+    # via pre-commit
+watchdog==6.0.0
+    # via -r requirements/dev.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

tools/flexmeasures-weather/requirements/test.in

@@ -0,0 +1,10 @@
+-c app.txt
+
+pytest
+pytest-flask
+pytest-sugar
+pytest-cov
+# lets tests run successfully in containers
+fakeredis
+# required with fakeredis, maybe because we use rq
+lupa

tools/flexmeasures-weather/requirements/test.txt

@@ -0,0 +1,88 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=requirements/test.txt requirements/test.in
+#
+async-timeout==5.0.1
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   redis
+blinker==1.9.0
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   flask
+click==8.2.1
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   flask
+coverage[toml]==7.8.2
+    # via pytest-cov
+exceptiongroup==1.3.0
+    # via pytest
+fakeredis==2.29.0
+    # via -r requirements/test.in
+flask==3.1.1
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   pytest-flask
+iniconfig==2.1.0
+    # via pytest
+itsdangerous==2.2.0
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   flask
+jinja2==3.1.6
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   flask
+lupa==2.4
+    # via -r requirements/test.in
+markupsafe==3.0.2
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   flask
+    #   jinja2
+    #   werkzeug
+packaging==25.0
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   pytest
+    #   pytest-sugar
+pluggy==1.6.0
+    # via pytest
+pytest==8.3.5
+    # via
+    #   -r requirements/test.in
+    #   pytest-cov
+    #   pytest-flask
+    #   pytest-sugar
+pytest-cov==6.1.1
+    # via -r requirements/test.in
+pytest-flask==1.3.0
+    # via -r requirements/test.in
+pytest-sugar==1.0.0
+    # via -r requirements/test.in
+redis==6.2.0
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   fakeredis
+sortedcontainers==2.4.0
+    # via fakeredis
+termcolor==3.1.0
+    # via pytest-sugar
+tomli==2.2.1
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   coverage
+    #   pytest
+typing-extensions==4.13.2
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   exceptiongroup
+    #   fakeredis
+werkzeug==3.1.3
+    # via
+    #   -c /home/nicolas/workspace/seita/flexmeasures-weather/requirements/app.txt
+    #   flask
+    #   pytest-flask

tools/flexmeasures-weather/run_mypy.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+pip install --upgrade mypy > 1.4
+pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools
+files=$(find flexmeasures_weather -name \*.py)
+mypy --follow-imports skip --ignore-missing-imports $files 

tools/flexmeasures-weather/scripts/load-psql-extensions.sql

@@ -0,0 +1,2 @@
+CREATE EXTENSION IF NOT EXISTS cube;
+CREATE EXTENSION IF NOT EXISTS earthdistance;

tools/flexmeasures-weather/scripts/solartest.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+
+"""
+Quick script to compare clear-sky irradiance computations
+from three different libraries.
+Among other considerations, this helped us to settle on pvlib.
+"""
+from typing import List, Dict
+from datetime import datetime, timedelta
+
+import solarpy
+import pvlib
+import pysolar
+import matplotlib.dates as mpl_dates
+import matplotlib.pyplot as plt
+import pytz
+from pandas import DatetimeIndex
+from tzwhere import tzwhere
+from astral import LocationInfo
+from astral.sun import sun
+
+
+DAY = datetime(2021, 2, 10, tzinfo=pytz.utc)
+tzwhere = tzwhere.tzwhere()
+
+locations = {
+    "Amsterdam": (52.370216, 4.895168),
+    "Tokyo": (35.6684415, 139.6007844),
+    "Dallas": (32.779167, -96.808891),
+    "Cape-Town": (-33.943707, 18.588740),  # check southern hemisphere, too
+}
+datetimes = [DAY + timedelta(minutes=i * 20) for i in range(24 * 3)]
+timezones = {k: tzwhere.tzNameAt(*v) for k, v in locations.items()}
+
+
+def irradiance_by_solarpy(
+    latitude: float, longitude: float, dt: datetime, z: str, metric: str = "dni"
+) -> float:
+    """Supports direct horizontal irradiance and direct normal irradiance."""
+    h = 0  # sea-level
+    dt = dt.astimezone(pytz.timezone(z)).replace(tzinfo=None)  # local time
+    dt = solarpy.standard2solar_time(dt, longitude)  # solar time
+    if metric == "dhi":  # direct horizontal irradiance
+        vnorm = [0, 0, -1]  # plane pointing up
+    elif metric == "dni":  # direct normal irradiance
+        vnorm = solarpy.solar_vector_ned(
+            dt, latitude
+        )  # plane pointing directly to the sun
+        vnorm[-1] = vnorm[-1] * 0.99999  # avoid floating point error
+    else:
+        return NotImplemented
+    return solarpy.irradiance_on_plane(vnorm, h, dt, latitude)
+
+
+def irradiance_by_pysolar(
+    latitude: float, longitude: float, dt: datetime, method: str = "dni"
+) -> float:
+    """Supports direct normal irradiance."""
+    altitude_deg = pysolar.solar.get_altitude(latitude, longitude, dt)
+    if method == "dni":
+        return pysolar.radiation.get_radiation_direct(dt, altitude_deg)
+    else:
+        return NotImplemented
+
+
+def irradiance_by_pvlib(
+    latitude: float, longitude: float, dt: datetime, method: str = "dni"
+) -> float:
+    """
+    Supports direct horizontal irradiance, direct normal irradiance and global horizontal irradiance.
+    https://firstgreenconsulting.wordpress.com/2012/04/26/differentiate-between-the-dni-dhi-and-ghi/
+    """
+    site = pvlib.location.Location(latitude, longitude, tz=pytz.utc)
+    solpos = site.get_solarposition(DatetimeIndex([dt]))
+    irradiance = site.get_clearsky(DatetimeIndex([dt]), solar_position=solpos).loc[dt]
+    if method in ("ghi", "dni", "dhi"):
+        return irradiance[method]
+    else:
+        return NotImplemented
+
+
+def plot_irradiance(
+    city: str,
+    datetimes: List[datetime],
+    values: Dict[str, List[float]],
+    sun_times: Dict[str, datetime],
+):
+
+    fig, ax = plt.subplots()
+
+    ax.set(
+        xlabel="Time (20m)",
+        ylabel="Direct Normal Irradiance (W/m²)",
+        title=f"Irradiance for {city} on {DAY.date()}",
+    )
+
+    # draw values
+    date_ticks = mpl_dates.date2num(datetimes)
+    for lib in ("pysolar", "solarpy", "pvlib"):
+        plt.plot_date(date_ticks, values[lib], "-", label=lib)
+
+    # make date ticks look okay
+    plt.gca().xaxis.set_major_locator(mpl_dates.HourLocator())
+    plt.setp(plt.gca().xaxis.get_majorticklabels(), "rotation", 40)
+
+    # draw day phases boxes
+    dawn_tick, sunrise_tick, noon_tick, sunset_tick, dusk_tick = mpl_dates.date2num(
+        (
+            sun_times["dawn"],
+            sun_times["sunrise"],
+            sun_times["noon"],
+            sun_times["sunset"],
+            sun_times["dusk"],
+        )
+    )
+    dawn_to_sunrise = plt.Rectangle(
+        (dawn_tick, -100),
+        sunrise_tick - dawn_tick,
+        1100,
+        fc="floralwhite",
+        ec="lemonchiffon",
+        label="Dawn to Sunrise",
+    )
+    plt.gca().add_patch(dawn_to_sunrise)
+
+    sunrise_to_sunset = plt.Rectangle(
+        (sunrise_tick, -100),
+        sunset_tick - sunrise_tick,
+        1100,
+        fc="lightyellow",
+        ec="lemonchiffon",
+        label="Sunrise to sunset",
+    )
+    plt.gca().add_patch(sunrise_to_sunset)
+
+    sunset_to_dusk = plt.Rectangle(
+        (sunset_tick, -100),
+        dusk_tick - sunset_tick,
+        1100,
+        fc="oldlace",
+        ec="lemonchiffon",
+        label="Sunset to dusk",
+    )
+    plt.gca().add_patch(sunset_to_dusk)
+
+    # draw noon
+    plt.axvline(x=noon_tick, color="gold", label="Noon")
+
+    plt.legend()
+
+    fig.savefig(f"test-irradiance-{city}.png")
+    plt.show()
+
+
+if __name__ == "__main__":
+    for city in locations:
+        values = dict(pysolar=[], solarpy=[], pvlib=[])
+        lat, lon = locations[city]
+        timezone = timezones[city]
+        loc_info = LocationInfo(timezone=timezone, latitude=lat, longitude=lon)
+        # this gives 'dawn', 'sunrise', 'noon', 'sunset' and 'dusk'
+        sun_times = sun(loc_info.observer, date=DAY.date(), tzinfo=loc_info.timezone)
+        local_datetimes = [
+            dt.replace(tzinfo=pytz.timezone(timezones[city])) for dt in datetimes
+        ]
+
+        for dt in local_datetimes:
+            irrad_pysolar = irradiance_by_pysolar(lat, lon, dt)
+            values["pysolar"].append(irrad_pysolar)
+            irrad_solarpy = irradiance_by_solarpy(lat, lon, dt, timezone)
+            values["solarpy"].append(irrad_solarpy)
+            irrad_pvlib = irradiance_by_pvlib(lat, lon, dt)
+            values["pvlib"].append(irrad_pvlib)
+            print(
+                f"For {city} at {dt} {timezones[city]} ― pysolar: {irrad_pysolar:.2f}, solarpy: {irrad_solarpy:.2f}, pvlib: {irrad_pvlib:.2f}"
+            )
+        plot_irradiance(city, local_datetimes, values, sun_times)

tools/flexmeasures-weather/setup.cfg

@@ -0,0 +1,11 @@
+[aliases]
+test = pytest
+flake8 = flake8
+
+[flake8]
+exclude = .git,__pycache__,documentation
+max-line-length = 160
+max-complexity = 13
+select = B,C,E,F,W,B9
+ignore = E501, W503, E203
+

tools/flexmeasures-weather/setup.py

@@ -0,0 +1,47 @@
+from setuptools import setup, find_packages
+
+
+def load_requirements(use_case):
+    """
+    Loading range requirements.
+    Packaging should be used for installing the package into existing stacks.
+    We therefore read the .in file for the use case.
+    .txt files include the exact pins, and are useful for deployments with
+    exactly comparable environments.
+    """
+    reqs = []
+    with open("requirements/%s.in" % use_case, "r") as f:
+        reqs = [
+            req
+            for req in f.read().splitlines()
+            if not req.strip() == ""
+            and not req.strip().startswith("#")
+            and not req.strip().startswith("-c")
+            and not req.strip().startswith("--find-links")
+        ]
+    return reqs
+
+
+setup(
+    name="flexmeasures-weather",
+    description="Integrating FlexMeasures with multiple API services",
+    author="Seita Energy Flexibility BV",
+    author_email="nicolas@seita.nl",
+    url="https://github.com/FlexMeasures/flexmeasures-weather",
+    keywords=["flexmeasures", "energy flexibility"],
+    install_requires=load_requirements("app"),
+    tests_require=load_requirements("test"),
+    setup_requires=["pytest-runner", "setuptools_scm"],
+    use_scm_version={"local_scheme": "no-local-version"},  # handled by setuptools_scm
+    packages=find_packages(),
+    include_package_data=True,  # setuptools_scm takes care of adding the files in SCM
+    classifiers=[
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3.9",
+        "Development Status :: 3 - Alpha",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    long_description="""\
+""",
+)