From fa9381ba6917e4682e7389a19d6ac24e6bc1ed30 Mon Sep 17 00:00:00 2001 From: Eric F Date: Mon, 8 Jun 2026 22:03:47 -0400 Subject: [PATCH] Fix simulator for cariflex.digitribe.fr domain - working ingestion --- scripts/cariflex_simulator.py | 136 ++++++++++------- scripts/configure_fm_ems.sh | 266 ++++++++++++++++++++++++++++++++++ 2 files changed, 348 insertions(+), 54 deletions(-) create mode 100644 scripts/configure_fm_ems.sh diff --git a/scripts/cariflex_simulator.py b/scripts/cariflex_simulator.py index 0b2c5e7..745d0e0 100644 --- a/scripts/cariflex_simulator.py +++ b/scripts/cariflex_simulator.py @@ -1,21 +1,15 @@ #!/usr/bin/env python3 -""" -Cariflex Simulator - Publishes simulated data to FlexMeasures API. -Uses flexmeasures_client for authentication and data posting. -Simulates 40 assets: 10 PV, 10 Battery, 10 EV Charger, 10 EV V2G -""" -import asyncio -import time +"""Cariflex Simulator - Posts data to FlexMeasures via session cookies.""" +import requests import random import math from datetime import datetime, timezone, timedelta -from flexmeasures_client import FlexMeasuresClient -FM_HOST = "https://flexmeasures.digitribe.fr" +FM_HOST = "https://cariflex.digitribe.fr" FM_EMAIL = "admin@digitribe.fr" -FM_PASSWORD = "Digitribe972" +with open("/tmp/fm_pass.txt") as f: + FM_PASSWORD=f.read().strip() -# Sensor mapping: sensor_id -> (name, type, unit, min, max) SENSORS = {} for i in range(1, 11): SENSORS[40 + i] = {"name": f"pv_{i:02d}_power", "type": "pv", "unit": "kW", "min": 0, "max": 5} @@ -27,7 +21,6 @@ for i in range(1, 11): SENSORS[70 + i] = {"name": f"ev_{i:02d}_power", "type": "ev_v2g", "unit": "kWh", "min": 15, "max": 75} def generate_value(cfg, hour): - """Generate realistic value based on asset type and time of day.""" t = cfg["type"] if t == "pv": if 6 <= hour <= 18: @@ -54,57 +47,92 @@ def generate_value(cfg, hour): return round(max(cfg["min"], min(cfg["max"], base + random.gauss(0, 3))), 2) return 0 -async def post_all_data(client): - """Post data for all 40 sensors.""" - now = datetime.now(timezone.utc) - hour = now.hour - start = now - timedelta(minutes=5) +def main(): + print(f"Cariflex Simulator -> {FM_HOST}", flush=True) + print(f" Sensors: {len(SENSORS)}", flush=True) - success = 0 - failed = 0 + # Create session with CSRF token handling + session = requests.Session() + session.verify = False # Self-signed cert - for sensor_id, cfg in SENSORS.items(): - value = generate_value(cfg, hour) - try: - await client.post_sensor_data( - sensor_id=sensor_id, - values=[value], - start=start.isoformat(), - duration="PT5M", - unit=cfg["unit"] - ) - success += 1 - except Exception as e: - failed += 1 - if failed <= 3: - print(f" ⚠️ Sensor {sensor_id}: {e}") + # Get login page to extract CSRF token + login_url = f"{FM_HOST}/login" + r = session.get(login_url) + print(f" Login page: {r.status_code}", flush=True) - return success, failed - -async def main(): - print("🚗 Cariflex Simulator → FlexMeasures API") - print(f" Sensors: {len(SENSORS)} (10 PV, 10 Bat, 10 Chg, 10 EV)") - print(f" FM API: {FM_HOST}") - print() + # Extract CSRF token + from html.parser import HTMLParser + class CSRFParser(HTMLParser): + def __init__(self): + super().__init__() + self.token = None + def handle_starttag(self, tag, attrs): + if tag == "input": + attrs = dict(attrs) + if attrs.get("name") == "csrf_token": + self.token = attrs.get("value") - client = FlexMeasuresClient( - email=FM_EMAIL, - password=FM_PASSWORD, - host="flexmeasures.digitribe.fr", - ssl=True, - request_timeout=60.0 - ) + parser = CSRFParser() + parser.feed(r.text) + csrf_token = parser.token + print(f" CSRF token: {csrf_token[:20] if csrf_token else 'NOT FOUND'}...", flush=True) - print("✅ Connected to FlexMeasures") + # Login + r = session.post(login_url, data={ + "email": FM_EMAIL, + "password": FM_PASSWORD, + "csrf_token": csrf_token, + "remember": "y", + }, allow_redirects=True) + print(f" Login: {r.status_code} (url: {r.url})", flush=True) + # Check if logged in + if "dashboard" in r.url or "login" not in r.url: + print(f" Logged in successfully!", flush=True) + else: + print(f" WARNING: May not be logged in (url: {r.url})", flush=True) + + # Post sensor data iteration = 0 while True: - success, failed = await post_all_data(client) - iteration += 1 now = datetime.now(timezone.utc) - print(f" 📊 Iteration {iteration}: {success} OK, {failed} failed (hour={now.hour})") + hour = now.hour - await asyncio.sleep(30) + success = 0 + failed = 0 + + for sensor_id, cfg in SENSORS.items(): + value = generate_value(cfg, hour) + start = (now - timedelta(minutes=5)).isoformat() + + try: + r = session.post( + f"{FM_HOST}/api/v3_0/sensors/{sensor_id}/data", + json={ + "values": [value], + "start": start, + "duration": "PT5M", + "unit": cfg["unit"], + "type": "PostSensorDataRequest", + }, + timeout=30, + ) + if r.status_code in [200, 201]: + success += 1 + else: + failed += 1 + if failed <= 3: + print(f" Sensor {sensor_id}: HTTP {r.status_code} - {r.text[:100]}", flush=True) + except Exception as e: + failed += 1 + if failed <= 3: + print(f" Sensor {sensor_id}: {e}", flush=True) + + iteration += 1 + print(f" Iteration {iteration}: {success} OK, {failed} failed (hour={hour})", flush=True) + + import time + time.sleep(30) if __name__ == "__main__": - asyncio.run(main()) + main() diff --git a/scripts/configure_fm_ems.sh b/scripts/configure_fm_ems.sh new file mode 100644 index 0000000..fca2476 --- /dev/null +++ b/scripts/configure_fm_ems.sh @@ -0,0 +1,266 @@ +#!/bin/bash +# Cariflex - FlexMeasures EMS Configuration Script +# This script configures FlexMeasures as a full EMS with: +# - Forecasting (PV, Load, Prices) +# - Scheduling (Batteries, EVs) +# - Ingestion (Sensors, Assets, Beliefs) + +set -e + +FM_EMAIL="admin@digitribe.fr" +FM_PASSWORD="Digi...n" +FM_HOST="https://cariflex.digitribe.fr" + +echo "╔══════════════════════════════════════════════════════════════════════════════╗" +echo "║ CARIFLEX - FlexMeasures EMS Configuration ║" +echo "╚══════════════════════════════════════════════════════════════════════════════╝" +echo "" + +# ═══════════════════════════════════════════════════════════════════════════════ +# 1. INITIAL STRUCTURE +# ═══════════════════════════════════════════════════════════════════════════════ +echo "═══════════════════════════════════════════════════════════════════════════════" +echo " 1. INITIAL STRUCTURE" +echo "═══════════════════════════════════════════════════════════════════════════════" + +echo " → Creating initial structure (account, roles, data sources)..." +docker exec flexmeasures-server bash -c " +cd /app && timeout 60 .venv/bin/flexmeasures add initial-structure 2>&1 | tail -5 +" 2>&1 + +echo "" + +# ═══════════════════════════════════════════════════════════════════════════════ +# 2. ASSETS AND SENSORS (if not already created) +# ═══════════════════════════════════════════════════════════════════════════════ +echo "═══════════════════════════════════════════════════════════════════════════════" +echo " 2. ASSETS AND SENSORS" +echo "═══════════════════════════════════════════════════════════════════════════════" + +# Check if assets already exist +ASSET_COUNT=$(docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -t -c " +SELECT COUNT(*) FROM generic_asset; +" 2>/dev/null | xargs) + +echo " → Current asset count: $ASSET_COUNT" + +if [ "$ASSET_COUNT" -lt 40 ]; then + echo " → Creating assets and sensors..." + + # Create PV assets (10) + for i in $(seq 1 10); do + echo " Creating PV_0$i..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 30 .venv/bin/flexmeasures add asset \ + --name 'PV_0$i' \ + --type 'solar' \ + --capacity 5 \ + --unit 'kW' \ + --latitude 14.6$(printf '%02d' $((i * 10))) \ + --longitude -61.2$(printf '%02d' $((i * 5))) \ + 2>&1 | tail -3 + " 2>/dev/null + done + + # Create Battery assets (10) + for i in $(seq 1 10); do + echo " Creating Bat_0$i..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 30 .venv/bin/flexmeasures add asset \ + --name 'Bat_0$i' \ + --type 'battery' \ + --capacity 100 \ + --unit 'kWh' \ + --latitude 14.5$(printf '%02d' $((i * 10))) \ + --longitude -61.1$(printf '%02d' $((i * 5))) \ + 2>&1 | tail -3 + " 2>/dev/null + done + + # Create EV Charger assets (10) + for i in $(seq 1 10); do + echo " Creating Chg_0$i..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 30 .venv/bin/flexmeasures add asset \ + --name 'Chg_0$i' \ + --type 'ev_charger' \ + --capacity 22 \ + --unit 'kW' \ + --latitude 14.6$(printf '%02d' $((i * 8))) \ + --longitude -61.0$(printf '%02d' $((i * 3))) \ + 2>&1 | tail -3 + " 2>/dev/null + done + + # Create EV assets (10) + for i in $(seq 1 10); do + echo " Creating EV_0$i..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 30 .venv/bin/flexmeasures add asset \ + --name 'EV_0$i' \ + --type 'ev' \ + --capacity 75 \ + --unit 'kWh' \ + --latitude 14.5$(printf '%02d' $((i * 7))) \ + --longitude -61.2$(printf '%02d' $((i * 4))) \ + 2>&1 | tail -3 + " 2>/dev/null + done +else + echo " → Assets already exist, skipping..." +fi + +echo "" + +# ═══════════════════════════════════════════════════════════════════════════════ +# 3. FORECASTING +# ═══════════════════════════════════════════════════════════════════════════════ +echo "═══════════════════════════════════════════════════════════════════════════════" +echo " 3. FORECASTING" +echo "═══════════════════════════════════════════════════════════════════════════════" + +echo " → Generating forecasts for PV sensors (41-50)..." +for sensor in $(seq 41 50); do + echo " Forecasting sensor $sensor..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \ + --sensor $sensor \ + --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \ + 2>&1 | grep -E 'Successfully|Error|SAVED' | head -1 + " 2>/dev/null +done + +echo " → Generating forecasts for Battery sensors (51-60)..." +for sensor in $(seq 51 60); do + echo " Forecasting sensor $sensor..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \ + --sensor $sensor \ + --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \ + 2>&1 | grep -E 'Successfully|Error|SAVED' | head -1 + " 2>/dev/null +done + +echo " → Generating forecasts for EV Charger sensors (61-70)..." +for sensor in $(seq 61 70); do + echo " Forecasting sensor $sensor..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \ + --sensor $sensor \ + --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \ + 2>&1 | grep -E 'Successfully|Error|SAVED' | head -1 + " 2>/dev/null +done + +echo " → Generating forecasts for EV sensors (71-80)..." +for sensor in $(seq 71 80); do + echo " Forecasting sensor $sensor..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \ + --sensor $sensor \ + --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \ + 2>&1 | grep -E 'Successfully|Error|SAVED' | head -1 + " 2>/dev/null +done + +echo "" + +# ═══════════════════════════════════════════════════════════════════════════════ +# 4. SCHEDULING +# ═══════════════════════════════════════════════════════════════════════════════ +echo "═══════════════════════════════════════════════════════════════════════════════" +echo " 4. SCHEDULING" +echo "═══════════════════════════════════════════════════════════════════════════════" + +echo " → Creating schedules for Battery sensors (51-60)..." +for sensor in $(seq 51 60); do + echo " Scheduling sensor $sensor..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 60 .venv/bin/flexmeasures add schedule \ + --sensor $sensor \ + --start \$(date -u +'%Y-%m-%dT%H:%M:%S+00:00') \ + --duration PT24H \ + --resolution PT5M \ + --soc-at-start 0.5 \ + 2>&1 | grep -E 'Successfully|Error|schedule' | head -1 + " 2>/dev/null +done + +echo " → Creating schedules for EV sensors (71-80)..." +for sensor in $(seq 71 80); do + echo " Scheduling sensor $sensor..." + docker exec flexmeasures-server bash -c " + cd /app && timeout 60 .venv/bin/flexmeasures add schedule \ + --sensor $sensor \ + --start \$(date -u +'%Y-%m-%dT%H:%M:%S+00:00') \ + --duration PT24H \ + --resolution PT5M \ + --soc-at-start 0.4 \ + 2>&1 | grep -E 'Successfully|Error|schedule' | head -1 + " 2>/dev/null +done + +echo "" + +# ═══════════════════════════════════════════════════════════════════════════════ +# 5. INGESTION STATUS +# ═══════════════════════════════════════════════════════════════════════════════ +echo "═══════════════════════════════════════════════════════════════════════════════" +echo " 5. INGESTION STATUS" +echo "═══════════════════════════════════════════════════════════════════════════════" + +echo " → Checking sensor data status..." +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT + s.name as sensor, + s.unit, + COUNT(t.id) as data_points, + MAX(t.event_start) as latest_data +FROM sensor s +LEFT JOIN timed_belief t ON t.sensor_id = s.id +WHERE s.id BETWEEN 41 AND 80 +GROUP BY s.name, s.unit +ORDER BY s.id; +" 2>/dev/null + +echo "" + +# ═══════════════════════════════════════════════════════════════════════════════ +# 6. SUMMARY +# ═══════════════════════════════════════════════════════════════════════════════ +echo "═══════════════════════════════════════════════════════════════════════════════" +echo " 6. EMS CONFIGURATION SUMMARY" +echo "═══════════════════════════════════════════════════════════════════════════════" + +echo "" +echo " Assets:" +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT gt.name as type, COUNT(*) as count +FROM generic_asset g +JOIN generic_asset_type gt ON g.generic_asset_type_id = gt.id +GROUP BY gt.name; +" 2>/dev/null + +echo "" +echo " Sensors:" +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT COUNT(*) as total_sensors FROM sensor; +" 2>/dev/null + +echo "" +echo " Data Points (Beliefs):" +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT COUNT(*) as total_beliefs FROM timed_belief; +" 2>/dev/null + +echo "" +echo " Forecasts:" +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT COUNT(*) as forecasts FROM timed_belief +WHERE source_id IN (SELECT id FROM data_source WHERE name LIKE '%forecast%'); +" 2>/dev/null + +echo "" +echo "╔══════════════════════════════════════════════════════════════════════════════╗" +echo "║ EMS Configuration Complete! ║" +echo "╚══════════════════════════════════════════════════════════════════════════════╝"