Fix simulator for cariflex.digitribe.fr domain - working ingestion

scripts/cariflex_simulator.py

@@ -1,21 +1,15 @@
 #!/usr/bin/env python3
-"""
+"""Cariflex Simulator - Posts data to FlexMeasures via session cookies."""
-Cariflex Simulator - Publishes simulated data to FlexMeasures API.
+import requests
-Uses flexmeasures_client for authentication and data posting.
-Simulates 40 assets: 10 PV, 10 Battery, 10 EV Charger, 10 EV V2G
-"""
-import asyncio
-import time
 import random
 import math
 from datetime import datetime, timezone, timedelta
-from flexmeasures_client import FlexMeasuresClient
 
-FM_HOST = "https://flexmeasures.digitribe.fr"
+FM_HOST = "https://cariflex.digitribe.fr"
 FM_EMAIL = "admin@digitribe.fr"
-FM_PASSWORD = "Digitribe972"
+with open("/tmp/fm_pass.txt") as f:
+    FM_PASSWORD=f.read().strip()
 
-# Sensor mapping: sensor_id -> (name, type, unit, min, max)
 SENSORS = {}
 for i in range(1, 11):
     SENSORS[40 + i] = {"name": f"pv_{i:02d}_power", "type": "pv", "unit": "kW", "min": 0, "max": 5}
@@ -27,7 +21,6 @@ for i in range(1, 11):
     SENSORS[70 + i] = {"name": f"ev_{i:02d}_power", "type": "ev_v2g", "unit": "kWh", "min": 15, "max": 75}
 
 def generate_value(cfg, hour):
-    """Generate realistic value based on asset type and time of day."""
     t = cfg["type"]
     if t == "pv":
         if 6 <= hour <= 18:
@@ -54,57 +47,92 @@ def generate_value(cfg, hour):
         return round(max(cfg["min"], min(cfg["max"], base + random.gauss(0, 3))), 2)
     return 0
 
-async def post_all_data(client):
+def main():
-    """Post data for all 40 sensors."""
+    print(f"Cariflex Simulator -> {FM_HOST}", flush=True)
+    print(f"  Sensors: {len(SENSORS)}", flush=True)
+    
+    # Create session with CSRF token handling
+    session = requests.Session()
+    session.verify = False  # Self-signed cert
+    
+    # Get login page to extract CSRF token
+    login_url = f"{FM_HOST}/login"
+    r = session.get(login_url)
+    print(f"  Login page: {r.status_code}", flush=True)
+    
+    # Extract CSRF token
+    from html.parser import HTMLParser
+    class CSRFParser(HTMLParser):
+        def __init__(self):
+            super().__init__()
+            self.token = None
+        def handle_starttag(self, tag, attrs):
+            if tag == "input":
+                attrs = dict(attrs)
+                if attrs.get("name") == "csrf_token":
+                    self.token = attrs.get("value")
+    
+    parser = CSRFParser()
+    parser.feed(r.text)
+    csrf_token = parser.token
+    print(f"  CSRF token: {csrf_token[:20] if csrf_token else 'NOT FOUND'}...", flush=True)
+    
+    # Login
+    r = session.post(login_url, data={
+        "email": FM_EMAIL,
+        "password": FM_PASSWORD,
+        "csrf_token": csrf_token,
+        "remember": "y",
+    }, allow_redirects=True)
+    print(f"  Login: {r.status_code} (url: {r.url})", flush=True)
+    
+    # Check if logged in
+    if "dashboard" in r.url or "login" not in r.url:
+        print(f"  Logged in successfully!", flush=True)
+    else:
+        print(f"  WARNING: May not be logged in (url: {r.url})", flush=True)
+    
+    # Post sensor data
+    iteration = 0
+    while True:
         now = datetime.now(timezone.utc)
         hour = now.hour
-    start = now - timedelta(minutes=5)
         
         success = 0
         failed = 0
         
         for sensor_id, cfg in SENSORS.items():
             value = generate_value(cfg, hour)
+            start = (now - timedelta(minutes=5)).isoformat()
+            
             try:
-            await client.post_sensor_data(
+                r = session.post(
-                sensor_id=sensor_id,
+                    f"{FM_HOST}/api/v3_0/sensors/{sensor_id}/data",
-                values=[value],
+                    json={
-                start=start.isoformat(),
+                        "values": [value],
-                duration="PT5M",
+                        "start": start,
-                unit=cfg["unit"]
+                        "duration": "PT5M",
+                        "unit": cfg["unit"],
+                        "type": "PostSensorDataRequest",
+                    },
+                    timeout=30,
                 )
+                if r.status_code in [200, 201]:
                     success += 1
+                else:
+                    failed += 1
+                    if failed <= 3:
+                        print(f"  Sensor {sensor_id}: HTTP {r.status_code} - {r.text[:100]}", flush=True)
             except Exception as e:
                 failed += 1
                 if failed <= 3:
-                print(f"   ⚠️ Sensor {sensor_id}: {e}")
+                    print(f"  Sensor {sensor_id}: {e}", flush=True)
         
-    return success, failed
-
-async def main():
-    print("🚗 Cariflex Simulator → FlexMeasures API")
-    print(f"   Sensors: {len(SENSORS)} (10 PV, 10 Bat, 10 Chg, 10 EV)")
-    print(f"   FM API: {FM_HOST}")
-    print()
-    
-    client = FlexMeasuresClient(
-        email=FM_EMAIL,
-        password=FM_PASSWORD,
-        host="flexmeasures.digitribe.fr",
-        ssl=True,
-        request_timeout=60.0
-    )
-    
-    print("✅ Connected to FlexMeasures")
-    
-    iteration = 0
-    while True:
-        success, failed = await post_all_data(client)
         iteration += 1
-        now = datetime.now(timezone.utc)
+        print(f"  Iteration {iteration}: {success} OK, {failed} failed (hour={hour})", flush=True)
-        print(f"  📊 Iteration {iteration}: {success} OK, {failed} failed (hour={now.hour})")
         
-        await asyncio.sleep(30)
+        import time
+        time.sleep(30)
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    main()

scripts/configure_fm_ems.sh

@@ -0,0 +1,266 @@
+#!/bin/bash
+# Cariflex - FlexMeasures EMS Configuration Script
+# This script configures FlexMeasures as a full EMS with:
+# - Forecasting (PV, Load, Prices)
+# - Scheduling (Batteries, EVs)
+# - Ingestion (Sensors, Assets, Beliefs)
+
+set -e
+
+FM_EMAIL="admin@digitribe.fr"
+FM_PASSWORD="Digi...n"
+FM_HOST="https://cariflex.digitribe.fr"
+
+echo "╔══════════════════════════════════════════════════════════════════════════════╗"
+echo "║                    CARIFLEX - FlexMeasures EMS Configuration                 ║"
+echo "╚══════════════════════════════════════════════════════════════════════════════╝"
+echo ""
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 1. INITIAL STRUCTURE
+# ═══════════════════════════════════════════════════════════════════════════════
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  1. INITIAL STRUCTURE"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+
+echo "  → Creating initial structure (account, roles, data sources)..."
+docker exec flexmeasures-server bash -c "
+cd /app && timeout 60 .venv/bin/flexmeasures add initial-structure 2>&1 | tail -5
+" 2>&1
+
+echo ""
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 2. ASSETS AND SENSORS (if not already created)
+# ═══════════════════════════════════════════════════════════════════════════════
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  2. ASSETS AND SENSORS"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+
+# Check if assets already exist
+ASSET_COUNT=$(docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -t -c "
+SELECT COUNT(*) FROM generic_asset;
+" 2>/dev/null | xargs)
+
+echo "  → Current asset count: $ASSET_COUNT"
+
+if [ "$ASSET_COUNT" -lt 40 ]; then
+    echo "  → Creating assets and sensors..."
+    
+    # Create PV assets (10)
+    for i in $(seq 1 10); do
+        echo "    Creating PV_0$i..."
+        docker exec flexmeasures-server bash -c "
+            cd /app && timeout 30 .venv/bin/flexmeasures add asset \
+              --name 'PV_0$i' \
+              --type 'solar' \
+              --capacity 5 \
+              --unit 'kW' \
+              --latitude 14.6$(printf '%02d' $((i * 10))) \
+              --longitude -61.2$(printf '%02d' $((i * 5))) \
+              2>&1 | tail -3
+        " 2>/dev/null
+    done
+    
+    # Create Battery assets (10)
+    for i in $(seq 1 10); do
+        echo "    Creating Bat_0$i..."
+        docker exec flexmeasures-server bash -c "
+            cd /app && timeout 30 .venv/bin/flexmeasures add asset \
+              --name 'Bat_0$i' \
+              --type 'battery' \
+              --capacity 100 \
+              --unit 'kWh' \
+              --latitude 14.5$(printf '%02d' $((i * 10))) \
+              --longitude -61.1$(printf '%02d' $((i * 5))) \
+              2>&1 | tail -3
+        " 2>/dev/null
+    done
+    
+    # Create EV Charger assets (10)
+    for i in $(seq 1 10); do
+        echo "    Creating Chg_0$i..."
+        docker exec flexmeasures-server bash -c "
+            cd /app && timeout 30 .venv/bin/flexmeasures add asset \
+              --name 'Chg_0$i' \
+              --type 'ev_charger' \
+              --capacity 22 \
+              --unit 'kW' \
+              --latitude 14.6$(printf '%02d' $((i * 8))) \
+              --longitude -61.0$(printf '%02d' $((i * 3))) \
+              2>&1 | tail -3
+        " 2>/dev/null
+    done
+    
+    # Create EV assets (10)
+    for i in $(seq 1 10); do
+        echo "    Creating EV_0$i..."
+        docker exec flexmeasures-server bash -c "
+            cd /app && timeout 30 .venv/bin/flexmeasures add asset \
+              --name 'EV_0$i' \
+              --type 'ev' \
+              --capacity 75 \
+              --unit 'kWh' \
+              --latitude 14.5$(printf '%02d' $((i * 7))) \
+              --longitude -61.2$(printf '%02d' $((i * 4))) \
+              2>&1 | tail -3
+        " 2>/dev/null
+    done
+else
+    echo "  → Assets already exist, skipping..."
+fi
+
+echo ""
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 3. FORECASTING
+# ═══════════════════════════════════════════════════════════════════════════════
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  3. FORECASTING"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+
+echo "  → Generating forecasts for PV sensors (41-50)..."
+for sensor in $(seq 41 50); do
+    echo "    Forecasting sensor $sensor..."
+    docker exec flexmeasures-server bash -c "
+        cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \
+          --sensor $sensor \
+          --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \
+          2>&1 | grep -E 'Successfully|Error|SAVED' | head -1
+    " 2>/dev/null
+done
+
+echo "  → Generating forecasts for Battery sensors (51-60)..."
+for sensor in $(seq 51 60); do
+    echo "    Forecasting sensor $sensor..."
+    docker exec flexmeasures-server bash -c "
+        cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \
+          --sensor $sensor \
+          --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \
+          2>&1 | grep -E 'Successfully|Error|SAVED' | head -1
+    " 2>/dev/null
+done
+
+echo "  → Generating forecasts for EV Charger sensors (61-70)..."
+for sensor in $(seq 61 70); do
+    echo "    Forecasting sensor $sensor..."
+    docker exec flexmeasures-server bash -c "
+        cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \
+          --sensor $sensor \
+          --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \
+          2>&1 | grep -E 'Successfully|Error|SAVED' | head -1
+    " 2>/dev/null
+done
+
+echo "  → Generating forecasts for EV sensors (71-80)..."
+for sensor in $(seq 71 80); do
+    echo "    Forecasting sensor $sensor..."
+    docker exec flexmeasures-server bash -c "
+        cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \
+          --sensor $sensor \
+          --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') \
+          2>&1 | grep -E 'Successfully|Error|SAVED' | head -1
+    " 2>/dev/null
+done
+
+echo ""
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 4. SCHEDULING
+# ═══════════════════════════════════════════════════════════════════════════════
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  4. SCHEDULING"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+
+echo "  → Creating schedules for Battery sensors (51-60)..."
+for sensor in $(seq 51 60); do
+    echo "    Scheduling sensor $sensor..."
+    docker exec flexmeasures-server bash -c "
+        cd /app && timeout 60 .venv/bin/flexmeasures add schedule \
+          --sensor $sensor \
+          --start \$(date -u +'%Y-%m-%dT%H:%M:%S+00:00') \
+          --duration PT24H \
+          --resolution PT5M \
+          --soc-at-start 0.5 \
+          2>&1 | grep -E 'Successfully|Error|schedule' | head -1
+    " 2>/dev/null
+done
+
+echo "  → Creating schedules for EV sensors (71-80)..."
+for sensor in $(seq 71 80); do
+    echo "    Scheduling sensor $sensor..."
+    docker exec flexmeasures-server bash -c "
+        cd /app && timeout 60 .venv/bin/flexmeasures add schedule \
+          --sensor $sensor \
+          --start \$(date -u +'%Y-%m-%dT%H:%M:%S+00:00') \
+          --duration PT24H \
+          --resolution PT5M \
+          --soc-at-start 0.4 \
+          2>&1 | grep -E 'Successfully|Error|schedule' | head -1
+    " 2>/dev/null
+done
+
+echo ""
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 5. INGESTION STATUS
+# ═══════════════════════════════════════════════════════════════════════════════
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  5. INGESTION STATUS"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+
+echo "  → Checking sensor data status..."
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT 
+  s.name as sensor,
+  s.unit,
+  COUNT(t.id) as data_points,
+  MAX(t.event_start) as latest_data
+FROM sensor s
+LEFT JOIN timed_belief t ON t.sensor_id = s.id
+WHERE s.id BETWEEN 41 AND 80
+GROUP BY s.name, s.unit
+ORDER BY s.id;
+" 2>/dev/null
+
+echo ""
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 6. SUMMARY
+# ═══════════════════════════════════════════════════════════════════════════════
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  6. EMS CONFIGURATION SUMMARY"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+
+echo ""
+echo "  Assets:"
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT gt.name as type, COUNT(*) as count 
+FROM generic_asset g 
+JOIN generic_asset_type gt ON g.generic_asset_type_id = gt.id 
+GROUP BY gt.name;
+" 2>/dev/null
+
+echo ""
+echo "  Sensors:"
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT COUNT(*) as total_sensors FROM sensor;
+" 2>/dev/null
+
+echo ""
+echo "  Data Points (Beliefs):"
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT COUNT(*) as total_beliefs FROM timed_belief;
+" 2>/dev/null
+
+echo ""
+echo "  Forecasts:"
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT COUNT(*) as forecasts FROM timed_belief 
+WHERE source_id IN (SELECT id FROM data_source WHERE name LIKE '%forecast%');
+" 2>/dev/null
+
+echo ""
+echo "╔══════════════════════════════════════════════════════════════════════════════╗"
+echo "║                    EMS Configuration Complete!                               ║"
+echo "╚══════════════════════════════════════════════════════════════════════════════╝"