EMS dashboard v2, branding Cariflex, MQTT config, EMS docs

config/docker-compose-mqtt.yml

@@ -0,0 +1,52 @@
+# Cariflex - MQTT Broker Configuration
+# Format compatible with FlexMeasures ingestion
+
+version: '3.8'
+
+services:
+  # ─── MQTT Broker for Cariflex ──────────────────────────────────────────
+  cariflex-mqtt:
+    image: eclipse-mosquitto:2
+    container_name: cariflex-mqtt
+    restart: unless-stopped
+    ports:
+      - "1886:1883"
+    volumes:
+      - ./config/mosquitto-cariflex.conf:/mosquitto/config/mosquitto.conf
+      - cariflex_mqtt_data:/mosquitto/data
+      - cariflex_mqtt_log:/mosquitto/log
+    networks:
+      - cariflex-internal
+
+  # ─── MQTT to FlexMeasures Bridge ──────────────────────────────────────
+  cariflex-mqtt-bridge:
+    image: cariflex/mqtt-bridge:latest
+    container_name: cariflex-mqtt-bridge
+    restart: unless-stopped
+    environment:
+      MQTT_BROKER: "cariflex-mqtt"
+      MQTT_PORT: "1883"
+      FM_API_URL: "http://flexmeasures-server:5000"
+      FM_EMAIL: "admin@digitribe.fr"
+      FM_PASSWORD: "${FM_PASSWORD}"
+      # Topic mapping: MQTT topic -> FM sensor ID
+      TOPIC_MAP: |
+        cariflex/sensors/pv/+/power: 41-50
+        cariflex/sensors/battery/+/soc: 51-60
+        cariflex/sensors/charger/+/power: 61-70
+        cariflex/sensors/ev/+/soc: 71-80
+    networks:
+      - cariflex-internal
+    depends_on:
+      - cariflex-mqtt
+      - flexmeasures-server
+
+volumes:
+  cariflex_mqtt_data:
+    driver: local
+  cariflex_mqtt_log:
+    driver: local
+
+networks:
+  cariflex-internal:
+    driver: bridge

docs/ems_operations.md

@@ -0,0 +1,230 @@
+# Cariflex - EMS Operations Guide
+
+> Guide d'opération du Energy Management System Cariflex
+
+## 1. Architecture EMS
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                              CARIFLEX EMS                                    │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                              │
+│  ┌──────────────────────────────────────────────────────────────────────┐   │
+│  │                         DATA INGESTION                                │   │
+│  │                                                                        │   │
+│  │  Sources:                                                              │   │
+│  │  • Cariflex Simulator (API REST, 30s interval)                        │   │
+│  │  • MQTT Broker (cariflex-mqtt:1886)                                   │   │
+│  │  • CitrineOS (OCPP 2.0.1)                                             │   │
+│  │  • Weather API (Solcast)                                              │   │
+│  │  • EPEX SPOT (Day-ahead prices)                                       │   │
+│  │                                                                        │   │
+│  │  Workers:                                                              │   │
+│  │  • cariflex-ingestion (RQ worker)                                     │   │
+│  │  • fm-worker-zedlkc (forecasting)                                     │   │
+│  │  • 235f252af21e4b63b412dfd6016c72d8 (scheduling)                      │   │
+│  └──────────────────────────────────────────────────────────────────────┘   │
+│                                                                              │
+│  ┌──────────────────────────────────────────────────────────────────────┐   │
+│  │                          FORECASTING                                  │   │
+│  │                                                                        │   │
+│  │  Model: TrainPredictPipeline (autoregressive)                         │   │
+│  │  Training window: 30 days                                              │   │
+│  │  Prediction window: 24 hours                                           │   │
+│  │  Resolution: 5 minutes                                                 │   │
+│  │                                                                        │   │
+│  │  Sensors:                                                              │   │
+│  │  • PV (41-50): 82 forecasts each                                       │   │
+│  │  • Battery (51-60): SOC forecasts                                      │   │
+│  │  • EV Charger (61-70): Power forecasts                                 │   │
+│  │  • EV (71-80): SOC forecasts                                           │   │
+│  └──────────────────────────────────────────────────────────────────────┘   │
+│                                                                              │
+│  ┌──────────────────────────────────────────────────────────────────────┐   │
+│  │                           SCHEDULING                                  │   │
+│  │                                                                        │   │
+│  │  Scheduler: StorageScheduler                                           │   │
+│  │  Optimization: Linear programming                                      │   │
+│  │  Duration: 24 hours                                                    │   │
+│  │  Resolution: 5 minutes                                                 │   │
+│  │                                                                        │   │
+│  │  Flex Model:                                                           │   │
+│  │  • soc_min: 10 kWh (batteries), 15 kWh (EVs)                          │   │
+│  │  • soc_max: 100 kWh (batteries), 75 kWh (EVs)                         │   │
+│  │  • power_capacity: 50 kW (batteries), 11 kW (EVs)                     │   │
+│  │  • charging_efficiency: 0.95                                           │   │
+│  │  • discharging_efficiency: 0.95                                        │   │
+│  └──────────────────────────────────────────────────────────────────────┘   │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
+## 2. Opérations Quotidiennes
+
+### 2.1 Démarrage des Services
+
+```bash
+# 1. Vérifier que FlexMeasures tourne
+docker ps | grep flexmeasures
+
+# 2. Vérifier que le worker RQ tourne
+docker exec flexmeasures-worker ps aux | grep rq
+
+# 3. Vérifier que le simulateur tourne
+ps aux | grep cariflex_simulator
+
+# 4. Vérifier les données en temps réel
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT MAX(event_start) FROM timed_belief;
+"
+```
+
+### 2.2 Forecasting
+
+```bash
+# Générer les forecasts pour tous les capteurs PV
+docker exec flexmeasures-server bash -c "
+cd /app && for sensor in \$(seq 41 50); do
+  .venv/bin/flexmeasures add forecasts --sensor \$sensor --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00')
+done
+"
+
+# Vérifier les forecasts
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT COUNT(*) FROM timed_belief WHERE source_id IN (SELECT id FROM data_source WHERE name LIKE '%forecast%');
+"
+```
+
+### 2.3 Scheduling
+
+```bash
+# Créer les schedules pour les batteries
+bash /home/eric/cariflex/scripts/fm_scheduling.sh
+
+# Vérifier les schedules
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT s.name, COUNT(t.id) as schedule_points
+FROM sensor s
+JOIN timed_belief t ON t.sensor_id = s.id
+WHERE s.id BETWEEN 51 AND 60
+  AND t.source_id IN (SELECT id FROM data_source WHERE name LIKE '%schedule%')
+GROUP BY s.name;
+"
+```
+
+### 2.4 Monitoring
+
+```bash
+# Vérifier les queues Redis
+docker exec flexmeasures-redis redis-cli -a Digitribe972 SMEMBERS rq:workers
+
+# Vérifier les logs du simulateur
+tail -f /tmp/simulator.log
+
+# Vérifier les logs du worker FM
+docker logs -f flexmeasures-worker --tail 50
+```
+
+## 3. Commandes Utiles
+
+### 3.1 Gestion des Workers
+
+```bash
+# Lancer un worker RQ pour l'ingestion
+docker exec -d flexmeasures-worker bash -c "
+cd /app && .venv/bin/flexmeasures jobs run-worker --queue ingestion --name cariflex-ingestion
+"
+
+# Lancer un worker RQ pour le forecasting
+docker exec -d flexmeasures-worker bash -c "
+cd /app && .venv/bin/flexmeasures jobs run-worker --queue forecasting --name cariflex-forecast
+"
+
+# Lancer un worker RQ pour le scheduling
+docker exec -d flexmeasures-worker bash -c "
+cd /app && .venv/bin/flexmeasures jobs run-worker --queue scheduling --name cariflex-scheduling
+"
+```
+
+### 3.2 Gestion du Simulateur
+
+```bash
+# Démarrer le simulateur
+bash /home/eric/cariflex/scripts/start_simulator.sh
+
+# Arrêter le simulateur
+pkill -f cariflex_simulator
+
+# Vérifier le statut
+ps aux | grep cariflex_simulator
+```
+
+### 3.3 Nettoyage
+
+```bash
+# Nettoyer les anciens workers RQ
+docker exec flexmeasures-redis redis-cli -a Digitribe972 DEL rq:workers
+
+# Nettoyer les anciennes données (plus de 30 jours)
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+DELETE FROM timed_belief WHERE event_start < NOW() - INTERVAL '30 days';
+"
+```
+
+## 4. Métriques et KPIs
+
+### 4.1 Données d'Ingestion
+- **Fréquence** : 30 secondes
+- **Capteurs** : 40 (10 PV, 10 Bat, 10 Chg, 10 EV)
+- **Données/jour** : ~115,000 beliefs
+
+### 4.2 Forecasting
+- **Fréquence** : Quotidien (cron)
+- **Horizon** : 24 heures
+- **Résolution** : 5 minutes
+- **Forecasts/jour** : 820 (82 per sensor × 10 sensors)
+
+### 4.3 Scheduling
+- **Fréquence** : Quotidien ou sur demande
+- **Horizon** : 24 hours
+- **Résolution** : 5 minutes
+- **Schedules/jour** : 20 (10 batteries + 10 EVs)
+
+## 5. Dépannage
+
+### 5.1 Le simulateur n'envoie plus de données
+```bash
+# Vérifier le processus
+ps aux | grep cariflex_simulator
+
+# Redémarrer
+pkill -f cariflex_simulator
+bash /home/eric/cariflex/scripts/start_simulator.sh
+
+# Vérifier les logs
+tail -f /tmp/simulator.log
+```
+
+### 5.2 Le worker RQ ne traite pas les jobs
+```bash
+# Vérifier les workers
+docker exec flexmeasures-redis redis-cli -a Digitribe972 SMEMBERS rq:workers
+
+# Redémarrer le worker
+docker exec -d flexmeasures-worker bash -c "
+cd /app && .venv/bin/flexmeasures jobs run-worker --queue ingestion --name cariflex-ingestion
+"
+```
+
+### 5.3 Les données ne s'affichent pas dans Grafana
+```bash
+# Vérifier la connexion DB
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "SELECT 1;"
+
+# Vérifier les données récentes
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+SELECT MAX(event_start) FROM timed_belief;
+"
+
+# Vérifier le datasource Grafana
+curl -s -u admin:admin "http://localhost:3001/api/datasources" | python3 -m json.tool | head -20
+```

scripts/daily_ems.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Cariflex - Daily EMS Automation
+# Run at 6:00 AM America/Martinique (10:00 UTC)
+# Crontab: 0 10 * * * /home/eric/cariflex/scripts/daily_ems.sh
+
+export FM_PASS="Digitribe972"
+LOG_FILE="/var/log/cariflex_daily_ems.log"
+
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
+}
+
+log "=== Starting Daily EMS Automation ==="
+
+# 1. Forecasting - Generate forecasts for all sensors
+log "Step 1: Forecasting..."
+for sensor in $(seq 41 80); do
+    result=$(docker exec flexmeasures-server bash -c "
+        cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \
+            --sensor $sensor \
+            --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') 2>&1 | tail -3
+    " 2>/dev/null)
+    log "  Sensor $sensor: $result"
+done
+
+# 2. Scheduling - Create schedules for batteries and EVs
+log "Step 2: Scheduling..."
+bash /home/eric/cariflex/scripts/fm_scheduling.sh >> "$LOG_FILE" 2>&1
+
+# 3. Cleanup old data (keep 30 days)
+log "Step 3: Cleanup..."
+docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "
+    DELETE FROM timed_belief WHERE event_start < NOW() - INTERVAL '30 days';
+    VACUUM timed_belief;
+" 2>/dev/null
+
+log "=== Daily EMS Automation Complete ==="

scripts/fm_scheduling.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# Cariflex - FlexMeasures EMS Scheduling Automation
+# This script schedules batteries and EVs via FM API
+
+FM_HOST="https://cariflex.digitribe.fr"
+FM_EMAIL="admin@digitribe.fr"
+FM_PASSWORD_FILE="/tmp/fm_pass.txt"
+
+# Get password
+FM_PASSWORD=$(cat "$FM_PASSWORD_FILE")
+
+# Login and get session
+login() {
+    local session=$(mktemp)
+    local csrf_token=$(curl -sk -c "$session" "$FM_HOST/login" | grep -oP 'id="csrf_token" value="\K[^"]+')
+    curl -sk -c "$session" -b "$session" -X POST "$FM_HOST/login" \
+        -d "email=$FM_EMAIL&password=$FM_PASSWORD&csrf_token=$csrf_token&remember=y" \
+        -L -o /dev/null
+    echo "$session"
+}
+
+# Create schedule for a sensor
+create_schedule() {
+    local session="$1"
+    local sensor_id="$2"
+    local duration="$3"
+    local flex_model="$4"
+    
+    local start=$(date -u +'%Y-%m-%dT%H:%M:%S+00:00')
+    
+    curl -sk -c "$session" -b "$session" -X POST \
+        "$FM_HOST/api/v3_0/sensors/$sensor_id/schedules/trigger" \
+        -H "Content-Type: application/json" \
+        -d "{
+            \"start\": \"$start\",
+            \"duration\": \"$duration\",
+            \"flex_model\": $flex_model
+        }"
+}
+
+echo "=== Cariflex EMS Scheduling Automation ==="
+echo "Date: $(date)"
+
+# Login
+SESSION=$(login)
+echo "Logged in successfully"
+
+# Schedule Batteries (sensors 51-60)
+echo ""
+echo "=== Scheduling Batteries (51-60) ==="
+for sensor in $(seq 51 60); do
+    echo "  Scheduling battery sensor $sensor..."
+    result=$(create_schedule "$SESSION" "$sensor" "PT24H" '{
+        "soc_min": "10 kWh",
+        "soc_max": "100 kWh",
+        "power_capacity": "50 kW",
+        "charging_efficiency": "0.95",
+        "discharging_efficiency": "0.95"
+    }')
+    echo "    Result: $(echo "$result" | grep -oP '"status":"[^"]*"' | head -1)"
+done
+
+# Schedule EVs (sensors 71-80)
+echo ""
+echo "=== Scheduling EVs (71-80) ==="
+for sensor in $(seq 71 80); do
+    echo "  Scheduling EV sensor $sensor..."
+    result=$(create_schedule "$SESSION" "$sensor" "PT12H" '{
+        "soc_min": "15 kWh",
+        "soc_max": "75 kWh",
+        "power_capacity": "11 kW",
+        "charging_efficiency": "0.95",
+        "discharging_efficiency": "0.95"
+    }')
+    echo "    Result: $(echo "$result" | grep -oP '"status":"[^"]*"' | head -1)"
+done
+
+# Cleanup
+rm -f "$SESSION"
+
+echo ""
+echo "=== Scheduling Complete ==="