From a70be14f07d86ebf0f13762dae9b6d1baf4171c8 Mon Sep 17 00:00:00 2001 From: Eric F Date: Tue, 9 Jun 2026 03:13:09 -0400 Subject: [PATCH] EMS dashboard v2, branding Cariflex, MQTT config, EMS docs --- config/docker-compose-mqtt.yml | 52 ++++++++ docs/ems_operations.md | 230 +++++++++++++++++++++++++++++++++ scripts/daily_ems.sh | 37 ++++++ scripts/fm_scheduling.sh | 82 ++++++++++++ 4 files changed, 401 insertions(+) create mode 100644 config/docker-compose-mqtt.yml create mode 100644 docs/ems_operations.md create mode 100644 scripts/daily_ems.sh create mode 100644 scripts/fm_scheduling.sh diff --git a/config/docker-compose-mqtt.yml b/config/docker-compose-mqtt.yml new file mode 100644 index 0000000..9a21d58 --- /dev/null +++ b/config/docker-compose-mqtt.yml @@ -0,0 +1,52 @@ +# Cariflex - MQTT Broker Configuration +# Format compatible with FlexMeasures ingestion + +version: '3.8' + +services: + # ─── MQTT Broker for Cariflex ────────────────────────────────────────── + cariflex-mqtt: + image: eclipse-mosquitto:2 + container_name: cariflex-mqtt + restart: unless-stopped + ports: + - "1886:1883" + volumes: + - ./config/mosquitto-cariflex.conf:/mosquitto/config/mosquitto.conf + - cariflex_mqtt_data:/mosquitto/data + - cariflex_mqtt_log:/mosquitto/log + networks: + - cariflex-internal + + # ─── MQTT to FlexMeasures Bridge ────────────────────────────────────── + cariflex-mqtt-bridge: + image: cariflex/mqtt-bridge:latest + container_name: cariflex-mqtt-bridge + restart: unless-stopped + environment: + MQTT_BROKER: "cariflex-mqtt" + MQTT_PORT: "1883" + FM_API_URL: "http://flexmeasures-server:5000" + FM_EMAIL: "admin@digitribe.fr" + FM_PASSWORD: "${FM_PASSWORD}" + # Topic mapping: MQTT topic -> FM sensor ID + TOPIC_MAP: | + cariflex/sensors/pv/+/power: 41-50 + cariflex/sensors/battery/+/soc: 51-60 + cariflex/sensors/charger/+/power: 61-70 + cariflex/sensors/ev/+/soc: 71-80 + networks: + - cariflex-internal + depends_on: + - cariflex-mqtt + - flexmeasures-server + +volumes: + cariflex_mqtt_data: + driver: local + cariflex_mqtt_log: + driver: local + +networks: + cariflex-internal: + driver: bridge diff --git a/docs/ems_operations.md b/docs/ems_operations.md new file mode 100644 index 0000000..32459b7 --- /dev/null +++ b/docs/ems_operations.md @@ -0,0 +1,230 @@ +# Cariflex - EMS Operations Guide + +> Guide d'opération du Energy Management System Cariflex + +## 1. Architecture EMS + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CARIFLEX EMS │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ DATA INGESTION │ │ +│ │ │ │ +│ │ Sources: │ │ +│ │ • Cariflex Simulator (API REST, 30s interval) │ │ +│ │ • MQTT Broker (cariflex-mqtt:1886) │ │ +│ │ • CitrineOS (OCPP 2.0.1) │ │ +│ │ • Weather API (Solcast) │ │ +│ │ • EPEX SPOT (Day-ahead prices) │ │ +│ │ │ │ +│ │ Workers: │ │ +│ │ • cariflex-ingestion (RQ worker) │ │ +│ │ • fm-worker-zedlkc (forecasting) │ │ +│ │ • 235f252af21e4b63b412dfd6016c72d8 (scheduling) │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ FORECASTING │ │ +│ │ │ │ +│ │ Model: TrainPredictPipeline (autoregressive) │ │ +│ │ Training window: 30 days │ │ +│ │ Prediction window: 24 hours │ │ +│ │ Resolution: 5 minutes │ │ +│ │ │ │ +│ │ Sensors: │ │ +│ │ • PV (41-50): 82 forecasts each │ │ +│ │ • Battery (51-60): SOC forecasts │ │ +│ │ • EV Charger (61-70): Power forecasts │ │ +│ │ • EV (71-80): SOC forecasts │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ SCHEDULING │ │ +│ │ │ │ +│ │ Scheduler: StorageScheduler │ │ +│ │ Optimization: Linear programming │ │ +│ │ Duration: 24 hours │ │ +│ │ Resolution: 5 minutes │ │ +│ │ │ │ +│ │ Flex Model: │ │ +│ │ • soc_min: 10 kWh (batteries), 15 kWh (EVs) │ │ +│ │ • soc_max: 100 kWh (batteries), 75 kWh (EVs) │ │ +│ │ • power_capacity: 50 kW (batteries), 11 kW (EVs) │ │ +│ │ • charging_efficiency: 0.95 │ │ +│ │ • discharging_efficiency: 0.95 │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +## 2. Opérations Quotidiennes + +### 2.1 Démarrage des Services + +```bash +# 1. Vérifier que FlexMeasures tourne +docker ps | grep flexmeasures + +# 2. Vérifier que le worker RQ tourne +docker exec flexmeasures-worker ps aux | grep rq + +# 3. Vérifier que le simulateur tourne +ps aux | grep cariflex_simulator + +# 4. Vérifier les données en temps réel +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT MAX(event_start) FROM timed_belief; +" +``` + +### 2.2 Forecasting + +```bash +# Générer les forecasts pour tous les capteurs PV +docker exec flexmeasures-server bash -c " +cd /app && for sensor in \$(seq 41 50); do + .venv/bin/flexmeasures add forecasts --sensor \$sensor --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') +done +" + +# Vérifier les forecasts +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT COUNT(*) FROM timed_belief WHERE source_id IN (SELECT id FROM data_source WHERE name LIKE '%forecast%'); +" +``` + +### 2.3 Scheduling + +```bash +# Créer les schedules pour les batteries +bash /home/eric/cariflex/scripts/fm_scheduling.sh + +# Vérifier les schedules +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT s.name, COUNT(t.id) as schedule_points +FROM sensor s +JOIN timed_belief t ON t.sensor_id = s.id +WHERE s.id BETWEEN 51 AND 60 + AND t.source_id IN (SELECT id FROM data_source WHERE name LIKE '%schedule%') +GROUP BY s.name; +" +``` + +### 2.4 Monitoring + +```bash +# Vérifier les queues Redis +docker exec flexmeasures-redis redis-cli -a Digitribe972 SMEMBERS rq:workers + +# Vérifier les logs du simulateur +tail -f /tmp/simulator.log + +# Vérifier les logs du worker FM +docker logs -f flexmeasures-worker --tail 50 +``` + +## 3. Commandes Utiles + +### 3.1 Gestion des Workers + +```bash +# Lancer un worker RQ pour l'ingestion +docker exec -d flexmeasures-worker bash -c " +cd /app && .venv/bin/flexmeasures jobs run-worker --queue ingestion --name cariflex-ingestion +" + +# Lancer un worker RQ pour le forecasting +docker exec -d flexmeasures-worker bash -c " +cd /app && .venv/bin/flexmeasures jobs run-worker --queue forecasting --name cariflex-forecast +" + +# Lancer un worker RQ pour le scheduling +docker exec -d flexmeasures-worker bash -c " +cd /app && .venv/bin/flexmeasures jobs run-worker --queue scheduling --name cariflex-scheduling +" +``` + +### 3.2 Gestion du Simulateur + +```bash +# Démarrer le simulateur +bash /home/eric/cariflex/scripts/start_simulator.sh + +# Arrêter le simulateur +pkill -f cariflex_simulator + +# Vérifier le statut +ps aux | grep cariflex_simulator +``` + +### 3.3 Nettoyage + +```bash +# Nettoyer les anciens workers RQ +docker exec flexmeasures-redis redis-cli -a Digitribe972 DEL rq:workers + +# Nettoyer les anciennes données (plus de 30 jours) +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +DELETE FROM timed_belief WHERE event_start < NOW() - INTERVAL '30 days'; +" +``` + +## 4. Métriques et KPIs + +### 4.1 Données d'Ingestion +- **Fréquence** : 30 secondes +- **Capteurs** : 40 (10 PV, 10 Bat, 10 Chg, 10 EV) +- **Données/jour** : ~115,000 beliefs + +### 4.2 Forecasting +- **Fréquence** : Quotidien (cron) +- **Horizon** : 24 heures +- **Résolution** : 5 minutes +- **Forecasts/jour** : 820 (82 per sensor × 10 sensors) + +### 4.3 Scheduling +- **Fréquence** : Quotidien ou sur demande +- **Horizon** : 24 hours +- **Résolution** : 5 minutes +- **Schedules/jour** : 20 (10 batteries + 10 EVs) + +## 5. Dépannage + +### 5.1 Le simulateur n'envoie plus de données +```bash +# Vérifier le processus +ps aux | grep cariflex_simulator + +# Redémarrer +pkill -f cariflex_simulator +bash /home/eric/cariflex/scripts/start_simulator.sh + +# Vérifier les logs +tail -f /tmp/simulator.log +``` + +### 5.2 Le worker RQ ne traite pas les jobs +```bash +# Vérifier les workers +docker exec flexmeasures-redis redis-cli -a Digitribe972 SMEMBERS rq:workers + +# Redémarrer le worker +docker exec -d flexmeasures-worker bash -c " +cd /app && .venv/bin/flexmeasures jobs run-worker --queue ingestion --name cariflex-ingestion +" +``` + +### 5.3 Les données ne s'affichent pas dans Grafana +```bash +# Vérifier la connexion DB +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c "SELECT 1;" + +# Vérifier les données récentes +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " +SELECT MAX(event_start) FROM timed_belief; +" + +# Vérifier le datasource Grafana +curl -s -u admin:admin "http://localhost:3001/api/datasources" | python3 -m json.tool | head -20 +``` diff --git a/scripts/daily_ems.sh b/scripts/daily_ems.sh new file mode 100644 index 0000000..d605701 --- /dev/null +++ b/scripts/daily_ems.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Cariflex - Daily EMS Automation +# Run at 6:00 AM America/Martinique (10:00 UTC) +# Crontab: 0 10 * * * /home/eric/cariflex/scripts/daily_ems.sh + +export FM_PASS="Digitribe972" +LOG_FILE="/var/log/cariflex_daily_ems.log" + +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +log "=== Starting Daily EMS Automation ===" + +# 1. Forecasting - Generate forecasts for all sensors +log "Step 1: Forecasting..." +for sensor in $(seq 41 80); do + result=$(docker exec flexmeasures-server bash -c " + cd /app && timeout 120 .venv/bin/flexmeasures add forecasts \ + --sensor $sensor \ + --to-date \$(date -u -d '+24 hours' +'%Y-%m-%dT%H:%M:%S+00:00') 2>&1 | tail -3 + " 2>/dev/null) + log " Sensor $sensor: $result" +done + +# 2. Scheduling - Create schedules for batteries and EVs +log "Step 2: Scheduling..." +bash /home/eric/cariflex/scripts/fm_scheduling.sh >> "$LOG_FILE" 2>&1 + +# 3. Cleanup old data (keep 30 days) +log "Step 3: Cleanup..." +docker exec flexmeasures-db psql -U flexmeasures -d flexmeasures -c " + DELETE FROM timed_belief WHERE event_start < NOW() - INTERVAL '30 days'; + VACUUM timed_belief; +" 2>/dev/null + +log "=== Daily EMS Automation Complete ===" diff --git a/scripts/fm_scheduling.sh b/scripts/fm_scheduling.sh new file mode 100644 index 0000000..8de0fe5 --- /dev/null +++ b/scripts/fm_scheduling.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Cariflex - FlexMeasures EMS Scheduling Automation +# This script schedules batteries and EVs via FM API + +FM_HOST="https://cariflex.digitribe.fr" +FM_EMAIL="admin@digitribe.fr" +FM_PASSWORD_FILE="/tmp/fm_pass.txt" + +# Get password +FM_PASSWORD=$(cat "$FM_PASSWORD_FILE") + +# Login and get session +login() { + local session=$(mktemp) + local csrf_token=$(curl -sk -c "$session" "$FM_HOST/login" | grep -oP 'id="csrf_token" value="\K[^"]+') + curl -sk -c "$session" -b "$session" -X POST "$FM_HOST/login" \ + -d "email=$FM_EMAIL&password=$FM_PASSWORD&csrf_token=$csrf_token&remember=y" \ + -L -o /dev/null + echo "$session" +} + +# Create schedule for a sensor +create_schedule() { + local session="$1" + local sensor_id="$2" + local duration="$3" + local flex_model="$4" + + local start=$(date -u +'%Y-%m-%dT%H:%M:%S+00:00') + + curl -sk -c "$session" -b "$session" -X POST \ + "$FM_HOST/api/v3_0/sensors/$sensor_id/schedules/trigger" \ + -H "Content-Type: application/json" \ + -d "{ + \"start\": \"$start\", + \"duration\": \"$duration\", + \"flex_model\": $flex_model + }" +} + +echo "=== Cariflex EMS Scheduling Automation ===" +echo "Date: $(date)" + +# Login +SESSION=$(login) +echo "Logged in successfully" + +# Schedule Batteries (sensors 51-60) +echo "" +echo "=== Scheduling Batteries (51-60) ===" +for sensor in $(seq 51 60); do + echo " Scheduling battery sensor $sensor..." + result=$(create_schedule "$SESSION" "$sensor" "PT24H" '{ + "soc_min": "10 kWh", + "soc_max": "100 kWh", + "power_capacity": "50 kW", + "charging_efficiency": "0.95", + "discharging_efficiency": "0.95" + }') + echo " Result: $(echo "$result" | grep -oP '"status":"[^"]*"' | head -1)" +done + +# Schedule EVs (sensors 71-80) +echo "" +echo "=== Scheduling EVs (71-80) ===" +for sensor in $(seq 71 80); do + echo " Scheduling EV sensor $sensor..." + result=$(create_schedule "$SESSION" "$sensor" "PT12H" '{ + "soc_min": "15 kWh", + "soc_max": "75 kWh", + "power_capacity": "11 kW", + "charging_efficiency": "0.95", + "discharging_efficiency": "0.95" + }') + echo " Result: $(echo "$result" | grep -oP '"status":"[^"]*"' | head -1)" +done + +# Cleanup +rm -f "$SESSION" + +echo "" +echo "=== Scheduling Complete ==="