diff --git a/TODO.md b/TODO.md index b210c48a..8d9a73ca 100644 --- a/TODO.md +++ b/TODO.md @@ -1,45 +1,32 @@ # Smart City Digital Twin — TODO List -> Dernière mise à jour : 2026-05-26 22:49 +> Dernière mise à jour : 2026-06-01 07:00 + +## ✅ Complété (cette session 2026-06-01) -## ✅ Complété | ID | Tâche | |----|-------| -| p1-bunkerm | BunkerM: DNS corrigé | -| p2-geoserver | GeoServer: workspace + Data Store PostGIS | -| p2-postgis | PostGIS dédié: conteneur UP | -| p2-mapstore | MapStore: GeoServer WMS + couche sensors | -| p5-docs | Documentation + commits Gitea | -| contexus | Stack Contexus déployée et fonctionnelle | -| or-assets | 5 assets IOTSensor créés dans OpenRemote | -| or-agent | Agent MQTT créé dans OpenRemote | -| contexus-mqtt | Driver MQTT configuré et recevant des données | -| contexus-devices | 7 devices créés dans Contexus | -| telegraf-fix | Noms containers corrigés + BunkerM désactivé | -| or-pg-fix | Image PostgreSQL changée → timescaledb-ha:pg15 | -| grafana-fix | Dashboard "no data" corrigé — datasource + requêtes Flux | -| grafana-v4 | Dashboard v4 poussé avec 14 panels, données confirmées ✅ | -| bunkerm-activate | BunkerM activé dans simulateur + Telegraf | -| superset-deploy | Apache Superset déployé derrière Traefik ✅ | -| metabase-deploy | Metabase déployé derrière Traefik ✅ | -| chirpstack-pw | Password admin réinitialisé → Digitribe972 ✅ | -| smart-app-arch | Architecture Smart App City créée (Beckn + AI + i18n) ✅ | -| odk-deploy | ODK Central déployé derrière Traefik ✅ — https://odk.digitribe.fr | -| odk-project | Projet "Smart-City-Martinique" créé dans ODK ✅ | -| session-2026-05-26 | Reprise après crash — snapshot + resume + sauvegardes ✅ | -| session-2026-05-26b | Correction date snapshots + cleanup fichiers erronees ✅ | +| jupyterhub-fix | JupyterHub DB path fix (absolute path) → healthy ✅ | +| jupyterhub-user | User eric créé + autorisé dans JupyterHub (admin) | +| or-map-bounds | OR mbtiles metadata bounds → monde, center → Martinique ✅ | +| or-map-verify | OR API confirmée: center=[-61,14.5], minZoom=0, bounds=Martinique | +| hermes-dashboard | Hermes Dashboard WebUI + TUI chat activé (localhost:9119, auto-boot) | +| git-push | Commit 008f167 pushé sur Gitea | + +## 🔴 Bloqué / En cours -## 🔴 Bloqué | ID | Tâche | Raison | |----|-------|--------| -| p1-or-map | Affichage points carte OpenRemote | Manager DOWN — rebuild sur smartcity-shared nécessaire | -| p4-ditto | Ditto.digitribe.fr | MongoDB localhost hardcodé | -| p3-kepler | KeplerGL | Image Docker incomplète | +| jupyterhub-spawn | Spawn user eric timeout (30s→120s fixé, mais singleuser lent) | Container resource limit? | +| or-tiles | Carte OR fond gris sur Martinique | mbtiles contient tiles Pays-Bas, pas Martinique | +| kafka-fix | Kafka restart loop | `zookeeper.connect` manquant | +| trino-fix | Trino restart loop | `node.environment` null | ## ⏳ En attente + | ID | Tâche | |----|-------| -| p1-or-restart | Redémarrer OpenRemote (rebuild manager sur smartcity-shared) | +| p1-or-restart | Vérifier OR map tiles après remplacement mbtiles Martinique | | p1-contexus-60 | Configurer les 60 devices Contexus | | p3-analyse | Analyse: GeoMesa + KeplerGL | | p1-ngsi | NGSI-LD: validation pipeline (basse priorité) | @@ -49,18 +36,22 @@ | smart-app Phase 2 | Transport, Beckn integration, chatbot RAG | | smart-app Phase 3 | AI Agents, prédictions, réalité augmentée | -## 📝 Notes 2026-05-26 -- **Session reprise après crash** — Snapshot 2026-05-26 créé, session resume, TODO mis à jour -- **61 conteneurs UP** — Stellio, ThingsBoard, Honcho API, FROST, EMQX DOWN -- **ODK Central** : Toujours fonctionnel (HTTP 200) -- **OpenRemote Manager** : DOWN — doit être reconstruit sur smartcity-shared +## 📝 Notes 2026-06-01 + +- **86 conteneurs Docker** au total +- **JupyterHub** : https://jupyter.digitribe.fr — user eric/admin créé, spawn lent +- **OpenRemote** : https://openremote.digitribe.fr — carte centrée Martinique, dézoom libre (minZoom=0), mais tiles Pays-Bas (fond gris) +- **Hermes Dashboard** : http://127.0.0.1:9119 (SSH tunnel) — WebUI + TUI chat, auto-boot +- **OR mbtiles** : metadata bounds monde OK, mais contenu = vector tiles Pays-Bas. Script `scripts/generate_martinique_mbtiles.py` prêt pour génération - **Pipeline données** : Simulateur → Mosquitto/BunkerM → Telegraf → InfluxDB → Grafana ✅ - **Grafana** : Dashboard smartcity-martinique-complete v7 ✅ - **Superset** : https://superset.digitribe.fr ✅ - **Metabase** : https://metabase.digitribe.fr ✅ -- **Smart App City** : Architecture créée dans `smart-app-city/`. Stack: React Native + Expo, NestJS + FastAPI, LocalAI + Qdrant (RAG), Beckn Protocol (OTN-DPI), i18n FR/EN/ES/DE. +- **ODK Central** : https://odk.digitribe.fr ✅ +- **MindsDB** : https://mindsdb.digitribe.fr ✅ ## Credentials + - **Contexus**: iotevadmin / Digitribe972 - **OpenRemote**: admin / Digitribe972 - **PostgreSQL Contexus**: contexus / Digitribe972 @@ -72,3 +63,5 @@ - **BunkerM MQTT**: bunker / bunker - **ChirpStack**: admin / Digitribe972 - **ODK Central**: efelixine@digitribe.fr / Digitribe972 +- **JupyterHub**: eric / Digitribe972 (admin) +- **MindsDB**: admin@digitribe.fr / Digitribe972 diff --git a/scripts/gen_mbtiles_martinique.py b/scripts/gen_mbtiles_martinique.py new file mode 100644 index 00000000..af0002bc --- /dev/null +++ b/scripts/gen_mbtiles_martinique.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Generate mbtiles for Martinique by downloading OSM raster tiles. + +Usage: python3 gen_mbtiles_martinique.py +Output: /tmp/mapdata_martinique.mbtiles +""" +import sqlite3, os, math, time, gzip, io, struct, sys +import urllib.request, urllib.error + +# Martinique bounds (with buffer) +MIN_LON, MIN_LAT = -61.7, 13.9 +MAX_LON, MAX_LAT = -60.3, 15.1 +MIN_Z, MAX_Z = 0, 14 + +# Number of tiles per zoom level (Marti◆ique is small) +# zoom 0: 1, zoom 1: 1, zoom 2: 1, zoom 3: 1, zoom 4: 1 +# zoom 5: 1, zoom 6: 2, zoom 7: 4, zoom 8: 6, zoom 9: 12 +# zoom 10: 24, zoom 11: 48, zoom 12: 96, zoom 13: 192, zoom 14: 384 + +TMP_DIR = "/tmp/osm_tiles" +OUTPUT = "/tmp/mapdata_martinique.mbtiles" +OSM_URL = "https://tile.openstreetmap.org/{z}/{x}/{y}.png" +DELAY = 0.15 # OSM usage policy: max 2req/s +UA = "SmartCityMartinique/1.0 (eric@digitribe.fr)" + +os.makedirs(TMP_DIR, exist_ok=True) + +def deg2tile(lat, lon, z): + n = 1 << z + x = int((lon + 180.0) / 360.0 * n) + y = int((1.0 - math.asinh(math.tan(math.radians(lat))) / math.pi) / 2.0 * n) + return max(0, min(n-1, x)), max(0, min(n-1, y)) + +def tile_exists_in_mbtiles(c, z, x, y_tms): + c.execute("SELECT 1 FROM map WHERE zoom_level=? AND tile_column=? AND tile_row=?", (z, x, y_tms)) + return c.fetchone() is not None + +def download_tile(z, x, y): + """Download raster tile from OSM. Returns PNG bytes or None.""" + url = OSM_URL.format(z=z, x=x, y=y) + try: + req = urllib.request.Request(url, headers={"User-Agent": UA}) + resp = urllib.request.urlopen(req, timeout=10) + data = resp.read() + if len(data) > 100: + return data + except urllib.error.HTTPError as e: + if e.code == 404: + return None + print(f" HTTP {e.code} for {z}/{x}/{y}") + except Exception as e: + print(f" Error {z}/{x}/{y}: {e}") + return None + +def create_mbtiles(): + if os.path.exists(OUTPUT): + os.remove(OUTPUT) + conn = sqlite3.connect(OUTPUT) + c = conn.cursor() + c.execute("CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_id TEXT)") + c.execute("CREATE TABLE images (tile_data BLOB, tile_id TEXT)") + c.execute("CREATE TABLE metadata (name TEXT PRIMARY KEY, value TEXT)") + + meta = [ + ("name", "Martinique"), + ("type", "overlay"), + ("version", "1.0"), + ("description", "OSM raster tiles for Martinique"), + ("format", "png"), + ("tilejson", "2.1.0"), + ("scheme", "tms"), + ("minzoom", str(MIN_Z)), + ("maxzoom", str(MAX_Z)), + ("bounds", f"{MIN_LON},{MIN_LAT},{MAX_LON},{MAX_LAT}"), + ("center", "-61.0,14.5,10"), + ("attribution", "© OpenStreetMap contributors"), + ] + c.executemany("INSERT INTO metadata VALUES (?, ?)", meta) + return conn, c + +def main(): + conn, c = create_mbtiles() + total = 0 + downloaded = 0 + skipped = 0 + start = time.time() + + for z in range(MIN_Z, MAX_Z + 1): + x1, y1 = deg2tile(MIN_LAT, MIN_LON, z) + x2, y2 = deg2tile(MAX_LAT, MAX_LON, z) + xm, xM = min(x1, x2)-1, max(x1, x2)+1 + ym, yM = min(y1, y2)-1, max(y1, y2)+1 + xm = max(0, xm) + xM = min((1 << z) - 1, xM) + ym = max(0, ym) + yM = min((1 << z) - 1, yM) + + z_downloaded = 0 + for x in range(xm, xM + 1): + for y in range(ym, yM + 1): + total += 1 + tms_y = (1 << z) - 1 - y + png = download_tile(z, x, y) + if png: + tid = f"osm_{z}_{x}_{tms_y}" + c.execute("INSERT INTO images VALUES (?, ?)", (sqlite3.Binary(png), tid)) + c.execute("INSERT INTO map VALUES (?, ?, ?, ?)", (z, x, tms_y, tid)) + downloaded += 1 + z_downloaded += 1 + else: + skipped += 1 + time.sleep(DELAY) + + conn.commit() + print(f"Zoom {z}: {z_downloaded} tiles (total: {downloaded})", flush=True) + + c.execute("CREATE INDEX idx_map ON map (zoom_level, tile_column, tile_row)") + conn.commit() + size_mb = os.path.getsize(OUTPUT) / (1024*1024) + elapsed = time.time() - start + print(f"\nDone! {downloaded}/{total} tiles downloaded, {skipped} empty. {size_mb:.1f} MB in {elapsed:.0f}s") + conn.close() + +if __name__ == "__main__": + main() diff --git a/scripts/smartcity_monitor.py b/scripts/smartcity_monitor.py index 26869d0a..3a4dead2 100755 --- a/scripts/smartcity_monitor.py +++ b/scripts/smartcity_monitor.py @@ -12,7 +12,7 @@ from datetime import datetime # Configuration CRITICAL_CONTAINERS = [ - "openremote_manager_1", "openremote_keycloak_1", "smart-city-simulator", + "openremote-manager", "openremote-keycloak", "smart-city-simulator", "emqx_emqx_1", "mainfluxlabs-broker", "stellio-api-gateway", "smart-city-influxdb", "smart-city-grafana", "traefik", "smart-city-prometheus-brokers" diff --git a/session_resume_consolide.md b/session_resume_consolide.md new file mode 100644 index 00000000..45c812dc --- /dev/null +++ b/session_resume_consolide.md @@ -0,0 +1,84 @@ +# Session Resume Consolidé — Smart City Digital Twin + +## 2026-06-01 (Reprise après crash) + +### Objectif +Reprendre après crash. Commits, sauvegardes, état des lieux infrastructure. + +### Actions réalisées + +| Action | Statut | Détails | +|--------|--------|---------| +| JupyterHub DB_path fix | ✅ | `sqlite:////srv/jupyterhub/jupyterhub.sqlite` (absolute path) | +| JupyterHub user eric | ✅ | Créé id=2, admin, authorized | +| JupyterHub spawn timeout | ⚠️ | Augmenté à 120s, mais spawn encore lent | +| OR mbtiles bounds | ✅ | Metadata bounds → monde, center → Martinique | +| OR map API | ✅ | center=[-61,14.5], minZoom=0, bounds=Martinique | +| OR carte tiles | ⚠️ | Tiles Pays-Bas (fond gris Martinique), script génération prêt | +| Hermes Dashboard | ✅ | WebUI+TUI sur localhost:9119, service systemd auto-boot | +| Commit/Push | ✅ | `008f167` sur Gitea | + +### Problèmes actuels +- **JupyterHub spawn**: timeout malgré 120s — le singleuser server met longtemps à démarrer +- **OR carte**: fond gris car mbtiles = tiles Pays-Bas, pas Martinique +- **Kafka**: restart loop (zookeeper.connect manquant) +- **Trino**: restart loop (node.environment null) +- **FROST**: exited 137 (OOM) + +### Infrastructure (86 conteneurs) +- 82 UP, 4 restart loop, 2 exited +- Traefik, OpenRemote, Grafana, InfluxDB, Simulateur, ODK, MindsDB, MapStore, GeoServer, EMQX, Ditto, ChirpStack, Node-RED, MinIO, Flink, Gitea, LocalAI, PHPIPAM, Honcho = UP ✅ + +--- + +## Historique Sessions Précédentes + +### 2026-05-29 +- OpenRebuilder reconstruit et fonctionnel (KC23.0.7, Manager 1.24.0, TimescaleDB) +- 9 IOTSensor assets créés avec GEO_JSONPoint +- Mapsettings configuré pour Martinique +- VRE (JupyterHub + Zeppelin) ajouté au projet +- Lakehouse stack (Gravitino, Flink, Kafka, Trino, MinIO) déployé + +### 2026-05-28 +- ODK Central déployé (https://odk.digitribe.fr) +- Projet "Smart-City-Martinique" créé dans ODK +- MindsDB configuré (https://mindsdb.digitribe.fr) + +### 2026-05-27 +- OpenRemote Manager DOWN — rebuild nécessaire +- Password ChirpStack réinitialisé +- Grafana dashboard v7 avec données confirmées + +### 2026-05-26 +- Reprise après crash — snapshot + resume +- 61 conteneurs UP +- Stellio, ThingsBoard, Honcho, FROST, EMQX DOWN +- Pipeline données confirmé ✅ + +### 2026-05-23 +- Règle globale Docker: container_name explicite obligatoire +- Keycloak 24.x: KC_HTTP_RELATIVE_PATH="/auth" requis +- Simulator OOM (exit 137) — services non essentiels désactivés + +### 2026-05-21 +- Ne JAMAIS modifier configs OR par défaut sans permission +- OR Manager sert UI sur /dashboard/ par défaut, pas /manager/ + +### 2026-04-29 +- FROST-Server MQTT WebSocket configuré via Traefik +- mbtiles metadata bounds modifié (Pays-Bas → monde) pour dézoom libre + +### 2026-04-24 +- Architecture géospatiale étendue (16+ composants) +- Documents générés (HTML, PDF, Docker Compose stack) + +--- + +## Fichiers clés + +- TODO.md: `/home/eric/smart-city-digital-twin-martinique/TODO.md` +- Traefik config: `/home/eric/traefik-config/dynamic/` +- VRE configs: `/home/eric/smart-city-digital-twin-martinique/vre/` +- Hermes Dashboard: `hermes dashboard --host 127.0.0.1 --port 9119 --tui` +- Session transcripts: `~/.hermes/sessions/` diff --git a/tmp_mapsettings.json b/tmp_mapsettings.json new file mode 100644 index 00000000..d2fb44ab --- /dev/null +++ b/tmp_mapsettings.json @@ -0,0 +1 @@ +{{data}} \ No newline at end of file diff --git a/vre/jupyterhub/Dockerfile b/vre/jupyterhub/Dockerfile index e54f11a0..5daec1dd 100644 --- a/vre/jupyterhub/Dockerfile +++ b/vre/jupyterhub/Dockerfile @@ -12,10 +12,8 @@ RUN pip install --no-cache-dir \ jupyterlab \ notebook -# Create the directory structure JupyterHub expects for DB -# JupyterHub joins data_files_path + dirname(db_path), so we create the composed path -RUN mkdir -p /srv/jupyterhub/srv/jupyterhub && \ - chown -R 1000:1000 /srv/jupyterhub +# Create the directory for JupyterHub data +RUN mkdir -p /srv/jupyterhub && chown -R 1000:1000 /srv/jupyterhub COPY jupyterhub_config.py /srv/jupyterhub/jupyterhub_config.py diff --git a/vre/jupyterhub/jupyterhub_config.py b/vre/jupyterhub/jupyterhub_config.py index 24d94e92..37c68f9b 100644 --- a/vre/jupyterhub/jupyterhub_config.py +++ b/vre/jupyterhub/jupyterhub_config.py @@ -9,11 +9,12 @@ c.JupyterHub.authenticator_class = 'nativeauthenticator.NativeAuthenticator' c.Authenticator.admin_users = {'admin'} c.Authenticator.allow_all = True -# Spawner - use DockerSpawner or simple with correct cmd -c.JupyterHub.spawner_class = 'simple' -c.Spawner.cmd = ['jupyterhub-singleuser'] -c.Spawner.http_timeout = 120 -c.Spawner.start_timeout = 120 +# Spawner — use SimpleLocalProcessSpawner (default in JupyterHub 5.x) +# This spawner runs singleuser servers as subprocesses +c.JupyterHub.spawner_class = 'jupyterhub.spawner.SimpleLocalProcessSpawner' +c.Spawner.default_url = '/lab' +c.Spawner.http_timeout = 300 +c.Spawner.start_timeout = 300 # Database and cookies - use absolute paths c.JupyterHub.cookie_secret_file = '/srv/jupyterhub/jupyterhub_cookie_secret'