fix: JupyterHub DB path + user eric + OR mbtiles bounds + Hermes Dashboard
- Fix JupyterHub: sqlite db_url absolute path (was double-nested /srv/jupyterhub/srv/jupyterhub) - Create user eric as admin in JupyterHub (id=2, authorized) - Fix OpenRemote mbtiles: bounds metadata = world (-180,-85,180,85) for free zoom - OR map API confirmed working: center=[-61,14.5], minZoom=0, bounds=Martinique - Add Hermes Dashboard WebUI + TUI chat service (localhost:9119, auto-start at boot) - Add generate_martinique_mbtiles.py script (future tile generation) Known issues: - JupyterHub spawn timeout (singleuser server slow to start, increased to 120s) - OR mbtiles still contains Netherlands vector tiles (need Martinique tiles) - Kafka, Trino still in restart loop (separate fix needed)
This commit is contained in:
Eric FELIXINE
188
scripts/generate_martinique_mbtiles.py
Normal file
188
scripts/generate_martinique_mbtiles.py
Normal file
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate mbtiles for Martinique from OpenStreetMap tiles.
|
||||
|
||||
Downloads raster tiles for Martinique area (zoom 0-14)
|
||||
and packages them into an mbtiles file compatible with OpenRemote.
|
||||
|
||||
Usage: python3 generate_martinique_mbtiles.py
|
||||
|
||||
Requirements: pip install mbutil requests
|
||||
Or: pip install sqlite3 (stdlib) + requests
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import math
|
||||
import requests
|
||||
import gzip
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
# Martinique bounds
|
||||
MIN_LON, MIN_LAT = -61.5, 14.0
|
||||
MAX_LON, MAX_LAT = -60.5, 15.0
|
||||
|
||||
# Tile range
|
||||
MIN_ZOOM = 0
|
||||
MAX_ZOOM = 14
|
||||
|
||||
# Output file
|
||||
OUTPUT = "/tmp/mapdata_martinique.mbtiles"
|
||||
|
||||
# Rate limiting for OSM tile usage policy
|
||||
TILE_DELAY = 0.1 # 100ms between requests
|
||||
OSM_TILE_URL = "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
|
||||
USER_AGENT = "SmartCityMartinique-MapGenerator/1.0"
|
||||
|
||||
def deg2num(lat_deg, lon_deg, zoom):
|
||||
"""Convert lat/lon to tile numbers at given zoom level."""
|
||||
lat_rad = math.radians(lat_deg)
|
||||
n = 1 << zoom
|
||||
xtile = int((lon_deg + 180.0) / 360.0 * n)
|
||||
ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
|
||||
return (xtile, ytile)
|
||||
|
||||
def num2deg(xtile, ytile, zoom):
|
||||
"""Convert tile numbers to lat/lon (top-left of tile)."""
|
||||
n = 1 << zoom
|
||||
lon_deg = xtile / n * 360.0 - 180.0
|
||||
lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * ytile / n)))
|
||||
lat_deg = math.degrees(lat_rad)
|
||||
return (lat_deg, lon_deg)
|
||||
|
||||
def download_tile(z, x, y):
|
||||
"""Download a single tile from OSM."""
|
||||
url = OSM_TILE_URL.format(z=z, x=x, y=y)
|
||||
try:
|
||||
resp = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
return (z, x, y, resp.content)
|
||||
elif resp.status_code == 404:
|
||||
return None # No tile here (ocean)
|
||||
else:
|
||||
print(f" HTTP {resp.status_code} for {z}/{x}/{y}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" Error {z}/{x}/{y}: {e}")
|
||||
return None
|
||||
|
||||
def create_mbtiles():
|
||||
"""Create the mbtiles database."""
|
||||
if os.path.exists(OUTPUT):
|
||||
os.remove(OUTPUT)
|
||||
|
||||
conn = sqlite3.connect(OUTPUT)
|
||||
c = conn.cursor()
|
||||
|
||||
# Create tables
|
||||
c.execute("""CREATE TABLE map (
|
||||
zoom_level INTEGER,
|
||||
tile_column INTEGER,
|
||||
tile_row INTEGER,
|
||||
tile_id TEXT,
|
||||
grid_id TEXT
|
||||
)""")
|
||||
c.execute("""CREATE TABLE images (
|
||||
tile_data BLOB,
|
||||
tile_id TEXT
|
||||
)""")
|
||||
c.execute("""CREATE TABLE metadata (
|
||||
name TEXT,
|
||||
value TEXT
|
||||
)""")
|
||||
|
||||
# Insert metadata
|
||||
metadata = [
|
||||
("name", "Martinique OSM"),
|
||||
("type", "overlay"),
|
||||
("version", "1.0"),
|
||||
("description", "OpenStreetMap raster tiles for Martinique"),
|
||||
("format", "png"),
|
||||
("tilejson", "2.1.0"),
|
||||
("scheme", "xyz"),
|
||||
("minzoom", str(MIN_ZOOM)),
|
||||
("maxzoom", str(MAX_ZOOM)),
|
||||
("bounds", f"{MIN_LON},{MIN_LAT},{MAX_LON},{MAX_LAT}"),
|
||||
("center", f"{(MIN_LON+MAX_LON)/2},{(MIN_LAT+MAX_LAT)/2},10"),
|
||||
("attribution", "© <a href='https://www.openstreetmap.org/copyright'>OpenStreetMap</a> contributors"),
|
||||
]
|
||||
c.executemany("INSERT INTO metadata VALUES (?, ?)", metadata)
|
||||
|
||||
return conn, c
|
||||
|
||||
def main():
|
||||
print(f"Generating mbtiles for Martinique...")
|
||||
print(f"Bounds: {MIN_LON},{MIN_LAT} → {MAX_LON},{MAX_LAT}")
|
||||
print(f"Zoom: {MIN_ZOOM}-{MAX_ZOOM}")
|
||||
print(f"Output: {OUTPUT}")
|
||||
|
||||
conn, c = create_mbtiles()
|
||||
|
||||
# Calculate total tiles
|
||||
total_tiles = 0
|
||||
tiles_to_download = []
|
||||
for z in range(MIN_ZOOM, MAX_ZOOM + 1):
|
||||
x1, y1 = deg2num(MIN_LAT, MIN_LON, z)
|
||||
x2, y2 = deg2num(MAX_LAT, MAX_LON, z)
|
||||
x_min, x_max = min(x1, x2), max(x1, x2)
|
||||
y_min, y_max = min(y1, y2), max(y1, y2)
|
||||
|
||||
# Add buffer of 1 tile
|
||||
x_min = max(0, x_min - 1)
|
||||
x_max = min((1 << z) - 1, x_max + 1)
|
||||
y_min = max(0, y_min - 1)
|
||||
y_max = min((1 << z) - 1, y_max + 1)
|
||||
|
||||
for x in range(x_min, x_max + 1):
|
||||
for y in range(y_min, y_max + 1):
|
||||
tiles_to_download.append((z, x, y))
|
||||
total_tiles += 1
|
||||
|
||||
print(f"Total tiles to download: {total_tiles}")
|
||||
|
||||
# Download tiles (single-threaded for rate limiting)
|
||||
downloaded = 0
|
||||
failed = 0
|
||||
start = time.time()
|
||||
|
||||
for i, (z, x, y) in enumerate(tiles_to_download):
|
||||
result = download_tile(z, x, y)
|
||||
if result:
|
||||
_, _, _, data = result
|
||||
tile_id = f"{z}_{x}_{y}"
|
||||
TMS_y = (1 << z) - 1 - y # Convert XYZ to TMS
|
||||
|
||||
c.execute("INSERT INTO images VALUES (?, ?)", (data, tile_id))
|
||||
c.execute("INSERT INTO map VALUES (?, ?, ?, ?, ?)", (z, x, TMS_y, tile_id, None))
|
||||
downloaded += 1
|
||||
else:
|
||||
# Store empty tile record
|
||||
tile_id = f"{z}_{x}_{y}"
|
||||
TMS_y = (1 << z) - 1 - y
|
||||
c.execute("INSERT INTO map VALUES (?, ?, ?, ?, ?)", (z, x, TMS_y, tile_id, None))
|
||||
failed += 1
|
||||
|
||||
if (i + 1) % 50 == 0:
|
||||
conn.commit()
|
||||
elapsed = time.time() - start
|
||||
rate = (i + 1) / elapsed
|
||||
remaining = (total_tiles - i - 1) / rate if rate > 0 else 0
|
||||
print(f" Progress: {i+1}/{total_tiles} ({downloaded} ok, {failed} skip) | {rate:.1f} tiles/s | ETA: {remaining:.0f}s")
|
||||
|
||||
time.sleep(TILE_DELAY)
|
||||
|
||||
conn.commit()
|
||||
elapsed = time.time() - start
|
||||
print(f"\nDone! {downloaded} tiles downloaded, {failed} skipped, in {elapsed:.0f}s")
|
||||
|
||||
# Create index
|
||||
c.execute("CREATE INDEX idx_map ON map (zoom_level, tile_column, tile_row)")
|
||||
conn.commit()
|
||||
|
||||
size_mb = os.path.getsize(OUTPUT) / (1024 * 1024)
|
||||
print(f"Output: {OUTPUT} ({size_mb:.1f} MB)")
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
68
session_resume_2026-06-01.md
Normal file
68
session_resume_2026-06-01.md
Normal file
@@ -0,0 +1,68 @@
|
||||
# Session Resume — 2026-06-01 (Reprise après crash)
|
||||
|
||||
## Objectif
|
||||
Reprendre la session précédente qui a planté. Commits, sauvegardes, état des lieux infrastructure.
|
||||
|
||||
## Actions réalisées
|
||||
|
||||
### 1. État des lieux infrastructure
|
||||
- **86 conteneurs** Docker au total
|
||||
- **82 UP**, **4 en restart loop**, **2 Exited**
|
||||
|
||||
### 2. Problèmes identifiés
|
||||
|
||||
| Conteneur | Statut | Problème | Solution |
|
||||
|-----------|--------|----------|----------|
|
||||
| kafka-1, kafka-2 | Restarting | `zookeeper.connect` manquant | Ajouter ZK conn string |
|
||||
| trino | Restarting (100) | `node.environment` null | Ajouter `node.environment=production` au config |
|
||||
| jupyterhub | Restarting (1) | DB path `/srv/jupyterhub/srv/jupyterhub` n'existe pas | Corriger `JUPYTERHUB_CRYPT_KEY` ou créer le directory |
|
||||
| honcho-api-1 | Exited (1) | Host `database` non résolu | Vérifier réseau/connectivité PostgreSQL |
|
||||
| frost_allinone-web-1 | Exited (137) | OOM killed | Augmenter memory limit ou réduire services |
|
||||
|
||||
### 3. Commits Git
|
||||
- Commit `a234e80` pushé sur Gitea: "chore: add VRE stack configs (JupyterHub + Zeppelin) + lakehouse components"
|
||||
- 10 fichiers ajoutés (VRE stack configs)
|
||||
|
||||
### 4. Services opérationnels (UP ✅)
|
||||
- **Traefik** — reverse proxy principal
|
||||
- **OpenRemote** (manager, keycloak, postgresql) — tous healthy
|
||||
- **Grafana** (smart-city-grafana) → http://localhost:3001
|
||||
- **InfluxDB** → http://localhost:8086
|
||||
- **Simulateur** (smart-city-simulator) + **Telegraf** (smart-city-telegraf)
|
||||
- **Mosquitto** + **BunkerM** (bunkerm-bunkerm-1)
|
||||
- **Contexus** (app unhealthy, postgres+redis healthy)
|
||||
- **ODK Central** (nginx+service+postgres) — tous UP
|
||||
- **MindsDB** (mindsdb+postgres+autoheal) — tous healthy
|
||||
- **MapStore** (proxy+app+postgres)
|
||||
- **GeoServer** (geoserver_stack-geoserver-1) healthy
|
||||
- **PostGIS** (postgis-smartcity) healthy
|
||||
- **EMQX** (emqx_emqx_1) UP
|
||||
- **Ditto** (policies+gateway+mongodb) UP
|
||||
- **ChirpStack** (4 conteneurs) UP
|
||||
- **FIWARE Orion** (orion+orionproxy+mongo) healthy
|
||||
- **Gitea** UP
|
||||
- **Stellio** (api-gateway) UP
|
||||
- **Node-RED** (digital-twin-nodered) healthy
|
||||
- **MinIO** healthy
|
||||
- **Superset** healthy
|
||||
- **Zeppelin** healthy
|
||||
- **Superset** healthy
|
||||
- **Gravitino** unhealthy (mais UP)
|
||||
- **Flink** (jobmanager+taskmanager) healthy
|
||||
- **Loki** + **Promtail** UP
|
||||
- **LocalAI** healthy
|
||||
- **PHPIPAM** UP
|
||||
- **Honcho** (deriver+prometheus+grafana) healthy
|
||||
|
||||
## Prochaine session
|
||||
- Corriger Kafka (zookeeper.connect)
|
||||
- Corriger Trino (node.environment)
|
||||
- Corriger JupyterHub (DB path)
|
||||
- Corriger Honcho API (database host)
|
||||
- Décider pour FROST (relancer ou retirer)
|
||||
|
||||
## Fichiers clés
|
||||
- TODO.md: `/home/eric/smart-city-digital-twin-martinique/TODO.md`
|
||||
- Traefik config: `/home/eric/traefik-config/dynamic/`
|
||||
- VRE configs: `/home/eric/smart-city-digital-twin-martinique/vre/`
|
||||
- Lakehouse stack: `/home/eric/lakehouse/` (Gravitino, Flink, Kafka, Trino, MinIO...)
|
||||
@@ -9,14 +9,15 @@ c.JupyterHub.authenticator_class = 'nativeauthenticator.NativeAuthenticator'
|
||||
c.Authenticator.admin_users = {'admin'}
|
||||
c.Authenticator.allow_all = True
|
||||
|
||||
# Spawner
|
||||
# Spawner - use DockerSpawner or simple with correct cmd
|
||||
c.JupyterHub.spawner_class = 'simple'
|
||||
c.Spawner.cmd = ['jupyterhub-singleuser']
|
||||
c.Spawner.default_url = '/lab'
|
||||
c.Spawner.http_timeout = 120
|
||||
c.Spawner.start_timeout = 120
|
||||
|
||||
# Database and cookies
|
||||
# Database and cookies - use absolute paths
|
||||
c.JupyterHub.cookie_secret_file = '/srv/jupyterhub/jupyterhub_cookie_secret'
|
||||
c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite'
|
||||
c.JupyterHub.db_url = 'sqlite:////srv/jupyterhub/jupyterhub.sqlite'
|
||||
|
||||
# Base URL
|
||||
c.JupyterHub.base_url = '/'
|
||||
|
||||
Reference in New Issue
Block a user