feat: Add Redpanda Console, Pulsar Distribution Service, and Grafana Dashboards
- Add Redpanda Console service (port 28080, Traefik integration) - Add Pulsar Distribution Service (Pulsar -> Brokers) - Create Grafana dashboards for Redpanda, Pulsar, and Smart City Ingestion - Configure Prometheus targets for Pulsar and Redpanda metrics - Fix FROST URL in distribution service - Create session resume for 2026-05-05
This commit is contained in:
Eric FELIXINE
118
grafana/provisioning/dashboards/pulsar-metrics.json
Normal file
118
grafana/provisioning/dashboards/pulsar-metrics.json
Normal file
@@ -0,0 +1,118 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"title": "Pulsar Overview",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"title": "JVM Memory Used",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "jvm_memory_used_bytes{area=\"heap\"}",
|
||||
"legendFormat": "Heap Memory"
|
||||
},
|
||||
{
|
||||
"expr": "jvm_memory_used_bytes{area=\"nonheap\"}",
|
||||
"legendFormat": "Non-Heap Memory"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 1}
|
||||
},
|
||||
{
|
||||
"title": "JVM GC Collection Seconds",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(jvm_gc_collection_seconds_sum[1m])",
|
||||
"legendFormat": "{{gc}} GC Rate"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 1}
|
||||
},
|
||||
{
|
||||
"title": "Pulsar Message Rates",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 9}
|
||||
},
|
||||
{
|
||||
"title": "Messages In/Sec",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(pulsar_in_bytes_total[1m])",
|
||||
"legendFormat": "In Bytes/sec"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 10}
|
||||
},
|
||||
{
|
||||
"title": "Messages Out/Sec",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(pulsar_out_bytes_total[1m])",
|
||||
"legendFormat": "Out Bytes/sec"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 10}
|
||||
},
|
||||
{
|
||||
"title": "Pulsar Topics",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(pulsar_topics_count)",
|
||||
"legendFormat": "Active Topics"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 18}
|
||||
},
|
||||
{
|
||||
"title": "Subscriptions",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(pulsar_subscriptions_count)",
|
||||
"legendFormat": "Active Subscriptions"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 18}
|
||||
},
|
||||
{
|
||||
"title": "Producers",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(pulsar_producers_count)",
|
||||
"legendFormat": "Active Producers"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 18}
|
||||
}
|
||||
],
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["pulsar", "smart-city"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"title": "Pulsar Metrics",
|
||||
"uid": "pulsar-metrics",
|
||||
"version": 1
|
||||
}
|
||||
102
grafana/provisioning/dashboards/redpanda-metrics.json
Normal file
102
grafana/provisioning/dashboards/redpanda-metrics.json
Normal file
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"title": "Redpanda Overview",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"title": "Kafka API Requests",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(redpanda_kafka_requests_total[1m])",
|
||||
"legendFormat": "{{method}} - {{topic}}"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 1}
|
||||
},
|
||||
{
|
||||
"title": "Under-Replicated Partitions",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "redpanda_cluster_under_replicated_partitions",
|
||||
"legendFormat": "Under-Replicated"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 1}
|
||||
},
|
||||
{
|
||||
"title": "Producer Latency (p99)",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, rate(redpanda_kafka_produce_latency_seconds_bucket[5m]))",
|
||||
"legendFormat": "p99 Latency"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 9}
|
||||
},
|
||||
{
|
||||
"title": "Consumer Fetch Latency (p99)",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, rate(redpanda_kafka_fetch_latency_seconds_bucket[5m]))",
|
||||
"legendFormat": "p99 Latency"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 9}
|
||||
},
|
||||
{
|
||||
"title": "Redpanda Resource Usage",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 17}
|
||||
},
|
||||
{
|
||||
"title": "Memory Usage",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "redpanda_memory_allocated_bytes",
|
||||
"legendFormat": "Allocated (bytes)"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 18}
|
||||
},
|
||||
{
|
||||
"title": "CPU Usage",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(redpanda_cpu_busy_seconds_total[1m])",
|
||||
"legendFormat": "CPU Busy Rate"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 18}
|
||||
}
|
||||
],
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["redpanda", "kafka", "smart-city"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"title": "Redpanda Metrics",
|
||||
"uid": "redpanda-metrics",
|
||||
"version": 1
|
||||
}
|
||||
103
grafana/provisioning/dashboards/smart-city-ingeston.json
Normal file
103
grafana/provisioning/dashboards/smart-city-ingeston.json
Normal file
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"title": "Smart City Data Ingeston",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"title": "Messages/sec by Type",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "influxdb", "uid": "InfluxDB-Simulator"},
|
||||
"targets": [
|
||||
{
|
||||
"query": "from(bucket: \"iot_data\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"sensor_data\") |> group(columns: [\"type\"]) |> count()",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 1}
|
||||
},
|
||||
{
|
||||
"title": "Temperature (Weather)",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "influxdb", "uid": "InfluxDB-Simulator"},
|
||||
"targets": [
|
||||
{
|
||||
"query": "from(bucket: \"iot_data\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"sensor_data\" and r[\"type\"] == \"weather\") |> filter(fn: (r) => r[\"_field\"] == \"temperature_celsius\") |> mean()",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 1}
|
||||
},
|
||||
{
|
||||
"title": "Air Quality (PM2.5)",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "influxdb", "uid": "InfluxDB-Simulator"},
|
||||
"targets": [
|
||||
{
|
||||
"query": "from(bucket: \"iot_data\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"sensor_data\" and r[\"type\"] == \"airquality\") |> filter(fn: (r) => r[\"_field\"] == \"pm25_ugm3\") |> mean()",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 9}
|
||||
},
|
||||
{
|
||||
"title": "Traffic Count",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "influxdb", "uid": "InfluxDB-Simulator"},
|
||||
"targets": [
|
||||
{
|
||||
"query": "from(bucket: \"iot_data\") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r[\"_measurement\"] == \"sensor_data\" and r[\"type\"] == \"traffic\") |> filter(fn: (r) => r[\"_field\"] == \"vehicle_count\") |> mean()",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 9}
|
||||
},
|
||||
{
|
||||
"title": "Pulsar Message Rates",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 17}
|
||||
},
|
||||
{
|
||||
"title": "Pulsar In/Out Bytes/sec",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(pulsar_in_bytes_total[1m])",
|
||||
"legendFormat": "In Bytes/sec"
|
||||
},
|
||||
{
|
||||
"expr": "rate(pulsar_out_bytes_total[1m])",
|
||||
"legendFormat": "Out Bytes/sec"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 18}
|
||||
}
|
||||
],
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["smart-city", "influxdb", "pulsar"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"title": "Smart City - Data Ingeston",
|
||||
"uid": "smart-city-ingeston",
|
||||
"version": 1
|
||||
}
|
||||
Reference in New Issue
Block a user