f45ac0cb6e
- Added defaults/main.yml with production-ready values for all 27 Ansible roles - Added meta/main.yml with role dependencies (DAG: prereq → namespaces → storage → traefik → cert-manager → services) - Created 4 missing Helm templates: flink-deployment, kafka-cluster, smartapp-web, smartapp-api - Fixed YAML syntax error in backup/tasks/main.yml (Velero backupStorageLocation) - Updated README with domain list, dependencies diagram, and corrected Helm chart names - All 81 YAML files pass validation
296 lines
7.6 KiB
Django/Jinja
296 lines
7.6 KiB
Django/Jinja
---
|
|
# Role: kafka
|
|
# Template: kafka-cluster.yml.j2
|
|
# Cluster Kafka via Strimzi KafkaOperator
|
|
# Variables:
|
|
# {{ kafka_namespace }} - Namespace Kubernetes (défaut: kafka)
|
|
# {{ kafka_replicas }} - Nombre de brokers Kafka (défaut: 3)
|
|
# {{ kafka_storage_size }} - Taille du stockage par broker (défaut: 100Gi)
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: {{ kafka_namespace | default('kafka') }}
|
|
labels:
|
|
app: kafka
|
|
version: "3.6"
|
|
|
|
---
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: Kafka
|
|
metadata:
|
|
name: kafka-cluster
|
|
namespace: {{ kafka_namespace | default('kafka') }}
|
|
labels:
|
|
app: kafka
|
|
version: "3.6"
|
|
spec:
|
|
kafka:
|
|
version: 3.6.0
|
|
replicas: {{ kafka_replicas | default(3) }}
|
|
listeners:
|
|
- name: plain
|
|
port: 9092
|
|
type: internal
|
|
tls: false
|
|
- name: tls
|
|
port: 9093
|
|
type: internal
|
|
tls: true
|
|
- name: external
|
|
port: 9094
|
|
type: ingress
|
|
tls: true
|
|
configuration:
|
|
bootstrap:
|
|
host: kafka-bootstrap.digitribe.fr
|
|
brokers:
|
|
- broker: 0
|
|
host: kafka-broker-0.digitribe.fr
|
|
- broker: 1
|
|
host: kafka-broker-1.digitribe.fr
|
|
- broker: 2
|
|
host: kafka-broker-2.digitribe.fr
|
|
config:
|
|
offsets.topic.replication.factor: 3
|
|
transaction.state.log.replication.factor: 3
|
|
transaction.state.log.min.isr: 2
|
|
default.replication.factor: 3
|
|
min.insync.replicas: 2
|
|
inter.broker.protocol.version: "3.6"
|
|
log.message.format.version: "3.6"
|
|
storage:
|
|
type: jbod
|
|
volumes:
|
|
- id: 0
|
|
type: persistent-claim
|
|
size: {{ kafka_storage_size | default('100Gi') }}
|
|
class: standard
|
|
deleteClaim: false
|
|
resources:
|
|
requests:
|
|
cpu: "1"
|
|
memory: "2Gi"
|
|
limits:
|
|
cpu: "2"
|
|
memory: "4Gi"
|
|
livenessProbe:
|
|
initialDelaySeconds: 30
|
|
timeoutSeconds: 5
|
|
readinessProbe:
|
|
initialDelaySeconds: 10
|
|
timeoutSeconds: 5
|
|
metricsConfig:
|
|
type: jmxPrometheusExporter
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: kafka-metrics
|
|
key: kafka-metrics-config.yml
|
|
template:
|
|
pod:
|
|
affinity:
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchExpressions:
|
|
- key: strimzi.io/name
|
|
operator: In
|
|
values:
|
|
- kafka-cluster-kafka
|
|
topologyKey: kubernetes.io/hostname
|
|
|
|
zookeeper:
|
|
replicas: 3
|
|
storage:
|
|
type: persistent-claim
|
|
size: 20Gi
|
|
class: standard
|
|
deleteClaim: false
|
|
resources:
|
|
requests:
|
|
cpu: "500m"
|
|
memory: "1Gi"
|
|
limits:
|
|
cpu: "1"
|
|
memory: "2Gi"
|
|
livenessProbe:
|
|
initialDelaySeconds: 30
|
|
timeoutSeconds: 5
|
|
readinessProbe:
|
|
initialDelaySeconds: 10
|
|
timeoutSeconds: 5
|
|
|
|
entityOperator:
|
|
topicOperator:
|
|
resources:
|
|
requests:
|
|
cpu: "250m"
|
|
memory: "512Mi"
|
|
limits:
|
|
cpu: "500m"
|
|
memory: "1Gi"
|
|
userOperator:
|
|
resources:
|
|
requests:
|
|
cpu: "250m"
|
|
memory: "512Mi"
|
|
limits:
|
|
cpu: "500m"
|
|
memory: "1Gi"
|
|
|
|
kafkaExporter:
|
|
topicRegex: ".*"
|
|
groupRegex: ".*"
|
|
resources:
|
|
requests:
|
|
cpu: "200m"
|
|
memory: "256Mi"
|
|
limits:
|
|
cpu: "500m"
|
|
memory: "512Mi"
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: kafka-metrics
|
|
namespace: {{ kafka_namespace | default('kafka') }}
|
|
labels:
|
|
app: kafka
|
|
version: "3.6"
|
|
data:
|
|
kafka-metrics-config.yml: |
|
|
# See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
|
|
lowercaseOutputName: true
|
|
rules:
|
|
# Special cases and very specific rules
|
|
- pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
|
|
name: kafka_server_$1_$2
|
|
type: GAUGE
|
|
labels:
|
|
clientId: "$3"
|
|
topic: "$4"
|
|
partition: "$5"
|
|
- pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
|
|
name: kafka_server_$1_$2
|
|
type: GAUGE
|
|
labels:
|
|
clientId: "$3"
|
|
broker: "$4:$5"
|
|
# Generic per-second counters with 0-2 key/value pairs
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
|
|
name: kafka_$1_$2_$3_total
|
|
type: COUNTER
|
|
labels:
|
|
"$4": "$5"
|
|
"$6": "$7"
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
|
|
name: kafka_$1_$2_$3_total
|
|
type: COUNTER
|
|
labels:
|
|
"$4": "$5"
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
|
|
name: kafka_$1_$2_$3_total
|
|
type: COUNTER
|
|
# Generic gauges with 0-2 key/value pairs
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
|
|
name: kafka_$1_$2_$3
|
|
type: GAUGE
|
|
labels:
|
|
"$4": "$5"
|
|
"$6": "$7"
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
|
|
name: kafka_$1_$2_$3
|
|
type: GAUGE
|
|
labels:
|
|
"$4": "$5"
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
|
|
name: kafka_$1_$2_$3
|
|
type: GAUGE
|
|
# Emulate Prometheus 'Summary' metrics for the exported 'Histogram's
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
|
|
name: kafka_$1_$2_$3_count
|
|
type: COUNTER
|
|
labels:
|
|
"$4": "$5"
|
|
"$6": "$7"
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
|
|
name: kafka_$1_$2_$3
|
|
type: SUMMARY
|
|
labels:
|
|
"$4": "$5"
|
|
"$6": "$7"
|
|
quantile: 0.95
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
|
|
name: kafka_$1_$2_$3_count
|
|
type: COUNTER
|
|
labels:
|
|
"$4": "$5"
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
|
|
name: kafka_$1_$2_$3
|
|
type: SUMMARY
|
|
labels:
|
|
"$4": "$5"
|
|
quantile: 0.95
|
|
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
|
|
name: kafka_$1_$2_$3_count
|
|
type: COUNTER
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kafka-bootstrap
|
|
namespace: {{ kafka_namespace | default('kafka') }}
|
|
labels:
|
|
app: kafka
|
|
component: bootstrap
|
|
version: "3.6"
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
strimzi.io/cluster: kafka-cluster
|
|
strimzi.io/name: kafka-cluster-kafka
|
|
ports:
|
|
- name: tcp-internal
|
|
port: 9092
|
|
targetPort: 9092
|
|
protocol: TCP
|
|
- name: tcp-tls
|
|
port: 9093
|
|
targetPort: 9093
|
|
protocol: TCP
|
|
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: kafka-external
|
|
namespace: {{ kafka_namespace | default('kafka') }}
|
|
labels:
|
|
app: kafka
|
|
component: external
|
|
version: "3.6"
|
|
annotations:
|
|
nginx.ingress.kubernetes.io/ssl-redirect: "true"
|
|
nginx.ingress.kubernetes.io/backend-protocol: "TCP"
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
spec:
|
|
ingressClassName: nginx
|
|
tls:
|
|
- hosts:
|
|
- kafka-bootstrap.digitribe.fr
|
|
secretName: kafka-bootstrap-tls
|
|
rules:
|
|
- host: kafka-bootstrap.digitribe.fr
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: kafka-cluster-kafka-external-bootstrap
|
|
port:
|
|
number: 9094
|