k8s部署kafka集群

k8s部署kafka集群

kafka（Kafka with KRaft）

mkdir -p ~/kafka-ymlkubectl create ns kafka

cat > ~/kafka-yml/kafka.yml << 'EOF'
apiVersion: v1
kind: Service
metadata:name: kafka-headlessnamespace: kafkalabels:app: kafka
spec:type: ClusterIPclusterIP: Noneports:- name: kafka-clientport: 9092targetPort: kafka-client- name: controllerport: 9093targetPort: controller   selector:app: kafka
---
#部署 Service，用于外部访问 Kafka
apiVersion: v1
kind: Service
metadata:name: kafka-servicenamespace: kafkalabels:app: kafka
spec:type: NodePortports:- name: kafka-clientport: 9092targetPort: kafka-clientnodePort: 30992selector:app: kafka
---
# 分别在 StatefulSet 中的每个 Pod 中获取相应的序号作为 KAFKA_CFG_NODE_ID（只能是整数），然后再执行启动脚本
apiVersion: v1
kind: ConfigMap
metadata:name: ldc-kafka-scriptsnamespace: kafka
data:setup.sh: |-#!/bin/bashexport KAFKA_CFG_NODE_ID=${MY_POD_NAME##*-} exec /opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh
---
apiVersion: apps/v1
kind: StatefulSet
metadata:name: kafkanamespace: kafkalabels:app: kafka
spec:selector:matchLabels:app: kafkaserviceName: kafka-headlesspodManagementPolicy: Parallelreplicas: 3 # 部署完成后，将会创建 3 个 Kafka 副本updateStrategy:type: RollingUpdatetemplate:metadata:labels:app: kafkaspec:affinity:podAntiAffinity: # 工作负载反亲和preferredDuringSchedulingIgnoredDuringExecution: # 尽量满足如下条件- weight: 1podAffinityTerm:labelSelector: # 选择Pod的标签，与工作负载本身反亲和matchExpressions:- key: "app"operator: Invalues:- kafkatopologyKey: "kubernetes.io/hostname"  # 在节点上起作用containers:- name: kafka#image: bitnami/kafka:3.4.1#image: bitnami/kafka:3.7.0image: ccr.ccs.tencentyun.com/huanghuanhui/bitnami-kafka:3.7.0imagePullPolicy: "IfNotPresent"command:- /opt/leaderchain/setup.shenv:- name: BITNAMI_DEBUGvalue: "true" # true 详细日志# KRaft settings - name: MY_POD_NAME # 用于生成 KAFKA_CFG_NODE_IDvalueFrom:fieldRef:fieldPath: metadata.name            - name: KAFKA_CFG_PROCESS_ROLESvalue: "controller,broker"- name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERSvalue: "[e-mail]-headless:9093,[e-mail]-headless:9093,[e-mail]-headless:9093"- name: KAFKA_KRAFT_CLUSTER_IDvalue: "Jc7hwCMorEyPprSI1Iw4sW"  # Listeners            - name: KAFKA_CFG_LISTENERSvalue: "PLAINTEXT://:9092,CONTROLLER://:9093"- name: KAFKA_CFG_ADVERTISED_LISTENERSvalue: "PLAINTEXT://:9092"- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAPvalue: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"- name: KAFKA_CFG_CONTROLLER_LISTENER_NAMESvalue: "CONTROLLER"- name: KAFKA_CFG_INTER_BROKER_LISTENER_NAMEvalue: "PLAINTEXT"ports:- containerPort: 9092name: kafka-client                  - containerPort: 9093name: controllerprotocol: TCP                     volumeMounts:- mountPath: /bitnami/kafkaname: kafka-data- mountPath: /opt/leaderchain/setup.shname: scriptssubPath: setup.shreadOnly: true      securityContext:fsGroup: 1001runAsUser: 1001volumes:    - configMap:defaultMode: 493name: ldc-kafka-scriptsname: scripts      volumeClaimTemplates:- metadata:name: kafka-dataspec:storageClassName: nfs-storageaccessModes: [ReadWriteOnce]resources:requests:storage: 2Ti
EOF

kubectl apply -f ~/kafka-yml/kafka.yml

kafka-ui

cat > ~/kafka-yml/kafka-ui.yml << 'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:name: kafka-uinamespace: kafkalabels:app: kafka-ui
spec:replicas: 1selector:matchLabels:app: kafka-uitemplate:metadata:labels:app: kafka-uispec:containers:- name: kafka-ui#image: provectuslabs/kafka-ui:v0.7.2image: ccr.ccs.tencentyun.com/huanghuanhui/kafka-ui:v0.7.2imagePullPolicy: IfNotPresentenv:- name: KAFKA_CLUSTERS_0_NAMEvalue: 'kafka-elk'- name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERSvalue: 'kafka-headless:9092'- name: DYNAMIC_CONFIG_ENABLEDvalue: "true"- name: AUTH_TYPE # https://docs.kafka-ui.provectus.io/configuration/authentication/basic-authenticationvalue: "LOGIN_FORM"- name: SPRING_SECURITY_USER_NAMEvalue: "admin"    - name: SPRING_SECURITY_USER_PASSWORDvalue: "Admin@2024"ports:- name: webcontainerPort: 8080
---
apiVersion: v1
kind: Service
metadata:name: kafka-uinamespace: kafka
spec:selector:app: kafka-uitype: NodePortports:- name: webport: 8080targetPort: 8080nodePort: 30088
EOF

kubectl apply -f ~/kafka-yml/kafka-ui.yml

cat > ~/kafka-yml/kafka-ui-Ingress.yml << 'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:name: kafka-ui-ingressnamespace: kafkaannotations:nginx.ingress.kubernetes.io/ssl-redirect: 'true'nginx.ingress.kubernetes.io/proxy-body-size: '4G'
spec:ingressClassName: nginxrules:- host: kafka-ui.openhhh.comhttp:paths:- path: /pathType: Prefixbackend:service:name: kafka-uiport:number: 8080tls:- hosts:- kafka-ui.openhhh.comsecretName: kafka-ui-ingress-tls
EOF

kubectl create secret -n kafka \
tls kafka-ui-ingress-tls \
--key=/root/ssl/openhhh.com.key \
--cert=/root/ssl/openhhh.com.pem

kubectl apply -f ~/kafka-yml/kafka-ui-Ingress.yml

访问地址：https://kafka-ui.openhhh.com

账号密码：admin、Admin@2024