17、ELK

17、ELK

helm 安装 elkfk（kafka 集群外可访问）

ES/Kibana <— Logstash <— Kafka <— Filebeat

部署顺序：

1、elasticsearch

2、kibana

3、kafka

4、logstash

5、filebeat

kubectl create ns elk

helm3部署elkfk

1、elasticsearch

helm repo add elastic https://helm.elastic.cohelm repo listhelm repo update

helm search repo elastic/elasticsearchcd && helm pull elastic/elasticsearch --untar --version 7.17.3cd elasticsearch

cat > values-prod.yaml << EOF
# 集群名称
clusterName: "elasticsearch"
# ElasticSearch 6.8+ 默认安装了 x-pack 插件，部分功能免费，这里选禁用
image: "docker.elastic.co/elasticsearch/elasticsearch"
imageTag: "7.17.3"
imagePullPolicy: "IfNotPresent"esConfig:elasticsearch.yml: |network.host: 0.0.0.0cluster.name: "elasticsearch"xpack.security.enabled: false
resources:limits:cpu: "2"memory: "4Gi"requests:cpu: "1"memory: "2Gi"
volumeClaimTemplate:storageClassName: "nfs-storage"accessModes: [ "ReadWriteOnce" ]resources:requests:storage: 2Ti
service:type: NodePortport: 9000nodePort: 31311
EOF

禁用 Kibana 安全提示（Elasticsearch built-in security features are not enabled）xpack.security.enabled: false

helm upgrade --install --namespace elk es -f ./values-prod.yaml .

验证

curl 192.168.1.200:31311/_cat/healthcurl 192.168.1.200:31311/_cat/nodes

2、kibana

helm search repo elastic/kibanacd && helm pull elastic/kibana --untar --version 7.17.3cd kibana

cat > values-prod.yaml << 'EOF'
kibanaConfig:kibana.yml: |server.port: 5601server.host: "0.0.0.0"elasticsearch.hosts: [ "http://elasticsearch-master-headless:9200" ]
resources:limits:cpu: "2"memory: "2Gi"requests:cpu: "1"memory: "1Gi"
kibanaConfig:kibana.yml: |i18n.locale: "zh-CN"
service:#type: ClusterIPtype: NodePortloadBalancerIP: ""port: 5601nodePort: "30026"
EOF

helm upgrade --install --namespace elk kibana -f ./values-prod.yaml .

cat > ~/kibana/kibana-Ingress.yml << 'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:name: kibana-ingressnamespace: elkannotations:nginx.ingress.kubernetes.io/ssl-redirect: 'true'nginx.ingress.kubernetes.io/proxy-body-size: '4G'nginx.ingress.kubernetes.io/auth-type: basicnginx.ingress.kubernetes.io/auth-secret: kibana-auth-secretnginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin'
spec:ingressClassName: nginxrules:- host: kibana.huanghuanhui.cloudhttp:paths:- path: /pathType: Prefixbackend:service:name: kibana-kibanaport:number: 5601tls:- hosts:- kibana.huanghuanhui.cloudsecretName: kibana-ingress-tls
EOF

yum -y install httpd-toolscd ~/kibana && htpasswd -bc auth admin Admin@2024kubectl create secret generic kibana-auth-secret --from-file=auth -n elk

kubectl create secret -n elk \
tls kibana-ingress-tls \
--key=/root/ssl/huanghuanhui.cloud.key \
--cert=/root/ssl/huanghuanhui.cloud.crt

kubectl apply -f ~/kibana/kibana-Ingress.yml 

访问地址：kibana.huanghuanhui.cloud

账号密码：admin、Admin@2024

http://192.168.1.201:30026/app/dev_tools#/consoleGET _cat/nodesGET _cat/healthGET _cat/indices

3、kafka（k8s部署kafka集群 ==》外部访问）

mkdir -p ~/kafka-yml && cd ~/kafka-yml

cat > ~/kafka-yml/zk.yml << 'EOF'
apiVersion: v1
kind: Service
metadata:labels:app: zookeeper-clusternamespace: elkname: zookeeper-cluster
spec:selector:app: zookeeper-clusterports:- name: clientport: 2181targetPort: 2181- name: followerport: 2888targetPort: 2888- name: leaderport: 3888targetPort: 3888clusterIP: None
---
apiVersion: v1
kind: Service
metadata:namespace: elkname: zookeeper-cs
spec:selector:app: zookeeper-clustertype: NodePortports:- name: clientport: 2181nodePort: 30152
---
apiVersion: apps/v1
kind: StatefulSet
metadata:namespace: elkname: crs-zookeeper
spec:replicas: 3podManagementPolicy: ParallelserviceName: zookeeper-clusterselector:matchLabels:app: zookeeper-clustertemplate:metadata:labels:component: zookeeper-clusterapp: zookeeper-clusterspec:containers:- name: zookeeperimage: bitnami/zookeeper:3.8.2imagePullPolicy: IfNotPresentsecurityContext:runAsUser: 0ports:- containerPort: 2181- containerPort: 2888- containerPort: 3888lifecycle:postStart:exec:command:- "sh"- "-c"- >echo $(( $(cat /etc/hosts | grep zookeeper | awk '{print($3)}' | awk '{split($0,array,"-")} END{print array[3]}') + 1 )) > /bitnami/zookeeper/data/myidenv:- name: ALLOW_ANONYMOUS_LOGINvalue: "yes"- name: ZOO_SERVERSvalue:  crs-zookeeper-0.zookeeper-cluster.elk.svc.cluster.local:2888:3888,crs-zookeeper-1.zookeeper-cluster.elk.svc.cluster.local:2888:3888,crs-zookeeper-2.zookeeper-cluster.elk.svc.cluster.local:2888:3888volumeMounts:- name: zoodata-outermountPath: /bitnami/zookeepervolumeClaimTemplates:- metadata:name: zoodata-outerspec:storageClassName: nfs-storageaccessModes:- "ReadWriteOnce"resources:requests:storage: 2Ti
EOF

kubectl apply -f ~/kafka-yml/zk.yml

cat > ~/kafka-yml/kafka.yml << 'EOF'
apiVersion: v1
kind: Service
metadata:namespace: elkname: kafka-headless
spec:selector:app: kafka-clusterports:- name: clientport: 9092targetPort: 9092clusterIP: None
---
apiVersion: v1
kind: Service
metadata:name: kafka-0namespace: elklabels:app: kafka-cluster
spec:ports:- port: 9092targetPort: 9092nodePort: 30127name: servertype: NodePortselector:statefulset.kubernetes.io/pod-name: crs-kafka-0
#    app: kafka-cluster---
apiVersion: v1
kind: Service
metadata:name: kafka-1namespace: elklabels:app: kafka-cluster
spec:ports:- port: 9092targetPort: 9092nodePort: 30128name: servertype: NodePortselector:statefulset.kubernetes.io/pod-name: crs-kafka-1---
apiVersion: v1
kind: Service
metadata:name: kafka-2namespace: elklabels:app: kafka-cluster
spec:ports:- port: 9092targetPort: 9092nodePort: 30129name: servertype: NodePortselector:statefulset.kubernetes.io/pod-name: crs-kafka-2
---
apiVersion: apps/v1
kind: StatefulSet
metadata:namespace: elkname: crs-kafka
spec:replicas: 3podManagementPolicy: ParallelserviceName: kafka-clusterselector:matchLabels:app: kafka-clustertemplate:metadata:labels:app: kafka-clusterspec:hostname: kafkacontainers:- name: kafkacommand:- bash- -ec- |HOSTNAME=`hostname -s`if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; thenORD=${BASH_REMATCH[2]}PORT=$((ORD + 30127))export KAFKA_CFG_ADVERTISED_LISTENERS="PLAINTEXT://192.168.1.200:$PORT"elseecho "Failed to get index from hostname $HOST"exit 1fiexec /entrypoint.sh /run.shimage: bitnami/kafka:3.5.1#        image: bitnami/kafka:latestimagePullPolicy: IfNotPresentsecurityContext:runAsUser: 0
#          resources:
#            requests:
#              memory: "1G"
#              cpu: "0.5"ports:- containerPort: 9092env:- name: KAFKA_CFG_ZOOKEEPER_CONNECTvalue: crs-zookeeper-0.zookeeper-cluster.elk.svc.cluster.local:2181,crs-zookeeper-1.zookeeper-cluster.elk.svc.cluster.local:2181,crs-zookeeper-2.zookeeper-cluster.elk.svc.cluster.local:2181#          value: zookeeper-cluster:2181- name: ALLOW_PLAINTEXT_LISTENERvalue: "yes"volumeMounts:- name: kafkadata-outermountPath: /bitnami/kafkavolumeClaimTemplates:- metadata:name: kafkadata-outerspec:storageClassName: nfs-storageaccessModes:- "ReadWriteOnce"resources:requests:storage: 2Ti
EOF

kubectl apply -f ~/kafka-yml/kafka.yml

注意修改yml文件98行里面的export的ip地址

这里修改为公网的ip：58.34.61.154（内网192.168.1.200）

kafka ui

docker pull provectuslabs/kafka-ui:latestdocker pull freakchicken/kafka-ui-lite

docker run -d \
--name kafka-ui1 \
--restart always \
--privileged=true \
-p 8888:8080 \
-e KAFKA_CLUSTERS_0_NAME=k8s-kafka \
-e KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=192.168.1.200:30127,192.168.1.200:30128,192.168.1.200:30129 \
provectuslabs/kafka-ui:latest

访问地址：192.168.1.200:8888

docker run -d \
--name kafka-ui2 \
--restart always \
--privileged=true \
-p 8889:8889 \
freakchicken/kafka-ui-lite

访问地址：192.168.1.200:8889

4、filebeat

1、k8s方式

helm search repo elastic/filebeatcd && helm pull elastic/filebeat --untar --version 7.17.3cd filebeat

cat > values-prod.yaml << 'EOF'
daemonset:filebeatConfig:filebeat.yml: |filebeat.inputs:- type: containerpaths:- /var/log/containers/*.logoutput.elasticsearch:enabled: falsehost: '${NODE_NAME}'hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'output.kafka:enabled: truehosts: ["192.168.1.200:30127","192.168.1.200:30128","192.168.1.200:30129"]topic: k8s-logs
EOF

helm upgrade --install --namespace elk filebeat -f ./values-prod.yaml .

2、docker方式

cat > filebeat.yml << 'EOF'
# 日志输入配置（可配置多个）
filebeat.inputs:
- type: logenabled: truepaths:- /mnt/nfs/logs/*/*.logtags: ["dev-c"]fields:server: dev-cfields_under_root: true
#日志输出配置
output.kafka:enabled: truehosts: ["192.168.1.200:30127","192.168.1.200:30128","192.168.1.200:30129"]topic: "dev-c"partition.round_robin:reachable_only: falserequired_acks: 1compression: gzipmax_message_bytes: 1000000
EOF

docker run -d --name filebeat \
--user=root \
--restart=always \
-v /mnt/nfs/logs/:/mnt/nfs/logs/ \
-v /root/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml \
-v /etc/localtime:/etc/localtime \
-v /etc/timezone:/etc/timezone \
elastic/filebeat:7.17.3 \

5、logstash

helm search repo elastic/logstashcd && helm pull elastic/logstash --untar --version 7.17.3cd logstash

cat > values-prod.yaml << 'EOF'
logstashConfig:logstash.yml: |xpack.monitoring.enabled: falselogstashPipeline: logstash.yml: |input {kafka {bootstrap_servers => "192.168.1.200:30127,192.168.1.200:30128,192.168.1.200:30129"topics => ["k8s-logs"]#group_id => "mygroup"#如果使用元数据就不能使用下面的byte字节序列化，否则会报错#key_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"#value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"consumer_threads => 1#默认为false，只有为true的时候才会获取到元数据decorate_events => trueauto_offset_reset => "earliest"}}filter {mutate {#从kafka的key中获取数据并按照逗号切割split => ["[@metadata][kafka][key]", ","]add_field => {#将切割后的第一位数据放入自定义的“index”字段中"index" => "%{[@metadata][kafka][key][0]}"}}}output { elasticsearch {pool_max => 1000pool_max_per_route => 200hosts => ["elasticsearch-master-headless.elk.svc.cluster.local:9200"]index => "k8s-logs-%{+YYYY.MM.dd}"}}# 资源限制
resources:requests:cpu: "100m"memory: "256Mi"limits:cpu: "1000m"memory: "1Gi"persistence:enabled: truevolumeClaimTemplate:accessModes: ["ReadWriteOnce"]storageClassName: nfs-storageresources:requests:storage: 2Ti
EOF

helm upgrade --install --namespace elk logstash -f ./values-prod.yaml .

手撕yml

mkdir -p ~/logstash-yml && cd ~/logstash-yml
cat > logstash.yaml << 'EOF'
apiVersion: v1
kind: ConfigMap
metadata:name: logstash-dev-configmapnamespace: elk
data:logstash.yml: |http.host: "0.0.0.0"path.config: /usr/share/logstash/pipelinelogstash.conf: |input {kafka {bootstrap_servers => "192.168.1.200:30127,192.168.1.200:30128,192.168.1.200:30129"topics => ["dev"]codec => "json"type => "dev"group_id => "dev"consumer_threads => 1}}filter {if [type] == "dev" {json {source => ["message"]remove_field => ["offset","host","beat","@version","event","agent","ecs"]}mutate {add_field => {project_path => "%{[log][file][path]}"}}mutate {split => ["project_path", "/"]add_field => {"project_name" => "%{[project_path][-3]}"}}date {match => ["time","yyyy-MM-dd HH:mm:ss.SSS"]timezone => "Asia/Shanghai"target => "@timestamp"}mutate {remove_field => ["log","project_path","time","input"]}}}output {elasticsearch {hosts => ["elasticsearch-master-headless.elk.svc.cluster.local:9200"]index => "dev-%{+YYYY.MM.dd}"}}
---
apiVersion: apps/v1
kind: Deployment
metadata:name: logstash-devnamespace: elk
spec:selector:matchLabels:app: logstash-devreplicas: 1template:metadata:labels:app: logstash-devspec:containers:- name: logstash-devimage: docker.elastic.co/logstash/logstash:7.17.3ports:- containerPort: 5044volumeMounts:- name: logstash-pipeline-volumemountPath: /usr/share/logstash/pipeline- mountPath: /etc/localtimename: localtimevolumes:- name: logstash-pipeline-volumeconfigMap:name: logstash-dev-configmapitems:- key: logstash.confpath: logstash.conf- hostPath:path: /etc/localtimename: localtime
---
kind: Service
apiVersion: v1
metadata:name: logstash-devnamespace: elk
spec:selector:app: logstashtype: ClusterIPports:- protocol: TCPport: 5044targetPort: 5044
EOF

kubectl apply -f logstash.yaml