Jenkins 基于Kubernetes 弹性构建池

流程：

• 创建Jenkins Agent；

• 获取Jenkins Agent的参数；

• 渲染yaml模板；

• 调用K8s API在固定的NS中创建一个Pod；

• 运行Jenkins pipeline到agent；

创建Agent

import hudson.model.Node.Mode
import hudson.slaves.*
import jenkins.model.Jenkins// 创建agent  下面是执行器数量的定义和label
String agentName = "__AGENT_NAME__"    
String executorNum = "1"     
String agentLabel = "JenkinsPod"agent_node = new DumbSlave(agentName, "Jenkins pod", "/opt/jenkins", executorNum,            Mode.EXCLUSIVE,   agentLabel,                       new JNLPLauncher(),         RetentionStrategy.INSTANCE) 
Jenkins.instance.addNode(agent_node)//获取agent的配置
node = Jenkins.instance.getNode(agentName)
computer = node.computer
jenkinsUrl = Jenkins.instance.rootUrl.trim().replaceAll('/$', '') return  """{
\"jenkinsUrl\" : \"${jenkinsUrl}\", 
\"jenkinsHome\": \"${node.remoteFS.trim()}\",
\"computerUrl\": \"${computer.url.trim().replaceAll('/$', '') as String}\",
\"computerSecret\": \"${computer.jnlpMac.trim()}\"
}"""

这里返回的是json类型的字符串，这样在调试的时候非常的方便。

这些信息最后在k8s里面需要以环境变量的方式指定出来。

其实手动创建好，然后将这串密钥拿下来，然后去启动他。但是现在自动化的去创建了。并且还将认证信息拿出来了。

后面在添加动态节点的时候先添加agent，然后拿到它的参数，最后渲染为pod的yaml，再去创建pod。

删除Agent

当资源不再使用了，就可以将节点删除。

import jenkins.model.JenkinsString agentName = "__AGENT_NAME__"Jenkins.instance.nodes.each { node ->String nodeName = node.nameif (nodeName.equals(agentName)) {Jenkins.instance.removeNode(node)}
}

ScriptConsole

// ScriptConsole运行脚本
def RunScriptConsole(scriptContent, crumb){response = sh returnStdout: true, script: """curl -s -d "script=\$(cat ${scriptContent})" \--header "Jenkins-Crumb:${crumb}" \-X  POST http://admin:112374bd5c557010386b55bb85a777aded@192.168.1.200:8080/scriptText"""try {response = readJSON text: response - "Result: "} catch(e){println(e)}return response
}// 获取Crumb值
def GetCrumb(){response = sh returnStdout: true, script: """ curl -s -u admin:admin \--location \--request GET 'http://192.168.1.200:8080/crumbIssuer/api/json' """response = readJSON text: responsereturn  response.crumb
}

http://192.168.11.128:8080/crumbIssuer/api/json{"_class":"hudson.security.csrf.DefaultCrumbIssuer","crumb":"f72990e2bc09a468effa69ec41f1432844bc709ca5ad6130d600da24ad16672a","crumbRequestField":"Jenkins-Crumb"}

curl -s -d "script=$(cat create_agent.groovy)" \--header "Jenkins-Crumb:f72990e2bc09a468effa69ec41f1432844bc709ca5ad6130d600da24ad16672a" \-X  POST http://admin:115fcd2e2d30505df7a18f23763c1332a0@192.168.11.128:8080/scriptText

[root@jenkins ~]# curl -s -d "script=$(cat create_agent.groovy)"   --header "Jenkins-Crumb:f72990e2bc09a468effa69ec41f1432844bc709ca5ad6130d600da24ad16672a"   -X  POST http://admin:115fcd2e2d30505df7a18f23763c1332a0@192.168.11.128:8080/scriptTextResult: {
"jenkinsUrl" : "http://192.168.11.128:8080", 
"jenkinsHome": "/opt/jenkins",
"computerUrl": "computer/%5F%5FAGENT%5FNAME%5F%5F",
"computerSecret": "6473dea92efa350d744450b91f010cd01880e0686597b8d33ef0876e698127c7"
}

Pod Yaml模板

apiVersion: v1
kind: Pod
metadata:labels:app: __AGENT_NAME__name: __AGENT_NAME__namespace: __NAMESPACE__
spec:containers:- name: dindimage: 'docker:stable-dind'command:- dockerd- --host=unix:///var/run/docker.sock- --host=tcp://0.0.0.0:8000- --insecure-registry=192.168.1.200:8088securityContext:privileged: truevolumeMounts:- mountPath: /var/runname: docker-dir- image: agenttest:6 #jenkins/inbound-agent:4.10-3-jdk8name: __AGENT_NAME__imagePullPolicy: IfNotPresentresources:limits:cpu: 1000mmemory: 8Girequests:cpu: 500mmemory: 2Gienv:- name: JENKINS_URLvalue: __JENKINS_URL__- name: JENKINS_SECRETvalue: __JENKINS_SECRET__- name: JENKINS_AGENT_NAMEvalue: __AGENT_NAME__- name: JENKINS_AGENT_WORKDIRvalue: /home/jenkins/workspacevolumeMounts:- mountPath: /var/runname: docker-dirdnsPolicy: ClusterFirstrestartPolicy: Alwaysvolumes:- name: docker-diremptyDir: {}

这里有两个容器，一个是docker in docker，因为containerd里面没有这个文件了。

因为docker共享了目录，在另外一个容器目录里面也可以看到，那么在另外的容器里面就可以运行docker命令了。

Kubernetes API准备工作

1. 创建一个NS名称空间， 专用于运行Pod；

kubectl create ns jenkins

2. 创建一个secret关联serviceaccount和role，操作K8s API；

# 创建role
kubectl -n jenkins create role jenkinsadmin \
--verb=create,delete,update,list,get,patch \
--resource=pods# 创建服务账户
kubectl -n jenkins create serviceaccount jenkinsadmin# 创建角色绑定
kubectl -n jenkins create rolebinding jenkinsadmin --role=jenkinsadmin --serviceaccount=jenkins:jenkinsadmin# 创建secret
kubectl apply -f - <<EOF
apiVersion: v1
kind: Secret
metadata:name: jenkinsadmin-tokennamespace: jenkinsannotations:kubernetes.io/service-account.name: jenkinsadmin
type: kubernetes.io/service-account-token
EOFwhile ! kubectl -n jenkins describe secret jenkinsadmin-token | grep -E '^token' >/dev/null; doecho "waiting for token..." >&2sleep 1
done# 获取令牌
TOKEN=$(kubectl -n jenkins get secret jenkinsadmin-token -o jsonpath='{.data.token}' | base64 --decode)echo $TOKEN# 获取API
kubectl config view -o jsonpath='{"Cluster name\tServer\n"}{range .clusters[*]}{.name}{"\t"}{.cluster.server}{"\n"}{end}'# 使用令牌玩转 API
curl -X GET https://127.0.0.1:35659/api --header "Authorization: Bearer $TOKEN" --insecure

[root@192 ~]# curl -X GET https://127.0.0.1:6443/api --header "Authorization: Bearer $TOKEN" --insecure
{"kind": "APIVersions","versions": ["v1"],"serverAddressByClientCIDRs": [{"clientCIDR": "0.0.0.0/0","serverAddress": "192.168.11.134:6443"}]
}

将token 以secret text类型存储到Jenkins

POD API

//删除POD
def DeletePod(namespace, podName){withCredentials([string(credentialsId: 'f66733bf-ef35-402d-87d1-a79510387d2b', variable: 'CICDTOKEN')]) {sh """curl --location --request DELETE "https://192.168.1.200:6443/api/v1/namespaces/${namespace}/pods/${podName}" \--header "Authorization: Bearer ${CICDTOKEN}" \--insecure >/dev/null"""}
}// 创建POD
def CreatePod(namespace, podYaml){withCredentials([string(credentialsId: 'f66733bf-ef35-402d-87d1-a79510387d2b', variable: 'CICDTOKEN')]) {sh """curl --location --request POST "https://192.168.1.200:6443/api/v1/namespaces/${namespace}/pods" \--header 'Content-Type: application/yaml' \--header "Authorization: Bearer ${CICDTOKEN}" \--data "${podYaml}" --insecure >/dev/null"""}
}

curl --location --request POST "https://192.168.11.134:6443/api/v1/namespaces/jenkins/pods" \
--header 'Content-Type: application/yaml' \
--header "Authorization: Bearer ${TOKEN}" \
--data "${podYaml}" --insecure >/dev/nullcurl --location --request DELETE "https://192.168.11.134:6443/api/v1/namespaces/jenkins/pods/nginx" \
--header "Authorization: Bearer ${TOKEN}" \
--insecure >/dev/null

curl --location --request GET 'https://192.168.11.134:6443/api/v1/namespaces/jenkins/pods' \
--header 'Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjBqSWxfOExtSW1pVmpKVkFUVGFQbHp1ZE5TcFo0SjlmOUtjMWFveWtrZGMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJqZW5raW5zIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6ImplbmtpbnNhZG1pbi10b2tlbi1jbTRsNyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJqZW5raW5zYWRtaW4iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJjMDAwZTVhMC00Yzg0LTQyZWEtYmJiNy1mZmM5MDBhYzUyMjIiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6amVua2luczpqZW5raW5zYWRtaW4ifQ.A-InywzLZZfwx_4-Na9iDDRGsIH0Bjm9Xk92VxKsnP7hflEbOU631EU0eHnewgg0UqAD1kkhW39XupN13oTXrOjHrzWdm_UUMe2cBIYCZD19VIpNnRZ6bBLZ3LFHA2F0gXO14HaVZiUvSYBXgIfyHydetIFQFBPWFtU7091u5iB4pcw_gE-VzGjX6NDHj-81j6Ap2Qr0gIrNvrPVAOpPO9uCSg3PNCgvQMq2ZNrY2te1w7QxmeFPEpOIPiK6VbUkRjhQlHmawULZFol2k5Wwv9z0m1hPQcc2Nten1f1__GR39hUjryWXfltJ8OLpbKWK-AtfBkHx8VqbiKH1vQOrRg' \
--header 'Content-Type: text/plain' \
--data-raw 'abc'

这里请求方法由get换成了delete

自定义构建镜像

• https://helm.sh/zh/docs/intro/install/

• https://github.com/jenkinsci/docker-inbound-agent/blob/master/8/alpine/Dockerfile

• https://kubernetes.io/zh-cn/docs/tasks/tools/install-kubectl-linux/#install-kubectl-binary-with-curl-on-linux

FROM centos:7USER root
ADD tools/jdk-8u201-linux-x64.tar.gz /usr/local
ENV JAVA_HOME=/usr/local/jdk1.8.0_201
ENV M2_HOME=/usr/local/apache-maven-3.8.1COPY tools/agent.jar /usr/share/jenkins/agent.jar
COPY tools/jenkins-agent /usr/local/bin/jenkins-agent
COPY tools/apache-maven-3.8.1 /usr/local/apache-maven-3.8.1
COPY tools/helm /usr/bin/helm
COPY tools/kubectl /usr/bin/kubectlRUN echo "export PATH=$PATH:$JAVA_HOME/bin:$M2_HOME/bin" >> /etc/profile  && \source /etc/profile && \java -version && \chmod +x /usr/local/bin/jenkins-agent && \ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave && \yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo && \yum -y install docker-ce && \mkdir -p /home/jenkins/workspace && chmod 777 /home/jenkins/workspace && \yum -y install git && \chmod +x /usr/bin/helm && helm version && chmod +x /usr/bin/kubectlCOPY tools/daemon.json  /etc/docker/daemon.json
ENTRYPOINT ["/usr/local/bin/jenkins-agent"]

上面就是整个流程,k8s动态创建 销毁pod，在pod上面创建构建任务，构建完任务就将这个pod给删除了。