pipeline业务发布

业务环境介绍

公司当前业务上线流程首先是通过nginx灰度，dubbo-admin操作禁用，然后发布上线主机，发布成功后，dubbo-admin启用，nginx启用主机；之前是通过手动操作，很不方便，本次优化为pipeline方式实现自动发布，需要saltstack api支持。

pipeline发布流程图

准备工作

1. 将saltstack端dubbo.py脚本部署好，可通过salt直接调用

2. 将所有脚本分别放到对应的主机上面（saltstack端、jenkins端）

3. jenkins提前准备好 gitlab、nginx ssh、saltapi 相关凭据，mvn、jdk工具

4. jenkins配置

1. pipeline脚本

// 此pipeline用于nginx + dubbo结构的应用，如没有dubbo，删除dubbo相关步骤即可
// 提前定义：
//     gitlab、nginx ssh、saltapi 相关凭据
//     mvn，jdk等工具定义
//     saltapi模板中servername地址// 以下所有变量根据实际情况修改
// 本项目的svn代码地址
def svnUrl = 'https://svn****'
// nginx ssh参数，多个以逗号未分隔符
def nginxHosts = '172.87.10.31,172.87.10.41'
// zookeeper主机端口，多个以逗号未分隔符
def zkHostPort = '172.87.40.14:2181,172.87.40.24:2181'
// jenkins发布机脚本绝对路径
def scriptAbsPath = '/app/jenkins_deploy/scripts'
// saltstack 脚本基于salt base路径
def saltScriptPath = 'jenkins_deploy/scripts'
// FTP存储路径映射到saltstack的路径
def ftpSaltPath = 'jenkins_deploy/jenkins_war_packages'
// 项目包名称
def packageName = 'app.war'
// 目包相对路径（相对于$workspace)
def source_file = 'target/' + "${packageName}"
// 应用重启命令
def appRestartCommand = '/app/apache-tomcat-9.0.37/bin/restart.sh app'
// 应用启动用户
def appStartUser = 'app'
// 构建命令
def buildCommand = 'mvn clean package -P product war:war'// saltapi模板
def Salt(salthost, saltfunc, saltargs) {result = salt(authtype: 'pam', clientInterface: local( arguments: saltargs,function: saltfunc, target: salthost, targettype: 'list'),credentialsId: "saltapi",saveFile: true,servername: "http://172.87.10.21:8000")return result
}pipeline {// 执行任务agent any// 定义工具tools {maven 'maven352'jdk 'jdk1.8'}stages {stage("Clone") {steps {checkout([$class: 'SubversionSCM', additionalCredentials: [], excludedCommitMessages: '', excludedRegions: '', excludedRevprop: '', excludedUsers: '', filterChangelog: false, ignoreDirPropChanges: false, includedRegions: '', locations: [[credentialsId: 'jenkinsprd', depthOption: 'infinity', ignoreExternalsOption: true, local: '.', remote: "${svnUrl}"]], workspaceUpdater: [$class: 'UpdateUpdater']])}}// nginx 禁用upstream主机stage("Nginx disable host") {steps {Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"disabled ${targetHosts} ${nginx_config_file}\"")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"}}// nginx 重新加载配置stage("Nginx reload 1") {input {message "是否重启nginx？"ok "确定"parameters {string(name: "nginx", defaultValue: "${nginx_config_file}", description: '此文件有所改动，请谨慎操作！！！')}}steps {Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_reload.sh")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"}}// dubbo 禁用生产者stage("Dubbo disable") {steps {script {// 获取主机注册的所有dubbo服务Salt("${targetHosts}","dubbo.ls","${dubbo_port}")dubbo_all_service = sh(script: "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} dubbo.ls", returnStdout:true).trim()// dubbo禁用sh "/usr/bin/python3 ${scriptAbsPath}/dubbo.py disable ${zkHostPort} ${targetHosts} ${dubbo_port} ${dubbo_all_service}"}}}// mvn构建、代码发布到应用服务器stage("mvn build and deploy") {steps {// 构建命令sh "${buildCommand}"// 需要修改war包所在的路径sh "python3 ${scriptAbsPath}/ftp_upload.py ${WORKSPACE}/${source_file} ${source_file}"// /tmp/路径不能修改，这个路径要匹配restart.sh中配置的Salt("${targetHosts}","state.sls","""jenkins_deploy.scripts.sync_app_package pillar='{"package_name": "${packageName}"}'""")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} cp.get_file"// 项目包权限修改为项目启动用户Salt("${targetHosts}","file.chown", "/tmp/${packageName} ${appStartUser} ${appStartUser}")}}// 重启应用服务stage("restart app service") {steps {// 应用重启命令Salt("${targetHosts}","cmd.run","'${appRestartCommand}' env=\'{\"LC_ALL\": \"en_US.UTF-8\",\"LANG\": \"en_US.UTF-8\"}\' runas=${appStartUser}")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} cmd.run"}}// 测试目标主机是否恢复stage("Test") {steps {println("Test ...")sleep 5}}// dubbo启用生产者stage("Dubbo enable") {input {message "是否启用dubbo？"ok "确定"parameters {string(name: "dubbo", defaultValue: "${targetHosts}-${dubbo_port}", description: '请确定应用正常启动，谨慎操作！！！')}}steps {script {// 获取主机注册的所有dubbo服务Salt("${targetHosts}","dubbo.ls","${dubbo_port}")dubbo_all_service = sh(script: "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} dubbo.ls", returnStdout:true).trim()// dubbo启用sh "/usr/bin/python3 ${scriptAbsPath}/dubbo.py enable ${zkHostPort} ${targetHosts} ${dubbo_port} ${dubbo_all_service}"}}}// nginx 启用upstream主机stage("Nginx enable host") {steps {Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"enabled ${targetHosts} ${nginx_config_file}\"")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"}}// nginx 重新加载配置stage("Nginx reload 2") {input {message "是否重启nginx？"ok "确定"parameters {string(name: "nginx", defaultValue: "${nginx_config_file}", description: '此文件有所改动，请谨慎操作！！！')}}steps {Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_reload.sh")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"}}}post {success {script {buildDescription "上次构建成功的主机：${params.targetHosts}"}}failure {script {// nginx 启用upstream主机Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"enabled ${targetHosts} ${nginx_config_file}\"")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"buildDescription "构建失败，请查看错误！"}}aborted {script {// nginx 启用upstream主机Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"enabled ${targetHosts} ${nginx_config_file}\"")sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"buildDescription "手动取消构建！"}}}
}

2. saltstack server端主机上的脚本

• nginx_reload.sh

#!/bin/bashsource /etc/profilenginx -t && nginx -s reload && sleep 3 ; ps -ef | grep nginx: | grep -v grep

• nginx_upstream.sh

#!/bin/bash
# Filename   :  nginx_upstream.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  本脚本用于jenkins pipeline发布代码过程中的禁用启用nginx主机操作SWITCH="$1"
HOSTS="$2"
FILE_NAME="$3"usage(){echo "$0: [enable|enabled|disable|disabled] [hosts] [NGINX_UPSTREAM_CONFIG_FILE]"
}# 启用nginx主机
enabled(){hosts="$1"file_name="$2"IFS=$','for h in ${hosts}dosed -ri "s/.*($h)/    server \1/g" ${file_name}donecat ${file_name}
}# 禁用nginx主机
disabled(){hosts="$1"file_name="$2"IFS=$','for h in ${hosts}dosed -ri "s/.*($h)/#    server \1/g" ${file_name}donecat ${file_name}
}# 传递参数不为3，或者任意参数为空则退出脚本
if [ "$#" -ne 3 -o -z "$SWITCH" -o -z "$HOSTS" -o -z "$FILE_NAME" ];thenusageexit 1
ficase $SWITCH inenable|enabled)enabled $HOSTS $FILE_NAME ;;disable|disabled)disabled $HOSTS $FILE_NAME;;*)usageexit 1;;
esac

• sync_app_package.sls

{% set package_name = pillar["package_name"] %}sync_app_package:file.managed:- name: /tmp/{{ package_name }}- source: salt://jenkins_deploy/jenkins_war_packages/target/{{ package_name }}- user: zf- group: zf- mode: 644

• dubbo.py （此脚本放在saltstack base/_modules目录下面）

# -*- coding: utf-8 -*-
# Filename   :  nginx_upstream.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  此脚本用于saltstack自定义模块from os.path import join as p_join
import telnetlib
import re__virtualname__ = 'dubbo'
finish = 'dubbo>'def __virtual__():return __virtualname__def ls(port, host='127.0.0.1', timeout=5):'''获取应用注册到dubbo的所有服务的详细信息'''tn = telnetlib.Telnet(host=host, port=port, timeout=timeout)tn.write('ls -l\n')res = tn.read_until(finish).strip(finish)tn.close()res_l = []# 遍历分割成列表的结果for service in res.strip('\r\n').split('\r\n'):serviceName = re.split('\s+->', service)[0]pattern = re.compile('group=(.*?)\&')d = pattern.search(service)if d:res_l.append(p_join(d.group(1), serviceName))continueres_l.append(serviceName)return ','.join(res_l)

• test_url.sh

#!/bin/bash
# Filename   :  test_url.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  此脚本用于测试url是否可以访问URL="$1"
N=0
MAX_N="${2-9999}"usage(){if [ $1 -gt 2 ] || [ $1 -lt 1 ];thenecho "$0 [URL] [REQUEST_NUMBER]"exit 1fi
}Curl(){code=$(curl -s -o /dev/null -w '%{http_code}' $URL)while truedoif [ "$N" -ge "$MAX_N" ];thenecho "Test Fail."exit 1elseif [ "$code" == 200 ];thenecho "Test $URL success, Code: $code."exit 0fifisleep 1N=$((N+1))done
}usage $#
Curl

3. jenkins server端主机上面的脚本

• dubbo.py

# -*- coding: utf-8 -*-
# Filename   :  dubbo.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  此脚本操作dubbo-admin中服务的启用和禁用from check_port_connect import check_port
from zookeeper import Zk
from os import path
import urllib.parse
import sysclass Dubbo:def __init__(self, zk_host, dubbo_host, dubbo_port, dubbo_all_service, dubbo_path='/dubbo'):self.zk_host = zk_hostself.dubbo_host = dubbo_hostself.dubbo_port = dubbo_portself.dubbo_host_port = [host + ':%s' % self.dubbo_port for host in self.dubbo_host]self.dubbo_path = dubbo_pathself.zk = Zk(self.zk_host)self.all_node = dubbo_all_service# 写入zookeeper对应dubbo的服务配置模板self.template = 'override://%s/%s?category=configurators&disabled=true&dynamic=false&enabled=true'self._check_port()def _check_port(self):# 检测dubbo主机的端口连通性for host in self.dubbo_host:dubbo_port_test = check_port(host, self.dubbo_port)if dubbo_port_test != 0: sys.exit(1) def _format(self, zk_node, dubbo_host_port):# 分割服务名，group/service 根号分割的名称是有组的服务l = zk_node.split('/')# 大于1就是有组的服务if len(l) > 1:zk_node = l[1]override = self.template % (dubbo_host_port, l[1])override += '&group=' + l[0]else:zk_node = l[0]override = self.template % (dubbo_host_port, l[0])return zk_node, overridedef disable(self):# 循环所有node并禁用所有dubbo主机的服务for zk_node in self.all_node:for host_port in self.dubbo_host_port:zk_node, override = self._format(zk_node, host_port)d = urllib.parse.urlencode({'name': override}).split('name=')[1]self.zk.create('%s/%s/configurators/%s' % (self.dubbo_path, zk_node, d), b'[]')print('dubbo %s的服务已禁用，请查看WEB页面！' % ','.join(self.dubbo_host_port))self.zk.stop()def enable(self):# 循环所有node并禁用所有dubbo主机的服务for zk_node in self.all_node:for host_port in self.dubbo_host_port:zk_node, override = self._format(zk_node, host_port)d = urllib.parse.urlencode({'name': override}).split('name=')[1]self.zk.delete('%s/%s/configurators/%s' % (self.dubbo_path, zk_node, d))print('dubbo %s的服务已启用，请查看WEB页面！' % ','.join(self.dubbo_host_port))self.zk.stop()def servic_status(self):for host_port in self.dubbo_host_port:n = 0for zk_node in self.all_node:zk_node, override = self._format(zk_node, host_port)d = urllib.parse.urlencode({'name': override}).split('name=')[1]res = self.zk.ls('%s/%s/configurators' % (self.dubbo_path, zk_node))# 如果格式化后的值存在于列表中，即为禁用if d not in res:n += 1print('Host: %s, Total: %s, Enable: %s' % (host_port, len(self.all_node), n))if __name__ == '__main__':usage = 'usgae: %s [enable|disable|service_status] [zk_host:port,...] [dubbo_host] [dubbo_port] [dubbo_all_service]' % sys.argv[0]if len(sys.argv) != 6:sys.exit(usage)method = sys.argv[1] zk_host = sys.argv[2]dubbo_host = sys.argv[3].split(',')dubbo_port = sys.argv[4]dubbo_all_service = sys.argv[5].split(',')dubbo = Dubbo(zk_host=zk_host, dubbo_host=dubbo_host, dubbo_port=dubbo_port, dubbo_all_service=dubbo_all_service)if method == 'disable':dubbo.disable()elif method == 'enable':dubbo.enable()elif method == 'service_status':dubbo.servic_status()else:sys.exit(usage)

• zookeeper.py

# -*- coding: utf-8 -*-
# Filename   :  zookeeper.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  此脚本用于连接、操作zookeeperfrom kazoo.client import KazooClient
from kazoo.exceptions import NoNodeError
from check_port_connect import check_port
from sys import exitclass Zk:def __init__(self, hosts, timeout=10):self.hosts = hostsself.timeout = timeoutself._zk = KazooClient(hosts=self.hosts, timeout=self.timeout)self._zk.start()def ls(self, path='/'):'''获取Zk执行path中所有node:return: 所有node'''try:all_node = self._zk.get_children(path)return all_nodeexcept Exception as e:print(repr(e))exit(1) def create(self, path, data):'''Zk创建:return:'''try:res = self._zk.create(path, data, makepath=True)return resexcept Exception as e:print(repr(e))exit(1)def delete(self, path):'''Zk 删除节点:param path: 要删除的节点:return:'''try:res = self._zk.delete(path)return resexcept Exception as e:return repr(e)def stop(self):self._zk.stop()

• view_saltoutput.py

# -*- coding: utf-8 -*-
# Filename   :  nginx_upstream.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  此脚本用于查看saltstack api的输出结果，因为saltapi中定义了saveFile: trueimport json, sysdef outPutJson(file_path, hosts, module):'''解析saltapi执行结果输出的json文件切记，不同模块输出格式有所不同'''with open(file_path, 'r') as read_f:data = json.load(read_f)for host in hosts:try:if module == 'cmd.script':res = (host, data[0][host]['ret']['stdout'])elif module == 'cmd.run':res = (host + '(stdout)', data[0][host]['ret'])elif module == 'cp.get_file':res = (host, data[0][host])if not res[1]:sys.exit(res)elif module == 'dubbo.ls':res = data[0][host]print(res)breakelse:res = ''print('%s {\n%s\n}' % res)except KeyError as e:print('KeyError: %s' % e)sys.exit(1)if __name__ == '__main__':usage = "usage: %s [SALT_OUTPUT_FILE] [HOSTS] [MODULE]" % sys.argv[0]if len(sys.argv) != 4:sys.exit(usage) file_path = sys.argv[1]hosts = sys.argv[2].split(',')module = sys.argv[3]outPutJson(file_path, hosts, module)

• ftp_upload.py

# -*- coding: utf-8 -*-
# Filename   :  ftp_upload.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  此脚本用于操作ftpfrom ftplib import FTP
from os import path
import sysclass FTPClient:def __init__(self, host, user, passwd, port=21):self.host = hostself.user = userself.passwd = passwdself.port = portself._ftp = Noneself._login()def _login(self):'''登录FTP服务器:return: 连接或登录出现异常时返回错误信息'''try:self._ftp = FTP()self._ftp.connect(self.host, self.port, timeout=30)self._ftp.login(self.user, self.passwd)except Exception as e:print(str(e))sys.exit(1)def upload(self, localpath, remotepath=None):'''上传ftp文件:param localpath: local file path:param remotepath: remote file path:return:'''if not localpath: return 'Please select a local file. '# 读取本地文件fp = open(localpath, 'rb')# 如果未传递远程文件路径，则上传到当前目录，文件名称同本地文件if not remotepath:remotepath = path.basename(localpath)# 上传文件try:self._ftp.storbinary('STOR ' + remotepath, fp)except Exception as e:print(str(e))def nlst(self, dir='/'):'''查看目录下的内容:return: 以列表形式返回目录下的所有内容'''files_list = self._ftp.nlst(dir)return files_listdef del_file(self, filename=None):'''删除文件:param filename: 文件名称:return: 执行结果'''if not filename: return 'Please input filename'try:del_f = self._ftp.delete(filename)except Exception as e:print(str(e))sys.exit(1)def close(self):'''退出ftp连接:return:'''try:# 向服务器发送quit命令self._ftp.quit()except Exception:print('No response from server')sys.exit(1)finally:# 客户端单方面关闭连接self._ftp.close()if __name__ == '__main__':usage = "usage: %s [FILE_NAME] [TARGET_PATH]" % sys.argv[0]ftp_host = '172.85.10.31'ftp_user = 'jenkins_deploy'ftp_password = 'jenkins@deploy_2021'if len(sys.argv) != 3:sys.exit(usage)ftp = FTPClient(host=ftp_host, user=ftp_user, passwd=ftp_password)file_path = sys.argv[1]target_path = sys.argv[2]all_file = ftp.nlst('target')if file_path.startswith('/'):file_path = path.join('/', file_path)f_path = 'target/' + path.basename(file_path)if f_path in all_file: ftp.del_file(f_path)ftp.upload(file_path, target_path)ftp.close()

• check_port_connect.py

# -*- coding: utf-8 -*-
# Filename   :  check_port_connect.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  [e-mail]
# Description:  此脚本用于检测端口连通性import socket
import syssocket.setdefaulttimeout(10) def check_port(ip, port):port = int(port)sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)res = sk.connect_ex((ip, port))if res == 0:# print('%s %s port is Open' % (ip, port))return 0else:print('%s %s port is Not Open' % (ip, port))sys.exit(1) if __name__ == '__main__':usage = 'usage: %s [IP|HSOTNAME] [PORT]'if len(sys.argv) != 3:sys.exit(usage) ip = sys.argv[1]    port = sys.argv[2]check_port(ip, port)