nginx平滑升级

1.平滑升级操作

1.1 备份安装目录下的nginx

cd /usr/local/nginx/sbin
mv nginx nginx.bak

1.2 复制objs目录下的nginx到当前sbin目录下

cp /opt/software/nginx/nginx-1.20.2/objs/nginx /usr/local/nginx/sbin/

1.3 发送信号user2给nginx老版本对应的进程

kill -user2 'more /usr/local/logs/nginx.pid'

1.4 发送信号quit给nginx老版本的进程

kill -quit 'more /usr/local/nginx/logs/nginx.pid.oldbin'

2. 源码解析

平滑升级中，在处理user2信号时，会将ngx_change_binary标识置为1，master进程在处理循环中，如果识别到ngx_change_binary为1，执行ngx_exec_new_binary，将监听套接字放到环境变量中，同时会重命令pid的文件名，便于下次终止老版本的进程，fork新进程，新进程执行新版本的逻辑。

2.1 如何处理信号

依托于ngx_signal_t数据结构

2.1.1 ngx_signal_t

typedef struct {int     signo;char   *signame;char   *name;void  (*handler)(int signo, siginfo_t *siginfo, void *ucontext);
} ngx_signal_t;

全局变量signals是ngx_signal_t的数组，记录了需要处理的信号，信号名以及对应的信号处理函数

2.1.2 singals

ngx_signal_t  signals[] = {{ ngx_signal_value(NGX_RECONFIGURE_SIGNAL),"SIG" ngx_value(NGX_RECONFIGURE_SIGNAL),"reload",ngx_signal_handler },{ ngx_signal_value(NGX_REOPEN_SIGNAL),"SIG" ngx_value(NGX_REOPEN_SIGNAL),"reopen",ngx_signal_handler },{ ngx_signal_value(NGX_NOACCEPT_SIGNAL),"SIG" ngx_value(NGX_NOACCEPT_SIGNAL),"",ngx_signal_handler },{ ngx_signal_value(NGX_TERMINATE_SIGNAL),"SIG" ngx_value(NGX_TERMINATE_SIGNAL),"stop",ngx_signal_handler },{ ngx_signal_value(NGX_SHUTDOWN_SIGNAL),"SIG" ngx_value(NGX_SHUTDOWN_SIGNAL),"quit",ngx_signal_handler },{ ngx_signal_value(NGX_CHANGEBIN_SIGNAL),"SIG" ngx_value(NGX_CHANGEBIN_SIGNAL),"",ngx_signal_handler },{ SIGALRM, "SIGALRM", "", ngx_signal_handler },{ SIGINT, "SIGINT", "", ngx_signal_handler },{ SIGIO, "SIGIO", "", ngx_signal_handler },{ SIGCHLD, "SIGCHLD", "", ngx_signal_handler },{ SIGSYS, "SIGSYS, SIG_IGN", "", NULL },{ SIGPIPE, "SIGPIPE, SIG_IGN", "", NULL },{ 0, NULL, "", NULL }
};

ngx_init_signals是在nginx启动时向内核注册信号处理

2.1.3 ngx_init_signals

ngx_int_t
ngx_init_signals(ngx_log_t *log)
{ngx_signal_t      *sig;struct sigaction   sa;for (sig = signals; sig->signo != 0; sig++) {ngx_memzero(&sa, sizeof(struct sigaction));if (sig->handler) {sa.sa_sigaction = sig->handler;sa.sa_flags = SA_SIGINFO;} else {sa.sa_handler = SIG_IGN;}sigemptyset(&sa.sa_mask);if (sigaction(sig->signo, &sa, NULL) == -1) {
#if (NGX_VALGRIND)ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,"sigaction(%s) failed, ignored", sig->signame);
#elsengx_log_error(NGX_LOG_EMERG, log, ngx_errno,"sigaction(%s) failed", sig->signame);return NGX_ERROR;
#endif}}return NGX_OK;
}

在处理USR2信号时，会将ngx_change_binary标识设置为1

2.1.4 ngx_signal_handler

static void
ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
{char            *action;ngx_int_t        ignore;ngx_err_t        err;ngx_signal_t    *sig;ignore = 0;err = ngx_errno;for (sig = signals; sig->signo != 0; sig++) {if (sig->signo == signo) {break;}}ngx_time_sigsafe_update();action = "";switch (ngx_process) {case NGX_PROCESS_MASTER:case NGX_PROCESS_SINGLE:switch (signo) {case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):ngx_quit = 1;action = ", shutting down";break;case ngx_signal_value(NGX_TERMINATE_SIGNAL):case SIGINT:ngx_terminate = 1;action = ", exiting";break;case ngx_signal_value(NGX_NOACCEPT_SIGNAL):if (ngx_daemonized) {ngx_noaccept = 1;action = ", stop accepting connections";}break;case ngx_signal_value(NGX_RECONFIGURE_SIGNAL):ngx_reconfigure = 1;action = ", reconfiguring";break;case ngx_signal_value(NGX_REOPEN_SIGNAL):ngx_reopen = 1;action = ", reopening logs";break;case ngx_signal_value(NGX_CHANGEBIN_SIGNAL):if (ngx_getppid() == ngx_parent || ngx_new_binary > 0) {/** Ignore the signal in the new binary if its parent is* not changed, i.e. the old binary's process is still* running.  Or ignore the signal in the old binary's* process if the new binary's process is already running.*/action = ", ignoring";ignore = 1;break;}ngx_change_binary = 1;action = ", changing binary";break;case SIGALRM:ngx_sigalrm = 1;break;case SIGIO:ngx_sigio = 1;break;case SIGCHLD:ngx_reap = 1;break;}break;case NGX_PROCESS_WORKER:case NGX_PROCESS_HELPER:switch (signo) {case ngx_signal_value(NGX_NOACCEPT_SIGNAL):if (!ngx_daemonized) {break;}ngx_debug_quit = 1;/* fall through */case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):ngx_quit = 1;action = ", shutting down";break;case ngx_signal_value(NGX_TERMINATE_SIGNAL):case SIGINT:ngx_terminate = 1;action = ", exiting";break;case ngx_signal_value(NGX_REOPEN_SIGNAL):ngx_reopen = 1;action = ", reopening logs";break;case ngx_signal_value(NGX_RECONFIGURE_SIGNAL):case ngx_signal_value(NGX_CHANGEBIN_SIGNAL):case SIGIO:action = ", ignoring";break;}break;}if (siginfo && siginfo->si_pid) {ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,"signal %d (%s) received from %P%s",signo, sig->signame, siginfo->si_pid, action);} else {ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,"signal %d (%s) received%s",signo, sig->signame, action);}if (ignore) {ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, 0,"the changing binary signal is ignored: ""you should shutdown or terminate ""before either old or new binary's process");}if (signo == SIGCHLD) {ngx_process_get_status();}ngx_set_errno(err);
}

2.2 master进程处理ngx_change_binary标识

master进程处理循环中会检测ngx_change_binary标识，如果置为1，会调用ngx_exec_new_binary，fork新进程，新进行执行execv来调用新的二进制文件

void
ngx_master_process_cycle(ngx_cycle_t *cycle)
{char              *title;u_char            *p;size_t             size;ngx_int_t          i;ngx_uint_t         sigio;sigset_t           set;struct itimerval   itv;ngx_uint_t         live;ngx_msec_t         delay;ngx_core_conf_t   *ccf;sigemptyset(&set);sigaddset(&set, SIGCHLD);sigaddset(&set, SIGALRM);sigaddset(&set, SIGIO);sigaddset(&set, SIGINT);sigaddset(&set, ngx_signal_value(NGX_RECONFIGURE_SIGNAL));sigaddset(&set, ngx_signal_value(NGX_REOPEN_SIGNAL));sigaddset(&set, ngx_signal_value(NGX_NOACCEPT_SIGNAL));sigaddset(&set, ngx_signal_value(NGX_TERMINATE_SIGNAL));sigaddset(&set, ngx_signal_value(NGX_SHUTDOWN_SIGNAL));sigaddset(&set, ngx_signal_value(NGX_CHANGEBIN_SIGNAL));if (sigprocmask(SIG_BLOCK, &set, NULL) == -1) {ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"sigprocmask() failed");}sigemptyset(&set);size = sizeof(master_process);for (i = 0; i < ngx_argc; i++) {size += ngx_strlen(ngx_argv[i]) + 1;}title = ngx_pnalloc(cycle->pool, size);if (title == NULL) {/* fatal */exit(2);}p = ngx_cpymem(title, master_process, sizeof(master_process) - 1);for (i = 0; i < ngx_argc; i++) {*p++ = ' ';p = ngx_cpystrn(p, (u_char *) ngx_argv[i], size);}ngx_setproctitle(title);ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);ngx_start_worker_processes(cycle, ccf->worker_processes,NGX_PROCESS_RESPAWN);ngx_start_cache_manager_processes(cycle, 0);ngx_new_binary = 0;delay = 0;sigio = 0;live = 1;for ( ;; ) {if (delay) {if (ngx_sigalrm) {sigio = 0;delay *= 2;ngx_sigalrm = 0;}ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,"termination cycle: %M", delay);itv.it_interval.tv_sec = 0;itv.it_interval.tv_usec = 0;itv.it_value.tv_sec = delay / 1000;itv.it_value.tv_usec = (delay % 1000 ) * 1000;if (setitimer(ITIMER_REAL, &itv, NULL) == -1) {ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"setitimer() failed");}}ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "sigsuspend");sigsuspend(&set);ngx_time_update();ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,"wake up, sigio %i", sigio);if (ngx_reap) {ngx_reap = 0;ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "reap children");live = ngx_reap_children(cycle);}if (!live && (ngx_terminate || ngx_quit)) {ngx_master_process_exit(cycle);}if (ngx_terminate) {if (delay == 0) {delay = 50;}if (sigio) {sigio--;continue;}sigio = ccf->worker_processes + 2 /* cache processes */;if (delay > 1000) {ngx_signal_worker_processes(cycle, SIGKILL);} else {ngx_signal_worker_processes(cycle,ngx_signal_value(NGX_TERMINATE_SIGNAL));}continue;}if (ngx_quit) {ngx_signal_worker_processes(cycle,ngx_signal_value(NGX_SHUTDOWN_SIGNAL));ngx_close_listening_sockets(cycle);continue;}if (ngx_reconfigure) {ngx_reconfigure = 0;if (ngx_new_binary) {ngx_start_worker_processes(cycle, ccf->worker_processes,NGX_PROCESS_RESPAWN);ngx_start_cache_manager_processes(cycle, 0);ngx_noaccepting = 0;continue;}ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reconfiguring");cycle = ngx_init_cycle(cycle);if (cycle == NULL) {cycle = (ngx_cycle_t *) ngx_cycle;continue;}ngx_cycle = cycle;ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,ngx_core_module);ngx_start_worker_processes(cycle, ccf->worker_processes,NGX_PROCESS_JUST_RESPAWN);ngx_start_cache_manager_processes(cycle, 1);/* allow new processes to start */ngx_msleep(100);live = 1;ngx_signal_worker_processes(cycle,ngx_signal_value(NGX_SHUTDOWN_SIGNAL));}if (ngx_restart) {ngx_restart = 0;ngx_start_worker_processes(cycle, ccf->worker_processes,NGX_PROCESS_RESPAWN);ngx_start_cache_manager_processes(cycle, 0);live = 1;}if (ngx_reopen) {ngx_reopen = 0;ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs");ngx_reopen_files(cycle, ccf->user);ngx_signal_worker_processes(cycle,ngx_signal_value(NGX_REOPEN_SIGNAL));}if (ngx_change_binary) {ngx_change_binary = 0;ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "changing binary");ngx_new_binary = ngx_exec_new_binary(cycle, ngx_argv);}if (ngx_noaccept) {ngx_noaccept = 0;ngx_noaccepting = 1;ngx_signal_worker_processes(cycle,ngx_signal_value(NGX_SHUTDOWN_SIGNAL));}}
}

2.2.1 ngx_exec_new_binary

• 将cycle中的listening监听套接字放到环境变量中

• 将pid文件生命为oldpid

• 创建新进程，执行新的二进制文件

ngx_pid_t
ngx_exec_new_binary(ngx_cycle_t *cycle, char *const *argv)
{char             **env, *var;u_char            *p;ngx_uint_t         i, n;ngx_pid_t          pid;ngx_exec_ctx_t     ctx;ngx_core_conf_t   *ccf;ngx_listening_t   *ls;ngx_memzero(&ctx, sizeof(ngx_exec_ctx_t));ctx.path = argv[0];ctx.name = "new binary process";ctx.argv = argv;n = 2;env = ngx_set_environment(cycle, &n);if (env == NULL) {return NGX_INVALID_PID;}var = ngx_alloc(sizeof(NGINX_VAR)+ cycle->listening.nelts * (NGX_INT32_LEN + 1) + 2,cycle->log);if (var == NULL) {ngx_free(env);return NGX_INVALID_PID;}p = ngx_cpymem(var, NGINX_VAR "=", sizeof(NGINX_VAR));ls = cycle->listening.elts;for (i = 0; i < cycle->listening.nelts; i++) {p = ngx_sprintf(p, "%ud;", ls[i].fd);}*p = '\0';env[n++] = var;#if (NGX_SETPROCTITLE_USES_ENV)/* allocate the spare 300 bytes for the new binary process title */env[n++] = "SPARE=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX""XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX""XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX""XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX""XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";#endifenv[n] = NULL;#if (NGX_DEBUG){char  **e;for (e = env; *e; e++) {ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, "env: %s", *e);}}
#endifctx.envp = (char *const *) env;ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);if (ngx_rename_file(ccf->pid.data, ccf->oldpid.data) == NGX_FILE_ERROR) {ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,ngx_rename_file_n " %s to %s failed ""before executing new binary process \"%s\"",ccf->pid.data, ccf->oldpid.data, argv[0]);ngx_free(env);ngx_free(var);return NGX_INVALID_PID;}pid = ngx_execute(cycle, &ctx);if (pid == NGX_INVALID_PID) {if (ngx_rename_file(ccf->oldpid.data, ccf->pid.data)== NGX_FILE_ERROR){ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,ngx_rename_file_n " %s back to %s failed after ""an attempt to execute new binary process \"%s\"",ccf->oldpid.data, ccf->pid.data, argv[0]);}}ngx_free(env);ngx_free(var);return pid;
}ngx_pid_t
ngx_execute(ngx_cycle_t *cycle, ngx_exec_ctx_t *ctx)
{return ngx_spawn_process(cycle, ngx_execute_proc, ctx, ctx->name,NGX_PROCESS_DETACHED);
}static void
ngx_execute_proc(ngx_cycle_t *cycle, void *data)
{ngx_exec_ctx_t  *ctx = data;if (execve(ctx->path, ctx->argv, ctx->envp) == -1) {ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"execve() failed while executing %s \"%s\"",ctx->name, ctx->path);}exit(1);
}

2.3 旧版本的master退出

此时，nginx会有两个进程，一个是新版本的，一个是旧版本的，需要将旧版本的结束，其在处理quit信号时，主进程会向work进程发送quit信号，现时主进程也会关闭监听套接字

if (ngx_quit) {ngx_signal_worker_processes(cycle,ngx_signal_value(NGX_SHUTDOWN_SIGNAL));ngx_close_listening_sockets(cycle);continue;
}

2.3.1 ngx_signal_worker_processes

向工作进程发送quit消息

static void
ngx_signal_worker_processes(ngx_cycle_t *cycle, int signo)
{ngx_int_t      i;ngx_err_t      err;ngx_channel_t  ch;ngx_memzero(&ch, sizeof(ngx_channel_t));#if (NGX_BROKEN_SCM_RIGHTS)ch.command = 0;#elseswitch (signo) {case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):ch.command = NGX_CMD_QUIT;break;case ngx_signal_value(NGX_TERMINATE_SIGNAL):ch.command = NGX_CMD_TERMINATE;break;case ngx_signal_value(NGX_REOPEN_SIGNAL):ch.command = NGX_CMD_REOPEN;break;default:ch.command = 0;}#endifch.fd = -1;for (i = 0; i < ngx_last_process; i++) {ngx_log_debug7(NGX_LOG_DEBUG_EVENT, cycle->log, 0,"child: %i %P e:%d t:%d d:%d r:%d j:%d",i,ngx_processes[i].pid,ngx_processes[i].exiting,ngx_processes[i].exited,ngx_processes[i].detached,ngx_processes[i].respawn,ngx_processes[i].just_spawn);if (ngx_processes[i].detached || ngx_processes[i].pid == -1) {continue;}if (ngx_processes[i].just_spawn) {ngx_processes[i].just_spawn = 0;continue;}if (ngx_processes[i].exiting&& signo == ngx_signal_value(NGX_SHUTDOWN_SIGNAL)){continue;}if (ch.command) {if (ngx_write_channel(ngx_processes[i].channel[0],&ch, sizeof(ngx_channel_t), cycle->log)== NGX_OK){if (signo != ngx_signal_value(NGX_REOPEN_SIGNAL)) {ngx_processes[i].exiting = 1;}continue;}}ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,"kill (%P, %d)", ngx_processes[i].pid, signo);if (kill(ngx_processes[i].pid, signo) == -1) {err = ngx_errno;ngx_log_error(NGX_LOG_ALERT, cycle->log, err,"kill(%P, %d) failed", ngx_processes[i].pid, signo);if (err == NGX_ESRCH) {ngx_processes[i].exited = 1;ngx_processes[i].exiting = 0;ngx_reap = 1;}continue;}if (signo != ngx_signal_value(NGX_REOPEN_SIGNAL)) {ngx_processes[i].exiting = 1;}}
}

2.3.2 ngx_close_listening_sockets

关闭监听套接字

void
ngx_close_listening_sockets(ngx_cycle_t *cycle)
{ngx_uint_t         i;ngx_listening_t   *ls;ngx_connection_t  *c;if (ngx_event_flags & NGX_USE_IOCP_EVENT) {return;}ngx_accept_mutex_held = 0;ngx_use_accept_mutex = 0;ls = cycle->listening.elts;for (i = 0; i < cycle->listening.nelts; i++) {c = ls[i].connection;if (c) {if (c->read->active) {if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {/** it seems that Linux-2.6.x OpenVZ sends events* for closed shared listening sockets unless* the events was explicitly deleted*/ngx_del_event(c->read, NGX_READ_EVENT, 0);} else {ngx_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);}}ngx_free_connection(c);c->fd = (ngx_socket_t) -1;}ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,"close listening %V #%d ", &ls[i].addr_text, ls[i].fd);if (ngx_close_socket(ls[i].fd) == -1) {ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,ngx_close_socket_n " %V failed", &ls[i].addr_text);}#if (NGX_HAVE_UNIX_DOMAIN)if (ls[i].sockaddr->sa_family == AF_UNIX&& ngx_process <= NGX_PROCESS_MASTER&& ngx_new_binary == 0&& (!ls[i].inherited || ngx_getppid() != ngx_parent)){u_char *name = ls[i].addr_text.data + sizeof("unix:") - 1;if (ngx_delete_file(name) == NGX_FILE_ERROR) {ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,ngx_delete_file_n " %s failed", name);}}#endifls[i].fd = (ngx_socket_t) -1;}cycle->listening.nelts = 0;
}