int wd_notice_server_down(void) { int rtn; wd_IP_down(); /* send notice server down packet */ rtn = wd_send_packet_no(WD_SERVER_DOWN); return rtn; }
/* * Check if pgpool is living */ int wd_lifecheck(void) { struct timeval tv; /* I'm in down.... */ if (WD_MYSELF->status == WD_DOWN) { ereport(NOTICE, (errmsg("watchdog lifecheck, watchdog status is DOWN. You need to restart pgpool"))); return WD_NG; } /* set startup time */ gettimeofday(&tv, NULL); /* check upper connection */ if (strlen(pool_config->trusted_servers)) { if(wd_is_upper_ok(pool_config->trusted_servers) != WD_OK) { ereport(WARNING, (errmsg("watchdog lifecheck, failed to connect to any trusted servers"))); if (WD_MYSELF->status == WD_MASTER && strlen(pool_config->delegate_IP) != 0) { wd_IP_down(); } wd_set_myself(&tv, WD_DOWN); wd_notice_server_down(); return WD_NG; } } /* skip lifecheck during recovery execution */ if (*InRecovery != RECOVERY_INIT) { return WD_OK; } /* check and update pgpool status */ check_pgpool_status(); return WD_OK; }
static int wd_send_response(int sock, WdPacket * recv_pack) { int rtn = WD_NG; WdInfo * p, *q; WdNodeInfo * node; WdLockInfo * lock; WdPacket send_packet; struct timeval tv; char pack_str[WD_MAX_PACKET_STRING]; int pack_str_len; char hash[(MD5_PASSWD_LEN+1)*2]; bool is_node_packet = false; if (recv_pack == NULL) { return rtn; } memset(&send_packet, 0, sizeof(WdPacket)); p = &(recv_pack->wd_body.wd_info); /* auhtentication */ if (strlen(pool_config->wd_authkey)) { /* calculate hash from packet */ pack_str_len = wd_packet_to_string(*recv_pack, pack_str, sizeof(pack_str)); wd_calc_hash(pack_str, pack_str_len, hash); if (strcmp(recv_pack->hash, hash)) { pool_log("wd_send_response: watchdog authentication failed"); rtn = wd_authentication_failed(sock); return rtn; } } /* set response packet no */ switch (recv_pack->packet_no) { /* add request into the watchdog list */ case WD_ADD_REQ: p = &(recv_pack->wd_body.wd_info); if (wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status) > 0) { send_packet.packet_no = WD_ADD_ACCEPT; } else { send_packet.packet_no = WD_ADD_REJECT; } memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); break; /* announce candidacy to be the new master */ case WD_STAND_FOR_MASTER: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status); /* check exist master */ if ((q = wd_is_alive_master()) != NULL) { /* vote against the candidate */ send_packet.packet_no = WD_MASTER_EXIST; memcpy(&(send_packet.wd_body.wd_info), q, sizeof(WdInfo)); } else { if (WD_MYSELF->tv.tv_sec <= p->tv.tv_sec ) { memcpy(&tv,&(p->tv),sizeof(struct timeval)); tv.tv_sec += 1; wd_set_myself(&tv, WD_NORMAL); } /* vote for the candidate */ send_packet.packet_no = WD_VOTE_YOU; memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); } break; /* announce assumption to be the new master */ case WD_DECLARE_NEW_MASTER: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status); if (WD_MYSELF->status == WD_MASTER) { /* resign master server */ pool_log("wd_declare_new_master: ifconfig down to resign master server"); wd_IP_down(); wd_set_myself(NULL, WD_NORMAL); } send_packet.packet_no = WD_READY; memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); break; /* annouce to assume lock holder */ case WD_STAND_FOR_LOCK_HOLDER: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status); /* only master handles lock holder privilege */ if (WD_MYSELF->status == WD_MASTER) { /* if theare are no lock holder yet */ if (wd_get_lock_holder() != NULL) { send_packet.packet_no = WD_LOCK_HOLDER_EXIST; } } memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); break; case WD_DECLARE_LOCK_HOLDER: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status); wd_set_lock_holder(p, true); send_packet.packet_no = WD_READY; memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); break; /* annouce to resigne lock holder */ case WD_RESIGN_LOCK_HOLDER: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status); wd_set_lock_holder(p, false); send_packet.packet_no = WD_READY; memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); break; case WD_START_INTERLOCK: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status); wd_set_interlocking(p, true); break; case WD_END_INTERLOCK: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), p->status); wd_set_interlocking(p, false); break; /* announce that server is down */ case WD_SERVER_DOWN: p = &(recv_pack->wd_body.wd_info); wd_set_wd_list(p->hostname,p->pgpool_port, p->wd_port, p->delegate_ip, &(p->tv), WD_DOWN); send_packet.packet_no = WD_READY; memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); if (wd_am_I_oldest() == WD_OK && WD_MYSELF->status != WD_MASTER) { wd_escalation(); } break; /* announce start online recovery */ case WD_START_RECOVERY: if (*InRecovery != RECOVERY_INIT) { send_packet.packet_no = WD_NODE_FAILED; } else { send_packet.packet_no = WD_NODE_READY; *InRecovery = RECOVERY_ONLINE; if (wait_connection_closed() != 0) { send_packet.packet_no = WD_NODE_FAILED; } } break; case WD_END_RECOVERY: send_packet.packet_no = WD_NODE_READY; *InRecovery = RECOVERY_INIT; kill(wd_ppid, SIGUSR2); break; case WD_FAILBACK_REQUEST: node = &(recv_pack->wd_body.wd_node_info); wd_set_node_mask(WD_FAILBACK_REQUEST,node->node_id_set,node->node_num); is_node_packet = true; send_packet.packet_no = WD_NODE_READY; break; case WD_DEGENERATE_BACKEND: node = &(recv_pack->wd_body.wd_node_info); wd_set_node_mask(WD_DEGENERATE_BACKEND,node->node_id_set, node->node_num); is_node_packet = true; send_packet.packet_no = WD_NODE_READY; break; case WD_PROMOTE_BACKEND: node = &(recv_pack->wd_body.wd_node_info); wd_set_node_mask(WD_PROMOTE_BACKEND,node->node_id_set, node->node_num); is_node_packet = true; send_packet.packet_no = WD_NODE_READY; break; case WD_UNLOCK_REQUEST: lock = &(recv_pack->wd_body.wd_lock_info); wd_set_lock(lock->lock_id, false); send_packet.packet_no = WD_LOCK_READY; break; default: send_packet.packet_no = WD_INVALID; memcpy(&(send_packet.wd_body.wd_info), WD_MYSELF, sizeof(WdInfo)); break; } /* send response packet */ rtn = wd_send_packet(sock, &send_packet); /* send node request signal. * wd_node_request_singnal() uses a semaphore lock internally, so should be * called after sending a response pakcet to prevent dead lock. */ if (is_node_packet) wd_node_request_signal(recv_pack->packet_no, node); return rtn; }
/* * fork de-escalation process */ pid_t fork_plunging_process(void) { pid_t pid; pid = fork(); if (pid != 0) { if (pid == -1) ereport(NOTICE, (errmsg("failed to fork a de-escalation process"))); return pid; } on_exit_reset(); processType = PT_WATCHDOG_UTILITY; POOL_SETMASK(&UnBlockSig); init_ps_display("", "", "", ""); pool_signal(SIGTERM, wd_exit); pool_signal(SIGINT, wd_exit); pool_signal(SIGQUIT, wd_exit); pool_signal(SIGCHLD, SIG_DFL); pool_signal(SIGHUP, SIG_IGN); pool_signal(SIGPIPE, SIG_IGN); MemoryContextSwitchTo(TopMemoryContext); set_ps_display("watchdog de-escalation", false); ereport(LOG, (errmsg("watchdog: de-escalation started"))); /* * STEP 1 execute de-escalation command provided by user in pgpool conf * file */ if (strlen(pool_config->wd_de_escalation_command)) { int r = system(pool_config->wd_de_escalation_command); if (WIFEXITED(r)) { if (WEXITSTATUS(r) == EXIT_SUCCESS) ereport(LOG, (errmsg("watchdog de-escalation successful"))); else { ereport(WARNING, (errmsg("watchdog de-escalation command failed with exit status: %d", WEXITSTATUS(r)))); } } else { ereport(WARNING, (errmsg("watchdog de-escalation command exit abnormally"))); } } /* * STEP 2 bring down the delegate IP */ if (strlen(pool_config->delegate_IP) != 0) { if (wd_IP_down() != WD_OK) ereport(WARNING, (errmsg("watchdog de-escalation failed to bring down delegate IP"))); } exit(0); }