static void
wd_exit(int exit_signo)
{
sigset_t mask;
sigemptyset(&mask);
sigaddset(&mask, SIGTERM);
sigaddset(&mask, SIGINT);
sigaddset(&mask, SIGQUIT);
sigaddset(&mask, SIGCHLD);
sigprocmask(SIG_BLOCK, &mask, NULL);
wd_notice_server_down();
exit(0);
}
/*
* Check if pgpool is living
*/
int
wd_lifecheck(void)
{
struct timeval tv;
/* I'm in down.... */
if (WD_MYSELF->status == WD_DOWN)
{
ereport(NOTICE,
(errmsg("watchdog lifecheck, watchdog status is DOWN. You need to restart pgpool")));
return WD_NG;
}
/* set startup time */
gettimeofday(&tv, NULL);
/* check upper connection */
if (strlen(pool_config->trusted_servers))
{
if(wd_is_upper_ok(pool_config->trusted_servers) != WD_OK)
{
ereport(WARNING,
(errmsg("watchdog lifecheck, failed to connect to any trusted servers")));
if (WD_MYSELF->status == WD_MASTER &&
strlen(pool_config->delegate_IP) != 0)
{
wd_IP_down();
}
wd_set_myself(&tv, WD_DOWN);
wd_notice_server_down();
return WD_NG;
}
}
/* skip lifecheck during recovery execution */
if (*InRecovery != RECOVERY_INIT)
{
return WD_OK;
}
/* check and update pgpool status */
check_pgpool_status();
return WD_OK;
}
static void
check_pgpool_status_by_query(void)
{
WdInfo * p = WD_List;
struct timeval tv;
pthread_attr_t attr;
pthread_t thread[MAX_WATCHDOG_NUM];
WdPgpoolThreadArg thread_arg[MAX_WATCHDOG_NUM];
int rc;
int i,cnt;
/* set startup time */
gettimeofday(&tv, NULL);
/* thread init */
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
/* send queries to all pgpools using threads */
cnt = 0;
while (p->status != WD_END)
{
if (p->status != WD_DOWN)
{
thread_arg[cnt].conn = create_conn(p->hostname, p->pgpool_port);
rc = watchdog_thread_create(&thread[cnt], &attr, thread_ping_pgpool, (void*)&thread_arg[cnt]);
}
p ++;
cnt ++;
if (cnt >= MAX_WATCHDOG_NUM)
{
ereport(WARNING,
(errmsg("checking pgpool status by query, pgpool num is out of range:%d",cnt)));
break;
}
}
pthread_attr_destroy(&attr);
/* check results of queries */
p = WD_List;
for (i = 0; i < cnt; )
{
int result;
ereport(DEBUG1,
(errmsg("checking pgpool status by query"),
errdetail("checking pgpool %d (%s:%d)",
i, p->hostname, p->pgpool_port)));
if (p->status == WD_DOWN)
{
ereport(LOG,
(errmsg("checking pgpool status by query"),
errdetail("pgpool %d (%s:%d) is in down status",
i, p->hostname, p->pgpool_port)));
i++;
p++;
continue;
}
else
{
rc = pthread_join(thread[i], (void **)&result);
if ((rc != 0) && (errno == EINTR))
{
usleep(100);
continue;
}
}
if (result == WD_OK)
{
ereport(DEBUG1,
(errmsg("checking pgpool status by query"),
errdetail("WD_OK: status: %d", p->status)));
/* life point init */
p->life = pool_config->wd_life_point;
}
else
{
ereport(DEBUG1,
(errmsg("checking pgpool status by query"),
errdetail("NG; status: %d life:%d", p->status, p->life)));
if (p->life > 0)
{
p->life --;
}
/* pgpool goes down */
if (p->life <= 0)
{
ereport(LOG,
(errmsg("checking pgpool status by query"),
errdetail("lifecheck failed %d times. pgpool %d (%s:%d) seems not to be working",
pool_config->wd_life_point, i, p->hostname, p->pgpool_port)));
/* It's me! */
if ((i == 0) &&
(WD_MYSELF->status != WD_DOWN))
{
wd_set_myself(&tv, WD_DOWN);
wd_notice_server_down();
}
/* It's other pgpool */
else if (p->status != WD_DOWN)
pgpool_down(p);
}
}
i++;
p++;
}
}
static void
check_pgpool_status_by_hb(void)
{
int cnt;
WdInfo *p = WD_List;
struct timeval tv;
gettimeofday(&tv, NULL);
cnt = 0;
while (p->status != WD_END)
{
ereport(DEBUG1,
(errmsg("watchdog life checking by heartbeat"),
errdetail("checking pgpool %d (%s:%d)",
cnt, p->hostname, p->pgpool_port)));
/* about myself */
if (p == WD_MYSELF)
{
/* parent is dead so it's orphan.... */
if (is_parent_alive() == WD_NG && WD_MYSELF->status != WD_DOWN)
{
ereport(LOG,
(errmsg("checking pgpool status by heartbeat"),
errdetail("lifecheck failed. pgpool %d (%s:%d) seems not to be working",
cnt, p->hostname, p->pgpool_port)));
wd_set_myself(&tv, WD_DOWN);
wd_notice_server_down();
}
/* otherwise, the parent would take care of children. */
else
{
ereport(DEBUG1,
(errmsg("watchdog life checking by heartbeat"),
errdetail("OK; status %d", p->status)));
}
}
/* about other pgpools, check the latest heartbeat. */
else
{
if (p->status == WD_DOWN)
{
ereport(LOG,
(errmsg("checking pgpool status by heartbeat"),
errdetail("pgpool: %d at \"%s:%d\" status is down",
cnt, p->hostname, p->pgpool_port)));
}
else if (wd_check_heartbeat(p) == WD_NG)
{
ereport(DEBUG1,
(errmsg("checking pgpool status by heartbeat"),
errdetail("NG; status %d", p->status)));
ereport(LOG,
(errmsg("checking pgpool status by heartbeat"),
errdetail("lifecheck failed. pgpool: %d at \"%s:%d\" seems not to be working",
cnt, p->hostname, p->pgpool_port)));
if (p->status != WD_DOWN)
pgpool_down(p);
}
else
{
ereport(DEBUG1,
(errmsg("checking pgpool status by heartbeat"),
errdetail("OK; status %d", p->status)));
}
}
p++;
cnt++;
if (cnt >= MAX_WATCHDOG_NUM)
{
ereport(WARNING,
(errmsg("checking pgpool status by heartbeat, pgpool num is out of range:%d",cnt)));
break;
}
}
}
static void
check_pgpool_status_by_hb(void)
{
int cnt;
WdInfo * p = WD_List;
struct timeval tv;
gettimeofday(&tv, NULL);
cnt = 0;
while (p->status != WD_END)
{
pool_debug("check_pgpool_status_by_hb: checking pgpool %d (%s:%d)",
cnt, p->hostname, p->pgpool_port);
/* about myself */
if (p == WD_MYSELF)
{
/* parent is dead so it's orphan.... */
if (is_parent_alive() == WD_NG && WD_MYSELF->status != WD_DOWN)
{
pool_debug("check_pgpool_status_by_hb: NG; the main pgpool process does't exist.");
pool_log("check_pgpool_status_by_hb: lifecheck failed. pgpool %d (%s:%d) seems not to be working",
cnt, p->hostname, p->pgpool_port);
wd_set_myself(&tv, WD_DOWN);
wd_notice_server_down();
}
/* otherwise, the parent would take care of children. */
else
{
pool_debug("check_pgpool_status_by_hb: OK; status %d", p->status);
}
}
/* about other pgpools, check the latest heartbeat. */
else
{
if (p->status == WD_DOWN)
{
pool_log("check_pgpool_status_by_hb: pgpool %d (%s:%d) is in down status",
cnt, p->hostname, p->pgpool_port);
}
else if (wd_check_heartbeat(p) == WD_NG)
{
pool_debug("check_pgpool_status_by_hb: NG; status %d", p->status);
pool_log("check_pgpool_status_by_hb: lifecheck failed. pgpool %d (%s:%d) seems not to be working",
cnt, p->hostname, p->pgpool_port);
if (p->status != WD_DOWN)
pgpool_down(p);
}
else
{
pool_debug("check_pgpool_status_by_hb: OK; status %d", p->status);
}
}
p++;
cnt++;
if (cnt >= MAX_WATCHDOG_NUM)
{
pool_error("check_pgpool_status_by_hb: pgpool num is out of range(%d)",cnt);
break;
}
}
}