/*
* Start online recovery.
* "recovery_node" is the node to be recovered.
* Master or primary node is chosen in this function.
*/
void start_recovery(int recovery_node)
{
int node_id;
BackendInfo *backend;
BackendInfo *recovery_backend;
PGconn *conn;
int failback_wait_count;
#define FAILBACK_WAIT_MAX_RETRY 5 /* 5 seconds should be enough for failback operation */
ereport(LOG,
(errmsg("starting recovering node %d", recovery_node)));
if ( (recovery_node < 0) || (recovery_node >= pool_config->backend_desc->num_backends) )
ereport(ERROR,
(errmsg("node recovery failed, node id: %d is not valid", recovery_node)));
if (VALID_BACKEND(recovery_node))
ereport(ERROR,
(errmsg("node recovery failed, node id: %d is alive", recovery_node)));
/* select master/primary node */
node_id = MASTER_SLAVE ? PRIMARY_NODE_ID : REAL_MASTER_NODE_ID;
backend = &pool_config->backend_desc->backend_info[node_id];
/* get node info to be recovered */
recovery_backend = &pool_config->backend_desc->backend_info[recovery_node];
conn = connect_backend_libpq(backend);
if (conn == NULL)
ereport(ERROR,
(errmsg("node recovery failed, unable to connect to master node: %d ", node_id)));
PG_TRY();
{
/* 1st stage */
if (REPLICATION)
{
exec_checkpoint(conn);
ereport(LOG,
(errmsg("node recovery, CHECKPOINT in the 1st stage done")));
}
exec_recovery(conn, backend, recovery_backend, FIRST_STAGE);
ereport(LOG,
(errmsg("node recovery, 1st stage is done")));
if (REPLICATION)
{
ereport(LOG,
(errmsg("node recovery, starting 2nd stage")));
/* 2nd stage */
*InRecovery = RECOVERY_ONLINE;
if (pool_config->use_watchdog)
{
/* announce start recovery */
if (WD_OK != wd_start_recovery())
ereport(ERROR,
(errmsg("node recovery failed, failed to send start recovery packet")));
}
if (wait_connection_closed() != 0)
ereport(ERROR,
(errmsg("node recovery failed, waiting connection closed in the other pgpools timeout")));
ereport(LOG,
(errmsg("node recovery, all connections from clients have been closed")));
exec_checkpoint(conn);
ereport(LOG,
(errmsg("node recovery"),
errdetail("CHECKPOINT in the 2nd stage done")));
exec_recovery(conn, backend, recovery_backend, SECOND_STAGE);
}
exec_remote_start(conn, recovery_backend);
check_postmaster_started(recovery_backend);
ereport(LOG,
(errmsg("node recovery, node: %d restarted", recovery_node)));
/*
* reset failover completion flag. this is necessary since
* previous failover/failback will set the flag to 1.
*/
pcp_wakeup_request = 0;
/* send failback request to pgpool parent */
send_failback_request(recovery_node);
/* wait for failback */
failback_wait_count = 0;
while (!pcp_wakeup_request)
{
struct timeval t = {1, 0};
/* polling SIGUSR2 signal every 1 sec */
select(0, NULL, NULL, NULL, &t);
failback_wait_count++;
if (failback_wait_count >= FAILBACK_WAIT_MAX_RETRY)
{
ereport(LOG,
(errmsg("node recovery"),
errdetail("waiting for wake up request is timeout(%d seconds)",
FAILBACK_WAIT_MAX_RETRY)));
break;
}
}
pcp_wakeup_request = 0;
}
PG_CATCH();
{
PQfinish(conn);
PG_RE_THROW();
}
PG_END_TRY();
PQfinish(conn);
ereport(LOG,
(errmsg("recovery done")));
}
/*
* Start online recovery.
* "recovery_node" is the node to be recovered.
* Master or primary node is chosen in this function.
*/
int start_recovery(int recovery_node)
{
int node_id;
BackendInfo *backend;
BackendInfo *recovery_backend;
PGconn *conn;
int failback_wait_count;
#define FAILBACK_WAIT_MAX_RETRY 5 /* 5 seconds should be enough for failback operation */
pool_log("starting recovering node %d", recovery_node);
if (VALID_BACKEND(recovery_node))
{
pool_error("start_recovery: backend node %d is alive", recovery_node);
return 1;
}
Req_info->kind = NODE_RECOVERY_REQUEST;
/* select master/primary node */
node_id = MASTER_SLAVE ? PRIMARY_NODE_ID : REAL_MASTER_NODE_ID;
backend = &pool_config->backend_desc->backend_info[node_id];
/* get node info to be recovered */
recovery_backend = &pool_config->backend_desc->backend_info[recovery_node];
conn = connect_backend_libpq(backend);
if (conn == NULL)
{
pool_error("start_recovery: could not connect master node (%d)", node_id);
return 1;
}
/* 1st stage */
if (REPLICATION)
{
if (exec_checkpoint(conn) != 0)
{
PQfinish(conn);
pool_error("start_recovery: CHECKPOINT failed");
return 1;
}
pool_log("CHECKPOINT in the 1st stage done");
}
if (exec_recovery(conn, recovery_backend, FIRST_STAGE) != 0)
{
PQfinish(conn);
return 1;
}
pool_log("1st stage is done");
if (REPLICATION)
{
pool_log("starting 2nd stage");
/* 2nd stage */
*InRecovery = RECOVERY_ONLINE;
if (pool_config->use_watchdog)
{
/* announce start recovery */
if (WD_OK != wd_start_recovery())
{
PQfinish(conn);
pool_error("start_recovery: timeover for waiting connection closed in the other pgpools");
return 1;
}
}
if (wait_connection_closed() != 0)
{
PQfinish(conn);
pool_error("start_recovery: timeover for waiting connection closed");
return 1;
}
pool_log("all connections from clients have been closed");
if (exec_checkpoint(conn) != 0)
{
PQfinish(conn);
pool_error("start_recovery: CHECKPOINT failed");
return 1;
}
pool_log("CHECKPOINT in the 2nd stage done");
if (exec_recovery(conn, recovery_backend, SECOND_STAGE) != 0)
{
PQfinish(conn);
return 1;
}
}
if (exec_remote_start(conn, recovery_backend) != 0)
{
PQfinish(conn);
pool_error("start_recovery: remote start failed");
return 1;
}
if (check_postmaster_started(recovery_backend))
{
PQfinish(conn);
pool_error("start_recovery: check start failed");
return 1;
}
pool_log("%d node restarted", recovery_node);
/*
* reset failover completion flag. this is necessary since
* previous failover/failback will set the flag to 1.
*/
pcp_wakeup_request = 0;
/* send failback request to pgpool parent */
send_failback_request(recovery_node);
/* wait for failback */
failback_wait_count = 0;
while (!pcp_wakeup_request)
{
struct timeval t = {1, 0};
/* polling SIGUSR2 signal every 1 sec */
select(0, NULL, NULL, NULL, &t);
failback_wait_count++;
if (failback_wait_count >= FAILBACK_WAIT_MAX_RETRY)
{
pool_log("start_recovery: waiting for wake up request is timeout(%d seconds)",
FAILBACK_WAIT_MAX_RETRY);
break;
}
}
pcp_wakeup_request = 0;
PQfinish(conn);
pool_log("recovery done");
return 0;
}