/*
* Insert monitor info, this is basically the time and xlog replayed,
* applied on standby and current xlog location in primary.
* Also do the math to see how far are we in bytes for being uptodate
*/
static void
MonitorExecute(void)
{
PGresult *res;
char monitor_standby_timestamp[MAXLEN];
char last_wal_primary_location[MAXLEN];
char last_wal_standby_received[MAXLEN];
char last_wal_standby_applied[MAXLEN];
unsigned long long int lsn_primary;
unsigned long long int lsn_standby_received;
unsigned long long int lsn_standby_applied;
int connection_retries;
/*
* Check if the master is still available, if after 5 minutes of retries
* we cannot reconnect, try to get a new master.
*/
for (connection_retries = 0; connection_retries < 15; connection_retries++)
{
if (PQstatus(primaryConn) != CONNECTION_OK)
{
log_warning(_("Connection to master has been lost, trying to recover...\n"));
/* wait 20 seconds between retries */
sleep(20);
PQreset(primaryConn);
}
else
{
if (connection_retries > 0)
{
log_notice(_("Connection to master has been restored, continue monitoring.\n"));
}
break;
}
}
if (PQstatus(primaryConn) != CONNECTION_OK)
{
log_err(_("We couldn't reconnect to master. Now checking if another node has been promoted.\n"));
for (connection_retries = 0; connection_retries < 6; connection_retries++)
{
primaryConn = getMasterConnection(myLocalConn, local_options.node,
local_options.cluster_name, &primary_options.node,NULL);
if (PQstatus(primaryConn) == CONNECTION_OK)
{
/* Connected, we can continue the process so break the loop */
log_err(_("Connected to node %d, continue monitoring.\n"), primary_options.node);
break;
}
else
{
log_err(_("We haven't found a new master, waiting before retry...\n"));
/* wait 5 minutes before retries, after 6 failures (30 minutes) we stop trying */
sleep(300);
}
}
}
if (PQstatus(primaryConn) != CONNECTION_OK)
{
log_err(_("We couldn't reconnect for long enough, exiting...\n"));
exit(ERR_DB_CON);
}
/* Check if we still are a standby, we could have been promoted */
if (!is_standby(myLocalConn))
{
log_err(_("It seems like we have been promoted, so exit from monitoring...\n"));
CloseConnections();
exit(ERR_PROMOTED);
}
/*
* first check if there is a command being executed,
* and if that is the case, cancel the query so i can
* insert the current record
*/
if (PQisBusy(primaryConn) == 1)
CancelQuery();
/* Get local xlog info */
sqlquery_snprintf(
sqlquery,
"SELECT CURRENT_TIMESTAMP, pg_last_xlog_receive_location(), "
"pg_last_xlog_replay_location()");
res = PQexec(myLocalConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err("PQexec failed: %s\n", PQerrorMessage(myLocalConn));
PQclear(res);
/* if there is any error just let it be and retry in next loop */
return;
}
strncpy(monitor_standby_timestamp, PQgetvalue(res, 0, 0), MAXLEN);
strncpy(last_wal_standby_received , PQgetvalue(res, 0, 1), MAXLEN);
strncpy(last_wal_standby_applied , PQgetvalue(res, 0, 2), MAXLEN);
PQclear(res);
/* Get primary xlog info */
sqlquery_snprintf(sqlquery, "SELECT pg_current_xlog_location() ");
res = PQexec(primaryConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err("PQexec failed: %s\n", PQerrorMessage(primaryConn));
PQclear(res);
return;
}
strncpy(last_wal_primary_location, PQgetvalue(res, 0, 0), MAXLEN);
PQclear(res);
/* Calculate the lag */
lsn_primary = walLocationToBytes(last_wal_primary_location);
lsn_standby_received = walLocationToBytes(last_wal_standby_received);
lsn_standby_applied = walLocationToBytes(last_wal_standby_applied);
if (only_one_entry && only_one_entry_desired)
{
sqlquery_snprintf(sqlquery,
"UPDATE %s.repl_monitor "
"VALUES(%d, %d, '%s'::timestamp with time zone, "
" '%s', '%s', "
" %lld, %lld)"
"WHERE primary_node=%d AND secondary_node=%d", repmgr_schema,
primary_options.node, local_options.node, monitor_standby_timestamp,
last_wal_primary_location,
last_wal_standby_received,
(lsn_primary - lsn_standby_received),
(lsn_standby_received - lsn_standby_applied));
res = PQexec(primaryConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err("PQexec failed: %s\n", PQerrorMessage(conn));
PQclear(res);
CloseConnections();
exit(ERR_DB_QUERY);
}
if (PQntuples(res) != 1)
{
only_one_entry = false;
}
PQclear(res);
}
else
{
/*
* Build and send insert
*/
sqlquery_snprintf(sqlquery,
"INSERT INTO %s.repl_monitor "
"VALUES(%d, %d, '%s'::timestamp with time zone, "
" '%s', '%s', "
" %lld, %lld)", repmgr_schema,
primary_options.node, local_options.node, monitor_standby_timestamp,
last_wal_primary_location,
last_wal_standby_received,
(lsn_primary - lsn_standby_received),
(lsn_standby_received - lsn_standby_applied));
res = PQexec(primaryConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err("PQexec failed: %s\n", PQerrorMessage(conn));
PQclear(res);
CloseConnections();
exit(ERR_DB_QUERY);
}
PQclear(res);
if (only_one_entry_desired)
{
/*
* Build the SQL to execute on primary
*/
sqlquery_snprintf(sqlquery,
"DELETE FROM %s.repl_monitor "
"WHERE primary_node=%d AND standby_node=%d AND last_monitor_time < '%s'::timestamp with time zone",
repmgr_schema, primary_options.node, local_options.node, monitor_standby_timestamp);
res = PQexec(primaryConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err("PQexec failed: %s\n", PQerrorMessage(conn));
PQclear(res);
CloseConnections();
exit(ERR_DB_QUERY);
}
PQclear(res);
only_one_entry = true;
}
}
}
static void
do_standby_promote(void)
{
PGconn *conn;
PGresult *res;
char sqlquery[QUERY_STR_LEN];
char script[QUERY_STR_LEN];
char myClusterName[MAXLEN];
int myLocalId = -1;
char conninfo[MAXLEN];
PGconn *old_master_conn;
int old_master_id;
int r;
char data_dir[MAXLEN];
char recovery_file_path[MAXLEN];
char recovery_done_path[MAXLEN];
char standby_version[MAXVERSIONSTR];
/*
* Read the configuration file: repmgr.conf
*/
parse_config(config_file, myClusterName, &myLocalId, conninfo);
if (myLocalId == -1)
{
fprintf(stderr, "Node information is missing. "
"Check the configuration file.\n");
exit(1);
}
/* We need to connect to check configuration */
conn = establishDBConnection(conninfo, true);
/* we need v9 or better */
pg_version(conn, standby_version);
if (strcmp(standby_version, "") == 0)
{
PQfinish(conn);
fprintf(stderr, _("%s needs standby to be PostgreSQL 9.0 or better\n"), progname);
return;
}
/* Check we are in a standby node */
if (!is_standby(conn))
{
fprintf(stderr, "repmgr: The command should be executed in a standby node\n");
return;
}
/* we also need to check if there isn't any master already */
old_master_conn = getMasterConnection(conn, myLocalId, myClusterName, &old_master_id);
if (old_master_conn != NULL)
{
PQfinish(old_master_conn);
fprintf(stderr, "There is a master already in this cluster");
return;
}
if (verbose)
printf(_("\n%s: Promoting standby...\n"), progname);
/* Get the data directory full path and the last subdirectory */
sprintf(sqlquery, "SELECT setting "
" FROM pg_settings WHERE name = 'data_directory'");
res = PQexec(conn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "Can't get info about data directory: %s\n", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return;
}
strcpy(data_dir, PQgetvalue(res, 0, 0));
PQclear(res);
PQfinish(conn);
sprintf(recovery_file_path, "%s/%s", data_dir, RECOVERY_FILE);
sprintf(recovery_done_path, "%s/%s", data_dir, RECOVERY_DONE_FILE);
rename(recovery_file_path, recovery_done_path);
/* We assume the pg_ctl script is in the PATH */
sprintf(script, "pg_ctl -D %s -m fast restart", data_dir);
r = system(script);
if (r != 0)
{
fprintf(stderr, "Can't restart service\n");
return;
}
/* reconnect to check we got promoted */
/*
* XXX i'm removing this because it gives an annoying message saying couldn't connect
* but is just the server starting up
* conn = establishDBConnection(conninfo, true);
* if (is_standby(conn))
* fprintf(stderr, "\n%s: STANDBY PROMOTE failed, this is still a standby node.\n", progname);
* else
* fprintf(stderr, "\n%s: you should REINDEX any hash indexes you have.\n", progname);
* PQfinish(conn);
*/
return;
}
int
main(int argc, char **argv)
{
static struct option long_options[] =
{
{"config", required_argument, NULL, 'f'},
{"verbose", no_argument, NULL, 'v'},
{"no-history", no_argument, NULL, 'H'},
{NULL, 0, NULL, 0}
};
int optindex;
int c;
char standby_version[MAXVERSIONSTR];
progname = get_progname(argv[0]);
if (argc > 1)
{
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
{
help(progname);
exit(SUCCESS);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
printf("%s (PostgreSQL) " PG_VERSION "\n", progname);
exit(SUCCESS);
}
}
while ((c = getopt_long(argc, argv, "f:vH", long_options, &optindex)) != -1)
{
switch (c)
{
case 'f':
config_file = optarg;
break;
case 'v':
verbose = true;
break;
case 'H': /* no-history */
only_one_entry_desired = true;
break;
default:
usage();
exit(ERR_BAD_CONFIG);
}
}
setup_cancel_handler();
/*
* Read the configuration file: repmgr.conf
*/
parse_config(config_file, &local_options);
if (local_options.node == -1)
{
log_err("Node information is missing. "
"Check the configuration file, or provide one if you have not done so.\n");
exit(ERR_BAD_CONFIG);
}
logger_init(progname, local_options.loglevel, local_options.logfacility);
if (verbose)
logger_min_verbose(LOG_INFO);
snprintf(repmgr_schema, MAXLEN, "%s%s", DEFAULT_REPMGR_SCHEMA_PREFIX, local_options.cluster_name);
log_info(_("%s Connecting to database '%s'\n"), progname, local_options.conninfo);
myLocalConn = establishDBConnection(local_options.conninfo, true);
/* should be v9 or better */
log_info(_("%s Connected to database, checking its state\n"), progname);
pg_version(myLocalConn, standby_version);
if (strcmp(standby_version, "") == 0)
{
PQfinish(myLocalConn);
log_err(_("%s needs standby to be PostgreSQL 9.0 or better\n"), progname);
exit(ERR_BAD_CONFIG);
}
/*
* Set my server mode, establish a connection to primary
* and start monitor
*/
myLocalMode = is_standby(myLocalConn) ? STANDBY_MODE : PRIMARY_MODE;
if (myLocalMode == PRIMARY_MODE)
{
primary_options.node = local_options.node;
strncpy(primary_options.conninfo, local_options.conninfo, MAXLEN);
primaryConn = myLocalConn;
}
else
{
/* I need the id of the primary as well as a connection to it */
log_info(_("%s Connecting to primary for cluster '%s'\n"),
progname, local_options.cluster_name);
primaryConn = getMasterConnection(myLocalConn, local_options.node,
local_options.cluster_name,
&primary_options.node,NULL);
if (primaryConn == NULL)
{
CloseConnections();
exit(ERR_BAD_CONFIG);
}
}
checkClusterConfiguration(myLocalConn,primaryConn);
checkNodeConfiguration(local_options.conninfo);
if (myLocalMode == STANDBY_MODE)
{
log_info(_("%s Starting continuous standby node monitoring\n"), progname);
MonitorCheck();
}
else
{
log_info(_("%s This is a primary node, program not needed here; exiting'\n"), progname);
}
/* Prevent a double-free */
if (primaryConn == myLocalConn)
myLocalConn = NULL;
/* close the connection to the database and cleanup */
CloseConnections();
/* Shuts down logging system */
logger_shutdown();
return 0;
}
static void
do_master_register(void)
{
PGconn *conn;
PGresult *res;
char sqlquery[QUERY_STR_LEN];
char myClusterName[MAXLEN];
int myLocalId = -1;
char conninfo[MAXLEN];
bool schema_exists = false;
char master_version[MAXVERSIONSTR];
/*
* Read the configuration file: repmgr.conf
*/
parse_config(config_file, myClusterName, &myLocalId, conninfo);
if (myLocalId == -1)
{
fprintf(stderr, "Node information is missing. "
"Check the configuration file.\n");
exit(1);
}
conn = establishDBConnection(conninfo, true);
/* master should be v9 or better */
pg_version(conn, master_version);
if (strcmp(master_version, "") == 0)
{
PQfinish(conn);
fprintf(stderr, _("%s needs master to be PostgreSQL 9.0 or better\n"), progname);
return;
}
/* Check we are a master */
if (is_standby(conn))
{
fprintf(stderr, "repmgr: This node should be a master\n");
PQfinish(conn);
return;
}
/* Check if there is a schema for this cluster */
sprintf(sqlquery, "SELECT 1 FROM pg_namespace WHERE nspname = 'repmgr_%s'", myClusterName);
res = PQexec(conn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "Can't get info about schemas: %s\n", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return;
}
if (PQntuples(res) > 0) /* schema exists */
{
if (!force) /* and we are not forcing so error */
{
fprintf(stderr, "Schema repmgr_%s already exists.", myClusterName);
PQclear(res);
PQfinish(conn);
return;
}
schema_exists = true;
}
PQclear(res);
if (!schema_exists)
{
/* ok, create the schema */
sprintf(sqlquery, "CREATE SCHEMA repmgr_%s", myClusterName);
if (!PQexec(conn, sqlquery))
{
fprintf(stderr, "Cannot create the schema repmgr_%s: %s\n",
myClusterName, PQerrorMessage(conn));
PQfinish(conn);
return;
}
/* ... the tables */
sprintf(sqlquery, "CREATE TABLE repmgr_%s.repl_nodes ( "
" id integer primary key, "
" cluster text not null, "
" conninfo text not null)", myClusterName);
if (!PQexec(conn, sqlquery))
{
fprintf(stderr, "Cannot create the table repmgr_%s.repl_nodes: %s\n",
myClusterName, PQerrorMessage(conn));
PQfinish(conn);
return;
}
sprintf(sqlquery, "CREATE TABLE repmgr_%s.repl_monitor ( "
" primary_node INTEGER NOT NULL, "
" standby_node INTEGER NOT NULL, "
" last_monitor_time TIMESTAMP WITH TIME ZONE NOT NULL, "
" last_wal_primary_location TEXT NOT NULL, "
" last_wal_standby_location TEXT NOT NULL, "
" replication_lag BIGINT NOT NULL, "
" apply_lag BIGINT NOT NULL) ", myClusterName);
if (!PQexec(conn, sqlquery))
{
fprintf(stderr, "Cannot create the table repmgr_%s.repl_monitor: %s\n",
myClusterName, PQerrorMessage(conn));
PQfinish(conn);
return;
}
/* and the view */
sprintf(sqlquery, "CREATE VIEW repmgr_%s.repl_status AS "
" WITH monitor_info AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY primary_node, standby_node "
" ORDER BY last_monitor_time desc) "
" FROM repmgr_%s.repl_monitor) "
" SELECT primary_node, standby_node, last_monitor_time, last_wal_primary_location, "
" last_wal_standby_location, pg_size_pretty(replication_lag) replication_lag, "
" pg_size_pretty(apply_lag) apply_lag, age(now(), last_monitor_time) AS time_lag "
" FROM monitor_info a "
" WHERE row_number = 1", myClusterName, myClusterName);
if (!PQexec(conn, sqlquery))
{
fprintf(stderr, "Cannot create the view repmgr_%s.repl_status: %s\n",
myClusterName, PQerrorMessage(conn));
PQfinish(conn);
return;
}
}
else
{
PGconn *master_conn;
int id;
/* Ensure there isn't any other master already registered */
master_conn = getMasterConnection(conn, myLocalId, myClusterName, &id);
if (master_conn != NULL)
{
PQfinish(master_conn);
fprintf(stderr, "There is a master already in this cluster");
return;
}
}
/* Now register the master */
if (force)
{
sprintf(sqlquery, "DELETE FROM repmgr_%s.repl_nodes "
" WHERE id = %d",
myClusterName, myLocalId);
if (!PQexec(conn, sqlquery))
{
fprintf(stderr, "Cannot delete node details, %s\n",
PQerrorMessage(conn));
PQfinish(conn);
return;
}
}
sprintf(sqlquery, "INSERT INTO repmgr_%s.repl_nodes "
"VALUES (%d, '%s', '%s')",
myClusterName, myLocalId, myClusterName, conninfo);
if (!PQexec(conn, sqlquery))
{
fprintf(stderr, "Cannot insert node details, %s\n",
PQerrorMessage(conn));
PQfinish(conn);
return;
}
PQfinish(conn);
return;
}
static void
do_standby_register(void)
{
PGconn *conn;
PGconn *master_conn;
int master_id;
PGresult *res;
char sqlquery[QUERY_STR_LEN];
char myClusterName[MAXLEN];
int myLocalId = -1;
char conninfo[MAXLEN];
char master_version[MAXVERSIONSTR];
char standby_version[MAXVERSIONSTR];
/*
* Read the configuration file: repmgr.conf
*/
parse_config(config_file, myClusterName, &myLocalId, conninfo);
if (myLocalId == -1)
{
fprintf(stderr, "Node information is missing. "
"Check the configuration file.\n");
exit(1);
}
conn = establishDBConnection(conninfo, true);
/* should be v9 or better */
pg_version(conn, standby_version);
if (strcmp(standby_version, "") == 0)
{
PQfinish(conn);
fprintf(stderr, _("%s needs standby to be PostgreSQL 9.0 or better\n"), progname);
return;
}
/* Check we are a standby */
if (!is_standby(conn))
{
fprintf(stderr, "repmgr: This node should be a standby\n");
PQfinish(conn);
return;
}
/* Check if there is a schema for this cluster */
sprintf(sqlquery, "SELECT 1 FROM pg_namespace WHERE nspname = 'repmgr_%s'", myClusterName);
res = PQexec(conn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "Can't get info about tablespaces: %s\n", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return;
}
if (PQntuples(res) == 0) /* schema doesn't exists */
{
fprintf(stderr, "Schema repmgr_%s doesn't exists.", myClusterName);
PQclear(res);
PQfinish(conn);
return;
}
PQclear(res);
/* check if there is a master in this cluster */
master_conn = getMasterConnection(conn, myLocalId, myClusterName, &master_id);
if (!master_conn)
return;
/* master should be v9 or better */
pg_version(master_conn, master_version);
if (strcmp(master_version, "") == 0)
{
PQfinish(conn);
PQfinish(master_conn);
fprintf(stderr, _("%s needs master to be PostgreSQL 9.0 or better\n"), progname);
return;
}
/* master and standby version should match */
if (strcmp(master_version, standby_version) != 0)
{
PQfinish(conn);
PQfinish(master_conn);
fprintf(stderr, _("%s needs versions of both master (%s) and standby (%s) to match.\n"),
progname, master_version, standby_version);
return;
}
/* Now register the standby */
if (force)
{
sprintf(sqlquery, "DELETE FROM repmgr_%s.repl_nodes "
" WHERE id = %d",
myClusterName, myLocalId);
if (!PQexec(master_conn, sqlquery))
{
fprintf(stderr, "Cannot delete node details, %s\n",
PQerrorMessage(master_conn));
PQfinish(master_conn);
PQfinish(conn);
return;
}
}
sprintf(sqlquery, "INSERT INTO repmgr_%s.repl_nodes "
"VALUES (%d, '%s', '%s')",
myClusterName, myLocalId, myClusterName, conninfo);
if (!PQexec(master_conn, sqlquery))
{
fprintf(stderr, "Cannot insert node details, %s\n",
PQerrorMessage(master_conn));
PQfinish(master_conn);
PQfinish(conn);
return;
}
PQfinish(master_conn);
PQfinish(conn);
return;
}
static void
do_standby_follow(void)
{
PGconn *conn;
PGresult *res;
char sqlquery[QUERY_STR_LEN];
char script[QUERY_STR_LEN];
char myClusterName[MAXLEN];
int myLocalId = -1;
char conninfo[MAXLEN];
PGconn *master_conn;
int master_id;
int r;
char data_dir[MAXLEN];
char master_version[MAXVERSIONSTR];
char standby_version[MAXVERSIONSTR];
/*
* Read the configuration file: repmgr.conf
*/
parse_config(config_file, myClusterName, &myLocalId, conninfo);
if (myLocalId == -1)
{
fprintf(stderr, "Node information is missing. "
"Check the configuration file.\n");
exit(1);
}
/* We need to connect to check configuration */
conn = establishDBConnection(conninfo, true);
/* Check we are in a standby node */
if (!is_standby(conn))
{
fprintf(stderr, "\n%s: The command should be executed in a standby node\n", progname);
return;
}
/* should be v9 or better */
pg_version(conn, standby_version);
if (strcmp(standby_version, "") == 0)
{
PQfinish(conn);
fprintf(stderr, _("\n%s needs standby to be PostgreSQL 9.0 or better\n"), progname);
return;
}
/* we also need to check if there is any master in the cluster */
master_conn = getMasterConnection(conn, myLocalId, myClusterName, &master_id);
if (master_conn == NULL)
{
PQfinish(conn);
fprintf(stderr, "There isn't a master to follow in this cluster");
return;
}
/* Check we are going to point to a master */
if (is_standby(master_conn))
{
PQfinish(conn);
fprintf(stderr, "%s: The node to follow should be a master\n", progname);
return;
}
/* should be v9 or better */
pg_version(master_conn, master_version);
if (strcmp(master_version, "") == 0)
{
PQfinish(conn);
PQfinish(master_conn);
fprintf(stderr, _("%s needs master to be PostgreSQL 9.0 or better\n"), progname);
return;
}
/* master and standby version should match */
if (strcmp(master_version, standby_version) != 0)
{
PQfinish(conn);
PQfinish(master_conn);
fprintf(stderr, _("%s needs versions of both master (%s) and standby (%s) to match.\n"),
progname, master_version, standby_version);
return;
}
/*
* set the host and masterport variables with the master ones
* before closing the connection because we will need them to
* recreate the recovery.conf file
*/
host = malloc(20);
masterport = malloc(10);
strcpy(host, PQhost(master_conn));
strcpy(masterport, PQport(master_conn));
PQfinish(master_conn);
if (verbose)
printf(_("\n%s: Changing standby's master...\n"), progname);
/* Get the data directory full path */
sprintf(sqlquery, "SELECT setting "
" FROM pg_settings WHERE name = 'data_directory'");
res = PQexec(conn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "Can't get info about data directory: %s\n", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return;
}
strcpy(data_dir, PQgetvalue(res, 0, 0));
PQclear(res);
PQfinish(conn);
/* write the recovery.conf file */
if (!create_recovery_file(data_dir))
return;
/* Finally, restart the service */
/* We assume the pg_ctl script is in the PATH */
sprintf(script, "pg_ctl -D %s -m fast restart", data_dir);
r = system(script);
if (r != 0)
{
fprintf(stderr, "Can't restart service\n");
return;
}
return;
}
/*
* Insert monitor info, this is basically the time and xlog replayed,
* applied on standby and current xlog location in primary.
* Also do the math to see how far are we in bytes for being uptodate
*/
static void
StandbyMonitor(void)
{
PGresult *res;
char monitor_standby_timestamp[MAXLEN];
char last_wal_primary_location[MAXLEN];
char last_wal_standby_received[MAXLEN];
char last_wal_standby_applied[MAXLEN];
unsigned long long int lsn_primary;
unsigned long long int lsn_standby_received;
unsigned long long int lsn_standby_applied;
int connection_retries;
/*
* Check if the master is still available, if after 5 minutes of retries
* we cannot reconnect, try to get a new master.
*/
CheckPrimaryConnection(); // this take up to NUM_RETRY * SLEEP_RETRY seconds
if (PQstatus(primaryConn) != CONNECTION_OK)
{
if (local_options.failover == MANUAL_FAILOVER)
{
log_err(_("We couldn't reconnect to master. Now checking if another node has been promoted.\n"));
for (connection_retries = 0; connection_retries < 6; connection_retries++)
{
primaryConn = getMasterConnection(myLocalConn, repmgr_schema, local_options.node,
local_options.cluster_name, &primary_options.node, NULL);
if (PQstatus(primaryConn) == CONNECTION_OK)
{
/* Connected, we can continue the process so break the loop */
log_err(_("Connected to node %d, continue monitoring.\n"), primary_options.node);
break;
}
else
{
log_err(_("We haven't found a new master, waiting before retry...\n"));
/* wait 5 minutes before retries, after 6 failures (30 minutes) we stop trying */
sleep(300);
}
}
if (PQstatus(primaryConn) != CONNECTION_OK)
{
log_err(_("We couldn't reconnect for long enough, exiting...\n"));
exit(ERR_DB_CON);
}
}
else if (local_options.failover == AUTOMATIC_FAILOVER)
{
/*
* When we returns from this function we will have a new primary and
* a new primaryConn
*/
do_failover();
}
}
/* Check if we still are a standby, we could have been promoted */
if (!is_standby(myLocalConn))
{
log_err(_("It seems like we have been promoted, so exit from monitoring...\n"));
CloseConnections();
exit(ERR_PROMOTED);
}
/*
* first check if there is a command being executed,
* and if that is the case, cancel the query so i can
* insert the current record
*/
if (PQisBusy(primaryConn) == 1)
CancelQuery();
/* Get local xlog info */
sqlquery_snprintf(
sqlquery,
"SELECT CURRENT_TIMESTAMP, pg_last_xlog_receive_location(), "
"pg_last_xlog_replay_location()");
res = PQexec(myLocalConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err(_("PQexec failed: %s\n"), PQerrorMessage(myLocalConn));
PQclear(res);
/* if there is any error just let it be and retry in next loop */
return;
}
strncpy(monitor_standby_timestamp, PQgetvalue(res, 0, 0), MAXLEN);
strncpy(last_wal_standby_received , PQgetvalue(res, 0, 1), MAXLEN);
strncpy(last_wal_standby_applied , PQgetvalue(res, 0, 2), MAXLEN);
PQclear(res);
/* Get primary xlog info */
sqlquery_snprintf(sqlquery, "SELECT pg_current_xlog_location() ");
res = PQexec(primaryConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
log_err(_("PQexec failed: %s\n"), PQerrorMessage(primaryConn));
PQclear(res);
return;
}
strncpy(last_wal_primary_location, PQgetvalue(res, 0, 0), MAXLEN);
PQclear(res);
/* Calculate the lag */
lsn_primary = walLocationToBytes(last_wal_primary_location);
lsn_standby_received = walLocationToBytes(last_wal_standby_received);
lsn_standby_applied = walLocationToBytes(last_wal_standby_applied);
/*
* Build the SQL to execute on primary
*/
sqlquery_snprintf(sqlquery,
"INSERT INTO %s.repl_monitor "
"VALUES(%d, %d, '%s'::timestamp with time zone, "
" '%s', '%s', "
" %lld, %lld)", repmgr_schema,
primary_options.node, local_options.node, monitor_standby_timestamp,
last_wal_primary_location,
last_wal_standby_received,
(lsn_primary - lsn_standby_received),
(lsn_standby_received - lsn_standby_applied));
/*
* Execute the query asynchronously, but don't check for a result. We
* will check the result next time we pause for a monitor step.
*/
if (PQsendQuery(primaryConn, sqlquery) == 0)
log_warning(_("Query could not be sent to primary. %s\n"),
PQerrorMessage(primaryConn));
}
int
main(int argc, char **argv)
{
static struct option long_options[] =
{
{"config", required_argument, NULL, 'f'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
};
int optindex;
int c;
char standby_version[MAXVERSIONSTR];
progname = get_progname(argv[0]);
if (argc > 1)
{
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
{
help(progname);
exit(SUCCESS);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
printf("%s (PostgreSQL) " PG_VERSION "\n", progname);
exit(SUCCESS);
}
}
while ((c = getopt_long(argc, argv, "f:v", long_options, &optindex)) != -1)
{
switch (c)
{
case 'f':
config_file = optarg;
break;
case 'v':
verbose = true;
break;
default:
usage();
exit(ERR_BAD_CONFIG);
}
}
setup_event_handlers();
/*
* Read the configuration file: repmgr.conf
*/
parse_config(config_file, &local_options);
if (local_options.node == -1)
{
log_err(_("Node information is missing. "
"Check the configuration file, or provide one if you have not done so.\n"));
exit(ERR_BAD_CONFIG);
}
logger_init(progname, local_options.loglevel, local_options.logfacility);
if (verbose)
logger_min_verbose(LOG_INFO);
snprintf(repmgr_schema, MAXLEN, "%s%s", DEFAULT_REPMGR_SCHEMA_PREFIX, local_options.cluster_name);
log_info(_("%s Connecting to database '%s'\n"), progname, local_options.conninfo);
myLocalConn = establishDBConnection(local_options.conninfo, true);
/* should be v9 or better */
log_info(_("%s Connected to database, checking its state\n"), progname);
pg_version(myLocalConn, standby_version);
if (strcmp(standby_version, "") == 0)
{
log_err(_("%s needs standby to be PostgreSQL 9.0 or better\n"), progname);
PQfinish(myLocalConn);
exit(ERR_BAD_CONFIG);
}
/*
* Set my server mode, establish a connection to primary
* and start monitor
*/
if (is_witness(myLocalConn, repmgr_schema, local_options.cluster_name, local_options.node))
myLocalMode = WITNESS_MODE;
else if (is_standby(myLocalConn))
myLocalMode = STANDBY_MODE;
else /* is the master */
myLocalMode = PRIMARY_MODE;
switch (myLocalMode)
{
case PRIMARY_MODE:
primary_options.node = local_options.node;
strncpy(primary_options.conninfo, local_options.conninfo, MAXLEN);
primaryConn = myLocalConn;
checkClusterConfiguration(myLocalConn, primaryConn);
checkNodeConfiguration(local_options.conninfo);
if (reload_configuration(config_file, &local_options))
{
PQfinish(myLocalConn);
myLocalConn = establishDBConnection(local_options.conninfo, true);
primaryConn = myLocalConn;
update_registration();
}
log_info(_("%s Starting continuous primary connection check\n"), progname);
/* Check that primary is still alive, and standbies are sending info */
/*
* Every SLEEP_MONITOR seconds, do master checks
* XXX
* Check that standbies are sending info
*/
for (;;)
{
if (CheckPrimaryConnection())
{
/*
CheckActiveStandbiesConnections();
CheckInactiveStandbies();
*/
sleep(SLEEP_MONITOR);
}
else
{
/* XXX
* May we do something more verbose ?
*/
exit (1);
}
if (got_SIGHUP)
{
/* if we can reload, then could need to change myLocalConn */
if (reload_configuration(config_file, &local_options))
{
PQfinish(myLocalConn);
myLocalConn = establishDBConnection(local_options.conninfo, true);
primaryConn = myLocalConn;
update_registration();
}
got_SIGHUP = false;
}
}
break;
case WITNESS_MODE:
case STANDBY_MODE:
/* I need the id of the primary as well as a connection to it */
log_info(_("%s Connecting to primary for cluster '%s'\n"),
progname, local_options.cluster_name);
primaryConn = getMasterConnection(myLocalConn, repmgr_schema, local_options.node,
local_options.cluster_name,
&primary_options.node, NULL);
if (primaryConn == NULL)
{
CloseConnections();
exit(ERR_BAD_CONFIG);
}
checkClusterConfiguration(myLocalConn, primaryConn);
checkNodeConfiguration(local_options.conninfo);
if (reload_configuration(config_file, &local_options))
{
PQfinish(myLocalConn);
myLocalConn = establishDBConnection(local_options.conninfo, true);
update_registration();
}
/*
* Every SLEEP_MONITOR seconds, do checks
*/
if (myLocalMode == WITNESS_MODE)
{
log_info(_("%s Starting continuous witness node monitoring\n"), progname);
}
else if (myLocalMode == STANDBY_MODE)
{
log_info(_("%s Starting continuous standby node monitoring\n"), progname);
}
for (;;)
{
if (myLocalMode == WITNESS_MODE)
WitnessMonitor();
else if (myLocalMode == STANDBY_MODE)
StandbyMonitor();
sleep(SLEEP_MONITOR);
if (got_SIGHUP)
{
/* if we can reload, then could need to change myLocalConn */
if (reload_configuration(config_file, &local_options))
{
PQfinish(myLocalConn);
myLocalConn = establishDBConnection(local_options.conninfo, true);
update_registration();
}
got_SIGHUP = false;
}
}
break;
default:
log_err(_("%s: Unrecognized mode for node %d\n"), progname, local_options.node);
}
/* Prevent a double-free */
if (primaryConn == myLocalConn)
myLocalConn = NULL;
/* close the connection to the database and cleanup */
CloseConnections();
/* Shuts down logging system */
logger_shutdown();
return 0;
}
int
main(int argc, char **argv)
{
static struct option long_options[] =
{
{"config", required_argument, NULL, 'f'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
};
int optindex;
int c;
char conninfo[MAXLEN];
char standby_version[MAXVERSIONSTR];
progname = get_progname(argv[0]);
if (argc > 1)
{
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
{
help(progname);
exit(0);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
printf("%s (PostgreSQL) " PG_VERSION "\n", progname);
exit(0);
}
}
while ((c = getopt_long(argc, argv, "f:v", long_options, &optindex)) != -1)
{
switch (c)
{
case 'f':
config_file = optarg;
break;
case 'v':
verbose = true;
break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
}
}
setup_cancel_handler();
if (config_file == NULL)
{
config_file = malloc(5 + sizeof(CONFIG_FILE));
sprintf(config_file, "./%s", CONFIG_FILE);
}
/*
* Read the configuration file: repmgr.conf
*/
parse_config(config_file, myClusterName, &myLocalId, conninfo);
if (myLocalId == -1)
{
fprintf(stderr, "Node information is missing. "
"Check the configuration file.\n");
exit(1);
}
myLocalConn = establishDBConnection(conninfo, true);
/* should be v9 or better */
pg_version(myLocalConn, standby_version);
if (strcmp(standby_version, "") == 0)
{
PQfinish(myLocalConn);
fprintf(stderr, _("%s needs standby to be PostgreSQL 9.0 or better\n"), progname);
exit(1);
}
/*
* Set my server mode, establish a connection to primary
* and start monitor
*/
myLocalMode = is_standby(myLocalConn) ? STANDBY_MODE : PRIMARY_MODE;
if (myLocalMode == PRIMARY_MODE)
{
primaryId = myLocalId;
strcpy(primaryConninfo, conninfo);
primaryConn = myLocalConn;
}
else
{
/* I need the id of the primary as well as a connection to it */
primaryConn = getMasterConnection(myLocalConn, myLocalId, myClusterName, &primaryId);
if (primaryConn == NULL)
exit(1);
}
checkClusterConfiguration();
checkNodeConfiguration(conninfo);
if (myLocalMode == STANDBY_MODE)
{
MonitorCheck();
}
/* close the connection to the database and cleanup */
CloseConnections();
return 0;
}
/*
* Insert monitor info, this is basically the time and xlog replayed,
* applied on standby and current xlog location in primary.
* Also do the math to see how far are we in bytes for being uptodate
*/
static void
MonitorExecute(void)
{
PGresult *res;
char monitor_standby_timestamp[MAXLEN];
char last_wal_primary_location[MAXLEN];
char last_wal_standby_received[MAXLEN];
char last_wal_standby_applied[MAXLEN];
unsigned long long int lsn_primary;
unsigned long long int lsn_standby_received;
unsigned long long int lsn_standby_applied;
int connection_retries;
/*
* Check if the master is still available, if after 5 minutes of retries
* we cannot reconnect, try to get a new master.
*/
for (connection_retries = 0; connection_retries < 15; connection_retries++)
{
if (PQstatus(primaryConn) != CONNECTION_OK)
{
fprintf(stderr, "\n%s: Connection to master has been lost, trying to recover...\n", progname);
/* wait 20 seconds between retries */
sleep(20);
PQreset(primaryConn);
}
else
{
fprintf(stderr, "\n%s: Connection to master has been restored, continue monitoring.\n", progname);
break;
}
}
if (PQstatus(primaryConn) != CONNECTION_OK)
{
fprintf(stderr, "\n%s: We couldn't reconnect to master, checking if ", progname);
fprintf(stderr, "%s: another node has been promoted.\n", progname);
for (connection_retries = 0; connection_retries < 6; connection_retries++)
{
primaryConn = getMasterConnection(myLocalConn, myLocalId, myClusterName, &primaryId);
if (PQstatus(primaryConn) == CONNECTION_OK)
{
/* Connected, we can continue the process so break the loop */
fprintf(stderr, "\n%s: Connected to node %d, continue monitoring.\n", progname, primaryId);
break;
}
else
{
fprintf(stderr, "\n%s: We haven't found a new master, waiting before retry...\n", progname);
/* wait 5 minutes before retries, after 6 failures (30 minutes) we stop trying */
sleep(300);
}
}
}
if (PQstatus(primaryConn) != CONNECTION_OK)
{
fprintf(stderr, "\n%s: We couldn't reconnect for long enough, exiting...\n", progname);
exit(1);
}
/* Check if we still are a standby, we could have been promoted */
if (!is_standby(myLocalConn))
{
fprintf(stderr, "\n%s: seems like we have been promoted, so exit from monitoring...\n",
progname);
CloseConnections();
exit(1);
}
/*
* first check if there is a command being executed,
* and if that is the case, cancel the query so i can
* insert the current record
*/
if (PQisBusy(primaryConn) == 1)
CancelQuery();
/* Get local xlog info */
sprintf(sqlquery,
"SELECT CURRENT_TIMESTAMP, pg_last_xlog_receive_location(), "
"pg_last_xlog_replay_location()");
res = PQexec(myLocalConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "PQexec failed: %s\n", PQerrorMessage(myLocalConn));
PQclear(res);
/* if there is any error just let it be and retry in next loop */
return;
}
strcpy(monitor_standby_timestamp, PQgetvalue(res, 0, 0));
strcpy(last_wal_standby_received , PQgetvalue(res, 0, 1));
strcpy(last_wal_standby_applied , PQgetvalue(res, 0, 2));
PQclear(res);
/* Get primary xlog info */
sprintf(sqlquery, "SELECT pg_current_xlog_location() ");
res = PQexec(primaryConn, sqlquery);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "PQexec failed: %s\n", PQerrorMessage(primaryConn));
PQclear(res);
return;
}
strcpy(last_wal_primary_location, PQgetvalue(res, 0, 0));
PQclear(res);
/* Calculate the lag */
lsn_primary = walLocationToBytes(last_wal_primary_location);
lsn_standby_received = walLocationToBytes(last_wal_standby_received);
lsn_standby_applied = walLocationToBytes(last_wal_standby_applied);
/*
* Build the SQL to execute on primary
*/
sprintf(sqlquery,
"INSERT INTO repmgr_%s.repl_monitor "
"VALUES(%d, %d, '%s'::timestamp with time zone, "
" '%s', '%s', "
" %lld, %lld)", myClusterName,
primaryId, myLocalId, monitor_standby_timestamp,
last_wal_primary_location,
last_wal_standby_received,
(lsn_primary - lsn_standby_received),
(lsn_standby_received - lsn_standby_applied));
/*
* Execute the query asynchronously, but don't check for a result. We
* will check the result next time we pause for a monitor step.
*/
if (PQsendQuery(primaryConn, sqlquery) == 0)
fprintf(stderr, "Query could not be sent to primary. %s\n",
PQerrorMessage(primaryConn));
}
