/* MultiClientConnectPoll returns the status of client connection. */
ConnectStatus
MultiClientConnectPoll(int32 connectionId)
{
PGconn *connection = NULL;
PostgresPollingStatusType pollingStatus = PGRES_POLLING_OK;
ConnectStatus connectStatus = CLIENT_INVALID_CONNECT;
Assert(connectionId != INVALID_CONNECTION_ID);
connection = ClientConnectionArray[connectionId];
Assert(connection != NULL);
pollingStatus = ClientPollingStatusArray[connectionId];
if (pollingStatus == PGRES_POLLING_OK)
{
connectStatus = CLIENT_CONNECTION_READY;
}
else if (pollingStatus == PGRES_POLLING_READING)
{
bool readReady = ClientConnectionReady(connection, PGRES_POLLING_READING);
if (readReady)
{
ClientPollingStatusArray[connectionId] = PQconnectPoll(connection);
connectStatus = CLIENT_CONNECTION_BUSY;
}
else
{
connectStatus = CLIENT_CONNECTION_BUSY_READ;
}
}
else if (pollingStatus == PGRES_POLLING_WRITING)
{
bool writeReady = ClientConnectionReady(connection, PGRES_POLLING_WRITING);
if (writeReady)
{
ClientPollingStatusArray[connectionId] = PQconnectPoll(connection);
connectStatus = CLIENT_CONNECTION_BUSY;
}
else
{
connectStatus = CLIENT_CONNECTION_BUSY_WRITE;
}
}
else if (pollingStatus == PGRES_POLLING_FAILED)
{
WarnRemoteError(connection, NULL);
connectStatus = CLIENT_CONNECTION_BAD;
}
return connectStatus;
}
static int
_conn_poll_connecting(connectionObject *self)
{
int res = PSYCO_POLL_ERROR;
const char *msg;
Dprintf("conn_poll: poll connecting");
switch (PQconnectPoll(self->pgconn)) {
case PGRES_POLLING_OK:
res = PSYCO_POLL_OK;
break;
case PGRES_POLLING_READING:
res = PSYCO_POLL_READ;
break;
case PGRES_POLLING_WRITING:
res = PSYCO_POLL_WRITE;
break;
case PGRES_POLLING_FAILED:
case PGRES_POLLING_ACTIVE:
msg = PQerrorMessage(self->pgconn);
if (!(msg && *msg)) {
msg = "asynchronous connection failed";
}
PyErr_SetString(OperationalError, msg);
res = PSYCO_POLL_ERROR;
break;
}
return res;
}
static void
evpg_connect_check(int sock, short which, void **usrdata)
{
struct evpg_cfg *config;
struct evpg_db_node *dbnode;
config = usrdata[0];
dbnode = usrdata[1];
switch(PQconnectPoll(dbnode->dbconn))
{
case PGRES_POLLING_WRITING:
event_set(&dbnode->event, sock, EV_WRITE,
(void *)evpg_connect_check, usrdata);
event_add(&dbnode->event, 0);
break;
case PGRES_POLLING_READING:
event_set(&dbnode->event, sock, EV_WRITE,
(void *)evpg_connect_check, usrdata);
event_add(&dbnode->event, 0);
break;
case PGRES_POLLING_OK:
PQsetnonblocking(dbnode->dbconn, 1);
evpg_set_ready(config, dbnode);
free(usrdata);
break;
default:
/* this is probably bad and should deal with it :) */
printf("ERROR %s\n", PQerrorMessage(dbnode->dbconn));
break;
}
}
static int
esql_postgresql_io(Esql *e)
{
if (PQstatus(e->backend.db) == CONNECTION_BAD)
{
ERR("%s", esql_postgresql_error_get(e));
return ECORE_FD_ERROR;
}
if (e->current == ESQL_CONNECT_TYPE_INIT)
{
switch (PQconnectPoll(e->backend.db))
{
case PGRES_POLLING_OK:
return 0;
case PGRES_POLLING_READING:
return ECORE_FD_READ;
case PGRES_POLLING_WRITING:
return ECORE_FD_WRITE;
default:
ERR("%s", esql_postgresql_error_get(e));
return ECORE_FD_ERROR;
}
}
if (!PQconsumeInput(e->backend.db))
{
ERR("%s", esql_postgresql_error_get(e));
return ECORE_FD_ERROR;
}
if (!PQisBusy(e->backend.db)) return 0;
return ECORE_FD_READ | ECORE_FD_WRITE; /* psql does not provide a method to get read/write mode :( */
}
static int
_conn_poll_connecting(connectionObject *self)
{
int res = PSYCO_POLL_ERROR;
Dprintf("conn_poll: poll connecting");
switch (PQconnectPoll(self->pgconn)) {
case PGRES_POLLING_OK:
res = PSYCO_POLL_OK;
break;
case PGRES_POLLING_READING:
res = PSYCO_POLL_READ;
break;
case PGRES_POLLING_WRITING:
res = PSYCO_POLL_WRITE;
break;
case PGRES_POLLING_FAILED:
case PGRES_POLLING_ACTIVE:
PyErr_SetString(OperationalError, "asynchronous connection failed");
res = PSYCO_POLL_ERROR;
break;
}
return res;
}
bool DoPoll()
{
switch(PQconnectPoll(sql))
{
case PGRES_POLLING_WRITING:
ServerInstance->SE->WantWrite(this);
status = CWRITE;
return true;
case PGRES_POLLING_READING:
status = CREAD;
return true;
case PGRES_POLLING_FAILED:
return false;
case PGRES_POLLING_OK:
status = WWRITE;
return DoConnectedPoll();
default:
return true;
}
}
bool DoPoll()
{
switch(PQconnectPoll(sql))
{
case PGRES_POLLING_WRITING:
ServerInstance->SE->ChangeEventMask(this, FD_WANT_POLL_WRITE | FD_WANT_NO_READ);
status = CWRITE;
return true;
case PGRES_POLLING_READING:
ServerInstance->SE->ChangeEventMask(this, FD_WANT_POLL_READ | FD_WANT_NO_WRITE);
status = CREAD;
return true;
case PGRES_POLLING_FAILED:
return false;
case PGRES_POLLING_OK:
ServerInstance->SE->ChangeEventMask(this, FD_WANT_POLL_READ | FD_WANT_NO_WRITE);
status = WWRITE;
DoConnectedPoll();
default:
return true;
}
}
/*
* Creates a new gang by logging on a session to each segDB involved.
*
* call this function in GangContext memory context.
* elog ERROR or return a non-NULL gang.
*/
static Gang*
createGang_async(GangType type, int gang_id, int size, int content)
{
Gang *newGangDefinition;
SegmentDatabaseDescriptor *segdbDesc = NULL;
int i = 0;
int create_gang_retry_counter = 0;
int in_recovery_mode_count = 0;
int successful_connections = 0;
bool retry = false;
int poll_timeout = 0;
struct timeval startTS;
PostgresPollingStatusType *pollingStatus = NULL;
/* true means connection status is confirmed, either established or in recovery mode */
bool *connStatusDone = NULL;
ELOG_DISPATCHER_DEBUG("createGang type = %d, gang_id = %d, size = %d, content = %d",
type, gang_id, size, content);
/* check arguments */
Assert(size == 1 || size == getgpsegmentCount());
Assert(CurrentResourceOwner != NULL);
Assert(CurrentMemoryContext == GangContext);
/* Writer gang is created before reader gangs. */
if (type == GANGTYPE_PRIMARY_WRITER)
Insist(!GangsExist());
/* Check writer gang firstly*/
if (type != GANGTYPE_PRIMARY_WRITER && !isPrimaryWriterGangAlive())
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("writer gang got broken before creating reader gangs")));
create_gang_retry:
/* If we're in a retry, we may need to reset our initial state, a bit */
newGangDefinition = NULL;
successful_connections = 0;
in_recovery_mode_count = 0;
retry = false;
/* allocate and initialize a gang structure */
newGangDefinition = buildGangDefinition(type, gang_id, size, content);
Assert(newGangDefinition != NULL);
Assert(newGangDefinition->size == size);
Assert(newGangDefinition->perGangContext != NULL);
MemoryContextSwitchTo(newGangDefinition->perGangContext);
/* allocate memory within perGangContext and will be freed automatically when gang is destroyed */
pollingStatus = palloc(sizeof(PostgresPollingStatusType) * size);
connStatusDone = palloc(sizeof(bool) * size);
struct pollfd *fds;
PG_TRY();
{
for (i = 0; i < size; i++)
{
char gpqeid[100];
char *options;
/*
* Create the connection requests. If we find a segment without a
* valid segdb we error out. Also, if this segdb is invalid, we must
* fail the connection.
*/
segdbDesc = &newGangDefinition->db_descriptors[i];
/*
* Build the connection string. Writer-ness needs to be processed
* early enough now some locks are taken before command line options
* are recognized.
*/
build_gpqeid_param(gpqeid, sizeof(gpqeid),
segdbDesc->segindex,
type == GANGTYPE_PRIMARY_WRITER,
gang_id);
options = makeOptions();
/* start connection in asynchronous way */
cdbconn_doConnectStart(segdbDesc, gpqeid, options);
if(cdbconn_isBadConnection(segdbDesc))
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("%s (%s)", PQerrorMessage(segdbDesc->conn), segdbDesc->whoami)));
connStatusDone[i] = false;
/*
* If connection status is not CONNECTION_BAD after PQconnectStart(), we must
* act as if the PQconnectPoll() had returned PGRES_POLLING_WRITING
*/
pollingStatus[i] = PGRES_POLLING_WRITING;
}
/*
* Ok, we've now launched all the connection attempts. Start the
* timeout clock (= get the start timestamp), and poll until they're
* all completed or we reach timeout.
*/
gettimeofday(&startTS, NULL);
fds = (struct pollfd *) palloc0(sizeof(struct pollfd) * size);
for(;;)
{
int nready;
int nfds = 0;
poll_timeout = getPollTimeout(&startTS);
for (i = 0; i < size; i++)
{
segdbDesc = &newGangDefinition->db_descriptors[i];
/* Skip established connections and in-recovery-mode connections*/
if (connStatusDone[i])
continue;
switch (pollingStatus[i])
{
case PGRES_POLLING_OK:
cdbconn_doConnectComplete(segdbDesc);
if (segdbDesc->motionListener == -1 || segdbDesc->motionListener == 0)
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("Internal error: No motion listener port (%s)", segdbDesc->whoami)));
successful_connections++;
connStatusDone[i] = true;
continue;
case PGRES_POLLING_READING:
fds[nfds].fd = PQsocket(segdbDesc->conn);
fds[nfds].events = POLLIN;
nfds++;
break;
case PGRES_POLLING_WRITING:
fds[nfds].fd = PQsocket(segdbDesc->conn);
fds[nfds].events = POLLOUT;
nfds++;
break;
case PGRES_POLLING_FAILED:
if (segment_failure_due_to_recovery(&segdbDesc->conn->errorMessage))
{
in_recovery_mode_count++;
connStatusDone[i] = true;
elog(LOG, "segment is in recovery mode (%s)", segdbDesc->whoami);
}
else
{
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("%s (%s)", PQerrorMessage(segdbDesc->conn), segdbDesc->whoami)));
}
break;
default:
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("unknow pollstatus (%s)", segdbDesc->whoami)));
break;
}
if (poll_timeout == 0)
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("timeout expired\n (%s)", segdbDesc->whoami)));
}
if (nfds == 0)
break;
CHECK_FOR_INTERRUPTS();
/* Wait until something happens */
nready = poll(fds, nfds, poll_timeout);
if (nready < 0)
{
int sock_errno = SOCK_ERRNO;
if (sock_errno == EINTR)
continue;
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("poll() failed: errno = %d", sock_errno)));
}
else if (nready > 0)
{
int currentFdNumber = 0;
for (i = 0; i < size; i++)
{
segdbDesc = &newGangDefinition->db_descriptors[i];
if (connStatusDone[i])
continue;
Assert(PQsocket(segdbDesc->conn) > 0);
Assert(PQsocket(segdbDesc->conn) == fds[currentFdNumber].fd);
if (fds[currentFdNumber].revents & fds[currentFdNumber].events)
pollingStatus[i] = PQconnectPoll(segdbDesc->conn);
currentFdNumber++;
}
}
}
ELOG_DISPATCHER_DEBUG("createGang: %d processes requested; %d successful connections %d in recovery",
size, successful_connections, in_recovery_mode_count);
MemoryContextSwitchTo(GangContext);
/* some segments are in recovery mode*/
if (successful_connections != size)
{
Assert(successful_connections + in_recovery_mode_count == size);
/* FTS shows some segment DBs are down */
if (isFTSEnabled() &&
FtsTestSegmentDBIsDown(newGangDefinition->db_descriptors, size))
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("FTS detected one or more segments are down")));
if ( gp_gang_creation_retry_count <= 0 ||
create_gang_retry_counter++ >= gp_gang_creation_retry_count ||
type != GANGTYPE_PRIMARY_WRITER)
ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
errmsg("failed to acquire resources on one or more segments"),
errdetail("segments is in recovery mode")));
ELOG_DISPATCHER_DEBUG("createGang: gang creation failed, but retryable.");
DisconnectAndDestroyGang(newGangDefinition);
newGangDefinition = NULL;
retry = true;
}
}
PG_CATCH();
{
MemoryContextSwitchTo(GangContext);
DisconnectAndDestroyGang(newGangDefinition);
newGangDefinition = NULL;
if (type == GANGTYPE_PRIMARY_WRITER)
{
DisconnectAndDestroyAllGangs(true);
CheckForResetSession();
}
PG_RE_THROW();
}
PG_END_TRY();
if (retry)
{
CHECK_FOR_INTERRUPTS();
pg_usleep(gp_gang_creation_retry_timer * 1000);
CHECK_FOR_INTERRUPTS();
goto create_gang_retry;
}
setLargestGangsize(size);
return newGangDefinition;
}
ngx_int_t
ngx_postgres_upstream_connect(ngx_http_request_t *r, ngx_connection_t *pgxc,
ngx_postgres_upstream_peer_data_t *pgdt)
{
PostgresPollingStatusType pgrc;
dd("entering");
pgrc = PQconnectPoll(pgdt->pgconn);
if (pgrc == PGRES_POLLING_READING || pgrc == PGRES_POLLING_WRITING) {
/*
* Fix for Linux issue found by chaoslawful (via agentzh):
* "According to the source of libpq (around fe-connect.c:1215), during
* the state switch from CONNECTION_STARTED to CONNECTION_MADE, there's
* no socket read/write operations (just a plain getsockopt call and a
* getsockname call). Therefore, for edge-triggered event model, we
* have to call PQconnectPoll one more time (immediately) when we see
* CONNECTION_MADE is returned, or we're very likely to wait for a
* writable event that has already appeared and will never appear
* again :)"
*/
if (PQstatus(pgdt->pgconn) == CONNECTION_MADE) {
dd("re-polling on connection made");
pgrc = PQconnectPoll(pgdt->pgconn);
if (pgrc == PGRES_POLLING_READING || pgrc == PGRES_POLLING_WRITING)
{
dd("returning NGX_AGAIN");
return NGX_AGAIN;
}
}
#if defined(DDEBUG) && (DDEBUG)
switch (PQstatus(pgdt->pgconn)) {
case CONNECTION_NEEDED:
dd("connecting (waiting for connect()))");
break;
case CONNECTION_STARTED:
dd("connecting (waiting for connection to be made)");
break;
case CONNECTION_MADE:
dd("connecting (connection established)");
break;
case CONNECTION_AWAITING_RESPONSE:
dd("connecting (credentials sent, waiting for response)");
break;
case CONNECTION_AUTH_OK:
dd("connecting (authenticated)");
break;
case CONNECTION_SETENV:
dd("connecting (negotiating envinroment)");
break;
case CONNECTION_SSL_STARTUP:
dd("connecting (negotiating SSL)");
break;
default:
/*
* This cannot happen, PQconnectPoll would return
* PGRES_POLLING_FAILED in that case.
*/
dd("connecting (unknown state: %d)", (int) PQstatus(pgdt->pgconn));
dd("returning NGX_ERROR");
return NGX_ERROR;
}
#endif /* DDEBUG */
dd("returning NGX_AGAIN");
return NGX_AGAIN;
}
/* remove connection timeout from new connection */
if (pgxc->write->timer_set) {
ngx_del_timer(pgxc->write);
}
if (pgrc != PGRES_POLLING_OK) {
dd("connection failed");
ngx_log_error(NGX_LOG_ERR, pgxc->log, 0,
"connection failed: %s",
PQerrorMessage(pgdt->pgconn));
dd("returning NGX_ERROR");
return NGX_ERROR;
}
dd("connected successfully");
pgxc->log->action = "sending query to PostgreSQL database";
pgdt->state = state_db_send_query;
dd("returning");
return ngx_postgres_upstream_send_query(r, pgxc, pgdt);
}
bool EpollPostgresql::epollEvent(const uint32_t &events)
{
if(conn==NULL)
{
std::cerr << "epollEvent() conn==NULL" << std::endl;
return false;
}
const ConnStatusType &connStatusType=PQstatus(conn);
if(connStatusType!=CONNECTION_OK)
{
if(connStatusType==CONNECTION_MADE)
{
started=true;
std::cout << "Connexion CONNECTION_MADE" << std::endl;
}
else if(connStatusType==CONNECTION_STARTED)
{
started=true;
std::cout << "Connexion CONNECTION_STARTED" << std::endl;
}
else if(connStatusType==CONNECTION_AWAITING_RESPONSE)
std::cout << "Connexion CONNECTION_AWAITING_RESPONSE" << std::endl;
else
{
if(connStatusType==CONNECTION_BAD)
{
started=false;
std::cerr << "Connexion not ok: CONNECTION_BAD" << std::endl;
//return false;
}
else if(connStatusType==CONNECTION_AUTH_OK)
std::cerr << "Connexion not ok: CONNECTION_AUTH_OK" << std::endl;
else if(connStatusType==CONNECTION_SETENV)
std::cerr << "Connexion not ok: CONNECTION_SETENV" << std::endl;
else if(connStatusType==CONNECTION_SSL_STARTUP)
std::cerr << "Connexion not ok: CONNECTION_SSL_STARTUP" << std::endl;
else if(connStatusType==CONNECTION_NEEDED)
std::cerr << "Connexion not ok: CONNECTION_NEEDED" << std::endl;
else
std::cerr << "Connexion not ok: " << connStatusType << std::endl;
}
}
if(connStatusType!=CONNECTION_BAD)
{
const PostgresPollingStatusType &postgresPollingStatusType=PQconnectPoll(conn);
if(postgresPollingStatusType==PGRES_POLLING_FAILED)
{
std::cerr << "Connexion status: PGRES_POLLING_FAILED" << std::endl;
return false;
}
}
if(events & EPOLLIN)
{
const int PQconsumeInputVar=PQconsumeInput(conn);
PGnotify *notify;
while((notify = PQnotifies(conn)) != NULL)
{
std::cerr << "ASYNC NOTIFY of '" << notify->relname << "' received from backend PID " << notify->be_pid << std::endl;
PQfreemem(notify);
}
if(/*PQisBusy(conn)==0, produce blocking, when server is unbusy this this never call*/true)
{
if(result!=NULL)
clear();
tuleIndex=-1;
ntuples=0;
result=PQgetResult(conn);
if(result==NULL)
std::cerr << "query async send failed: " << errorMessage() << ", PQgetResult(conn) have returned NULL" << std::endl;
else
{
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> elapsed = end-start;
const uint32_t &ms=elapsed.count();
if(ms>5000)
{
if(queriesList.empty())
std::cerr << "query too slow, take " << ms << "ms" << std::endl;
else
std::cerr << queriesList.front().query << ": query too slow, take " << ms << "ms" << std::endl;
}
#ifdef DEBUG_MESSAGE_CLIENT_SQL
else
{
if(queriesList.empty())
std::cout << "query take " << ms << "ms" << std::endl;
else
std::cout << queriesList.front().query << ": query take " << ms << "ms" << std::endl;
}
#endif
start = std::chrono::high_resolution_clock::now();
while(result!=NULL)
{
const ExecStatusType &execStatusType=PQresultStatus(result);
if(execStatusType!=PGRES_TUPLES_OK && execStatusType!=PGRES_COMMAND_OK)
{
#ifdef DEBUG_MESSAGE_CLIENT_SQL
std::cerr << simplifiedstrCoPG << ", ";
#endif
if(queriesList.empty())
std::cerr << "Query to database failed: " << errorMessage() << std::endl;
else
std::cerr << "Query to database failed: " << errorMessage() << queriesList.front().query << std::endl;
abort();//prevent continue running to prevent data corruption
tuleIndex=0;
}
else
ntuples=PQntuples(result);
if(!queue.empty())
{
CallBack callback=queue.front();
if(callback.method!=NULL)
callback.method(callback.object);
queue.erase(queue.cbegin());
}
if(result!=NULL)
clear();
if(!queriesList.empty())
queriesList.erase(queriesList.cbegin());
if(!queriesList.empty())
if(!sendNextQuery())
return false;
result=PQgetResult(conn);
}
}
}
else
std::cout << "PostgreSQL events with EPOLLIN: PQisBusy: " << std::to_string(PQconsumeInputVar) << std::endl;
}
if(events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR))
{
started=false;
if(events == EPOLLRDHUP)
{
std::cerr << "Database disconnected, try reconnect: " << errorMessage() << std::endl;
syncDisconnect();
conn=NULL;
syncReconnect();
}
}
return true;
}