/*
* cdbdisp_makeDispatchResults:
* Allocates a CdbDispatchResults object in the current memory context.
* Will be freed in function cdbdisp_destroyDispatcherState by deleting the
* memory context.
*/
CdbDispatchResults *
cdbdisp_makeDispatchResults(int sliceCapacity,
bool cancelOnError)
{
CdbDispatchResults *results = palloc0(sizeof(*results));
int resultCapacity = largestGangsize() * sliceCapacity;
int nbytes = resultCapacity * sizeof(results->resultArray[0]);
results->resultArray = palloc0(nbytes);
results->resultCapacity = resultCapacity;
results->resultCount = 0;
results->iFirstError = -1;
results->errcode = 0;
results->cancelOnError = cancelOnError;
results->sliceMap = NULL;
results->sliceCapacity = sliceCapacity;
if (sliceCapacity > 0)
{
nbytes = sliceCapacity * sizeof(results->sliceMap[0]);
results->sliceMap = palloc0(nbytes);
}
return results;
}
static int
getMaxThreadsPerGang(void)
{
int maxThreads = 0;
if (gp_connections_per_thread == 0)
maxThreads = 1; /* one, not zero, because we need to allocate one param block */
else
maxThreads = 1 + (largestGangsize() - 1) / gp_connections_per_thread;
return maxThreads;
}
/*
* cdbdisp_makeDispatchThreads:
* Allocates memory for a CdbDispatchCmdThreads structure and the memory
* needed inside. Do the initialization.
* Will be freed in function cdbdisp_destroyDispatcherState by deleting the
* memory context.
*/
CdbDispatchCmdThreads *
cdbdisp_makeDispatchThreads(int maxSlices)
{
int maxThreadsPerGang = getMaxThreadsPerGang();
/*
* the maximum number of command parameter blocks we'll possibly need is
* * one for each slice on the primary gang. Max sure that we
* * have enough -- once we've created the command block we're stuck with it
* * for the duration of this statement (including CDB-DTM ).
* * X 2 for good measure ?
*/
int maxThreads = maxThreadsPerGang * 4 * Max(maxSlices, 5);
int maxConn = gp_connections_per_thread == 0 ? largestGangsize() : gp_connections_per_thread;
int size = 0;
int i = 0;
CdbDispatchCmdThreads *dThreads = palloc0(sizeof(*dThreads));
size = maxThreads * sizeof(DispatchCommandParms);
dThreads->dispatchCommandParmsAr = (DispatchCommandParms *) palloc0(size);
dThreads->dispatchCommandParmsArSize = maxThreads;
dThreads->threadCount = 0;
for (i = 0; i < maxThreads; i++)
{
DispatchCommandParms *pParms = &dThreads->dispatchCommandParmsAr[i];
pParms->nfds = maxConn;
MemSet(&pParms->thread, 0, sizeof(pthread_t));
size = maxConn * sizeof(CdbDispatchResult *);
pParms->dispatchResultPtrArray = (CdbDispatchResult **) palloc0(size);
size = sizeof(struct pollfd) * maxConn;
pParms->fds = (struct pollfd *) palloc0(size);
}
return dThreads;
}
void
cdbdisp_dispatchToGang_internal(struct CdbDispatcherState *ds,
struct Gang *gp,
int sliceIndex,
CdbDispatchDirectDesc * disp_direct)
{
struct CdbDispatchResults *dispatchResults = ds->primaryResults;
SegmentDatabaseDescriptor *segdbDesc;
int i,
max_threads,
segdbs_in_thread_pool = 0,
newThreads = 0;
int gangSize = 0;
SegmentDatabaseDescriptor *db_descriptors;
char *newQueryText = NULL;
DispatchCommandParms *pParms = NULL;
gangSize = gp->size;
Assert(gangSize <= largestGangsize());
db_descriptors = gp->db_descriptors;
/*
* The most threads we could have is segdb_count / gp_connections_per_thread, rounded up.
* This is equivalent to 1 + (segdb_count-1) / gp_connections_per_thread.
* We allocate enough memory for this many DispatchCommandParms structures,
* even though we may not use them all.
*
* We can only use gp->size here if we're not dealing with a
* singleton gang. It is safer to always use the max number of segments we are
* controlling (largestGangsize).
*/
Assert(gp_connections_per_thread >= 0);
Assert(ds->dispatchThreads != NULL);
/*
* If we attempt to reallocate, there is a race here: we
* know that we have threads running using the
* dispatchCommandParamsAr! If we reallocate we
* potentially yank it out from under them! Don't do
* it!
*/
max_threads = getMaxThreadsPerGang();
if (ds->dispatchThreads->dispatchCommandParmsArSize <
(ds->dispatchThreads->threadCount + max_threads))
{
elog(ERROR,
"Attempted to reallocate dispatchCommandParmsAr while other threads still running size %d new threadcount %d",
ds->dispatchThreads->dispatchCommandParmsArSize,
ds->dispatchThreads->threadCount + max_threads);
}
pParms = &ds->dispatchThreads->dispatchCommandParmsAr[0];
newQueryText =
dupQueryTextAndSetSliceId(ds->dispatchStateContext, pParms->query_text,
pParms->query_text_len, sliceIndex);
/*
* Create the thread parms structures based targetSet parameter.
* This will add the segdbDesc pointers appropriate to the
* targetSet into the thread Parms structures, making sure that each thread
* handles gp_connections_per_thread segdbs.
*/
for (i = 0; i < gangSize; i++)
{
CdbDispatchResult *qeResult;
segdbDesc = &db_descriptors[i];
int parmsIndex = 0;
Assert(segdbDesc != NULL);
if (disp_direct->directed_dispatch)
{
Assert(disp_direct->count == 1); /* currently we allow direct-to-one dispatch, only */
if (disp_direct->content[0] !=
segdbDesc->segment_database_info->segindex)
continue;
}
/*
* Initialize the QE's CdbDispatchResult object.
*/
qeResult = cdbdisp_makeResult(dispatchResults, segdbDesc, sliceIndex);
if (qeResult == NULL)
{
/*
* writer_gang could be NULL if this is an extended query.
*/
if (dispatchResults->writer_gang)
dispatchResults->writer_gang->dispatcherActive = true;
elog(FATAL, "could not allocate resources for segworker communication");
}
/*
* Transfer any connection errors from segdbDesc.
*/
if (segdbDesc->errcode || segdbDesc->error_message.len)
cdbdisp_mergeConnectionErrors(qeResult, segdbDesc);
parmsIndex = gp_connections_per_thread == 0 ? 0 : segdbs_in_thread_pool / gp_connections_per_thread;
pParms =
ds->dispatchThreads->dispatchCommandParmsAr +
ds->dispatchThreads->threadCount + parmsIndex;
pParms->dispatchResultPtrArray[pParms->db_count++] = qeResult;
if (newQueryText != NULL)
pParms->query_text = newQueryText;
/*
* This CdbDispatchResult/SegmentDatabaseDescriptor pair will be
* dispatched and monitored by a thread to be started below. Only that
* thread should touch them until the thread is finished with them and
* resets the stillRunning flag. Caller must CdbCheckDispatchResult()
* to wait for completion.
*/
qeResult->stillRunning = true;
segdbs_in_thread_pool++;
}
/*
* Compute the thread count based on how many segdbs were added into the
* thread pool, knowing that each thread handles gp_connections_per_thread
* segdbs.
*/
if (segdbs_in_thread_pool == 0)
newThreads = 0;
else if (gp_connections_per_thread == 0)
newThreads = 1;
else
newThreads = 1
+ (segdbs_in_thread_pool - 1) / gp_connections_per_thread;
/*
* Create the threads. (which also starts the dispatching).
*/
for (i = 0; i < newThreads; i++)
{
DispatchCommandParms *pParms =
&(ds->dispatchThreads->dispatchCommandParmsAr +
ds->dispatchThreads->threadCount)[i];
Assert(pParms != NULL);
if (gp_connections_per_thread == 0)
{
Assert(newThreads <= 1);
thread_DispatchOut(pParms);
}
else
{
int pthread_err = 0;
pParms->thread_valid = true;
pthread_err =
gp_pthread_create(&pParms->thread, thread_DispatchCommand,
pParms, "dispatchToGang");
if (pthread_err != 0)
{
int j;
pParms->thread_valid = false;
/*
* Error during thread create (this should be caused by
* resource constraints). If we leave the threads running,
* they'll immediately have some problems -- so we need to
* join them, and *then* we can issue our FATAL error
*/
pParms->waitMode = DISPATCH_WAIT_CANCEL;
for (j = 0; j < ds->dispatchThreads->threadCount + (i - 1); j++)
{
DispatchCommandParms *pParms;
pParms = &ds->dispatchThreads->dispatchCommandParmsAr[j];
pParms->waitMode = DISPATCH_WAIT_CANCEL;
pParms->thread_valid = false;
pthread_join(pParms->thread, NULL);
}
ereport(FATAL,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("could not create thread %d of %d", i + 1, newThreads),
errdetail ("pthread_create() failed with err %d", pthread_err)));
}
}
}
ds->dispatchThreads->threadCount += newThreads;
elog(DEBUG4, "dispatchToGang: Total threads now %d",
ds->dispatchThreads->threadCount);
}
