void initCGroupThreads(void)
{
/* We don't initialize CGroup thread if CPU enforcement is not enabled*/
if (!rm_enforce_cpu_enable)
{
return;
}
/* Initialize queue for CPU enforcement tasks */
g_queue_cgroup = queue_create();
if (g_queue_cgroup == NULL)
{
elog(ERROR, "%s Function initCGroupThreads failed with out of memory",
ENFORCER_MESSAGE_HEAD);
}
/* Create thread to handle CPU enforcement tasks in queue */
if (pthread_create(&t_move_cgroup, NULL, cgroupService, NULL))
{
elog(FATAL, "%s Function initCGroupThreads failed to create worker thread",
ENFORCER_MESSAGE_HEAD);
}
/* Set current time as latest cleanup time for CGroup */
setCGroupLastCleanupTime(gettime_microsec());
}
int MainHandlerLoop_RMSEG(void)
{
int res = FUNC_RETURN_OK;
uint64_t curtime = 0;
int errorcode = FUNC_RETURN_OK;
char errorbuf[1024];
while( DRMGlobalInstance->ResManagerMainKeepRun ) {
if (!PostmasterIsAlive(true)) {
DRMGlobalInstance->ResManagerMainKeepRun = false;
elog(LOG, "Postmaster is not alive, resource manager exits");
break;
}
/* PART1. Handle socket server inputs. */
res = processAllCommFileDescs();
if ( res != FUNC_RETURN_OK ) {
/*
* The possible error here is the failure of poll(), we won't keep
* running HAWQ RM any longer, graceful quit is requested.
*/
DRMGlobalInstance->ResManagerMainKeepRun = false;
elog(LOG, "System error cause resource manager not possible to track "
"network communications.");
}
/* PART2. Handle all BE submitted requests. */
processSubmittedRequests();
/* PART3. Fresh local host info and send IMAlive message to resource
* manager server. */
curtime = gettime_microsec();
if ( DRMGlobalInstance->LocalHostStat == NULL ||
curtime - DRMGlobalInstance->LocalHostLastUpdateTime >
SEGMENT_HOSTCHECK_INTERVAL ) {
refreshLocalHostInstance();
checkLocalPostmasterStatus();
}
if ( DRMGlobalInstance->SendIMAlive ) {
if (DRMGlobalInstance->LocalHostStat != NULL &&
curtime - DRMGlobalInstance->HeartBeatLastSentTime >
SEGMENT_HEARTBEAT_INTERVAL ) {
sendIMAlive(&errorcode, errorbuf, sizeof(errorbuf));
DRMGlobalInstance->HeartBeatLastSentTime = gettime_microsec();
}
}
/* PART4. Send responses back to the clients. */
sendResponseToClients();
/* PART5. Resource enforcement work thread quit */
if (g_enforcement_thread_quited) {
elog(ERROR, "Resource enforcement thread quited");
}
}
elog(RMLOG, "Resource manager main event handler exits.");
return res;
}
/**
* Move QE process into CGroup for resource enforcement purpose.
*/
int MoveToCGroup(uint32 pid, const char *cgroup_name)
{
int res = FUNC_RETURN_OK;
CGroupInfo *cgi = NULL;
uint32 *pid_add = NULL;
GSimpStringPtr pkey = stringToGSimpString(cgroup_name);
if (pkey == NULL)
{
write_log("%s Prepare CGroup name %s failed with out of memory",
ENFORCER_MESSAGE_HEAD,
cgroup_name);
return RESENFORCER_ERROR_INSUFFICIENT_MEMORY;
}
Pair cgroup = getGHashNode(g_ghash_cgroup, (void *)pkey);
if (cgroup == NULL)
{
if (rm_enforce_cpu_enable)
{
res = createCGroup(cgroup_name, "cpu");
if (res != FUNC_RETURN_OK)
{
write_log("%s Create CGroup %s failed",
ENFORCER_MESSAGE_HEAD,
cgroup_name);
return res;
}
}
cgi = (CGroupInfo *)malloc(sizeof(CGroupInfo));
if (cgi == NULL)
{
write_log("%s Create CGroup %s failed with out of memory",
ENFORCER_MESSAGE_HEAD,
cgroup_name);
return RESENFORCER_ERROR_INSUFFICIENT_MEMORY;
}
Assert(strlen(cgroup_name) < sizeof(cgi->name));
strncpy(cgi->name, cgroup_name, strlen(cgroup_name)+1);
cgi->creation_time = gettime_microsec();
cgi->pids = llist_create();
if (cgi->pids == NULL)
{
write_log("%s Add PID %d to CGroup %s failed",
ENFORCER_MESSAGE_HEAD,
pid,
cgroup_name);
res = RESENFORCER_ERROR_INSUFFICIENT_MEMORY;
goto exit;
}
cgi->vcore_current = 0;
cgi->vdisk_current = 0;
cgi->to_be_deleted = 0;
void *oldvalue = NULL;
#ifdef DEBUG_GHASH
write_log("%s ########## Before add CGroup %s in hash in MoveToCGroup ##########",
ENFORCER_MESSAGE_HEAD,
cgroup_name);
dumpGHash(g_ghash_cgroup);
#endif
if (setGHashNode(g_ghash_cgroup,
(void *)pkey,
(void *)cgi,
false,
&oldvalue) != FUNC_RETURN_OK)
{
write_log("%s Add CGroup to list failed with out of memory",
ENFORCER_MESSAGE_HEAD);
res = RESENFORCER_ERROR_INSUFFICIENT_MEMORY;
goto exit;
}
#ifdef DEBUG_GHASH
write_log("%s ########## After add CGroup %s in hash in MoveToCGroup ##########",
ENFORCER_MESSAGE_HEAD,
cgroup_name);
dumpGHash(g_ghash_cgroup);
#endif
}
else
{
cgi = (CGroupInfo *)(cgroup->Value);
/* revert the delete operation */
if (cgi == NULL)
{
write_log("%s CGroup %s found in hash but its content is inaccessible",
ENFORCER_MESSAGE_HEAD,
cgroup_name);
goto exit;
}
else if (cgi->to_be_deleted > 0)
{
cgi->to_be_deleted = 0;
}
}
#ifdef DEBUG_GHASH
write_log("%s ########## Before add PID %d in CGroup %s in hash in MoveToCGroup ##########",
ENFORCER_MESSAGE_HEAD,
pid,
cgroup_name);
dumpGHash(g_ghash_cgroup);
#endif
pid_add = (uint32 *)malloc(sizeof(uint32));
if (pid_add == NULL)
{
write_log("%s Create PID %d failed with out of memory",
ENFORCER_MESSAGE_HEAD,
pid);
res = RESENFORCER_ERROR_INSUFFICIENT_MEMORY;
goto exit;
}
*pid_add = pid;
llist_insert(cgi->pids, pid_add);
#ifdef DEBUG_GHASH
write_log("%s ########## After add PID %d in CGroup %s in hash in MoveToCGroup ##########",
ENFORCER_MESSAGE_HEAD,
pid,
cgroup_name);
dumpGHash(g_ghash_cgroup);
#endif
/* Process CGroup for cpu sub-system */
if (rm_enforce_cpu_enable)
{
res = setCGroupProcess(cgroup_name, "cpu", pid);
if (res != FUNC_RETURN_OK)
{
write_log("%s Add PID %d to CPU CGroup %s failed",
ENFORCER_MESSAGE_HEAD,
pid,
cgroup_name);
goto exit;
}
}
return FUNC_RETURN_OK;
exit:
if (pkey)
{
free(pkey);
}
if (pid_add)
{
free(pid_add);
}
if (cgi)
{
freeCGroupInfo(cgi);
}
return res;
}
/**
* Clean up CGroup directories in a periodical fashion when HAWQ is up and running.
*/
int CleanUpCGroupAtRuntime(void)
{
int res = FUNC_RETURN_OK;
if (gettime_microsec() - getCGroupLastCleanupTime() < getCGroupCleanupThreshold())
{
return FUNC_RETURN_OK;
}
for (int i = 0; i < g_ghash_cgroup->SlotVolume; ++i)
{
if (g_ghash_cgroup->Slots[i] == NULL)
{
continue;
}
lnode *pairnode = (g_ghash_cgroup->Slots[i])->head;
lnode *nextnode = NULL;
while (pairnode)
{
nextnode = pairnode->next;
Pair pair = (Pair)llist_lfirst(pairnode);
GSimpString *key = (GSimpString *)(pair->Key);
CGroupInfo *cgi = (CGroupInfo *)(pair->Value);
/* Delay remove CGroup directory to workaround CGroup panic bug */
if (cgi->to_be_deleted == 1)
{
cgi->to_be_deleted++;
}
else if (cgi->to_be_deleted > 1)
{
res = deleteCGroup(cgi->name, "cpu");
if (res == RESENFORCER_ERROR_INSUFFICIENT_MEMORY)
{
write_log("%s Cannot remove CPU CGroup directory %s due to out of memory",
ENFORCER_MESSAGE_HEAD,
cgi->name);
return RESENFORCER_ERROR_INSUFFICIENT_MEMORY;
}
if (res == FUNC_RETURN_OK)
{
if (pairnode->prev)
{
pairnode->prev->next = pairnode->next;
if (pairnode->next)
{
pairnode->next->prev = pairnode->prev;
}
}
else
{
(g_ghash_cgroup->Slots[i])->head = pairnode->next;
if (pairnode->next)
{
pairnode->next->prev = NULL;
}
}
(g_ghash_cgroup->Slots[i])->size--;
if (key)
{
g_ghash_cgroup->KeyFreeFunction((void *)key);
}
if (g_ghash_cgroup->ValFreeFunction)
{
g_ghash_cgroup->ValFreeFunction((void *)cgi);
}
free(pairnode->data);
free(pairnode);
}
else
{
write_log("%s Failed to remove CGroup %s with errno %d",
ENFORCER_MESSAGE_HEAD,
cgi->name,
res);
}
}
pairnode = nextnode;
}
}
setCGroupLastCleanupTime(gettime_microsec());
return FUNC_RETURN_OK;
}
