int main(int argc, char *argv[])
{
char *inName, *name;
int chunkSize = 4048*1024;
FILE *in;
int accSize = 0;
int newAccSize;
int oneSize;
char line[512];
int lineCount;
char *words[16];
int wordCount;
struct slName *bacs = NULL, *bn;
char *dirName;
char *outDir;
if (argc != 4)
usage();
inName = argv[1];
dirName = argv[2];
outDir = argv[3];
in = mustOpen(inName, "r");
while (fgets(line, sizeof(line), in))
{
char *sizeString;
++lineCount;
wordCount = chopLine(line, words);
if (wordCount == 0)
continue;
if (wordCount != 9)
errAbort("Line %d of %s doesn't look like an ls -l line", lineCount, inName);
sizeString = words[4];
if (!isdigit(sizeString[0]))
errAbort("Line %d of %s doesn't look like an ls - l line", lineCount, inName);
name = words[8];
oneSize = atoi(sizeString);
newAccSize = accSize + oneSize;
if (newAccSize > chunkSize)
{
finishJob(&bacs, accSize);
accSize = oneSize;
if (oneSize > chunkSize)
warn("Size %d of %s exceed chunk size %d", oneSize, name, chunkSize);
}
else
{
accSize = newAccSize;
}
bn = newSlName(name);
slAddHead(&bacs, bn);
}
if (bacs != NULL)
finishJob(&bacs, accSize);
printf("%d total jobs\n", jobCount);
writeInLists(outDir, dirName);
//writeJobs("job", "in", startMachine, stopMachine, "cc");
}
void FractalGenerator::executeJob()
{
QMutexLocker locker( &m_mutex );
if ( m_enabled && m_regions.count() > 0 )
calculateRegion( m_regions.takeFirst() );
finishJob();
handleState();
}
//--------------------------------------------------------------------------
//
// Runs the next appropriate waiting Job.
//
// Pre-conditions:
// A RunnableJob must exist in the JobSet
//
// Post-conditions:
// The chosen RunnableJob will have Job::doJob() called.
//
// Invariants:
// <none>
//
void processTask ()
{
Job job;
{
ScopedLock lock (m_mutex);
getNextJob (job, lock);
++m_processCount;
}
JobTypeData& data (getJobTypeData (job.getType ()));
// Skip the job if we are stopping and the
// skipOnStop flag is set for the job type
//
if (!isStopping() || !data.info.skip ())
{
beast::Thread::setCurrentThreadName (data.name ());
m_journal.trace << "Doing " << data.name () << " job";
Job::clock_type::time_point const start_time (
Job::clock_type::now());
on_dequeue (job.getType (), start_time - job.queue_time ());
job.doJob ();
on_execute (job.getType (), Job::clock_type::now() - start_time);
}
else
{
m_journal.trace << "Skipping processTask ('" << data.name () << "')";
}
{
ScopedLock lock (m_mutex);
finishJob (job, lock);
--m_processCount;
checkStopped (lock);
}
// Note that when Job::~Job is called, the last reference
// to the associated LoadEvent object (in the Job) may be destroyed.
}
/**
* Called when submit TransferJob has finished and data is received.
*/
void ScrobblerSubmitter::audioScrobblerSubmitResult( KIO::Job* job ) //SLOT
{
m_prevSubmitTime = QDateTime::currentDateTime( Qt::UTC ).toTime_t();
m_inProgress = false;
if ( job->error() ) {
warning() << "KIO error! errno: " << job->error() << endl;
enqueueJob( job );
return;
}
// debug()
// << "Submit result received: "
// << endl << m_submitResultBuffer << endl;
// OK
// INTERVAL n (protocol 1.1)
if (m_submitResultBuffer.startsWith( "OK" ) )
{
debug() << "Submit successful" << endl;
QString interval = m_submitResultBuffer.section( "\n", 1, 1 );
if ( interval.startsWith( "INTERVAL" ) )
m_interval = interval.mid( 9 ).toUInt();
finishJob( job );
}
// FAILED <reason (optional)>
// INTERVAL n (protocol 1.1)
else if ( m_submitResultBuffer.startsWith( "FAILED" ) )
{
QString reason = m_submitResultBuffer.mid( 0, m_submitResultBuffer.find( "\n" ) );
if ( reason.length() > 6 )
reason = reason.mid( 7 ).stripWhiteSpace();
warning() << "Submit failed (" << reason << ")" << endl;
QString interval = m_submitResultBuffer.section( "\n", 1, 1 );
if ( interval.startsWith( "INTERVAL" ) )
m_interval = interval.mid( 9 ).toUInt();
enqueueJob( job );
}
// BADAUTH
// INTERVAL n (protocol 1.1)
else if ( m_submitResultBuffer.startsWith( "BADAUTH" ) )
{
warning() << "Submit failed (Authentication failed)" << endl;
QString interval = m_submitResultBuffer.section( "\n", 1, 1 );
if ( interval.startsWith( "INTERVAL" ) )
m_interval = interval.mid( 9 ).toUInt();
m_challenge = QString::null;
enqueueJob( job );
}
else
{
warning() << "Unknown submit response" << endl;
enqueueJob( job );
}
}
void CThreadPool::handlePipeReadable() const { finishJob(getJobFromPipe()); }
void CThreadPool::cancelJobs(const std::set<CJob*>& jobs) {
// Thanks to the mutex, jobs cannot change state anymore. There are
// three different states which can occur:
//
// READY: The job is still in our list of pending jobs and no threads
// got it yet. Just clean up.
//
// DONE: The job finished running and was already written to the pipe
// that is used for waking up finished jobs. We can just read from the
// pipe until we see this job.
//
// RUNNING: This is the complicated case. The job is currently being
// executed. We change its state to CANCELLED so that wasCancelled()
// returns true. Afterwards we wait on a CV for the job to have finished
// running. This CV is signaled by jobDone() which checks the job's
// status and sees that the job was cancelled. It signals to us that
// cancellation is done by changing the job's status to DONE.
CMutexLocker guard(m_mutex);
std::set<CJob*> wait, finished, deleteLater;
std::set<CJob*>::const_iterator it;
// Start cancelling all jobs
for (it = jobs.begin(); it != jobs.end(); ++it) {
switch ((*it)->m_eState) {
case CJob::READY: {
(*it)->m_eState = CJob::CANCELLED;
// Job wasn't started yet, must be in the queue
std::list<CJob*>::iterator it2 =
std::find(m_jobs.begin(), m_jobs.end(), *it);
assert(it2 != m_jobs.end());
m_jobs.erase(it2);
deleteLater.insert(*it);
continue;
}
case CJob::RUNNING:
(*it)->m_eState = CJob::CANCELLED;
wait.insert(*it);
continue;
case CJob::DONE:
(*it)->m_eState = CJob::CANCELLED;
finished.insert(*it);
continue;
case CJob::CANCELLED:
default:
assert(0);
}
}
// Now wait for cancellation to be done
// Collect jobs that really were cancelled. Finished cancellation is
// signaled by changing their state to DONE.
while (!wait.empty()) {
it = wait.begin();
while (it != wait.end()) {
if ((*it)->m_eState != CJob::CANCELLED) {
assert((*it)->m_eState == CJob::DONE);
// Re-set state for the destructor
(*it)->m_eState = CJob::CANCELLED;
deleteLater.insert(*it);
wait.erase(it++);
} else
it++;
}
if (wait.empty()) break;
// Then wait for more to be done
m_cancellationCond.wait(m_mutex);
}
// We must call destructors with m_mutex unlocked so that they can call
// wasCancelled()
guard.unlock();
// Handle finished jobs. They must already be in the pipe.
while (!finished.empty()) {
CJob* job = getJobFromPipe();
if (finished.erase(job) > 0) {
assert(job->m_eState == CJob::CANCELLED);
delete job;
} else
finishJob(job);
}
// Delete things that still need to be deleted
while (!deleteLater.empty()) {
delete *deleteLater.begin();
deleteLater.erase(deleteLater.begin());
}
}
int main() {
char *rmHost = "master";
char *rmPort = "8032";
char *schedHost = "master";
char *schedPort = "8030";
char *amHost = "master";
int32_t amPort = 8090;
char *am_tracking_url = "url";
int heartbeatInterval = 1000000;
int i=0;
//0. new client
LibYarnClient_t *client = NULL;
int result = newLibYarnClient("postgres", rmHost, rmPort, schedHost, schedPort,
amHost, amPort, am_tracking_url,&client,heartbeatInterval);
printf("newLibYarnClient Result Code:%d\n",result);
//1. createJob
char *jobName = "libyarn";
char *queue = "default";
char *jobId = NULL;
result = createJob(client, jobName, queue,&jobId);
printf("1. createJob, jobid:%s The createJob Result Code:%d\n", jobId,result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf("1. createJob, errorMessage:%s\n",errorMessage);
}
char *blackListAdditions[0];
char *blackListRemovals[0];
//1. allocate resource
LibYarnResource_t *allocatedResourcesArray;
int allocatedResourceArraySize;
result = allocateResources(client, jobId, 1, 1, 1024, 5,
blackListAdditions, 0, blackListRemovals, 0, NULL, 0,
&allocatedResourcesArray, &allocatedResourceArraySize);
printf("The allocateResources Result Code:%d\n",result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf("2. allocateResources, errorMessage:%s\n",errorMessage);
}
int64_t activeContainerIds[allocatedResourceArraySize];
int64_t releaseContainerIds[allocatedResourceArraySize];
int64_t statusContainerIds[allocatedResourceArraySize];
printf("2. allocateResources, allocatedResourceArraySize:%d\n", allocatedResourceArraySize);
for (i = 0 ; i < allocatedResourceArraySize; i++) {
puts("----------------------------");
printf("allocatedResourcesArray[i].containerId:%ld\n", allocatedResourcesArray[i].containerId);
activeContainerIds[i] = allocatedResourcesArray[i].containerId;
releaseContainerIds[i] = allocatedResourcesArray[i].containerId;
statusContainerIds[i] = allocatedResourcesArray[i].containerId;
printf("allocatedResourcesArray[i].host:%s\n", allocatedResourcesArray[i].host);
printf("allocatedResourcesArray[i].port:%d\n", allocatedResourcesArray[i].port);
printf("allocatedResourcesArray[i].nodeHttpAddress:%s\n", allocatedResourcesArray[i].nodeHttpAddress);
printf("allocatedResourcesArray[i].vCores:%d\n", allocatedResourcesArray[i].vCores);
printf("allocatedResourcesArray[i].memory:%d\n", allocatedResourcesArray[i].memory);
}
//3. active
result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize);
printf("The activeResources Result Code:%d\n",result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf("2. activeResources, errorMessage:%s\n",errorMessage);
}
sleep(10);
int64_t *activeFailIds;
int activeFailSize;
result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
printf("Active Fail Container Size:%d\n",activeFailSize);
for (i = 0; i < activeFailSize; i++) {
printf("Active Fail Container Id:%ld\n",activeFailIds[i]);
}
//4. getContainerReport
LibYarnContainerReport_t *containerReportArray;
int containerReportArraySize;
result = getContainerReports(client, jobId, &containerReportArray,
&containerReportArraySize);
printf("The getContainerReports Result Code:%d\n", result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getContainerReports, errorMessage:%s\n", errorMessage);
}
printf("containerReportArraySize=%d\n", containerReportArraySize);
for (i = 0; i < containerReportArraySize; i++) {
printf("-------------container: %d--------------------------\n", i);
printf("containerId:%ld\n", containerReportArray[i].containerId);
printf("vCores:%d\n", containerReportArray[i].vCores);
printf("memory:%d\n", containerReportArray[i].memory);
printf("host:%s\n", containerReportArray[i].host);
printf("port:%d\n", containerReportArray[i].port);
printf("exitStatus:%d\n", containerReportArray[i].exitStatus);
printf("state:%d\n", containerReportArray[i].state);
}
freeContainerReportArray(containerReportArray, containerReportArraySize);
//4. getContainerReport
LibYarnContainerStatus_t *containerStatusArray;
int containerStatusArraySize;
result = getContainerStatuses(client, jobId, statusContainerIds,
allocatedResourceArraySize, &containerStatusArray,
&containerStatusArraySize);
printf("The getContainerStatus Result Code:%d\n", result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getContainerStatus, errorMessage:%s\n", errorMessage);
}
printf("containerStatusArraySize=%d\n", containerStatusArraySize);
for (i = 0; i < containerStatusArraySize; i++) {
printf("-------------container: %d--------------------------\n", i);
printf("containerId:%ld\n", containerStatusArray[i].containerId);
printf("exitStatus:%d\n", containerStatusArray[i].exitStatus);
printf("state:%d\n", containerStatusArray[i].state);
printf("diagnostics:%s\n", containerStatusArray[i].diagnostics);
}
freeContainerStatusArray(containerStatusArray,containerStatusArraySize);
//6. getQueueInfo
LibYarnQueueInfo_t *queueInfo = NULL;
result = getQueueInfo(client, queue, true, true, true, &queueInfo);
printf("The getQueueInfo Result Code:%d\n",result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getQueueInfo, errorMessage:%s\n",errorMessage);
}
printf("QueueInfo: queueInfo->queueName:%s, queueInfo->capacity:%f, queueInfo->maximumCapacity:%f, queueInfo->currentCapacity:%f, queueInfo->state:%d\n",
queueInfo->queueName, queueInfo->capacity, queueInfo->maximumCapacity,
queueInfo->currentCapacity, queueInfo->state);
puts("---------chilldQueue:");
for (i = 0; i < queueInfo->childQueueNameArraySize; i++) {
printf("QueueInfo: queueInfo->childQueueNameArray[%d]:%s\n", i, queueInfo->childQueueNameArray[i]);
}
freeMemQueueInfo(queueInfo);
//7. getCluster
LibYarnNodeReport_t *nodeReportArray;
int nodeReportArraySize;
result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize);
printf("The getClusterNodes Result Code:%d\n",result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getClusterNodes, errorMessage:%s\n",errorMessage);
}
printf("nodeReportArraySize=%d\n", nodeReportArraySize);
for (i = 0; i < nodeReportArraySize; i++) {
printf("-------------node %d--------------------------\n", i);
printf("host:%s\n", nodeReportArray[i].host);
printf("port:%d\n", nodeReportArray[i].port);
printf("httpAddress:%s\n", nodeReportArray[i].httpAddress);
printf("rackName:%s\n", nodeReportArray[i].rackName);
printf("memoryUsed:%d\n", nodeReportArray[i].memoryUsed);
printf("vcoresUsed:%d\n", nodeReportArray[i].vcoresUsed);
printf("memoryCapability:%d\n", nodeReportArray[i].memoryCapability);
printf("vcoresCapability:%d\n", nodeReportArray[i].vcoresCapability);
printf("numContainers:%d\n", nodeReportArray[i].numContainers);
printf("nodeState:%d\n", nodeReportArray[i].nodeState);
printf("healthReport:%s\n", nodeReportArray[i].healthReport);
printf("lastHealthReportTime:%lld\n", nodeReportArray[i].lastHealthReportTime);
}
freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
//8. getApplicationReport
LibYarnApplicationReport_t *applicationReport = NULL;
result = getApplicationReport(client, jobId, &applicationReport);
printf("The getApplicationReport Result Code:%d\n", result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getApplicationReport, errorMessage:%s\n",
errorMessage);
}
printf("-------------ApplicationReport-------------------------\n");
printf("appId:%d\n", applicationReport->appId);
printf("user:%s\n", applicationReport->user);
printf("queue:%s\n", applicationReport->queue);
printf("name:%s\n", applicationReport->name);
printf("host:%s\n", applicationReport->host);
printf("port:%d\n", applicationReport->port);
printf("status:%d\n", applicationReport->status);
freeApplicationReport(applicationReport);
//5. release
result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize);
printf("The releaseResources Result Code:%d\n",result);
result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
printf("Active Fail Container Size:%d\n",activeFailSize);
for (i = 0; i < activeFailSize; i++) {
printf("Active Fail Container Id:%d\n",activeFailIds[i]);
}
free(activeFailIds);
freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize);
printf("freeMemAllocated is OK\n");
//9. finish
printf("jobId:%s\n", jobId);
result = finishJob(client, jobId, APPLICATION_SUCCEEDED);
printf("The finishJob Result Code:%d\n", result);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The finishJob, errorMessage:%s\n", errorMessage);
}
free(jobId);
//10. free client
deleteLibYarnClient(client);
return 0;
}
int main() {
struct timeval tpstart,tpend;
float allocateTime = 0;
float activeTime = 0;
float releaseTime = 0;
float timeuse;
char *rmHost = "localhost";
char *rmPort = "8032";
char *schedHost = "localhost";
char *schedPort = "8030";
char *amHost = "localhost";
int32_t amPort = 10;
char *am_tracking_url = "url";
int heartbeatInterval = 1000;
//0. new client
LibYarnClient_t *client = NULL;
int result = newLibYarnClient(rmHost, rmPort, schedHost, schedPort,
amHost, amPort, am_tracking_url,&client,heartbeatInterval);
//1. createJob
char *jobName = "libyarn";
char *queue = "default";
char *jobId = NULL;
result = createJob(client, jobName, queue,&jobId);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf("1. createJob, errorMessage:%s\n",errorMessage);
}
int i,j;
for (j = 0; j < 10; j++) {
LibYarnResourceRequest_t resRequest;
resRequest.priority = 1;
resRequest.host = "*";
resRequest.vCores = 1;
resRequest.memory = 1024;
resRequest.num_containers = 2;
resRequest.relax_locality = 1;
char *blackListAdditions[0];
char *blackListRemovals[0];
//1. allocate resource
LibYarnResource_t *allocatedResourcesArray;
int allocatedResourceArraySize;
gettimeofday(&tpstart,NULL);
result = allocateResources(client, jobId, &resRequest, blackListAdditions,
0, blackListRemovals, 0, &allocatedResourcesArray, &allocatedResourceArraySize,5);
gettimeofday(&tpend,NULL);
timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
allocateTime += timeuse;
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf("2. allocateResources, errorMessage:%s\n",errorMessage);
}
int32_t activeContainerIds[allocatedResourceArraySize];
int32_t releaseContainerIds[allocatedResourceArraySize];
int32_t statusContainerIds[allocatedResourceArraySize];
for (i = 0 ; i < allocatedResourceArraySize; i++) {
activeContainerIds[i] = allocatedResourcesArray[i].containerId;
releaseContainerIds[i] = allocatedResourcesArray[i].containerId;
statusContainerIds[i] = allocatedResourcesArray[i].containerId;
}
//3. active
gettimeofday(&tpstart,NULL);
result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize);
gettimeofday(&tpend,NULL);
timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
activeTime += timeuse;
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf("2. activeResources, errorMessage:%s\n",errorMessage);
}
int *activeFailIds;
int activeFailSize;
result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
for (i = 0; i < activeFailSize; i++) {
printf("Active Fail Container Id:%d\n",activeFailIds[i]);
}
//sleep(160);
//4. getContainerReport
LibYarnContainerReport_t *containerReportArray;
int containerReportArraySize;
result = getContainerReports(client, jobId, &containerReportArray,
&containerReportArraySize);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getContainerReports, errorMessage:%s\n", errorMessage);
}
freeContainerReportArray(containerReportArray, containerReportArraySize);
//4. getContainerReport
LibYarnContainerStatus_t *containerStatusArray;
int containerStatusArraySize;
result = getContainerStatuses(client, jobId, statusContainerIds,
allocatedResourceArraySize, &containerStatusArray,
&containerStatusArraySize);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getContainerStatus, errorMessage:%s\n", errorMessage);
}
freeContainerStatusArray(containerStatusArray,containerStatusArraySize);
//5. release
gettimeofday(&tpstart,NULL);
result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize);
gettimeofday(&tpend,NULL);
timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
releaseTime += timeuse;
result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
for (i = 0; i < activeFailSize; i++) {
printf("Active Fail Container Id:%d\n",activeFailIds[i]);
}
free(activeFailIds);
freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize);
//6. getQueueInfo
LibYarnQueueInfo_t *queueInfo = NULL;
result = getQueueInfo(client, queue, true, true, true, &queueInfo);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getQueueInfo, errorMessage:%s\n",errorMessage);
}
freeMemQueueInfo(queueInfo);
//7. getCluster
LibYarnNodeReport_t *nodeReportArray;
int nodeReportArraySize;
result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getClusterNodes, errorMessage:%s\n",errorMessage);
}
freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
//8. getApplicationReport
LibYarnApplicationReport_t *applicationReport = NULL;
result = getApplicationReport(client, jobId, &applicationReport);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The getApplicationReport, errorMessage:%s\n",
errorMessage);
}
freeApplicationReport(applicationReport);
}
printf("allocateTime:%f s\n",allocateTime/1000000);
printf("activeTime:%f s\n",activeTime/1000000);
printf("releaseTime:%f s\n",releaseTime/1000000);
//9. finish
result = finishJob(client, jobId, APPLICATION_SUCCEEDED);
if (result != FUNCTION_SUCCEEDED) {
const char* errorMessage = getErrorMessage();
printf(" The finishJob, errorMessage:%s\n", errorMessage);
}
free(jobId);
//10. free client
deleteLibYarnClient(client);
return 0;
}
