int main(int argc, char *argv[]) { char *inName, *name; int chunkSize = 4048*1024; FILE *in; int accSize = 0; int newAccSize; int oneSize; char line[512]; int lineCount; char *words[16]; int wordCount; struct slName *bacs = NULL, *bn; char *dirName; char *outDir; if (argc != 4) usage(); inName = argv[1]; dirName = argv[2]; outDir = argv[3]; in = mustOpen(inName, "r"); while (fgets(line, sizeof(line), in)) { char *sizeString; ++lineCount; wordCount = chopLine(line, words); if (wordCount == 0) continue; if (wordCount != 9) errAbort("Line %d of %s doesn't look like an ls -l line", lineCount, inName); sizeString = words[4]; if (!isdigit(sizeString[0])) errAbort("Line %d of %s doesn't look like an ls - l line", lineCount, inName); name = words[8]; oneSize = atoi(sizeString); newAccSize = accSize + oneSize; if (newAccSize > chunkSize) { finishJob(&bacs, accSize); accSize = oneSize; if (oneSize > chunkSize) warn("Size %d of %s exceed chunk size %d", oneSize, name, chunkSize); } else { accSize = newAccSize; } bn = newSlName(name); slAddHead(&bacs, bn); } if (bacs != NULL) finishJob(&bacs, accSize); printf("%d total jobs\n", jobCount); writeInLists(outDir, dirName); //writeJobs("job", "in", startMachine, stopMachine, "cc"); }
void FractalGenerator::executeJob() { QMutexLocker locker( &m_mutex ); if ( m_enabled && m_regions.count() > 0 ) calculateRegion( m_regions.takeFirst() ); finishJob(); handleState(); }
//-------------------------------------------------------------------------- // // Runs the next appropriate waiting Job. // // Pre-conditions: // A RunnableJob must exist in the JobSet // // Post-conditions: // The chosen RunnableJob will have Job::doJob() called. // // Invariants: // <none> // void processTask () { Job job; { ScopedLock lock (m_mutex); getNextJob (job, lock); ++m_processCount; } JobTypeData& data (getJobTypeData (job.getType ())); // Skip the job if we are stopping and the // skipOnStop flag is set for the job type // if (!isStopping() || !data.info.skip ()) { beast::Thread::setCurrentThreadName (data.name ()); m_journal.trace << "Doing " << data.name () << " job"; Job::clock_type::time_point const start_time ( Job::clock_type::now()); on_dequeue (job.getType (), start_time - job.queue_time ()); job.doJob (); on_execute (job.getType (), Job::clock_type::now() - start_time); } else { m_journal.trace << "Skipping processTask ('" << data.name () << "')"; } { ScopedLock lock (m_mutex); finishJob (job, lock); --m_processCount; checkStopped (lock); } // Note that when Job::~Job is called, the last reference // to the associated LoadEvent object (in the Job) may be destroyed. }
/** * Called when submit TransferJob has finished and data is received. */ void ScrobblerSubmitter::audioScrobblerSubmitResult( KIO::Job* job ) //SLOT { m_prevSubmitTime = QDateTime::currentDateTime( Qt::UTC ).toTime_t(); m_inProgress = false; if ( job->error() ) { warning() << "KIO error! errno: " << job->error() << endl; enqueueJob( job ); return; } // debug() // << "Submit result received: " // << endl << m_submitResultBuffer << endl; // OK // INTERVAL n (protocol 1.1) if (m_submitResultBuffer.startsWith( "OK" ) ) { debug() << "Submit successful" << endl; QString interval = m_submitResultBuffer.section( "\n", 1, 1 ); if ( interval.startsWith( "INTERVAL" ) ) m_interval = interval.mid( 9 ).toUInt(); finishJob( job ); } // FAILED <reason (optional)> // INTERVAL n (protocol 1.1) else if ( m_submitResultBuffer.startsWith( "FAILED" ) ) { QString reason = m_submitResultBuffer.mid( 0, m_submitResultBuffer.find( "\n" ) ); if ( reason.length() > 6 ) reason = reason.mid( 7 ).stripWhiteSpace(); warning() << "Submit failed (" << reason << ")" << endl; QString interval = m_submitResultBuffer.section( "\n", 1, 1 ); if ( interval.startsWith( "INTERVAL" ) ) m_interval = interval.mid( 9 ).toUInt(); enqueueJob( job ); } // BADAUTH // INTERVAL n (protocol 1.1) else if ( m_submitResultBuffer.startsWith( "BADAUTH" ) ) { warning() << "Submit failed (Authentication failed)" << endl; QString interval = m_submitResultBuffer.section( "\n", 1, 1 ); if ( interval.startsWith( "INTERVAL" ) ) m_interval = interval.mid( 9 ).toUInt(); m_challenge = QString::null; enqueueJob( job ); } else { warning() << "Unknown submit response" << endl; enqueueJob( job ); } }
void CThreadPool::handlePipeReadable() const { finishJob(getJobFromPipe()); }
void CThreadPool::cancelJobs(const std::set<CJob*>& jobs) { // Thanks to the mutex, jobs cannot change state anymore. There are // three different states which can occur: // // READY: The job is still in our list of pending jobs and no threads // got it yet. Just clean up. // // DONE: The job finished running and was already written to the pipe // that is used for waking up finished jobs. We can just read from the // pipe until we see this job. // // RUNNING: This is the complicated case. The job is currently being // executed. We change its state to CANCELLED so that wasCancelled() // returns true. Afterwards we wait on a CV for the job to have finished // running. This CV is signaled by jobDone() which checks the job's // status and sees that the job was cancelled. It signals to us that // cancellation is done by changing the job's status to DONE. CMutexLocker guard(m_mutex); std::set<CJob*> wait, finished, deleteLater; std::set<CJob*>::const_iterator it; // Start cancelling all jobs for (it = jobs.begin(); it != jobs.end(); ++it) { switch ((*it)->m_eState) { case CJob::READY: { (*it)->m_eState = CJob::CANCELLED; // Job wasn't started yet, must be in the queue std::list<CJob*>::iterator it2 = std::find(m_jobs.begin(), m_jobs.end(), *it); assert(it2 != m_jobs.end()); m_jobs.erase(it2); deleteLater.insert(*it); continue; } case CJob::RUNNING: (*it)->m_eState = CJob::CANCELLED; wait.insert(*it); continue; case CJob::DONE: (*it)->m_eState = CJob::CANCELLED; finished.insert(*it); continue; case CJob::CANCELLED: default: assert(0); } } // Now wait for cancellation to be done // Collect jobs that really were cancelled. Finished cancellation is // signaled by changing their state to DONE. while (!wait.empty()) { it = wait.begin(); while (it != wait.end()) { if ((*it)->m_eState != CJob::CANCELLED) { assert((*it)->m_eState == CJob::DONE); // Re-set state for the destructor (*it)->m_eState = CJob::CANCELLED; deleteLater.insert(*it); wait.erase(it++); } else it++; } if (wait.empty()) break; // Then wait for more to be done m_cancellationCond.wait(m_mutex); } // We must call destructors with m_mutex unlocked so that they can call // wasCancelled() guard.unlock(); // Handle finished jobs. They must already be in the pipe. while (!finished.empty()) { CJob* job = getJobFromPipe(); if (finished.erase(job) > 0) { assert(job->m_eState == CJob::CANCELLED); delete job; } else finishJob(job); } // Delete things that still need to be deleted while (!deleteLater.empty()) { delete *deleteLater.begin(); deleteLater.erase(deleteLater.begin()); } }
int main() { char *rmHost = "master"; char *rmPort = "8032"; char *schedHost = "master"; char *schedPort = "8030"; char *amHost = "master"; int32_t amPort = 8090; char *am_tracking_url = "url"; int heartbeatInterval = 1000000; int i=0; //0. new client LibYarnClient_t *client = NULL; int result = newLibYarnClient("postgres", rmHost, rmPort, schedHost, schedPort, amHost, amPort, am_tracking_url,&client,heartbeatInterval); printf("newLibYarnClient Result Code:%d\n",result); //1. createJob char *jobName = "libyarn"; char *queue = "default"; char *jobId = NULL; result = createJob(client, jobName, queue,&jobId); printf("1. createJob, jobid:%s The createJob Result Code:%d\n", jobId,result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf("1. createJob, errorMessage:%s\n",errorMessage); } char *blackListAdditions[0]; char *blackListRemovals[0]; //1. allocate resource LibYarnResource_t *allocatedResourcesArray; int allocatedResourceArraySize; result = allocateResources(client, jobId, 1, 1, 1024, 5, blackListAdditions, 0, blackListRemovals, 0, NULL, 0, &allocatedResourcesArray, &allocatedResourceArraySize); printf("The allocateResources Result Code:%d\n",result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf("2. allocateResources, errorMessage:%s\n",errorMessage); } int64_t activeContainerIds[allocatedResourceArraySize]; int64_t releaseContainerIds[allocatedResourceArraySize]; int64_t statusContainerIds[allocatedResourceArraySize]; printf("2. allocateResources, allocatedResourceArraySize:%d\n", allocatedResourceArraySize); for (i = 0 ; i < allocatedResourceArraySize; i++) { puts("----------------------------"); printf("allocatedResourcesArray[i].containerId:%ld\n", allocatedResourcesArray[i].containerId); activeContainerIds[i] = allocatedResourcesArray[i].containerId; releaseContainerIds[i] = allocatedResourcesArray[i].containerId; statusContainerIds[i] = allocatedResourcesArray[i].containerId; printf("allocatedResourcesArray[i].host:%s\n", allocatedResourcesArray[i].host); printf("allocatedResourcesArray[i].port:%d\n", allocatedResourcesArray[i].port); printf("allocatedResourcesArray[i].nodeHttpAddress:%s\n", allocatedResourcesArray[i].nodeHttpAddress); printf("allocatedResourcesArray[i].vCores:%d\n", allocatedResourcesArray[i].vCores); printf("allocatedResourcesArray[i].memory:%d\n", allocatedResourcesArray[i].memory); } //3. active result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize); printf("The activeResources Result Code:%d\n",result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf("2. activeResources, errorMessage:%s\n",errorMessage); } sleep(10); int64_t *activeFailIds; int activeFailSize; result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize); printf("Active Fail Container Size:%d\n",activeFailSize); for (i = 0; i < activeFailSize; i++) { printf("Active Fail Container Id:%ld\n",activeFailIds[i]); } //4. getContainerReport LibYarnContainerReport_t *containerReportArray; int containerReportArraySize; result = getContainerReports(client, jobId, &containerReportArray, &containerReportArraySize); printf("The getContainerReports Result Code:%d\n", result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getContainerReports, errorMessage:%s\n", errorMessage); } printf("containerReportArraySize=%d\n", containerReportArraySize); for (i = 0; i < containerReportArraySize; i++) { printf("-------------container: %d--------------------------\n", i); printf("containerId:%ld\n", containerReportArray[i].containerId); printf("vCores:%d\n", containerReportArray[i].vCores); printf("memory:%d\n", containerReportArray[i].memory); printf("host:%s\n", containerReportArray[i].host); printf("port:%d\n", containerReportArray[i].port); printf("exitStatus:%d\n", containerReportArray[i].exitStatus); printf("state:%d\n", containerReportArray[i].state); } freeContainerReportArray(containerReportArray, containerReportArraySize); //4. getContainerReport LibYarnContainerStatus_t *containerStatusArray; int containerStatusArraySize; result = getContainerStatuses(client, jobId, statusContainerIds, allocatedResourceArraySize, &containerStatusArray, &containerStatusArraySize); printf("The getContainerStatus Result Code:%d\n", result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getContainerStatus, errorMessage:%s\n", errorMessage); } printf("containerStatusArraySize=%d\n", containerStatusArraySize); for (i = 0; i < containerStatusArraySize; i++) { printf("-------------container: %d--------------------------\n", i); printf("containerId:%ld\n", containerStatusArray[i].containerId); printf("exitStatus:%d\n", containerStatusArray[i].exitStatus); printf("state:%d\n", containerStatusArray[i].state); printf("diagnostics:%s\n", containerStatusArray[i].diagnostics); } freeContainerStatusArray(containerStatusArray,containerStatusArraySize); //6. getQueueInfo LibYarnQueueInfo_t *queueInfo = NULL; result = getQueueInfo(client, queue, true, true, true, &queueInfo); printf("The getQueueInfo Result Code:%d\n",result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getQueueInfo, errorMessage:%s\n",errorMessage); } printf("QueueInfo: queueInfo->queueName:%s, queueInfo->capacity:%f, queueInfo->maximumCapacity:%f, queueInfo->currentCapacity:%f, queueInfo->state:%d\n", queueInfo->queueName, queueInfo->capacity, queueInfo->maximumCapacity, queueInfo->currentCapacity, queueInfo->state); puts("---------chilldQueue:"); for (i = 0; i < queueInfo->childQueueNameArraySize; i++) { printf("QueueInfo: queueInfo->childQueueNameArray[%d]:%s\n", i, queueInfo->childQueueNameArray[i]); } freeMemQueueInfo(queueInfo); //7. getCluster LibYarnNodeReport_t *nodeReportArray; int nodeReportArraySize; result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize); printf("The getClusterNodes Result Code:%d\n",result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getClusterNodes, errorMessage:%s\n",errorMessage); } printf("nodeReportArraySize=%d\n", nodeReportArraySize); for (i = 0; i < nodeReportArraySize; i++) { printf("-------------node %d--------------------------\n", i); printf("host:%s\n", nodeReportArray[i].host); printf("port:%d\n", nodeReportArray[i].port); printf("httpAddress:%s\n", nodeReportArray[i].httpAddress); printf("rackName:%s\n", nodeReportArray[i].rackName); printf("memoryUsed:%d\n", nodeReportArray[i].memoryUsed); printf("vcoresUsed:%d\n", nodeReportArray[i].vcoresUsed); printf("memoryCapability:%d\n", nodeReportArray[i].memoryCapability); printf("vcoresCapability:%d\n", nodeReportArray[i].vcoresCapability); printf("numContainers:%d\n", nodeReportArray[i].numContainers); printf("nodeState:%d\n", nodeReportArray[i].nodeState); printf("healthReport:%s\n", nodeReportArray[i].healthReport); printf("lastHealthReportTime:%lld\n", nodeReportArray[i].lastHealthReportTime); } freeMemNodeReportArray(nodeReportArray, nodeReportArraySize); //8. getApplicationReport LibYarnApplicationReport_t *applicationReport = NULL; result = getApplicationReport(client, jobId, &applicationReport); printf("The getApplicationReport Result Code:%d\n", result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getApplicationReport, errorMessage:%s\n", errorMessage); } printf("-------------ApplicationReport-------------------------\n"); printf("appId:%d\n", applicationReport->appId); printf("user:%s\n", applicationReport->user); printf("queue:%s\n", applicationReport->queue); printf("name:%s\n", applicationReport->name); printf("host:%s\n", applicationReport->host); printf("port:%d\n", applicationReport->port); printf("status:%d\n", applicationReport->status); freeApplicationReport(applicationReport); //5. release result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize); printf("The releaseResources Result Code:%d\n",result); result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize); printf("Active Fail Container Size:%d\n",activeFailSize); for (i = 0; i < activeFailSize; i++) { printf("Active Fail Container Id:%d\n",activeFailIds[i]); } free(activeFailIds); freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize); printf("freeMemAllocated is OK\n"); //9. finish printf("jobId:%s\n", jobId); result = finishJob(client, jobId, APPLICATION_SUCCEEDED); printf("The finishJob Result Code:%d\n", result); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The finishJob, errorMessage:%s\n", errorMessage); } free(jobId); //10. free client deleteLibYarnClient(client); return 0; }
int main() { struct timeval tpstart,tpend; float allocateTime = 0; float activeTime = 0; float releaseTime = 0; float timeuse; char *rmHost = "localhost"; char *rmPort = "8032"; char *schedHost = "localhost"; char *schedPort = "8030"; char *amHost = "localhost"; int32_t amPort = 10; char *am_tracking_url = "url"; int heartbeatInterval = 1000; //0. new client LibYarnClient_t *client = NULL; int result = newLibYarnClient(rmHost, rmPort, schedHost, schedPort, amHost, amPort, am_tracking_url,&client,heartbeatInterval); //1. createJob char *jobName = "libyarn"; char *queue = "default"; char *jobId = NULL; result = createJob(client, jobName, queue,&jobId); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf("1. createJob, errorMessage:%s\n",errorMessage); } int i,j; for (j = 0; j < 10; j++) { LibYarnResourceRequest_t resRequest; resRequest.priority = 1; resRequest.host = "*"; resRequest.vCores = 1; resRequest.memory = 1024; resRequest.num_containers = 2; resRequest.relax_locality = 1; char *blackListAdditions[0]; char *blackListRemovals[0]; //1. allocate resource LibYarnResource_t *allocatedResourcesArray; int allocatedResourceArraySize; gettimeofday(&tpstart,NULL); result = allocateResources(client, jobId, &resRequest, blackListAdditions, 0, blackListRemovals, 0, &allocatedResourcesArray, &allocatedResourceArraySize,5); gettimeofday(&tpend,NULL); timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec; allocateTime += timeuse; if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf("2. allocateResources, errorMessage:%s\n",errorMessage); } int32_t activeContainerIds[allocatedResourceArraySize]; int32_t releaseContainerIds[allocatedResourceArraySize]; int32_t statusContainerIds[allocatedResourceArraySize]; for (i = 0 ; i < allocatedResourceArraySize; i++) { activeContainerIds[i] = allocatedResourcesArray[i].containerId; releaseContainerIds[i] = allocatedResourcesArray[i].containerId; statusContainerIds[i] = allocatedResourcesArray[i].containerId; } //3. active gettimeofday(&tpstart,NULL); result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize); gettimeofday(&tpend,NULL); timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec; activeTime += timeuse; if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf("2. activeResources, errorMessage:%s\n",errorMessage); } int *activeFailIds; int activeFailSize; result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize); for (i = 0; i < activeFailSize; i++) { printf("Active Fail Container Id:%d\n",activeFailIds[i]); } //sleep(160); //4. getContainerReport LibYarnContainerReport_t *containerReportArray; int containerReportArraySize; result = getContainerReports(client, jobId, &containerReportArray, &containerReportArraySize); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getContainerReports, errorMessage:%s\n", errorMessage); } freeContainerReportArray(containerReportArray, containerReportArraySize); //4. getContainerReport LibYarnContainerStatus_t *containerStatusArray; int containerStatusArraySize; result = getContainerStatuses(client, jobId, statusContainerIds, allocatedResourceArraySize, &containerStatusArray, &containerStatusArraySize); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getContainerStatus, errorMessage:%s\n", errorMessage); } freeContainerStatusArray(containerStatusArray,containerStatusArraySize); //5. release gettimeofday(&tpstart,NULL); result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize); gettimeofday(&tpend,NULL); timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec; releaseTime += timeuse; result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize); for (i = 0; i < activeFailSize; i++) { printf("Active Fail Container Id:%d\n",activeFailIds[i]); } free(activeFailIds); freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize); //6. getQueueInfo LibYarnQueueInfo_t *queueInfo = NULL; result = getQueueInfo(client, queue, true, true, true, &queueInfo); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getQueueInfo, errorMessage:%s\n",errorMessage); } freeMemQueueInfo(queueInfo); //7. getCluster LibYarnNodeReport_t *nodeReportArray; int nodeReportArraySize; result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getClusterNodes, errorMessage:%s\n",errorMessage); } freeMemNodeReportArray(nodeReportArray, nodeReportArraySize); //8. getApplicationReport LibYarnApplicationReport_t *applicationReport = NULL; result = getApplicationReport(client, jobId, &applicationReport); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The getApplicationReport, errorMessage:%s\n", errorMessage); } freeApplicationReport(applicationReport); } printf("allocateTime:%f s\n",allocateTime/1000000); printf("activeTime:%f s\n",activeTime/1000000); printf("releaseTime:%f s\n",releaseTime/1000000); //9. finish result = finishJob(client, jobId, APPLICATION_SUCCEEDED); if (result != FUNCTION_SUCCEEDED) { const char* errorMessage = getErrorMessage(); printf(" The finishJob, errorMessage:%s\n", errorMessage); } free(jobId); //10. free client deleteLibYarnClient(client); return 0; }