/**
* \brief The listQueues function gets queues information
* \param sessionKey : The session key
* \param machineId : The id of the machine
* \param listofQueues : The list of queues
* \param queueName The option value, if it is given, listQueues gives information only of this queue
* \return int : an error code
*/
int
vishnu::listQueues(const std::string& sessionKey,
const std::string& machineId,
TMS_Data::ListQueues& listofQueues,
const std::string& queueName)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
checkEmptyString(machineId, "The machine id");
std::string serviceName = (boost::format("%1%@%2%") % SERVICES_TMS[GETLISTOFQUEUES] % machineId).str();
SessionProxy sessionProxy(sessionKey);
QueryProxy<std::string, TMS_Data::ListQueues>
query(queueName, sessionProxy, serviceName, machineId);
TMS_Data::ListQueues* listQueues_ptr = NULL ;
try {
listQueues_ptr = query.list();
} catch(...) {
throw ;
}
if(listQueues_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for(unsigned int i = 0; i < listQueues_ptr->getQueues().size(); i++) {
TMS_Data::Queue_ptr queue = ecoreFactory->createQueue();
//To copy the content and not the pointer
*queue = *listQueues_ptr->getQueues().get(i);
listofQueues.getQueues().push_back(queue);
}
listofQueues.setNbQueues(listofQueues.getNbQueues()+listQueues_ptr->getQueues().size());
delete listQueues_ptr;
}
return 0;
}
/**
* \brief The getCompletedJobsOutput() function gets standard output and error output files
* of completed jobs (applies only once for each job)
* \param sessionKey : The session key
* \param machineId : The id of the machine
* \param listOfResults : Is the list of jobs results
* \param outDir : The output directory where the files will be stored (default is current directory)
* \return int : an error code
*/
int
vishnu::getCompletedJobsOutput(const std::string& sessionKey,
const std::string& machineId,
ListJobResults& listOfResults,
const std::string& outDir)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
checkEmptyString(machineId, "The machine id");
if((outDir.size()!=0)&&(!boost::filesystem::exists(outDir))) {
throw UMSVishnuException(ERRCODE_INVALID_PARAM, "The directory "+outDir+" does not exist");
}
SessionProxy sessionProxy(sessionKey);
JobOutputProxy jobOutputProxy(sessionProxy, machineId, outDir);
TMS_Data::ListJobResults_ptr listJobResults_ptr = jobOutputProxy.getCompletedJobsOutput();
if(listJobResults_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for(unsigned int i = 0; i < listJobResults_ptr->getResults().size(); i++) {
TMS_Data::JobResult_ptr jobResult = ecoreFactory->createJobResult();
//To copy the content and not the pointer
*jobResult = *listJobResults_ptr->getResults().get(i);
listOfResults.getResults().push_back(jobResult);
}
listOfResults.setNbJobs(listJobResults_ptr->getNbJobs());
delete listJobResults_ptr;
}
return 0;
}
/**
* \brief The listQueues function gets queues information
* \param sessionKey : The session key
* \param machineId : The id of the machine
* \param listofQueues : The list of queues
* \param queueName The option value, if it is given, listQueues gives information only of this queue
* \return int : an error code
*/
int
vishnu::listQueues(const std::string& sessionKey,
const std::string& machineId,
ListQueues& listofQueues,
const std::string& queueName)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
checkEmptyString(machineId, "The machine id");
std::string serviceName = "getListOfQueues_";
serviceName.append(machineId);
SessionProxy sessionProxy(sessionKey);
QueryProxy<std::string, TMS_Data::ListQueues>
query(queueName, sessionProxy, serviceName, machineId);
TMS_Data::ListQueues* listQueues_ptr = query.list();
if(listQueues_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for(unsigned int i = 0; i < listQueues_ptr->getQueues().size(); i++) {
TMS_Data::Queue_ptr queue = ecoreFactory->createQueue();
//To copy the content and not the pointer
*queue = *listQueues_ptr->getQueues().get(i);
listofQueues.getQueues().push_back(queue);
}
listofQueues.setNbQueues(listQueues_ptr->getQueues().size());
delete listQueues_ptr;
}
return 0;
}
/**
* \brief The getJobProgress function gets the progression status of jobs
* \param sessionKey : The session key
* \param machineId : Is the id of the machine to get the jobs progression.
* \param listOfProgress : Is the object containing jobs progression information
* \param options : Is an object containing the available options jobs for progression .
* \return int : an error code
*/
int
vishnu::getJobProgress(const std::string& sessionKey,
const std::string& machineId,
ListProgression& listOfProgress,
const ProgressOptions& options)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
checkEmptyString(machineId, "The machine id");
std::string serviceName = "getJobsProgression_";
serviceName.append(machineId);
SessionProxy sessionProxy(sessionKey);
QueryProxy<TMS_Data::ProgressOptions, TMS_Data::ListProgression>
query(options, sessionProxy, serviceName, machineId);
TMS_Data::ListProgression* listProgression_ptr = query.list();
if(listProgression_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for(unsigned int i = 0; i < listProgression_ptr->getProgress().size(); i++) {
TMS_Data::Progression_ptr progression = ecoreFactory->createProgression();
//To copy the content and not the pointer
*progression = *listProgression_ptr->getProgress().get(i);
listOfProgress.getProgress().push_back(progression);
}
listOfProgress.setNbJobs(listProgression_ptr->getProgress().size());
delete listProgression_ptr;
}
return 0;
}
/**
* \brief getJobProgress: function gets the progression status of jobs
* \param sessionKey: The session key
* \param listOfProgress: The object containing jobs progression information
* \param options: The object containing the available options jobs for progression .
* \return int: an error code
*/
int
vishnu::getJobProgress(const std::string& sessionKey,
TMS_Data::ListProgression& listOfProgress,
const TMS_Data::ProgressOptions& options)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
std::string machineId = options.getMachineId();
UMS_Data::ListMachines machines;
if (! machineId.empty() && machineId != ALL_KEYWORD) {
// Use the specified machine
UMS_Data::Machine_ptr machine = new UMS_Data::Machine();
machine->setMachineId(machineId);
machines.getMachines().push_back(machine);
} else {
listMachinesWithUserLocalAccount(sessionKey, machines);
}
// Now perform the request
listOfProgress.setNbJobs(0);
for(int i=0; i< machines.getMachines().size(); i++) {
UMS_Data::Machine_ptr machine = machines.getMachines().get(i);
std::string serviceName = (boost::format("%1%@%2%") % SERVICES_TMS[GETJOBSPROGRESSION] % machine->getMachineId()).str();
SessionProxy sessionProxy(sessionKey);
QueryProxy<TMS_Data::ProgressOptions, TMS_Data::ListProgression>
query(options, sessionProxy, serviceName, machine->getMachineId());
TMS_Data::ListProgression* listProgression_ptr = NULL ;
try {
listProgression_ptr = query.list();
} catch(...) {
// Means that the machine is not active or the user doesn't have a local account on it
if(machineId != ALL_KEYWORD) {
throw ;
} else {
continue ;
}
}
if (listProgression_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for(unsigned int i = 0; i < listProgression_ptr->getProgress().size(); i++) {
TMS_Data::Progression_ptr progression = ecoreFactory->createProgression();
//To copy the content and not the pointer
*progression = *listProgression_ptr->getProgress().get(i);
listOfProgress.getProgress().push_back(progression);
}
listOfProgress.setNbJobs(listOfProgress.getNbJobs()+listProgression_ptr->getProgress().size());
delete listProgression_ptr;
}
}
return 0;
}
/**
* \brief The listJobs function gets a list of all submitted jobs
* \param sessionKey : The session key
* \param listOfJobs : The constructed object list of jobs
* \param options : Additional options for jobs listing
* \return int : an error code
*/
int
vishnu::listJobs(const std::string& sessionKey,
TMS_Data::ListJobs& listOfJobs,
const TMS_Data::ListJobsOptions& options)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
UMS_Data::Machine_ptr machine = new UMS_Data::Machine();
const std::string machineId = ALL_KEYWORD;
// Here the list of machine should contain only a single machine
machine->setMachineId(machineId);
listOfJobs.setNbJobs(0);
listOfJobs.setNbRunningJobs(0);
listOfJobs.setNbWaitingJobs(0);
std::string serviceName = (boost::format("%1%") % SERVICES_TMS[GETLISTOFJOBS_ALL]).str();
SessionProxy sessionProxy(sessionKey);
checkJobStatus(options.getStatus()); // check the job status options
checkJobPriority(options.getPriority()); //check the job priority options
QueryProxy<TMS_Data::ListJobsOptions, TMS_Data::ListJobs>
query(options, sessionProxy, serviceName, machine->getMachineId());
TMS_Data::ListJobs* listJobs_ptr = NULL;
try {
listJobs_ptr = query.list();
} catch(...) {
throw ;
}
if (listJobs_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for (unsigned int j = 0; j < listJobs_ptr->getJobs().size(); j++) {
TMS_Data::Job_ptr job = ecoreFactory->createJob();
//copy the content and not the pointer
*job = *listJobs_ptr->getJobs().get(j);
listOfJobs.getJobs().push_back(job);
}
listOfJobs.setNbJobs(listOfJobs.getNbJobs()+listJobs_ptr->getJobs().size());
listOfJobs.setNbRunningJobs(listOfJobs.getNbRunningJobs()+listJobs_ptr->getNbRunningJobs());
listOfJobs.setNbWaitingJobs(listOfJobs.getNbWaitingJobs()+listJobs_ptr->getNbWaitingJobs());
delete listJobs_ptr;
}
return 0;
}
/**
* \brief Gets standard output files of all completed jobs (applies only once for each job)
* \param sessionKey : The session key
* \param options: object containing options
* \param listOfResults : Is the list of jobs results
* \param options: Object containing options
* \return int : an error code
*/
int
vishnu::getCompletedJobsOutput(const std::string& sessionKey,
TMS_Data::ListJobResults& listOfResults,
const TMS_Data::JobOutputOptions& options)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
std::string outputDir = options.getOutputDir();
if (! outputDir.empty() && ! boost::filesystem::exists(outputDir)) {
throw UMSVishnuException(ERRCODE_INVALID_PARAM, "The ouput directory does not exist: "+outputDir);
}
UMS_Data::ListMachines machines;
if (options.getMachineId().empty()) {
vishnu::listMachinesWithUserLocalAccount(sessionKey, machines);
} else {
UMS_Data::Machine_ptr machine = new UMS_Data::Machine(); // delete by EMF
machine->setMachineId(options.getMachineId());
machines.getMachines().push_back(machine);
}
int machineCount = machines.getMachines().size();
for (int index = 0; index < machineCount; ++index) {
SessionProxy sessionProxy(sessionKey);
JobOutputProxy jobOutputProxy(sessionProxy, machines.getMachines().get(index)->getMachineId());
TMS_Data::ListJobResults_ptr listJobResults_ptr = jobOutputProxy.getCompletedJobsOutput(options);
if (listJobResults_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for(unsigned int i = 0; i < listJobResults_ptr->getResults().size(); i++) {
TMS_Data::JobResult_ptr jobResult = ecoreFactory->createJobResult();
//To copy the content and not the pointer
*jobResult = *listJobResults_ptr->getResults().get(i);
listOfResults.getResults().push_back(jobResult);
}
listOfResults.setNbJobs(listJobResults_ptr->getNbJobs());
delete listJobResults_ptr;
}
}
return 0;
}
/**
* \brief The listJobs function gets a list of all submitted jobs
* \param sessionKey : The session key
* \param machineId : The id of the machine
* \param listOfJobs : The constructed object list of jobs
* \param options : Additional options for jobs listing
* \return int : an error code
*/
int
vishnu::listJobs(const std::string& sessionKey,
const std::string& machineId,
ListJobs& listOfJobs,
const ListJobsOptions& options)
throw (UMSVishnuException, TMSVishnuException, UserException, SystemException) {
checkEmptyString(sessionKey, "The session key");
checkEmptyString(machineId, "The machine id");
std::string serviceName = "getListOfJobs_";
serviceName.append(machineId);
SessionProxy sessionProxy(sessionKey);
//To check the job status options
checkJobStatus(options.getStatus());
//To check the job priority options
checkJobPriority(options.getPriority());
QueryProxy<TMS_Data::ListJobsOptions, TMS_Data::ListJobs>
query(options, sessionProxy, serviceName, machineId);
TMS_Data::ListJobs* listJobs_ptr = query.list();
if(listJobs_ptr != NULL) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
for(unsigned int i = 0; i < listJobs_ptr->getJobs().size(); i++) {
TMS_Data::Job_ptr job = ecoreFactory->createJob();
//To copy the content and not the pointer
*job = *listJobs_ptr->getJobs().get(i);
listOfJobs.getJobs().push_back(job);
}
listOfJobs.setNbJobs(listJobs_ptr->getJobs().size());
listOfJobs.setNbRunningJobs(listJobs_ptr->getNbRunningJobs());
listOfJobs.setNbWaitingJobs(listJobs_ptr->getNbWaitingJobs());
delete listJobs_ptr;
}
return 0;
}
/**
* \brief Function to request the status of queues
* \param optQueueName (optional) the name of the queue to request
* \return The requested status in to ListQueues data structure
*/
TMS_Data::ListQueues*
SGEServer::listQueues(const std::string& optqueueName) {
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
mlistQueues = ecoreFactory->createListQueues();
std::string exe = boost::process::find_executable_in_path("qconf");
std::vector<std::string> args;
std::vector<std::string> argss;
std::vector<std::string> queueNames;
boost::process::context ctx;
ctx.streams[boost::process::stdout_id] = boost::process::behavior::pipe();
args.push_back("-sq");
char diagnosis[DRMAA_ERROR_STRING_BUFFER];
int drmaa_errno;
drmaa_errno = drmaa_init(NULL, diagnosis, sizeof(diagnosis)-1);
if ((drmaa_errno!= DRMAA_ERRNO_SUCCESS)&&
(drmaa_errno!= DRMAA_ERRNO_ALREADY_ACTIVE_SESSION)){
throw TMSVishnuException(ERRCODE_BATCH_SCHEDULER_ERROR,
"SGE ERROR: "+std::string(diagnosis));
}
if(optqueueName.size()!=0) {
queueNames.push_back(optqueueName.c_str());
} else {
argss.push_back("-sql");
boost::process::child c = boost::process::create_child(exe, argss, ctx);
boost::process::pistream is(c.get_handle(boost::process::stdout_id));
std::string line;
while (std::getline(is, line)){
queueNames.push_back(line.c_str());
}
}
vector<std::string>::iterator it;
for ( it=queueNames.begin() ; it < queueNames.end(); ++it ){
args.push_back((*it).c_str());
boost::process::child c1 = boost::process::create_child(exe, args, ctx);
boost::process::pistream iss(c1.get_handle(boost::process::stdout_id));
std::string line;
if (iss){
TMS_Data::Queue_ptr queue = ecoreFactory->createQueue();
while (std::getline(iss, line)){
if(boost::algorithm::starts_with(line, "qname")){
line = line.substr(5);
boost::algorithm::trim_left(line);
queue->setName(line);
}
if(boost::algorithm::starts_with(line, "priority")){
line = line.substr(8);
boost::algorithm::trim_left(line);
int priority = boost::lexical_cast<int>(line);
queue->setPriority(priority);
}
if(boost::algorithm::starts_with(line, "slots")){
line = line.substr(5);
boost::algorithm::trim_left(line);
size_t pos = line.find(",");
int MaxJob =0;
while(pos!=std::string::npos){
std::string jobnb = line.substr(0,pos);
size_t position;
position = jobnb.find("=");
if(position!=std::string::npos ){
jobnb = jobnb.substr(position +1);
}
MaxJob += boost::lexical_cast<int>(jobnb);
line = line.substr(pos+1);
pos = line.find(",");
}
queue->setMaxJobCpu(MaxJob);
}
if(boost::algorithm::starts_with(line, "hostlist")){
line = line.substr(8);
boost::algorithm::trim_left(line);
int nodes =0 ;
if (line != "NONE"){
nodes = (int) std::count(line.begin(),line.end(),',') +1;
}
queue->setNode(nodes);
}
if(boost::algorithm::starts_with(line, "h_rt")){
line = line.substr(4);
boost::algorithm::trim_left(line);
if (line != "INFINITY"){
queue->setWallTime(vishnu::convertStringToWallTime(line));
}
}
}
/*get Queue Status*/
std::vector<std::string> argument;
argument.push_back("-f");
argument.push_back("-q");
argument.push_back((*it).c_str());
argument.push_back("-u");
argument.push_back("*");
std::string exec = boost::process::find_executable_in_path("qstat");
boost::process::child c3 = boost::process::create_child(exec, argument,
ctx);
boost::process::pistream isstream(c3.get_handle(boost::process::stdout_id));
argument.pop_back();
std::string cline;
if (isstream){
while (std::getline(isstream, cline)){
int state =0;
if (boost::algorithm::starts_with(cline,(*it).c_str() )){
std::vector< std::string > SplitVec;
boost::algorithm::split( SplitVec, cline,
boost::algorithm::is_any_of(" "),
boost::algorithm::token_compress_on );
std::vector<std::string>::iterator itt;
itt = SplitVec.end()-1;
if (boost::algorithm::contains(*itt, "u")){
queue->setState(0);
state =0;
} else if (boost::algorithm::contains(*itt, "a")) {
queue->setState(2);
state =1;
} else if (boost::algorithm::contains(*itt, "A") ||
boost::algorithm::contains(*itt, "C") ||
boost::algorithm::contains(*itt, "D") ||
boost::algorithm::contains(*itt, "s")){
queue->setState(1);
state =0;
} else if (boost::algorithm::contains(*itt, "S") ||
boost::algorithm::contains(*itt, "d") ||
boost::algorithm::contains(*itt, "E") ||
boost::algorithm::contains(*itt, "o")){
state =0;
} else if (boost::algorithm::contains(*itt, "")){
queue->setState(2);
state =1;
}
}
if(state){
break;
}
}
}
std::vector<std::string> arg;
arg.push_back("-q");
arg.push_back((*it).c_str());
std::string ex = boost::process::find_executable_in_path("qstat");
boost::process::child c2 = boost::process::create_child(ex, arg, ctx);
boost::process::pistream isss(c2.get_handle(boost::process::stdout_id));
arg.pop_back();
std::string lines;
int nbrunningjobs = 0;
int nbjobsinqueue = 0;
if (isss){
while (std::getline(isss, lines)){
if (boost::algorithm::contains(lines, " r ") ||
boost::algorithm::contains(lines, " t ") ||
boost::algorithm::contains(lines, " R ")){
nbrunningjobs++;
}
if (boost::algorithm::contains(lines, " qw ") ||
boost::algorithm::contains(lines, " s ") ||
boost::algorithm::contains(lines, " w ")){
nbjobsinqueue++;
}
}
}
queue->setNbRunningJobs(nbrunningjobs);
queue->setNbJobsInQueue(nbjobsinqueue);
queue->setMemory(-1);//UNDEFINED in SGE
queue->setDescription("");//UNDEFINED in SGE
queue->setMaxProcCpu(-1);//UNDEFINED in SGE
mlistQueues->getQueues().push_back(queue);
}
args.pop_back();
}
mlistQueues->setNbQueues(mlistQueues->getQueues().size());
drmaa_exit(NULL, 0);
return mlistQueues;
}
/**
* \brief Function to list sessions information
* \return The pointer to the UMS_Data::ListSessions containing sessions information
* \return raises an exception on error
*/
TMS_Data::ListProgression*
list(TMS_Data::ProgressOptions_ptr options) {
std::vector<std::string> results;
std::vector<std::string>::iterator iter;
std::string batchJobId;
int status;
long startTime;
long walltime;
TMS_Data::TMS_DataFactory_ptr ecoreFactory = TMS_Data::TMS_DataFactory::_instance();
mlistObject = ecoreFactory->createListProgression();
std::string sqlRequest = "SELECT jobId, jobName, wallClockLimit, endDate, status, batchJobId "
" FROM vsession, job "
" WHERE vsession.numsessionid=job.vsession_numsessionid ";
std::string machineId = options->getMachineId();
if (!machineId.empty()){
checkMachineId(machineId);
sqlRequest += " AND submitMachineId='"+mdatabaseInstance->escapeData(machineId)+"'";
}
if (! options->getJobId().empty()) {
std::string jobId = options->getJobId();
sqlRequest.append(" and jobId='"+mdatabaseInstance->escapeData(jobId)+"'");
boost::scoped_ptr<DatabaseResult> sqlResult(ServerXMS::getInstance()->getDatabaseVishnu()->getResult(sqlRequest.c_str()));
if(sqlResult->getNbTuples() == 0) {
throw TMSVishnuException(ERRCODE_UNKNOWN_JOBID);
}
} else {
sqlRequest.append(" and owner='"+mdatabaseInstance->escapeData(options->getUser())+"'");
}
sqlRequest.append(" and status < 5 order by jobId");
boost::scoped_ptr<DatabaseResult> sqlResult(ServerXMS::getInstance()->getDatabaseVishnu()->getResult(sqlRequest.c_str()));
if (sqlResult->getNbTuples() != 0){
for (size_t i = 0; i < sqlResult->getNbTuples(); ++i) {
results.clear();
results = sqlResult->get(i);
iter = results.begin();
TMS_Data::Progression_ptr job = ecoreFactory->createProgression();
job->setJobId(*iter);
job->setJobName(*(++iter));
walltime = vishnu::convertToInt(*(++iter));
job->setWallTime(walltime);
job->setEndTime(vishnu::convertToLong(*(++iter)));
status = vishnu::convertToInt(*(++iter));
job->setStatus(status);
batchJobId = *(++iter);
BatchFactory factory;
BatchType batchType = ServerXMS::getInstance()->getBatchType();
std::string batchVersion = ServerXMS::getInstance()->getBatchVersion();
boost::scoped_ptr<BatchServer> batchServer(factory.getBatchServerInstance(batchType,
batchVersion));
startTime = batchServer->getJobStartTime(batchJobId);
if(startTime!=0) {
job->setStartTime(startTime);
if (status == vishnu::STATE_COMPLETED) {
job->setPercent(100);
} else if(status == vishnu::STATE_RUNNING) {
time_t currentTime = vishnu::getCurrentTimeInUTC();
int percent = 0;
time_t gap = currentTime-startTime;
if (walltime == 0) {
walltime = 60;
}
if (gap < walltime) {
double ratio = 100*(double(gap)/walltime);
if(ratio > 0.0 && ratio <= 1.0) {
percent = 1;
} else {
percent = static_cast<int>(ratio);
}
} else {
percent = 99;
}
job->setPercent(percent);
} else {
job->setPercent(0);
}
} else {
job->setPercent(0);
}
mlistObject->getProgress().push_back(job);
}
}
mlistObject->setNbJobs(mlistObject->getProgress().size());
return mlistObject;
}
