/****** SCHEDD/remove_immediate_jobs()******************************************
* NAME
* remove_immediate_jobs() -- test for and remove immediate jobs which can't
* be scheduled
*
* SYNOPSIS
* int remove_immediate_jobs(lList *pending_job_list,
lList *running_job_list, order_t *orders)
*
* FUNCTION
* Goes through all jobs in the pending list to see if any are immediate and
* not idle. If any are, they are removed. This is done by generating an
* order of type ORT_remove_immediate_job. If any array jobs are removed,
* the running list is checked for tasks belonging to the job, which are
* also removed. This is done by removing the ORT_start_job orders and
* adding an order of type ORT_remove_immediate_job.
*
* INPUTS
* lList *pending_job_list - The list of pending jobs for this scheduler
* pass (JB_Type)
* lList *running_job_list - The list of running jobs for this scheduler
* pass (JB_Type)
* order_t *orders - The order structure for this scheduler pass
*
* RESULT
* int - Error code: 0 = OK, 1 = Errors -- always returns 0
*
* NOTES
* MT-NOTE: remove_immediate_jobs() is MT safe
*
*******************************************************************************/
int remove_immediate_jobs(lList *pending_job_list, lList *running_job_list, order_t *orders)
{
lListElem *next_job, *job, *ep;
lList* lp;
DENTER (TOP_LAYER, "remove_immediate_jobs");
next_job = lFirst (pending_job_list);
while ((job = next_job)) {
lCondition *where = NULL;
next_job = lNext(job);
/* skip non immediate .. */
if (!JOB_TYPE_IS_IMMEDIATE(lGetUlong(job, JB_type))) {
continue;
}
/* .. and non idle jobs */
if ((lp = lGetList(job, JB_ja_tasks)) &&
(ep = lFirst(lp)) &&
lGetUlong(ep, JAT_status)==JIDLE) {
continue;
}
/* Prepare search condition for running list */
where = lWhere("%T(%I==%u)", JB_Type, JB_job_number, lGetUlong(job, JB_job_number));
/* Remove the job from the pending list */
remove_immediate_job(pending_job_list, job, orders, 0);
/* If the job also exists in the running list, we need to remove it there
* as well since array jobs are all or nothing. */
if ((job = lFindFirst(running_job_list, where)) != NULL) {
remove_immediate_job(running_job_list, job, orders, 1);
}
lFreeWhere(&where);
}
DEXIT;
return 0;
}
/****** category/sge_build_job_category_dstring() ******************************
* NAME
* sge_build_job_category_dstring() -- build the category string
*
* SYNOPSIS
* void sge_build_job_category_dstring(dstring *category_str, lListElem
* *job, lList *acl_list)
*
* FUNCTION
* The following parameter are put into the category:
* hard_queue_list
* master_hard_queue_list
* hard_resource_list
* soft_resource_list
* checkpoint_name
* type
*
* owner/group: -U user_lists
* Omitted, if user_lists/xuser_lists were not used in
* host_conf(5), sge_pe(5) and queue_conf(5). In sge_conf(5)
* user_lists/xuser_lists still can be used, as it causes
* jobs already be rejected at submit time.
*
* project: -P user_lists
* Omitted, if projects/xprojects were not used in
* host_conf(5), sge_pe(5) and queue_conf(5). In sge_conf(5)
* projects/xprojects still can be used, as it cuases
* jobs already be rejected at submit time.
*
* pe
*
* INPUTS
* dstring *category_str - target string, contains the category or nothing
* lListElem *job - the job for the category creating
* lList *acl_list - global access list
*
* NOTES
* MT-NOTE: sge_build_job_category_dstring() is MT safe as long as the caller is
*
*******************************************************************************/
void sge_build_job_category_dstring(dstring *category_str, lListElem *job, lList *acl_list, const lList *prj_list, bool *did_project, const lList *rqs_list)
{
const char *owner = NULL;
const char *group = NULL;
DENTER(TOP_LAYER, "sge_build_job_category_dstring");
DTRACE;
#if 0
sge_mutex_lock("cull_order_mutex", SGE_FUNC, __LINE__, &Category_Control.cull_order_mutex);
if (Category_Control.cull_order_pos.JB_hard_queue_list_pos == -1) {
Category_Control.cull_order_pos.JB_checkpoint_name_pos = lGetPosViaElem(job, JB_checkpoint_name, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_soft_resource_list_pos = lGetPosViaElem(job, JB_soft_resource_list, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_master_hard_queue_list_pos = lGetPosViaElem(job, JB_master_hard_queue_list, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_hard_queue_list_pos = lGetPosViaElem(job, JB_hard_queue_list, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_owner_pos = lGetPosViaElem(job, JB_owner, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_group_pos = lGetPosViaElem(job, JB_group, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_hard_resource_list_pos = lGetPosViaElem(job, JB_hard_resource_list, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_type_pos = lGetPosViaElem(job, JB_type, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_project_pos = lGetPosViaElem(job, JB_project, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_ar_pos = lGetPosViaElem(job, JB_ar, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_pe_pos = lGetPosViaElem(job, JB_pe, SGE_NO_ABORT);
Category_Control.cull_order_pos.JB_range_pos = lGetPosViaElem(job, JB_pe_range, SGE_NO_ABORT);
}
sge_mutex_unlock("cull_order_mutex", SGE_FUNC, __LINE__, &Category_Control.cull_order_mutex);
#endif
DTRACE;
/*
** owner -> acl
*/
owner = lGetPosString(job, lGetPosViaElem(job, JB_owner, SGE_NO_ABORT));
DTRACE;
group = lGetPosString(job, lGetPosViaElem(job, JB_group, SGE_NO_ABORT));
DTRACE;
sge_unparse_acl_dstring(category_str, owner, group, acl_list, "-U");
DTRACE;
/*
** -u if referenced in resource quota sets
*/
/* RD TODO: A possible performance enhancement is to split user and group inside category.
Some users are only referenced by the unix group. Their jobs could be grouped
together by referencing only the group in the category string
*/
if (sge_user_is_referenced_in_rqs(rqs_list, lGetString(job, JB_owner), lGetString(job, JB_group), acl_list)) {
if (sge_dstring_strlen(category_str) > 0) {
sge_dstring_append(category_str, " ");
}
sge_dstring_append(category_str, "-u ");
sge_dstring_append(category_str, lGetString(job, JB_owner));
}
DTRACE;
/*
** -hard -q qlist
*/
sge_unparse_queue_list_dstring(category_str, job, lGetPosViaElem(job, JB_hard_queue_list, SGE_NO_ABORT), "-q");
DTRACE;
/*
** -masterq qlist
*/
sge_unparse_queue_list_dstring(category_str, job, lGetPosViaElem(job, JB_master_hard_queue_list, SGE_NO_ABORT), "-masterq");
DTRACE;
/*
** -l rlist (hard resource list)
*/
sge_unparse_resource_list_dstring(category_str, job, lGetPosViaElem(job, JB_hard_resource_list, SGE_NO_ABORT), "-l");
DTRACE;
/*
** -soft -l rlist
*/
sge_unparse_resource_list_dstring(category_str, job, lGetPosViaElem(job, JB_soft_resource_list, SGE_NO_ABORT), "-soft -l");
DTRACE;
/*
** -pe pe_name pe_range
*/
sge_unparse_pe_dstring(category_str, job, lGetPosViaElem(job, JB_pe, SGE_NO_ABORT),
lGetPosViaElem(job, JB_pe_range, SGE_NO_ABORT), "-pe");
DTRACE;
/*
** -ckpt ckpt_name
*/
sge_unparse_string_option_dstring(category_str, job, lGetPosViaElem(job, JB_checkpoint_name, SGE_NO_ABORT), "-ckpt");
DTRACE;
/*
** interactive jobs
*/
if (JOB_TYPE_IS_IMMEDIATE(lGetPosUlong(job, lGetPosViaElem(job, JB_type, SGE_NO_ABORT)))) {
if (sge_dstring_strlen(category_str) > 0) {
sge_dstring_append(category_str, " -I y");
}
else {
sge_dstring_append(category_str, "-I y");
}
}
DTRACE;
/*
** project
*/
{
const char *project = lGetPosString(job, lGetPosViaElem(job, JB_project, SGE_NO_ABORT));
const lListElem *prj;
if (project && (prj=lGetElemStr(prj_list, PR_name, project)) && lGetBool(prj, PR_consider_with_categories)) {
if (did_project)
*did_project = true;
sge_unparse_string_option_dstring(category_str, job, lGetPosViaElem(job, JB_project, SGE_NO_ABORT), "-P");
} else
if (did_project)
*did_project = false;
}
DTRACE;
/*
** -ar ar_id
*/
sge_unparse_ulong_option_dstring(category_str, job, lGetPosViaElem(job, JB_ar, SGE_NO_ABORT), "-ar");
DRETURN_VOID;
}
int
main(int argc, char **argv)
{
lList *opts_cmdline = NULL;
lList *opts_defaults = NULL;
lList *opts_scriptfile = NULL;
lList *opts_all = NULL;
lListElem *job = NULL;
lList *alp = NULL;
lListElem *ep;
int exit_status = 0;
int just_verify;
int tmp_ret;
int wait_for_job = 0, is_immediate = 0;
dstring session_key_out = DSTRING_INIT;
dstring diag = DSTRING_INIT;
dstring jobid = DSTRING_INIT;
u_long32 start, end, step;
u_long32 num_tasks;
int count, stat;
char *jobid_string = NULL;
bool has_terse;
drmaa_attr_values_t *jobids = NULL;
u_long32 prog_number = 0;
u_long32 myuid = 0;
const char *sge_root = NULL;
const char *cell_root = NULL;
const char *username = NULL;
const char *qualified_hostname = NULL;
const char *unqualified_hostname = NULL;
const char *mastername = NULL;
DENTER_MAIN(TOP_LAYER, "qsub");
prof_mt_init();
/* Set up the program information name */
sge_setup_sig_handlers(QSUB);
DPRINTF(("Initializing JAPI\n"));
if (japi_init(NULL, NULL, NULL, QSUB, false, NULL, &diag)
!= DRMAA_ERRNO_SUCCESS) {
fprintf(stderr, "\n");
fprintf(stderr, MSG_QSUB_COULDNOTINITIALIZEENV_S,
sge_dstring_get_string(&diag));
fprintf(stderr, "\n");
DEXIT;
SGE_EXIT((void**)&ctx, 1);
}
prog_number = ctx->get_who(ctx);
myuid = ctx->get_uid(ctx);
sge_root = ctx->get_sge_root(ctx);
cell_root = ctx->get_cell_root(ctx);
username = ctx->get_username(ctx);
qualified_hostname = ctx->get_qualified_hostname(ctx);
unqualified_hostname = ctx->get_unqualified_hostname(ctx);
mastername = ctx->get_master(ctx, false);
/*
* read switches from the various defaults files
*/
opt_list_append_opts_from_default_files(prog_number, cell_root, username, &opts_defaults, &alp, environ);
tmp_ret = answer_list_print_err_warn(&alp, NULL, NULL, MSG_WARNING);
if (tmp_ret > 0) {
DEXIT;
SGE_EXIT((void**)&ctx, tmp_ret);
}
/*
* append the commandline switches to the list
*/
opt_list_append_opts_from_qsub_cmdline(prog_number, &opts_cmdline, &alp,
argv + 1, environ);
tmp_ret = answer_list_print_err_warn(&alp, NULL, "qsub: ", MSG_QSUB_WARNING_S);
if (tmp_ret > 0) {
DEXIT;
SGE_EXIT((void**)&ctx, tmp_ret);
}
/*
* show usage if -help was in commandline
*/
if (opt_list_has_X(opts_cmdline, "-help")) {
sge_usage(QSUB, stdout);
DEXIT;
SGE_EXIT((void**)&ctx, 0);
}
/*
* We will only read commandline options from scripfile if the script
* itself should not be handled as binary
*/
if (opt_list_is_X_true(opts_cmdline, "-b") ||
(!opt_list_has_X(opts_cmdline, "-b") &&
opt_list_is_X_true(opts_defaults, "-b"))) {
DPRINTF(("Skipping options from script due to -b option\n"));
} else {
opt_list_append_opts_from_script(prog_number,
&opts_scriptfile, &alp,
opts_cmdline, environ);
tmp_ret = answer_list_print_err_warn(&alp, NULL, MSG_QSUB_COULDNOTREADSCRIPT_S,
MSG_WARNING);
if (tmp_ret > 0) {
DEXIT;
SGE_EXIT((void**)&ctx, tmp_ret);
}
}
/*
* Merge all commandline options and interprete them
*/
opt_list_merge_command_lines(&opts_all, &opts_defaults,
&opts_scriptfile, &opts_cmdline);
/*
* Check if -terse is requested
*/
has_terse = opt_list_has_X(opts_all, "-terse");
/* If "-sync y" is set, wait for the job to end. */
/* Remove all -sync switches since cull_parse_job_parameter()
* doesn't know what to do with them. */
while ((ep = lGetElemStr(opts_all, SPA_switch, "-sync"))) {
if (lGetInt(ep, SPA_argval_lIntT) == TRUE) {
wait_for_job = 1;
}
lRemoveElem(opts_all, &ep);
}
if (wait_for_job) {
DPRINTF(("Wait for job end\n"));
}
alp = cull_parse_job_parameter(myuid, username, cell_root, unqualified_hostname,
qualified_hostname, opts_all, &job);
tmp_ret = answer_list_print_err_warn(&alp, NULL, "qsub: ", MSG_WARNING);
if (tmp_ret > 0) {
DEXIT;
SGE_EXIT((void**)&ctx, tmp_ret);
}
if (set_sec_cred(sge_root, mastername, job, &alp) != 0) {
answer_list_output(&alp);
DEXIT;
SGE_EXIT((void**)&ctx, 1);
}
/* Check if job is immediate */
is_immediate = (int)JOB_TYPE_IS_IMMEDIATE(lGetUlong(job, JB_type));
DPRINTF(("Job is%s immediate\n", is_immediate ? "" : " not"));
DPRINTF(("Everything ok\n"));
if (lGetUlong(job, JB_verify)) {
cull_show_job(job, 0, false);
DEXIT;
SGE_EXIT((void**)&ctx, 0);
}
if (is_immediate || wait_for_job) {
pthread_t sigt;
qsub_setup_sig_handlers();
if (pthread_create(&sigt, NULL, sig_thread, (void *)NULL) != 0) {
fprintf(stderr, "\n");
fprintf(stderr, MSG_QSUB_COULDNOTINITIALIZEENV_S,
" error preparing signal handling thread");
fprintf(stderr, "\n");
exit_status = 1;
goto Error;
}
if (japi_enable_job_wait(username, unqualified_hostname, NULL, &session_key_out, error_handler, &diag) ==
DRMAA_ERRNO_DRM_COMMUNICATION_FAILURE) {
const char *msg = sge_dstring_get_string(&diag);
fprintf(stderr, "\n");
fprintf(stderr, MSG_QSUB_COULDNOTINITIALIZEENV_S,
msg?msg:" error starting event client thread");
fprintf(stderr, "\n");
exit_status = 1;
goto Error;
}
}
job_get_submit_task_ids(job, &start, &end, &step);
num_tasks = (end - start) / step + 1;
if (num_tasks > 1) {
int error = japi_run_bulk_jobs(&jobids, &job, start, end, step, &diag);
if (error != DRMAA_ERRNO_SUCCESS) {
/* No active session here means that japi_enable_job_wait() was
* interrupted by the signal handler, in which case we just break out
* quietly. */
if (error != DRMAA_ERRNO_NO_ACTIVE_SESSION) {
fprintf(stderr, MSG_QSUB_COULDNOTRUNJOB_S,
sge_dstring_get_string(&diag));
fprintf(stderr, "\n");
}
/* BUGFIX: Issuezilla #1013
* To quickly fix this issue, I'm mapping the JAPI/DRMAA error code
* back into a GDI error code. This is the easy solution. The
* correct solution would be to address issue #859, presumably by
* having JAPI reuse the GDI error codes instead of the JAPI error
* codes. */
if (error == DRMAA_ERRNO_TRY_LATER) {
exit_status = STATUS_NOTOK_DOAGAIN;
}
else {
exit_status = 1;
}
goto Error;
}
DPRINTF(("job id is: %ld\n", jobids->it.ji.jobid));
jobid_string = get_bulk_jobid_string((long)jobids->it.ji.jobid, start, end, step);
}
else if (num_tasks == 1) {
int error = japi_run_job(&jobid, &job, &diag);
if (error != DRMAA_ERRNO_SUCCESS) {
if (error != DRMAA_ERRNO_NO_ACTIVE_SESSION) {
fprintf(stderr, MSG_QSUB_COULDNOTRUNJOB_S,
sge_dstring_get_string(&diag));
fprintf(stderr, "\n");
}
/* BUGFIX: Issuezilla #1013
* To quickly fix this issue, I'm mapping the JAPI/DRMAA error code
* back into a GDI error code. This is the easy solution. The
* correct solution would be to address issue #859, presumably by
* having JAPI reuse the GDI error codes instead of the DRMAA error
* codes. */
if (error == DRMAA_ERRNO_TRY_LATER) {
exit_status = STATUS_NOTOK_DOAGAIN;
}
else {
exit_status = 1;
}
goto Error;
}
jobid_string = strdup(sge_dstring_get_string(&jobid));
DPRINTF(("job id is: %s\n", jobid_string));
sge_dstring_free(&jobid);
}
else {
fprintf(stderr, MSG_QSUB_COULDNOTRUNJOB_S, "invalid task structure");
fprintf(stderr, "\n");
exit_status = 1;
goto Error;
}
/* only success message is printed to stdout */
just_verify = (lGetUlong(job, JB_verify_suitable_queues)==JUST_VERIFY ||
lGetUlong(job, JB_verify_suitable_queues)==POKE_VERIFY);
DPRINTF(("Just verifying job\n"));
if (!just_verify) {
const char *output = sge_dstring_get_string(&diag);
/* print the tersed output */
if (has_terse) {
printf("%s", jobid_string);
} else if (output != NULL) {
printf("%s", output);
} else {
printf(MSG_QSUB_YOURJOBHASBEENSUBMITTED_SS, jobid_string, lGetString(job, JB_job_name));
}
printf("\n");
} else {
printf(MSG_JOB_VERIFYFOUNDQ);
printf("\n");
}
if ((wait_for_job || is_immediate) && !just_verify) {
int event;
if (is_immediate) {
fprintf(stderr, "%s\n", MSG_QSUB_WAITINGFORIMMEDIATEJOBTOBESCHEDULED);
/* We only need to wait for the first task to be scheduled to be able
* to say that the job is running. */
tmp_ret = japi_wait(DRMAA_JOB_IDS_SESSION_ANY, &jobid, &stat,
DRMAA_TIMEOUT_WAIT_FOREVER, JAPI_JOB_START, &event,
NULL, &diag);
if ((tmp_ret == DRMAA_ERRNO_SUCCESS) && (event == JAPI_JOB_START)) {
fprintf(stderr, "\n");
fprintf(stderr, MSG_QSUB_YOURIMMEDIATEJOBXHASBEENSUCCESSFULLYSCHEDULED_S,
jobid_string);
fprintf(stderr, "\n");
}
/* A job finish event here means that the job was rejected. */
else if ((tmp_ret == DRMAA_ERRNO_SUCCESS) &&
(event == JAPI_JOB_FINISH)) {
fprintf(stderr, "\n%s\n", MSG_QSUB_YOURQSUBREQUESTCOULDNOTBESCHEDULEDDTRYLATER);
exit_status = 1;
goto Error;
}
else {
/* Since we told japi_wait to wait forever, we know that if it gets
* a timeout, it's because it's been interrupted to exit, in which
* case we don't complain. Same for no active session. */
if ((tmp_ret != DRMAA_ERRNO_EXIT_TIMEOUT) &&
(tmp_ret != DRMAA_ERRNO_NO_ACTIVE_SESSION)) {
fprintf(stderr, "\n");
fprintf(stderr, MSG_QSUB_COULDNOTWAITFORJOB_S,
sge_dstring_get_string(&diag));
fprintf(stderr, "\n");
}
exit_status = 1;
goto Error;
}
}
if (wait_for_job) {
/* Rather than using japi_synchronize on ALL for bulk jobs, we use
* japi_wait on ANY num_tasks times because with synchronize, we would
* have to wait for all the tasks to finish before we know if any
* finished. */
for (count = 0; count < num_tasks; count++) {
/* Since there's only one running job in the session, we can just
* wait for ANY. */
if ((tmp_ret = japi_wait(DRMAA_JOB_IDS_SESSION_ANY, &jobid, &stat,
DRMAA_TIMEOUT_WAIT_FOREVER, JAPI_JOB_FINISH, &event,
NULL, &diag)) != DRMAA_ERRNO_SUCCESS) {
if ((tmp_ret != DRMAA_ERRNO_EXIT_TIMEOUT) &&
(tmp_ret != DRMAA_ERRNO_NO_ACTIVE_SESSION)) {
fprintf(stderr, "\n");
fprintf(stderr, MSG_QSUB_COULDNOTWAITFORJOB_S, sge_dstring_get_string(&diag));
fprintf(stderr, "\n");
}
exit_status = 1;
goto Error;
}
/* report how job finished */
/* If the job is an array job, use the first non-zero exit code as
* the exit code for qsub. */
if (exit_status == 0) {
exit_status = report_exit_status(stat,
sge_dstring_get_string(&jobid));
}
/* If we've already found a non-zero exit code, just print the exit
* info for the task. */
else {
report_exit_status(stat, sge_dstring_get_string(&jobid));
}
}
}
}
Error:
FREE(jobid_string);
lFreeList(&alp);
lFreeList(&opts_all);
if ((tmp_ret = japi_exit(JAPI_EXIT_NO_FLAG, &diag)) != DRMAA_ERRNO_SUCCESS) {
if (tmp_ret != DRMAA_ERRNO_NO_ACTIVE_SESSION) {
fprintf(stderr, "\n");
fprintf(stderr, MSG_QSUB_COULDNOTFINALIZEENV_S, sge_dstring_get_string(&diag));
fprintf(stderr, "\n");
}
else {
struct timespec ts;
/* We know that if we get a DRMAA_ERRNO_NO_ACTIVE_SESSION here, it's
* because the signal handler thread called japi_exit(). We know this
* because if the call to japi_init() fails, we just exit directly.
* If the call to japi_init() succeeds, then we have an active session,
* so coming here because of an error would not result in the
* DRMAA_ERRNO_NO_ACTIVE_SESSION error. */
DPRINTF(("Sleeping for 15 seconds to wait for the exit to finish.\n"));
sge_relative_timespec(15, &ts);
sge_mutex_lock("qsub_exit_mutex", SGE_FUNC, __LINE__, &exit_mutex);
while (!exited) {
if (pthread_cond_timedwait(&exit_cv, &exit_mutex, &ts) == ETIMEDOUT) {
DPRINTF(("Exit has not finished after 15 seconds. Exiting.\n"));
break;
}
}
sge_mutex_unlock("qsub_exit_mutex", SGE_FUNC, __LINE__, &exit_mutex);
}
}
sge_prof_cleanup();
/* This is an exit() instead of an SGE_EXIT() because when the qmaster is
* supended, SGE_EXIT() hangs. */
exit(exit_status);
DEXIT;
return exit_status;
}
