// Check if any of the active tasks have exceeded their
// resource limits on disk, CPU time or memory
//
bool ACTIVE_TASK_SET::check_rsc_limits_exceeded() {
//LOGD("app_control: ACTIVE_TASK_SET::check_rsc_limits_exceeded");
unsigned int i;
ACTIVE_TASK *atp;
static double last_disk_check_time = 0;
bool do_disk_check = false;
bool did_anything = false;
double ram_left = gstate.available_ram();
double max_ram = gstate.max_available_ram();
// Some slot dirs have lots of files,
// so only check every min(disk_interval, 300) secs
//
double min_interval = gstate.global_prefs.disk_interval;
if (min_interval < 300) min_interval = 300;
if (gstate.now > last_disk_check_time + min_interval) {
do_disk_check = true;
}
for (i=0; i<active_tasks.size(); i++) {
atp = active_tasks[i];
if (atp->task_state() != PROCESS_EXECUTING) continue;
if (!atp->result->non_cpu_intensive() && (atp->elapsed_time > atp->max_elapsed_time)) {
msg_printf(atp->result->project, MSG_INFO,
"Aborting task %s: exceeded elapsed time limit %.2f (%.2fG/%.2fG)",
atp->result->name, atp->max_elapsed_time,
atp->result->wup->rsc_fpops_bound/1e9,
atp->result->avp->flops/1e9
);
atp->abort_task(ERR_RSC_LIMIT_EXCEEDED, "Maximum elapsed time exceeded");
did_anything = true;
continue;
}
if (atp->procinfo.working_set_size_smoothed > max_ram) {
msg_printf(atp->result->project, MSG_INFO,
"Aborting task %s: exceeded memory limit %.2fMB > %.2fMB\n",
atp->result->name,
atp->procinfo.working_set_size_smoothed/MEGA, max_ram/MEGA
);
atp->abort_task(ERR_RSC_LIMIT_EXCEEDED, "Maximum memory exceeded");
did_anything = true;
continue;
}
if (do_disk_check && atp->check_max_disk_exceeded()) {
did_anything = true;
continue;
}
ram_left -= atp->procinfo.working_set_size_smoothed;
}
if (ram_left < 0) {
gstate.request_schedule_cpus("RAM usage limit exceeded");
}
if (do_disk_check) {
last_disk_check_time = gstate.now;
}
return did_anything;
}
static void handle_result_op(GUI_RPC_CONN& grc, const char* op) {
RESULT* rp;
char result_name[256];
ACTIVE_TASK* atp;
string project_url;
strcpy(result_name, "");
while (!grc.xp.get_tag()) {
if (grc.xp.parse_str("name", result_name, sizeof(result_name))) continue;
if (grc.xp.parse_string("project_url", project_url)) continue;
}
PROJECT* p = get_project(grc, project_url);
if (!p) return;
if (!strlen(result_name)) {
grc.mfout.printf("<error>Missing result name</error>\n");
return;
}
rp = gstate.lookup_result(p, result_name);
if (!rp) {
grc.mfout.printf("<error>no such result</error>\n");
return;
}
if (!strcmp(op, "abort")) {
msg_printf(p, MSG_INFO, "task %s aborted by user", result_name);
atp = gstate.lookup_active_task_by_result(rp);
if (atp) {
atp->abort_task(EXIT_ABORTED_VIA_GUI, "aborted by user");
} else {
rp->abort_inactive(EXIT_ABORTED_VIA_GUI);
}
gstate.request_work_fetch("result aborted by user");
} else if (!strcmp(op, "suspend")) {
msg_printf(p, MSG_INFO, "task %s suspended by user", result_name);
rp->suspended_via_gui = true;
gstate.request_work_fetch("result suspended by user");
} else if (!strcmp(op, "resume")) {
msg_printf(p, MSG_INFO, "task %s resumed by user", result_name);
rp->suspended_via_gui = false;
}
gstate.request_schedule_cpus("result suspended, resumed or aborted by user");
gstate.set_client_state_dirty("Result RPC");
grc.mfout.printf("<success/>\n");
}
static void handle_result_op(char* buf, MIOFILE& fout, const char* op) {
RESULT* rp;
char result_name[256];
ACTIVE_TASK* atp;
PROJECT* p = get_project(buf, fout);
if (!p) {
fout.printf("<error>No such project</error>\n");
return;
}
if (!parse_str(buf, "<name>", result_name, sizeof(result_name))) {
fout.printf("<error>Missing result name</error>\n");
return;
}
rp = gstate.lookup_result(p, result_name);
if (!rp) {
fout.printf("<error>no such result</error>\n");
return;
}
if (!strcmp(op, "abort")) {
msg_printf(p, MSG_INFO, "task %s aborted by user", result_name);
atp = gstate.lookup_active_task_by_result(rp);
if (atp) {
atp->abort_task(ERR_ABORTED_VIA_GUI, "aborted by user");
} else {
rp->abort_inactive(ERR_ABORTED_VIA_GUI);
}
gstate.request_work_fetch("result aborted by user");
} else if (!strcmp(op, "suspend")) {
msg_printf(p, MSG_INFO, "task %s suspended by user", result_name);
rp->suspended_via_gui = true;
gstate.request_work_fetch("result suspended by user");
} else if (!strcmp(op, "resume")) {
msg_printf(p, MSG_INFO, "task %s resumed by user", result_name);
rp->suspended_via_gui = false;
}
gstate.request_schedule_cpus("result suspended, resumed or aborted by user");
gstate.set_client_state_dirty("Result RPC");
fout.printf("<success/>\n");
}
// Handle the reply from a scheduler
//
int CLIENT_STATE::handle_scheduler_reply(
PROJECT* project, char* scheduler_url
) {
SCHEDULER_REPLY sr;
FILE* f;
int retval;
unsigned int i;
bool signature_valid, update_global_prefs=false, update_project_prefs=false;
char buf[1024], filename[256];
std::string old_gui_urls = project->gui_urls;
PROJECT* p2;
vector<RESULT*>new_results;
project->last_rpc_time = now;
if (requested_work()) {
had_or_requested_work = true;
}
get_sched_reply_filename(*project, filename, sizeof(filename));
f = fopen(filename, "r");
if (!f) return ERR_FOPEN;
retval = sr.parse(f, project);
fclose(f);
if (retval) return retval;
if (log_flags.sched_ops) {
if (requested_work()) {
sprintf(buf, ": got %d new tasks", (int)sr.results.size());
} else {
strcpy(buf, "");
}
msg_printf(project, MSG_INFO, "Scheduler request completed%s", buf);
}
if (log_flags.sched_op_debug) {
if (sr.scheduler_version) {
msg_printf(project, MSG_INFO,
"[sched_op] Server version %d",
sr.scheduler_version
);
}
}
// check that master URL is correct
//
if (strlen(sr.master_url)) {
canonicalize_master_url(sr.master_url);
string url1 = sr.master_url;
string url2 = project->master_url;
downcase_string(url1);
downcase_string(url2);
if (url1 != url2) {
p2 = lookup_project(sr.master_url);
if (p2) {
msg_printf(project, MSG_USER_ALERT,
"You are attached to this project twice. Please remove projects named %s, then add %s",
project->project_name,
sr.master_url
);
} else {
msg_printf(project, MSG_INFO,
_("You used the wrong URL for this project. When convenient, remove this project, then add %s"),
sr.master_url
);
}
}
}
// make sure we don't already have a project of same name
//
bool dup_name = false;
for (i=0; i<projects.size(); i++) {
p2 = projects[i];
if (project == p2) continue;
if (!strcmp(p2->project_name, project->project_name)) {
dup_name = true;
break;
}
}
if (dup_name) {
msg_printf(project, MSG_INFO,
"Already attached to a project named %s (possibly with wrong URL)",
project->project_name
);
msg_printf(project, MSG_INFO,
"Consider detaching this project, then trying again"
);
}
// show messages from server
//
for (i=0; i<sr.messages.size(); i++) {
USER_MESSAGE& um = sr.messages[i];
int prio = (!strcmp(um.priority.c_str(), "notice"))?MSG_SCHEDULER_ALERT:MSG_INFO;
string_substitute(um.message.c_str(), buf, sizeof(buf), "%", "%%");
msg_printf(project, prio, "%s", buf);
}
if (log_flags.sched_op_debug && sr.request_delay) {
msg_printf(project, MSG_INFO,
"Project requested delay of %.0f seconds", sr.request_delay
);
}
// if project is down, return error (so that we back off)
// and don't do anything else
//
if (sr.project_is_down) {
if (sr.request_delay) {
double x = now + sr.request_delay;
project->set_min_rpc_time(x, "project is down");
}
return ERR_PROJECT_DOWN;
}
// if the scheduler reply includes global preferences,
// insert extra elements, write to disk, and parse
//
if (sr.global_prefs_xml) {
// skip this if we have host-specific prefs
// and we're talking to an old scheduler
//
if (!global_prefs.host_specific || sr.scheduler_version >= 507) {
retval = save_global_prefs(
sr.global_prefs_xml, project->master_url, scheduler_url
);
if (retval) {
return retval;
}
update_global_prefs = true;
} else {
if (log_flags.sched_op_debug) {
msg_printf(project, MSG_INFO,
"ignoring prefs from old server; we have host-specific prefs"
);
}
}
}
// see if we have a new venue from this project
// (this must go AFTER the above, since otherwise
// global_prefs_source_project() is meaningless)
//
if (strcmp(project->host_venue, sr.host_venue)) {
safe_strcpy(project->host_venue, sr.host_venue);
msg_printf(project, MSG_INFO, "New computer location: %s", sr.host_venue);
update_project_prefs = true;
if (project == global_prefs_source_project()) {
strcpy(main_host_venue, sr.host_venue);
update_global_prefs = true;
}
}
if (update_global_prefs) {
read_global_prefs();
}
// deal with project preferences (should always be there)
// If they've changed, write to account file,
// then parse to get our venue, and pass to running apps
//
if (sr.project_prefs_xml) {
if (strcmp(project->project_prefs.c_str(), sr.project_prefs_xml)) {
project->project_prefs = string(sr.project_prefs_xml);
update_project_prefs = true;
}
}
// the account file has GUI URLs and project prefs.
// rewrite if either of these has changed
//
if (project->gui_urls != old_gui_urls || update_project_prefs) {
retval = project->write_account_file();
if (retval) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Can't write account file: %s", boincerror(retval)
);
return retval;
}
}
if (update_project_prefs) {
project->parse_account_file();
if (strlen(project->host_venue)) {
project->parse_account_file_venue();
}
project->parse_preferences_for_user_files();
active_tasks.request_reread_prefs(project);
}
// if the scheduler reply includes a code-signing key,
// accept it if we don't already have one from the project.
// Otherwise verify its signature, using the key we already have.
//
if (sr.code_sign_key) {
if (!strlen(project->code_sign_key)) {
safe_strcpy(project->code_sign_key, sr.code_sign_key);
} else {
if (sr.code_sign_key_signature) {
retval = check_string_signature2(
sr.code_sign_key, sr.code_sign_key_signature,
project->code_sign_key, signature_valid
);
if (!retval && signature_valid) {
safe_strcpy(project->code_sign_key, sr.code_sign_key);
} else {
msg_printf(project, MSG_INTERNAL_ERROR,
"New code signing key doesn't validate"
);
}
} else {
msg_printf(project, MSG_INTERNAL_ERROR,
"Missing code sign key signature"
);
}
}
}
// copy new entities to client state
//
for (i=0; i<sr.apps.size(); i++) {
APP* app = lookup_app(project, sr.apps[i].name);
if (app) {
strcpy(app->user_friendly_name, sr.apps[i].user_friendly_name);
} else {
app = new APP;
*app = sr.apps[i];
retval = link_app(project, app);
if (retval) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Can't handle application %s in scheduler reply", app->name
);
delete app;
} else {
apps.push_back(app);
}
}
}
FILE_INFO* fip;
for (i=0; i<sr.file_infos.size(); i++) {
fip = lookup_file_info(project, sr.file_infos[i].name);
if (fip) {
fip->merge_info(sr.file_infos[i]);
} else {
fip = new FILE_INFO;
*fip = sr.file_infos[i];
retval = link_file_info(project, fip);
if (retval) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Can't handle file %s in scheduler reply", fip->name
);
delete fip;
} else {
file_infos.push_back(fip);
}
}
}
for (i=0; i<sr.file_deletes.size(); i++) {
fip = lookup_file_info(project, sr.file_deletes[i].c_str());
if (fip) {
if (log_flags.file_xfer_debug) {
msg_printf(project, MSG_INFO,
"[file_xfer_debug] Got server request to delete file %s",
fip->name
);
}
fip->sticky = false;
}
}
for (i=0; i<sr.app_versions.size(); i++) {
if (project->anonymous_platform) {
msg_printf(project, MSG_INTERNAL_ERROR,
"App version returned from anonymous platform project; ignoring"
);
continue;
}
APP_VERSION& avpp = sr.app_versions[i];
if (strlen(avpp.platform) == 0) {
strcpy(avpp.platform, get_primary_platform());
} else {
if (!is_supported_platform(avpp.platform)) {
msg_printf(project, MSG_INTERNAL_ERROR,
"App version has unsupported platform %s", avpp.platform
);
continue;
}
}
if (avpp.missing_coproc) {
msg_printf(project, MSG_INTERNAL_ERROR,
"App version uses non-existent %s GPU",
avpp.missing_coproc_name
);
}
APP* app = lookup_app(project, avpp.app_name);
if (!app) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Missing app %s", avpp.app_name
);
continue;
}
APP_VERSION* avp = lookup_app_version(
app, avpp.platform, avpp.version_num, avpp.plan_class
);
if (avp) {
// update performance-related info;
// generally this shouldn't change,
// but if it does it's better to use the new stuff
//
avp->avg_ncpus = avpp.avg_ncpus;
avp->max_ncpus = avpp.max_ncpus;
avp->flops = avpp.flops;
strcpy(avp->cmdline, avpp.cmdline);
avp->gpu_usage = avpp.gpu_usage;
strlcpy(avp->api_version, avpp.api_version, sizeof(avp->api_version));
avp->dont_throttle = avpp.dont_throttle;
avp->needs_network = avpp.needs_network;
// if we had download failures, clear them
//
avp->clear_errors();
continue;
}
avp = new APP_VERSION;
*avp = avpp;
retval = link_app_version(project, avp);
if (retval) {
delete avp;
continue;
}
app_versions.push_back(avp);
}
for (i=0; i<sr.workunits.size(); i++) {
if (lookup_workunit(project, sr.workunits[i].name)) continue;
WORKUNIT* wup = new WORKUNIT;
*wup = sr.workunits[i];
wup->project = project;
retval = link_workunit(project, wup);
if (retval) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Can't handle task %s in scheduler reply", wup->name
);
delete wup;
continue;
}
wup->clear_errors();
workunits.push_back(wup);
}
double est_rsc_runtime[MAX_RSC];
for (int j=0; j<coprocs.n_rsc; j++) {
est_rsc_runtime[j] = 0;
}
for (i=0; i<sr.results.size(); i++) {
if (lookup_result(project, sr.results[i].name)) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Already have task %s\n", sr.results[i].name
);
continue;
}
RESULT* rp = new RESULT;
*rp = sr.results[i];
retval = link_result(project, rp);
if (retval) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Can't handle task %s in scheduler reply", rp->name
);
delete rp;
continue;
}
if (strlen(rp->platform) == 0) {
strcpy(rp->platform, get_primary_platform());
rp->version_num = latest_version(rp->wup->app, rp->platform);
}
rp->avp = lookup_app_version(
rp->wup->app, rp->platform, rp->version_num, rp->plan_class
);
if (!rp->avp) {
msg_printf(project, MSG_INTERNAL_ERROR,
"No app version found for app %s platform %s ver %d class %s; discarding %s",
rp->wup->app->name, rp->platform, rp->version_num, rp->plan_class, rp->name
);
delete rp;
continue;
}
if (rp->avp->missing_coproc) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Missing coprocessor for task %s; aborting", rp->name
);
rp->abort_inactive(EXIT_MISSING_COPROC);
} else {
rp->set_state(RESULT_NEW, "handle_scheduler_reply");
int rt = rp->avp->gpu_usage.rsc_type;
if (rt > 0) {
est_rsc_runtime[rt] += rp->estimated_runtime();
gpus_usable = true;
// trigger a check of whether GPU is actually usable
} else {
est_rsc_runtime[0] += rp->estimated_runtime();
}
}
rp->wup->version_num = rp->version_num;
rp->received_time = now;
new_results.push_back(rp);
results.push_back(rp);
}
sort_results();
if (log_flags.sched_op_debug) {
if (sr.results.size()) {
for (int j=0; j<coprocs.n_rsc; j++) {
msg_printf(project, MSG_INFO,
"[sched_op] estimated total %s task duration: %.0f seconds",
rsc_name(j),
est_rsc_runtime[j]/time_stats.availability_frac(j)
);
}
}
}
// update records for ack'ed results
//
for (i=0; i<sr.result_acks.size(); i++) {
if (log_flags.sched_op_debug) {
msg_printf(project, MSG_INFO,
"[sched_op] handle_scheduler_reply(): got ack for task %s\n",
sr.result_acks[i].name
);
}
RESULT* rp = lookup_result(project, sr.result_acks[i].name);
if (rp) {
rp->got_server_ack = true;
} else {
msg_printf(project, MSG_INTERNAL_ERROR,
"Got ack for task %s, but can't find it", sr.result_acks[i].name
);
}
}
// handle result abort requests
//
for (i=0; i<sr.result_abort.size(); i++) {
RESULT* rp = lookup_result(project, sr.result_abort[i].name);
if (rp) {
ACTIVE_TASK* atp = lookup_active_task_by_result(rp);
if (atp) {
atp->abort_task(EXIT_ABORTED_BY_PROJECT,
"aborted by project - no longer usable"
);
} else {
rp->abort_inactive(EXIT_ABORTED_BY_PROJECT);
}
} else {
msg_printf(project, MSG_INTERNAL_ERROR,
"Server requested abort of unknown task %s",
sr.result_abort[i].name
);
}
}
for (i=0; i<sr.result_abort_if_not_started.size(); i++) {
RESULT* rp = lookup_result(project, sr.result_abort_if_not_started[i].name);
if (!rp) {
msg_printf(project, MSG_INTERNAL_ERROR,
"Server requested conditional abort of unknown task %s",
sr.result_abort_if_not_started[i].name
);
continue;
}
if (rp->not_started) {
rp->abort_inactive(EXIT_ABORTED_BY_PROJECT);
}
}
// remove acked trickle files
//
if (sr.message_ack) {
remove_trickle_files(project);
}
if (sr.send_full_workload) {
project->send_full_workload = true;
}
project->dont_use_dcf = sr.dont_use_dcf;
project->send_time_stats_log = sr.send_time_stats_log;
project->send_job_log = sr.send_job_log;
project->trickle_up_pending = false;
// The project returns a hostid only if it has created a new host record.
// In that case reset RPC seqno
//
if (sr.hostid) {
if (project->hostid) {
// if we already have a host ID for this project,
// we must have sent it a stale seqno,
// which usually means our state file was copied from another host.
// So generate a new host CPID.
//
generate_new_host_cpid();
msg_printf(project, MSG_INFO,
"Generated new computer cross-project ID: %s",
host_info.host_cpid
);
}
//msg_printf(project, MSG_INFO, "Changing host ID from %d to %d", project->hostid, sr.hostid);
project->hostid = sr.hostid;
project->rpc_seqno = 0;
}
#ifdef ENABLE_AUTO_UPDATE
if (sr.auto_update.present) {
if (!sr.auto_update.validate_and_link(project)) {
auto_update = sr.auto_update;
}
}
#endif
project->project_files = sr.project_files;
project->link_project_files();
project->create_project_file_symlinks();
if (log_flags.state_debug) {
msg_printf(project, MSG_INFO,
"[state] handle_scheduler_reply(): State after handle_scheduler_reply():"
);
print_summary();
}
// the following must precede the backoff and request_delay checks,
// since it overrides them
//
if (sr.next_rpc_delay) {
project->next_rpc_time = now + sr.next_rpc_delay;
} else {
project->next_rpc_time = 0;
}
work_fetch.handle_reply(project, &sr, new_results);
project->nrpc_failures = 0;
project->min_rpc_time = 0;
if (sr.request_delay) {
double x = now + sr.request_delay;
project->set_min_rpc_time(x, "requested by project");
}
if (sr.got_rss_feeds) {
handle_sr_feeds(sr.sr_feeds, project);
}
update_trickle_up_urls(project, sr.trickle_up_urls);
// garbage collect in case the project sent us some irrelevant FILE_INFOs;
// avoid starting transfers for them
//
gstate.garbage_collect_always();
return 0;
}
// Do periodic checks on running apps:
// - get latest CPU time and % done info
// - check if any has exited, and clean up
// - see if any has exceeded its CPU or disk space limits, and abort it
//
bool ACTIVE_TASK_SET::poll() {
bool action;
unsigned int i;
static double last_time = 0;
if (!gstate.clock_change && gstate.now - last_time < TASK_POLL_PERIOD) return false;
last_time = gstate.now;
action = check_app_exited();
send_heartbeats();
send_trickle_downs();
process_control_poll();
action |= check_rsc_limits_exceeded();
get_msgs();
for (i=0; i<active_tasks.size(); i++) {
ACTIVE_TASK* atp = active_tasks[i];
if (atp->task_state() == PROCESS_ABORT_PENDING) {
if (gstate.now > atp->abort_time + ABORT_TIMEOUT) {
if (log_flags.task_debug) {
msg_printf(atp->result->project, MSG_INFO,
"[task] abort request timed out, killing task %s",
atp->result->name
);
}
atp->kill_task(false);
}
}
if (atp->task_state() == PROCESS_QUIT_PENDING) {
if (gstate.now > atp->quit_time + QUIT_TIMEOUT) {
if (log_flags.task_debug) {
msg_printf(atp->result->project, MSG_INFO,
"[task] quit request timed out, killing task %s",
atp->result->name
);
}
atp->kill_task(true);
}
}
}
// Check for finish files every 10 sec.
// If we already found a finish file, abort the app;
// it must be hung somewhere in boinc_finish();
//
static double last_finish_check_time = 0;
if (gstate.clock_change || gstate.now - last_finish_check_time > 10) {
last_finish_check_time = gstate.now;
for (i=0; i<active_tasks.size(); i++) {
ACTIVE_TASK* atp = active_tasks[i];
if (atp->task_state() == PROCESS_UNINITIALIZED) continue;
if (atp->finish_file_time) {
// process is still there 10 sec after it wrote finish file.
// abort the job
atp->abort_task(EXIT_ABORTED_BY_CLIENT, "finish file present too long");
} else if (atp->finish_file_present()) {
atp->finish_file_time = gstate.now;
}
}
}
if (action) {
gstate.set_client_state_dirty("ACTIVE_TASK_SET::poll");
}
return action;
}
// Check if any of the active tasks have exceeded their
// resource limits on disk, CPU time or memory
//
// TODO: this gets called ever 1 sec,
// but mem and disk usage are computed less often.
// refactor.
//
bool ACTIVE_TASK_SET::check_rsc_limits_exceeded() {
unsigned int i;
ACTIVE_TASK *atp;
static double last_disk_check_time = 0;
bool do_disk_check = false;
bool did_anything = false;
char buf[256];
double ram_left = gstate.available_ram();
double max_ram = gstate.max_available_ram();
// Some slot dirs have lots of files,
// so only check every min(disk_interval, 300) secs
//
double min_interval = gstate.global_prefs.disk_interval;
if (min_interval < 300) min_interval = 300;
if (gstate.clock_change || gstate.now > last_disk_check_time + min_interval) {
do_disk_check = true;
}
for (i=0; i<active_tasks.size(); i++) {
atp = active_tasks[i];
if (atp->task_state() != PROCESS_EXECUTING) continue;
if (!atp->result->non_cpu_intensive() && (atp->elapsed_time > atp->max_elapsed_time)) {
sprintf(buf, "exceeded elapsed time limit %.2f (%.2fG/%.2fG)",
atp->max_elapsed_time,
atp->result->wup->rsc_fpops_bound/1e9,
atp->result->avp->flops/1e9
);
msg_printf(atp->result->project, MSG_INFO,
"Aborting task %s: %s", atp->result->name, buf
);
atp->abort_task(EXIT_TIME_LIMIT_EXCEEDED, buf);
did_anything = true;
continue;
}
#if 0
// removing this for now because most projects currently
// have too-low values of workunit.rsc_memory_bound
// (causing lots of aborts)
// and I don't think we can expect projects to provide
// accurate bounds.
//
if (atp->procinfo.working_set_size_smoothed > atp->max_mem_usage) {
sprintf(buf, "working set size > workunit.rsc_memory_bound: %.2fMB > %.2fMB",
atp->procinfo.working_set_size_smoothed/MEGA, atp->max_mem_usage/MEGA
);
msg_printf(atp->result->project, MSG_INFO,
"Aborting task %s: %s",
atp->result->name, buf
);
atp->abort_task(EXIT_MEM_LIMIT_EXCEEDED, buf);
did_anything = true;
continue;
}
#endif
if (atp->procinfo.working_set_size_smoothed > max_ram) {
sprintf(buf, "working set size > client RAM limit: %.2fMB > %.2fMB",
atp->procinfo.working_set_size_smoothed/MEGA, max_ram/MEGA
);
msg_printf(atp->result->project, MSG_INFO,
"Aborting task %s: %s",
atp->result->name, buf
);
atp->abort_task(EXIT_MEM_LIMIT_EXCEEDED, buf);
did_anything = true;
continue;
}
if (do_disk_check || atp->peak_disk_usage == 0) {
if (atp->check_max_disk_exceeded()) {
did_anything = true;
continue;
}
}
// don't count RAM usage of non-CPU-intensive jobs
//
if (!atp->result->non_cpu_intensive()) {
ram_left -= atp->procinfo.working_set_size_smoothed;
}
}
if (ram_left < 0) {
gstate.request_schedule_cpus("RAM usage limit exceeded");
}
if (do_disk_check) {
last_disk_check_time = gstate.now;
}
return did_anything;
}
