void HR_INFO::scan_db() { DB_HOST host; int retval; int i, n=0; double sum=0, sum_sqr=0; while (1) { retval = host.enumerate("where expavg_credit>1"); if (retval) break; if (host.p_fpops > 1e7 && host.p_fpops < 1e13) { n++; sum += host.p_fpops; sum_sqr += host.p_fpops*host.p_fpops; } //printf("host %d: %s | %s | %s\n", host.id, host.os_name, host.p_vendor, host.p_model); for (i=1; i<HR_NTYPES; i++) { if (hr_unknown_class(host, i)) { //printf("type %d: unknown\n", i); continue; } int hrc = hr_class(host, i); //printf("type %d: class %d\n", i, hrc); if (!hrc) continue; rac_per_class[i][hrc] += host.expavg_credit; } } if (retval != ERR_DB_NOT_FOUND) { fprintf(stderr, "host enum: %d", retval); exit(1); } }
int handle_trickle_init(int argc, char** argv) { int retval; for (int i=1; i<argc; i++) { if (!strcmp(argv[i], "--max_runtime")) { max_runtime = atof(argv[++i]); } else { log_messages.printf(MSG_CRITICAL, "unknown arg %s\n", argv[i]); return ERR_XML_PARSE; } } if (!max_runtime) { log_messages.printf(MSG_CRITICAL, "missing --max_runtime arg\n"); return ERR_NULL; } DB_HOST host; retval = host.fpops_percentile(50, flops_50_percentile); if (retval) { log_messages.printf(MSG_CRITICAL, "fpops_percentile failed: %d\n", retval); return retval; } retval = host.fpops_percentile(95, flops_95_percentile); if (retval) { log_messages.printf(MSG_CRITICAL, "fpops_percentile failed: %d\n", retval); return retval; } log_messages.printf(MSG_NORMAL, "default FLOPS: %f\n", flops_50_percentile); log_messages.printf(MSG_NORMAL, "max FLOPS: %f\n", flops_95_percentile); log_messages.printf(MSG_NORMAL, "max runtime: %f\n", max_runtime); return 0; }
// handle timed-out hosts // bool scan_hosts() { DB_HOST h; char buf[256]; int retval; bool found = false; sprintf(buf, "where cpu_efficiency < %f", dtime()); while (1) { retval = h.enumerate(buf); if (retval == ERR_DB_NOT_FOUND) break; if (retval) { log_messages.printf(MSG_CRITICAL, "host.enumerate() failed\n"); exit(1); } found = true; retval = handle_host(h); if (retval) { log_messages.printf(MSG_CRITICAL, "handle_host() failed: %d\n", retval); exit(1); } retval = h.update_field("cpu_efficiency=1e12"); if (retval) { log_messages.printf(MSG_CRITICAL, "h.update_field() failed: %d\n", retval); exit(1); } } return found; }
// update the DB record to the values in "xhost" // "initial_host" stores the current DB values; // update only those fields that have changed // static int update_host_record(HOST& initial_host, HOST& xhost, USER& user) { DB_HOST host; int retval; char buf[1024]; host = xhost; // hash the CPID reported by the host with the user's email address. // This prevents one user from spoofing another one's host. // if (strlen(host.host_cpid)) { sprintf(buf, "%s%s", host.host_cpid, user.email_addr); md5_block((const unsigned char*)buf, strlen(buf), host.host_cpid); } char* p = getenv("REMOTE_ADDR"); if (p) { strlcpy(host.external_ip_addr, p, sizeof(host.external_ip_addr)); } retval = host.update_diff_sched(initial_host); if (retval) { log_messages.printf(MSG_CRITICAL, "host.update() failed: %s\n", boincerror(retval) ); } return 0; }
int main() { if ( boinc_db.open("predictor", "boinc", NULL, NULL) ) { printf("Open failed\n"); return 0; } DB_WORKUNIT workunit; char buf[256]; while (!workunit.enumerate()) { printf("workunit %d wsn %d\n", workunit.id, workunit.workseq_next); DB_RESULT result; sprintf(buf, "where workunitid=%d", workunit.id); if ( !result.enumerate(buf) ) { DB_HOST host; sprintf(buf, "where id=%d", result.hostid); if ( !host.enumerate(buf) ) { workunit.workseq_next = OS(host) + CPU(host); if ( workunit.update() ) printf("Update failed!\n"); } } } };
int update_hosts() { DB_HOST host; int retval; char buf[256]; while (1) { retval = host.enumerate("where expavg_credit>0.1"); if (retval) { if (retval != ERR_DB_NOT_FOUND) { log_messages.printf(MSG_CRITICAL, "lost DB conn\n"); exit(1); } break; } if (host.expavg_time > update_time_cutoff) continue; update_average(0, 0, CREDIT_HALF_LIFE, host.expavg_credit, host.expavg_time); sprintf( buf,"expavg_credit=%f, expavg_time=%f", host.expavg_credit, host.expavg_time ); retval = host.update_field(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't update host %d\n", host.id); return retval; } } return 0; }
int is_invalid(WORKUNIT& wu, RESULT& result) { char buf[256]; int retval; DB_HOST host; retval = host.lookup_id(result.hostid); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d] lookup of host %d failed %d\n", result.id, result.hostid, retval ); return retval; } double old_error_rate = host.error_rate; if (!is_unreplicated(wu)) { update_error_rate(host, false); } sprintf(buf, "error_rate=%f", host.error_rate); retval = host.update_field(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d] update of host %d failed %d\n", result.id, result.hostid, retval ); return retval; } log_messages.printf(MSG_DEBUG, "[HOST#%d] invalid result; error rate %f->%f\n", host.id, old_error_rate, host.error_rate ); return 0; }
// Here when a result has been validated and its granted_credit has been set. // Grant credit to host, user and team, and update host error rate. // int is_valid(RESULT& result, WORKUNIT& wu) { DB_HOST host; DB_CREDITED_JOB credited_job; int retval; char buf[256]; retval = host.lookup_id(result.hostid); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d] lookup of host %d failed %d\n", result.id, result.hostid, retval ); return retval; } grant_credit(host, result.sent_time, result.cpu_time, result.granted_credit); double turnaround = result.received_time - result.sent_time; compute_avg_turnaround(host, turnaround); double old_error_rate = host.error_rate; if (!is_unreplicated(wu)) { update_error_rate(host, true); } sprintf( buf, "avg_turnaround=%f, error_rate=%f", host.avg_turnaround, host.error_rate ); retval = host.update_field(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d] update of host %d failed %d\n", result.id, result.hostid, retval ); } log_messages.printf(MSG_DEBUG, "[HOST#%d] error rate %f->%f\n", host.id, old_error_rate, host.error_rate ); if (update_credited_job) { credited_job.userid = host.userid; credited_job.workunitid = long(wu.opaque); retval = credited_job.insert(); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d] Warning: credited_job insert failed (userid: %d workunit: %f err: %d)\n", result.id, host.userid, wu.opaque, retval ); } else { log_messages.printf(MSG_DEBUG, "[RESULT#%d %s] added credited_job record [WU#%d OPAQUE#%f USER#%d]\n", result.id, result.name, wu.id, wu.opaque, host.userid ); } } return 0; }
int handle_trickle(MSG_FROM_HOST& msg) { double runtime = 0; MIOFILE mf; mf.init_buf_read(msg.xml); XML_PARSER xp(&mf); while (!xp.get_tag()) { if (xp.parse_double("runtime", runtime)) break; log_messages.printf(MSG_NORMAL, "unexpected tag: %s\n", xp.parsed_tag); } if (runtime <= 0) { log_messages.printf(MSG_NORMAL, "unexpected nonpositive runtime: %f\n", runtime ); return ERR_XML_PARSE; } DB_HOST host; int retval = host.lookup_id(msg.hostid); if (retval) return retval; HOST old_host = host; double flops_sec = host.p_fpops; // sanity checks - customize as needed // if (runtime > max_runtime) { log_messages.printf(MSG_NORMAL, "Reported runtime exceeds bound: %f>%f\n", runtime, max_runtime ); runtime = max_runtime; } if (flops_sec < 0) { log_messages.printf(MSG_NORMAL, "host CPU speed %f < 0. Using %f instead\n", flops_sec, flops_50_percentile ); flops_sec = flops_50_percentile; } if (flops_sec > flops_95_percentile) { log_messages.printf(MSG_NORMAL, "host CPU speed %f exceeds %f. Using %f instead\n", flops_sec, flops_95_percentile, flops_95_percentile ); flops_sec = flops_95_percentile; } double credit = cpu_time_to_credit(runtime, flops_sec); grant_credit(host, dtime()-86400, credit); log_messages.printf(MSG_DEBUG, "granting %f credit to host %lu\n", credit, host.id ); // update the host's credit fields // retval = host.update_diff_validator(old_host); return 0; }
// find the user's most recently-created host with given host CPID // static bool find_host_by_cpid(DB_USER& user, char* host_cpid, DB_HOST& host) { char buf[1024], buf2[256]; sprintf(buf, "%s%s", host_cpid, user.email_addr); md5_block((const unsigned char*)buf, strlen(buf), buf2); sprintf(buf, "where userid=%d and host_cpid='%s' order by id desc", user.id, buf2 ); if (!host.enumerate(buf)) { host.end_enumerate(); return true; } return false; }
int PERF_INFO::get_from_db() { int retval, n; DB_HOST host; host_fpops_mean = 2.2e9; host_fpops_stddev = .7e9; host_fpops_50_percentile = 3.3e9; host_fpops_95_percentile = 3.3e9; retval = host.count(n); if (retval) return retval; if (n < 10) return 0; retval = host.fpops_mean(host_fpops_mean); retval = host.fpops_stddev(host_fpops_stddev); retval = host.fpops_percentile(50, host_fpops_50_percentile); retval = host.fpops_percentile(95, host_fpops_95_percentile); return 0; }
// find the user's most recently-created host with given various characteristics // static bool find_host_by_other(DB_USER& user, HOST req_host, DB_HOST& host) { char buf[2048]; char dn[512], ip[512], os[512], pm[512]; // don't dig through hosts of these users // prevents flooding the DB with slow queries from users with many hosts // for (unsigned int i=0; i < config.dont_search_host_for_userid.size(); i++) { if (user.id == config.dont_search_host_for_userid[i]) { return false; } } // Only check if all the fields are populated // if (strlen(req_host.domain_name) && strlen(req_host.last_ip_addr) && strlen(req_host.os_name) && strlen(req_host.p_model)) { safe_strcpy(dn, req_host.domain_name); escape_string(dn, sizeof(dn)); safe_strcpy(ip, req_host.last_ip_addr); escape_string(ip, sizeof(ip)); safe_strcpy(os, req_host.os_name); escape_string(os, sizeof(os)); safe_strcpy(pm, req_host.p_model); escape_string(pm, sizeof(pm)); sprintf(buf, "where userid=%d and id>%d and domain_name='%s' and last_ip_addr = '%s' and os_name = '%s' and p_model = '%s'" " and m_nbytes = %lf order by id desc", user.id, req_host.id, dn, ip, os, pm, req_host.m_nbytes ); if (!host.enumerate(buf)) { host.end_enumerate(); return true; } } return false; }
// Based on the info in the request message, // look up the host and its user, and make sure the authenticator matches. // Some special cases: // 1) If no host ID is supplied, or if RPC seqno mismatch, // create a new host record // 2) If the host record specified by g_request->hostid is a "zombie" // (i.e. it was merged with another host via the web site) // then follow links to find the proper host // // POSTCONDITION: // If this function returns zero, then: // - reply.host contains a valid host record (possibly new) // - reply.user contains a valid user record // - if user belongs to a team, reply.team contains team record // int authenticate_user() { int retval; char buf[1024]; DB_HOST host; DB_USER user; DB_TEAM team; if (g_request->hostid) { retval = host.lookup_id(g_request->hostid); while (!retval && host.userid==0) { // if host record is zombie, follow link to new host // retval = host.lookup_id(host.rpc_seqno); if (!retval) { g_reply->hostid = host.id; log_messages.printf(MSG_NORMAL, "[HOST#%d] forwarding to new host ID %d\n", g_request->hostid, host.id ); } } if (retval) { g_reply->insert_message("Can't find host record", "low"); log_messages.printf(MSG_NORMAL, "[HOST#%d?] can't find host\n", g_request->hostid ); g_request->hostid = 0; goto lookup_user_and_make_new_host; } g_reply->host = host; // look up user based on the ID in host record, // and see if the authenticator matches (regular or weak) // g_request->using_weak_auth = false; sprintf(buf, "where id=%d", host.userid); retval = user.lookup(buf); if (!retval && !strcmp(user.authenticator, g_request->authenticator)) { // req auth matches user auth - go on } else { if (!retval) { // user for host.userid exists - check weak auth // get_weak_auth(user, buf); if (!strcmp(buf, g_request->authenticator)) { g_request->using_weak_auth = true; log_messages.printf(MSG_DEBUG, "[HOST#%d] accepting weak authenticator\n", host.id ); } } if (!g_request->using_weak_auth) { // weak auth failed - look up user based on authenticator // strlcpy( user.authenticator, g_request->authenticator, sizeof(user.authenticator) ); escape_string(user.authenticator, sizeof(user.authenticator)); sprintf(buf, "where authenticator='%s'", user.authenticator); retval = user.lookup(buf); if (retval) { g_reply->insert_message( _("Invalid or missing account key. To fix, remove and add this project."), "notice" ); g_reply->set_delay(DELAY_MISSING_KEY); g_reply->nucleus_only = true; log_messages.printf(MSG_CRITICAL, "[HOST#%d] [USER#%d] Bad authenticator '%s'\n", host.id, user.id, g_request->authenticator ); return ERR_AUTHENTICATOR; } } } g_reply->user = user; if (host.userid != user.id) { // If the request's host ID isn't consistent with the authenticator, // create a new host record. // log_messages.printf(MSG_NORMAL, "[HOST#%d] [USER#%d] inconsistent host ID; creating new host\n", host.id, user.id ); goto make_new_host; } // If the seqno from the host is less than what we expect, // the user must have copied the state file to a different host. // Make a new host record. // if (!batch && g_request->rpc_seqno < g_reply->host.rpc_seqno) { g_request->hostid = 0; log_messages.printf(MSG_NORMAL, "[HOST#%d] [USER#%d] RPC seqno %d less than expected %d; creating new host\n", g_reply->host.id, user.id, g_request->rpc_seqno, g_reply->host.rpc_seqno ); goto make_new_host; } } else { // Here no hostid was given, or the ID was bad. // Look up the user, then create a new host record // lookup_user_and_make_new_host: // if authenticator contains _, it's a weak auth // if (strchr(g_request->authenticator, '_')) { int userid = atoi(g_request->authenticator); retval = user.lookup_id(userid); if (!retval) { get_weak_auth(user, buf); if (strcmp(buf, g_request->authenticator)) { retval = ERR_AUTHENTICATOR; } } } else { strlcpy( user.authenticator, g_request->authenticator, sizeof(user.authenticator) ); escape_string(user.authenticator, sizeof(user.authenticator)); sprintf(buf, "where authenticator='%s'", user.authenticator); retval = user.lookup(buf); } if (retval) { g_reply->insert_message( "Invalid or missing account key. To fix, remove and add this project .", "low" ); g_reply->set_delay(DELAY_MISSING_KEY); log_messages.printf(MSG_CRITICAL, "[HOST#<none>] Bad authenticator '%s': %s\n", g_request->authenticator, boincerror(retval) ); return ERR_AUTHENTICATOR; } g_reply->user = user; // If host CPID is present, // scan backwards through this user's hosts, // looking for one with the same host CPID. // If we find one, it means the user detached and reattached. // Use the existing host record, // and mark in-progress results as over. // if (strlen(g_request->host.host_cpid)) { if (find_host_by_cpid(user, g_request->host.host_cpid, host)) { log_messages.printf(MSG_NORMAL, "[HOST#%d] [USER#%d] No host ID in request, but host with matching CPID found.\n", host.id, host.userid ); if ((g_request->allow_multiple_clients != 1) && (g_request->other_results.size() == 0) ) { mark_results_over(host); } goto got_host; } } make_new_host: // One final attempt to locate an existing host record: // scan backwards through this user's hosts, // looking for one with the same host name, // IP address, processor and amount of RAM. // If found, use the existing host record, // and mark in-progress results as over. // // NOTE: If the client was run with --allow_multiple_clients, skip this. // if ((g_request->allow_multiple_clients != 1) && find_host_by_other(user, g_request->host, host) ) { log_messages.printf(MSG_NORMAL, "[HOST#%d] [USER#%d] Found similar existing host for this user - assigned.\n", host.id, host.userid ); mark_results_over(host); goto got_host; } // either of the above cases, // or host ID didn't match user ID, // or RPC seqno was too low. // // Create a new host. // g_reply->user is filled in and valid at this point // host = g_request->host; host.id = 0; host.create_time = time(0); host.userid = g_reply->user.id; host.rpc_seqno = 0; host.expavg_time = time(0); safe_strcpy(host.venue, g_reply->user.venue); host.fix_nans(); retval = host.insert(); if (retval) { g_reply->insert_message( "Couldn't create host record in database", "low" ); boinc_db.print_error("host.insert()"); log_messages.printf(MSG_CRITICAL, "host.insert() failed\n"); return retval; } host.id = boinc_db.insert_id(); got_host: g_reply->host = host; g_reply->hostid = g_reply->host.id; // this tells client to updates its host ID g_request->rpc_seqno = 0; // this value eventually gets written to host DB record; // for new hosts it must be zero. // This kludge forces this. } // have user record in g_reply->user at this point // if (g_reply->user.teamid) { retval = team.lookup_id(g_reply->user.teamid); if (!retval) g_reply->team = team; } // compute email hash // md5_block( (unsigned char*)g_reply->user.email_addr, strlen(g_reply->user.email_addr), g_reply->email_hash ); // if new user CPID, update user record // if (!g_request->using_weak_auth && strlen(g_request->cross_project_id)) { if (strcmp(g_request->cross_project_id, g_reply->user.cross_project_id)) { user.id = g_reply->user.id; escape_string(g_request->cross_project_id, sizeof(g_request->cross_project_id)); sprintf(buf, "cross_project_id='%s'", g_request->cross_project_id); unescape_string(g_request->cross_project_id, sizeof(g_request->cross_project_id)); user.update_field(buf); } } return 0; }
// handle a workunit which has new results // int handle_wu( DB_VALIDATOR_ITEM_SET& validator, std::vector<VALIDATOR_ITEM>& items ) { int canonical_result_index = -1; bool update_result, retry; TRANSITION_TIME transition_time = NO_CHANGE; int retval = 0, canonicalid = 0, x; double credit = 0; unsigned int i; WORKUNIT& wu = items[0].wu; g_wup = &wu; vector<RESULT> results; vector<DB_HOST_APP_VERSION> host_app_versions, host_app_versions_orig; int nsuccess_results; // Here if WU doesn't have a canonical result yet. // Try to get one log_messages.printf(MSG_NORMAL, "[WU#%d %s] handle_wu(): No canonical result yet\n", wu.id, wu.name ); ++log_messages; // make a vector of the successful results, // and a parallel vector of host_app_versions // for (i=0; i<items.size(); i++) { RESULT& result = items[i].res; if ((result.server_state == RESULT_SERVER_STATE_OVER) && (result.outcome == RESULT_OUTCOME_SUCCESS) ) { results.push_back(result); DB_HOST_APP_VERSION hav; retval = hav_lookup(hav, result.hostid, generalized_app_version_id(result.app_version_id, result.appid) ); if (retval) { hav.host_id=0; // flag that it's missing } host_app_versions.push_back(hav); host_app_versions_orig.push_back(hav); } } log_messages.printf(MSG_DEBUG, "[WU#%d %s] Found %d successful results\n", wu.id, wu.name, (int)results.size() ); if (results.size() >= (unsigned int)wu.min_quorum) { log_messages.printf(MSG_DEBUG, "[WU#%d %s] Enough for quorum, checking set.\n", wu.id, wu.name ); double dummy; retval = check_set( results, wu, canonicalid, dummy, retry ); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%d %s] check_set returned %d, exiting\n", wu.id, wu.name, retval ); return retval; } if (retry) transition_time = DELAYED; if (credit_from_wu) { retval = get_credit_from_wu(wu, results, credit); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%d %s] get_credit_from_wu returned %d\n", wu.id, wu.name, retval ); return retval; } } if (canonicalid) { retval = assign_credit_set( wu, results, app, app_versions, host_app_versions, max_granted_credit, credit ); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%d %s] assign_credit_set() returned %d\n", wu.id, wu.name, retval ); transition_time = DELAYED; goto leave; } } if (max_granted_credit && credit>max_granted_credit) { credit = max_granted_credit; } // scan results. // update as needed, and count the # of results // that are still outcome=SUCCESS // (some may have changed to VALIDATE_ERROR) // nsuccess_results = 0; for (i=0; i<results.size(); i++) { RESULT& result = results[i]; DB_HOST_APP_VERSION& hav = host_app_versions[i]; DB_HOST_APP_VERSION& hav_orig = host_app_versions_orig[i]; update_result = false; bool update_host = false; if (result.outcome == RESULT_OUTCOME_VALIDATE_ERROR) { transition_time = IMMEDIATE; update_result = true; } else { nsuccess_results++; } DB_HOST host; HOST host_initial; switch (result.validate_state) { case VALIDATE_STATE_VALID: case VALIDATE_STATE_INVALID: retval = host.lookup_id(result.hostid); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d] lookup of host %d failed %d\n", result.id, result.hostid, retval ); continue; } host_initial = host; } switch (result.validate_state) { case VALIDATE_STATE_VALID: update_result = true; update_host = true; retval = is_valid(host, result, wu, host_app_versions[i]); if (retval) { log_messages.printf(MSG_DEBUG, "[RESULT#%d %s] is_valid() failed: %d\n", result.id, result.name, retval ); } grant_credit( host, result.sent_time, result.cpu_time, result.granted_credit ); log_messages.printf(MSG_NORMAL, "[RESULT#%d %s] Valid; granted %f credit [HOST#%d]\n", result.id, result.name, result.granted_credit, result.hostid ); break; case VALIDATE_STATE_INVALID: update_result = true; update_host = true; log_messages.printf(MSG_NORMAL, "[RESULT#%d %s] Invalid [HOST#%d]\n", result.id, result.name, result.hostid ); is_invalid(host_app_versions[i]); break; case VALIDATE_STATE_INIT: log_messages.printf(MSG_NORMAL, "[RESULT#%d %s] Inconclusive [HOST#%d]\n", result.id, result.name, result.hostid ); result.validate_state = VALIDATE_STATE_INCONCLUSIVE; update_result = true; break; } if (hav.host_id) { retval = hav.update_validator(hav_orig); } if (update_host) { retval = host.update_diff_validator(host_initial); } if (update_result) { retval = validator.update_result(result); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d %s] result.update() failed: %d\n", result.id, result.name, retval ); } } } if (canonicalid) { // if we found a canonical result, // trigger the assimilator, but do NOT trigger // the transitioner - doing so creates a race condition // transition_time = NEVER; log_messages.printf(MSG_DEBUG, "[WU#%d %s] Found a canonical result: id=%d\n", wu.id, wu.name, canonicalid ); wu.canonical_resultid = canonicalid; wu.canonical_credit = credit; wu.assimilate_state = ASSIMILATE_READY; // don't need to send any more results // for (i=0; i<items.size(); i++) { RESULT& result = items[i].res; if (result.server_state != RESULT_SERVER_STATE_UNSENT) { continue; } result.server_state = RESULT_SERVER_STATE_OVER; result.outcome = RESULT_OUTCOME_DIDNT_NEED; retval = validator.update_result(result); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%d %s] result.update() failed: %d\n", result.id, result.name, retval ); } } } else { // here if no consensus. // check if #success results is too large // if (nsuccess_results > wu.max_success_results) { wu.error_mask |= WU_ERROR_TOO_MANY_SUCCESS_RESULTS; transition_time = IMMEDIATE; } // if #success results >= target_nresults, // we need more results, so bump target_nresults // NOTE: nsuccess_results should never be > target_nresults, // but accommodate that if it should happen // if (nsuccess_results >= wu.target_nresults) { wu.target_nresults = nsuccess_results+1; transition_time = IMMEDIATE; } } } leave: --log_messages; switch (transition_time) { case IMMEDIATE: wu.transition_time = time(0); break; case DELAYED: x = time(0) + 6*3600; if (x < wu.transition_time) wu.transition_time = x; break; case NEVER: wu.transition_time = INT_MAX; break; case NO_CHANGE: break; } wu.need_validate = 0; retval = validator.update_workunit(wu); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%d %s] update_workunit() failed: %d; exiting\n", wu.id, wu.name, retval ); return retval; } return 0; }
// Pick a host to send a chunk of this file to. // The host must: // 1) be alive (recent RPC time) // 2) not have any chunks of this file // // We maintain a cache of such hosts // The policy is: // // - scan the cache, removing hosts that are no longer alive; // return if find a live host // - pick a random starting point in host ID space, // and enumerate 100 live hosts; wrap around if needed. // Return one and put the rest in cache // int VDA_FILE_AUX::choose_host() { int retval; DB_HOST host; return 467; // replenish cache if needed // if (!available_hosts.size()) { int nhosts_scanned = 0; int rand_id; for (int i=0; i<2; i++) { char buf[256]; if (i == 0) { retval = host.max_id(rand_id, ""); if (retval) { log_messages.printf(MSG_CRITICAL, "host.max_id() failed\n"); return 0; } rand_id = (int)(((double)id)*drand()); sprintf(buf, "where %s and id>=%d order by id limit 100", host_alive_clause(), rand_id ); } else { sprintf(buf, "where %s and id<%d order by id limit %d", host_alive_clause(), rand_id, 100-nhosts_scanned ); } while (1) { retval = host.enumerate(buf); if (retval == ERR_DB_NOT_FOUND) break; if (retval) { log_messages.printf(MSG_CRITICAL, "host enum failed\n"); return 0; } nhosts_scanned++; DB_VDA_CHUNK_HOST ch; char buf2[256]; int count; sprintf(buf2, "where vda_file_id=%d and host_id=%d", id, host.id); #if 0 retval = ch.count(count, buf2); if (retval) { log_messages.printf(MSG_CRITICAL, "ch.count failed\n"); return 0; } #else count = 0; #endif if (count == 0) { available_hosts.push_back(host.id); } if (nhosts_scanned == 100) break; } if (nhosts_scanned == 100) break; } } while (available_hosts.size()) { int hostid = available_hosts.back(); available_hosts.pop_back(); retval = host.lookup_id(hostid); if (retval || !alive(host)) { continue; } return hostid; } log_messages.printf(MSG_CRITICAL, "No hosts available\n"); return 0; }
// handle a workunit which has new results // int handle_wu( DB_VALIDATOR_ITEM_SET& validator, std::vector<VALIDATOR_ITEM>& items ) { int canonical_result_index = -1; bool update_result, retry; TRANSITION_TIME transition_time = NO_CHANGE; int retval = 0, x; DB_ID_TYPE canonicalid = 0; double credit = 0; unsigned int i; WORKUNIT& wu = items[0].wu; g_wup = &wu; if (wu.canonical_resultid) { log_messages.printf(MSG_NORMAL, "[WU#%lu %s] Already has canonical result %lu\n", wu.id, wu.name, wu.canonical_resultid ); ++log_messages; // Here if WU already has a canonical result. // Get unchecked results and see if they match the canonical result // for (i=0; i<items.size(); i++) { RESULT& result = items[i].res; if (result.id == wu.canonical_resultid) { canonical_result_index = i; } } if (canonical_result_index == -1) { log_messages.printf(MSG_CRITICAL, "[WU#%lu %s] Can't find canonical result %lu\n", wu.id, wu.name, wu.canonical_resultid ); return 0; } RESULT& canonical_result = items[canonical_result_index].res; // scan this WU's results, and check the unchecked ones // for (i=0; i<items.size(); i++) { RESULT& result = items[i].res; if (result.server_state != RESULT_SERVER_STATE_OVER) continue; if (result.outcome != RESULT_OUTCOME_SUCCESS) continue; switch (result.validate_state) { case VALIDATE_STATE_INIT: case VALIDATE_STATE_INCONCLUSIVE: break; default: continue; } log_messages.printf(MSG_NORMAL, "[WU#%lu] handle_wu(): testing result %lu\n", wu.id, result.id ); check_pair(result, canonical_result, retry); if (retry) { // this usually means an NFS mount has failed; // arrange to try again later. // transition_time = DELAYED; goto leave; } update_result = false; if (result.outcome == RESULT_OUTCOME_VALIDATE_ERROR) { update_result = true; } // this might be last result, so let transitioner // trigger file delete etc. if needed // transition_time = IMMEDIATE; DB_HOST host; retval = host.lookup_id(result.hostid); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%lu] lookup of host %lu failed: %s\n", result.id, result.hostid, boincerror(retval) ); continue; } HOST host_initial = host; bool update_hav = false; DB_HOST_APP_VERSION hav; retval = hav_lookup(hav, result.hostid, generalized_app_version_id(result.app_version_id, result.appid) ); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%lu %s] hav_lookup returned %d\n", result.id, result.name, retval ); hav.host_id = 0; } DB_HOST_APP_VERSION hav_orig = hav; vector<DB_HOST_APP_VERSION> havv; havv.push_back(hav); vector<RESULT> rv; switch (result.validate_state) { case VALIDATE_STATE_VALID: update_result = true; update_hav = true; log_messages.printf(MSG_NORMAL, "[RESULT#%lu %s] pair_check() matched: setting result to valid\n", result.id, result.name ); retval = is_valid(host, result, wu, havv[0]); if (retval) { log_messages.printf(MSG_NORMAL, "[RESULT#%lu %s] is_valid() error: %s\n", result.id, result.name, boincerror(retval) ); } // do credit computation, but grant credit of canonical result // rv.push_back(result); assign_credit_set( wu, rv, app, app_versions, havv, max_granted_credit, credit ); if (!no_credit) { result.granted_credit = canonical_result.granted_credit; grant_credit(host, result.sent_time, result.granted_credit); if (config.credit_by_app) { grant_credit_by_app(result, result.granted_credit); } } break; case VALIDATE_STATE_INVALID: update_result = true; update_hav = true; log_messages.printf(MSG_NORMAL, "[RESULT#%lu %s] pair_check() didn't match: setting result to invalid\n", result.id, result.name ); is_invalid(havv[0]); } if (hav.host_id && update_hav) { if (dry_run) { log_messages.printf(MSG_NORMAL, "DB not updated (dry run)\n"); } else { log_messages.printf(MSG_NORMAL, "[HOST#%lu AV#%lu] [outlier=%d] Updating HAV in DB. pfc.n=%f->%f\n", havv[0].host_id, havv[0].app_version_id, result.runtime_outlier, hav_orig.pfc.n, havv[0].pfc.n ); retval=havv[0].update_validator(hav_orig); if (retval) { log_messages.printf(MSG_CRITICAL, "[HOST#%lu AV%lu] hav.update_validator() failed: %s\n", hav.host_id, hav.app_version_id, boincerror(retval) ); } } } host.update_diff_validator(host_initial); if (update_result) { log_messages.printf(MSG_NORMAL, "[RESULT#%lu %s] granted_credit %f\n", result.id, result.name, result.granted_credit ); if (dry_run) { log_messages.printf(MSG_NORMAL, "DB not updated (dry run)\n"); } else { retval = validator.update_result(result); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%lu %s] Can't update result: %s\n", result.id, result.name, boincerror(retval) ); } } } } } else { // Here if WU doesn't have a canonical result yet. // Try to get one vector<RESULT> viable_results; vector<DB_HOST_APP_VERSION> host_app_versions, host_app_versions_orig; log_messages.printf(MSG_NORMAL, "[WU#%lu %s] handle_wu(): No canonical result yet\n", wu.id, wu.name ); ++log_messages; // make a vector of the "viable" (i.e. possibly canonical) results, // and a parallel vector of host_app_versions // for (i=0; i<items.size(); i++) { RESULT& result = items[i].res; if (result.server_state != RESULT_SERVER_STATE_OVER) continue; if (result.outcome != RESULT_OUTCOME_SUCCESS) continue; if (result.validate_state == VALIDATE_STATE_INVALID) continue; viable_results.push_back(result); DB_HOST_APP_VERSION hav; retval = hav_lookup(hav, result.hostid, generalized_app_version_id(result.app_version_id, result.appid) ); if (retval) { hav.host_id=0; // flag that it's missing } host_app_versions.push_back(hav); host_app_versions_orig.push_back(hav); } log_messages.printf(MSG_DEBUG, "[WU#%lu %s] Found %d viable results\n", wu.id, wu.name, (int)viable_results.size() ); if (viable_results.size() >= (unsigned int)wu.min_quorum) { log_messages.printf(MSG_DEBUG, "[WU#%lu %s] Enough for quorum, checking set.\n", wu.id, wu.name ); double dummy; retval = check_set(viable_results, wu, canonicalid, dummy, retry); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%lu %s] check_set() error: %s\n", wu.id, wu.name, boincerror(retval) ); return retval; } if (retry) transition_time = DELAYED; // if we found a canonical instance, decide on credit // if (canonicalid) { // always do the credit calculation, to update statistics, // even if we're granting credit a different way // retval = assign_credit_set( wu, viable_results, app, app_versions, host_app_versions, max_granted_credit, credit ); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%lu %s] assign_credit_set(): %s\n", wu.id, wu.name, boincerror(retval) ); transition_time = DELAYED; goto leave; } if (credit_from_wu) { retval = get_credit_from_wu(wu, viable_results, credit); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%lu %s] get_credit_from_wu(): credit not specified in WU\n", wu.id, wu.name ); credit = 0; } } else if (credit_from_runtime) { credit = 0; for (i=0; i<viable_results.size(); i++) { RESULT& result = viable_results[i]; if (result.id == canonicalid) { DB_HOST host; retval = host.lookup_id(result.hostid); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%lu %s] host %lu lookup failed\n", wu.id, wu.name, result.hostid ); break; } double runtime = result.elapsed_time; if (runtime <=0 || runtime > max_runtime) { runtime = max_runtime; } credit = result.flops_estimate * runtime * COBBLESTONE_SCALE; log_messages.printf(MSG_NORMAL, "[WU#%lu][RESULT#%lu] credit_from_runtime %.2f = %.0fs * %.2fGFLOPS\n", wu.id, result.id, credit, runtime, result.flops_estimate/1e9 ); break; } } } else if (no_credit) { credit = 0; } if (max_granted_credit && credit>max_granted_credit) { credit = max_granted_credit; } } // scan the viable results. // update as needed, // and count the # of results that are still viable // (some may now have outcome VALIDATE_ERROR, // or validate_state INVALID) // int n_viable_results = 0; for (i=0; i<viable_results.size(); i++) { RESULT& result = viable_results[i]; DB_HOST_APP_VERSION& hav = host_app_versions[i]; DB_HOST_APP_VERSION& hav_orig = host_app_versions_orig[i]; update_result = false; bool update_host = false; if (result.outcome != RESULT_OUTCOME_SUCCESS || result.validate_state == VALIDATE_STATE_INVALID ) { transition_time = IMMEDIATE; update_result = true; } else { n_viable_results++; } DB_HOST host; HOST host_initial; switch (result.validate_state) { case VALIDATE_STATE_VALID: case VALIDATE_STATE_INVALID: retval = host.lookup_id(result.hostid); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%lu] lookup of host %lu: %s\n", result.id, result.hostid, boincerror(retval) ); continue; } host_initial = host; } switch (result.validate_state) { case VALIDATE_STATE_VALID: update_result = true; update_host = true; retval = is_valid(host, result, wu, host_app_versions[i]); if (retval) { log_messages.printf(MSG_DEBUG, "[RESULT#%lu %s] is_valid() failed: %s\n", result.id, result.name, boincerror(retval) ); } if (!no_credit) { result.granted_credit = credit; grant_credit(host, result.sent_time, credit); log_messages.printf(MSG_NORMAL, "[RESULT#%lu %s] Valid; granted %f credit [HOST#%lu]\n", result.id, result.name, result.granted_credit, result.hostid ); if (config.credit_by_app) { grant_credit_by_app(result, credit); } } break; case VALIDATE_STATE_INVALID: update_result = true; update_host = true; log_messages.printf(MSG_NORMAL, "[RESULT#%lu %s] Invalid [HOST#%lu]\n", result.id, result.name, result.hostid ); is_invalid(host_app_versions[i]); break; case VALIDATE_STATE_INIT: log_messages.printf(MSG_NORMAL, "[RESULT#%lu %s] Inconclusive [HOST#%lu]\n", result.id, result.name, result.hostid ); result.validate_state = VALIDATE_STATE_INCONCLUSIVE; update_result = true; break; } if (dry_run) { log_messages.printf(MSG_NORMAL, "DB not updated (dry run)\n"); } else { if (hav.host_id) { log_messages.printf(MSG_NORMAL, "[HOST#%lu AV#%lu] [outlier=%d] Updating HAV in DB. pfc.n=%f->%f\n", hav.host_id, hav.app_version_id, result.runtime_outlier, hav_orig.pfc.n, hav.pfc.n ); retval = hav.update_validator(hav_orig); if (retval) { log_messages.printf(MSG_CRITICAL, "[HOST#%lu AV%lu] hav.update_validator() failed: %s\n", hav.host_id, hav.app_version_id, boincerror(retval) ); } } if (update_host) { retval = host.update_diff_validator(host_initial); if (retval) { log_messages.printf(MSG_CRITICAL, "[HOST#%lu] host.update_diff_validator() failed: %s\n", host.id, boincerror(retval) ); } } if (update_result) { retval = validator.update_result(result); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%lu %s] result.update() failed: %s\n", result.id, result.name, boincerror(retval) ); } } } } if (canonicalid) { // if we found a canonical result, // trigger the assimilator, but do NOT trigger // the transitioner - doing so creates a race condition // transition_time = NEVER; log_messages.printf(MSG_DEBUG, "[WU#%lu %s] Found a canonical result: id=%lu\n", wu.id, wu.name, canonicalid ); wu.canonical_resultid = canonicalid; wu.canonical_credit = credit; wu.assimilate_state = ASSIMILATE_READY; // don't need to send any more results // for (i=0; i<items.size(); i++) { RESULT& result = items[i].res; if (result.server_state != RESULT_SERVER_STATE_UNSENT) { continue; } result.server_state = RESULT_SERVER_STATE_OVER; result.outcome = RESULT_OUTCOME_DIDNT_NEED; if (dry_run) { log_messages.printf(MSG_NORMAL, "DB not updated (dry run)\n"); } else { retval = validator.update_result(result); if (retval) { log_messages.printf(MSG_CRITICAL, "[RESULT#%lu %s] result.update() failed: %s\n", result.id, result.name, boincerror(retval) ); } } } } else { // here if no consensus. // check if #viable results is too large // if (n_viable_results > wu.max_success_results) { wu.error_mask |= WU_ERROR_TOO_MANY_SUCCESS_RESULTS; transition_time = IMMEDIATE; } // if #viable results >= target_nresults, // we need more results, so bump target_nresults // NOTE: n_viable_results should never be > target_nresults, // but accommodate that if it should happen // if (n_viable_results >= wu.target_nresults) { wu.target_nresults = n_viable_results+1; transition_time = IMMEDIATE; } } } } leave: --log_messages; switch (transition_time) { case IMMEDIATE: wu.transition_time = time(0); break; case DELAYED: x = time(0) + 6*3600; if (x < wu.transition_time) wu.transition_time = x; break; case NEVER: wu.transition_time = INT_MAX; break; case NO_CHANGE: break; } wu.need_validate = 0; if (dry_run) { log_messages.printf(MSG_NORMAL, "DB not updated (dry run)\n"); } else { retval = validator.update_workunit(wu); if (retval) { log_messages.printf(MSG_CRITICAL, "[WU#%lu %s] update_workunit() failed: %s\n", wu.id, wu.name, boincerror(retval) ); return retval; } } return 0; }