int handle_status(const char* name) { DB_VDA_FILE dvf; char buf[1024]; sprintf(buf, "where file_name='%s'", name); int retval = dvf.lookup(buf); if (retval) return retval; VDA_FILE_AUX vf = dvf; sprintf(buf, "%s/boinc_meta.txt", vf.dir); retval = vf.policy.parse(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't parse policy file %s\n", buf); return retval; } retval = vf.get_state(); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't get file state: %d\n", retval); return retval; } printf("status for file %s:\n", vf.file_name); vf.meta_chunk->recovery_plan(); vf.meta_chunk->compute_min_failures(); vf.meta_chunk->print_status(0); printf("fault tolerance level: %d\n", vf.meta_chunk->min_failures-1); if (vf.retrieving) { if (vf.retrieved) { printf("Retrieving: completed\n"); } else { printf("Retrieving: in progress\n"); } } return 0; }
// if project is using more than its share of disk space, // remove some chunks and mark vda_files for update // static int enforce_quota(CHUNK_LIST& chunks) { if (!g_request->host.d_boinc_max) return 0; double x = g_request->host.d_boinc_used_project; if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] share: %f used: %f\n", g_request->host.d_boinc_max, x ); } CHUNK_LIST::iterator it = chunks.begin(); while (x > g_request->host.d_boinc_max && it != chunks.end()) { DB_VDA_CHUNK_HOST& ch = it->second; if (!ch.found) continue; FILE_INFO fi; strcpy(fi.name, ch.physical_file_name); if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] deleting: %s\n", ch.physical_file_name ); } DB_VDA_FILE vf; vf.lookup_id(ch.vda_file_id); x -= vf.chunk_size; g_reply->file_deletes.push_back(fi); it++; } return 0; }
// this host is declared dead; deal with the loss of data // int handle_host(DB_HOST& h) { DB_VDA_CHUNK_HOST ch; char buf[256]; int retval; log_messages.printf(MSG_NORMAL, "processing dead host %d\n", h.id); sprintf(buf, "where host_id=%d", h.id); while (1) { retval = ch.enumerate(buf); if (retval == ERR_DB_NOT_FOUND) break; if (retval) return retval; log_messages.printf(MSG_NORMAL, " updating file%d\n", ch.vda_file_id); DB_VDA_FILE vf; retval = vf.lookup_id(ch.vda_file_id); if (retval) { log_messages.printf(MSG_CRITICAL, " file lookup failed%d\n", ch.vda_file_id ); return retval; } retval = vf.update_field("need_update=1"); if (retval) { log_messages.printf(MSG_CRITICAL, " file update failed%d\n", ch.vda_file_id ); return retval; } } return 0; }
int handle_remove(const char* name) { DB_VDA_FILE vf; char buf[1024]; sprintf(buf, "where file_name='%s'", name); int retval = vf.lookup(buf); if (retval) return retval; // delete DB records // DB_VDA_CHUNK_HOST ch; sprintf(buf, "vda_file_id=%d", vf.id); ch.delete_from_db_multi(buf); vf.delete_from_db(); // remove symlink from download hier // dir_hier_path(name, config.download_dir, config.uldl_dir_fanout, buf); unlink(buf); // remove encoded data and directories // retval = chdir(vf.dir); if (retval) perror("chdir"); retval = system("/bin/rm -r [0-9]* Coding data.vda"); if (retval) perror("system"); return 0; }
// handle files // bool scan_files() { DB_VDA_FILE vf; bool found = false; int retval; while (1) { retval = vf.enumerate("where need_update<>0"); if (retval == ERR_DB_NOT_FOUND) break; if (retval) { log_messages.printf(MSG_CRITICAL, "VDA_FILE enumerate failed\n"); exit(1); } VDA_FILE_AUX vfa(vf); found = true; retval = handle_file(vfa, vf); if (retval) { log_messages.printf( MSG_CRITICAL, "handle_file() failed: %d\n", retval ); exit(1); } else { retval = vf.update_field("need_update=0"); if (retval) { log_messages.printf( MSG_CRITICAL, "update_field() failed: %d\n", retval ); exit(1); } } } return found; }
int handle_update(const char* name) { DB_VDA_FILE dvf; char buf[1024]; sprintf(buf, "where file_name='%s'", name); int retval = dvf.lookup(buf); if (retval) return retval; return dvf.update_field("need_update=1"); }
int handle_status(const char* name) { DB_VDA_FILE vf; char buf[1024]; sprintf(buf, "where name='%s'", name); int retval = vf.lookup(buf); if (retval) return retval; return 0; }
// Process a present file; possibilities: // - a download finished // - this host hasn't communicated in a while, and we deleted the // VDA_CHUNK_HOST record // So: // - create a vda_chunk_host record if needed // - set present_on_host flag in vda_chunk_host // - mark our in-memory vda_chunk_host record as "found" // - mark vda_file for update // static void process_present_file(FILE_INFO& fi, CHUNK_LIST& chunks) { char fname[256], chunk_name[256], buf[256]; int hostid, retval; retval = parse_physical_filename(fi.name, hostid, chunk_name, fname); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't parse VDA filename %s\n", fi.name ); return; } DB_VDA_FILE vf; sprintf(buf, "where file_name='%s'", fname); retval = vf.lookup(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "No VDA file %s\n", fname); return; } if (fi.nbytes != vf.chunk_size) { log_messages.printf(MSG_CRITICAL, "wrong chunk size: %.0f != %.0f\n", fi.nbytes, vf.chunk_size ); return; } CHUNK_LIST::iterator cli = chunks.find(string(fi.name)); if (cli == chunks.end()) { // don't have a record of this chunk on this host; make one // DB_VDA_CHUNK_HOST ch; ch.create_time = dtime(); ch.vda_file_id = vf.id; ch.host_id = g_reply->host.id; strcpy(ch.physical_file_name, fi.name); ch.present_on_host = true; ch.transfer_in_progress = false; ch.transfer_wait = false; ch.transfer_request_time = 0; ch.transfer_send_time = 0; retval = ch.insert(); if (retval) { log_messages.printf(MSG_CRITICAL, "ch.insert() failed\n"); return; } } else { // update the existing record // DB_VDA_CHUNK_HOST* chp = &(cli->second); chp->transfer_in_progress = false; chp->transfer_wait = false; chp->present_on_host = true; chp->update(); } mark_for_update(vf.id); }
int handle_retrieve(const char* name) { DB_VDA_FILE vf; char buf[1024]; sprintf(buf, "where name='%s'", name); int retval = vf.lookup(buf); if (retval) return retval; retval = vf.update_field("retrieving=1"); return retval; }
// process a completed upload: // if vda_chunk_host found // verify md5 of upload // move it from upload dir to vda_file dir // mark vda_file for update // clear transfer_in_progress flag in vda_chunk_host // else // delete from upload dir // static int process_completed_upload(char* chunk_name, CHUNK_LIST& chunks) { char path[1024], client_filename[1024], dir[1024]; int retval; physical_file_name(g_reply->host.id, chunk_name, client_filename); dir_hier_path( client_filename, config.upload_dir, config.uldl_dir_fanout, dir, false ); sprintf(path, "%s/%s", dir, client_filename); CHUNK_LIST::iterator i2 = chunks.find(string(chunk_name)); if (i2 == chunks.end()) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] chunk_host not found for %s\n", chunk_name ); } boinc_delete_file(path); } else { char client_md5[64], server_md5[64]; char chunk_dir[1024]; DB_VDA_CHUNK_HOST& ch = i2->second; DB_VDA_FILE vf; double size; retval = vf.lookup_id(ch.vda_file_id); get_chunk_dir(vf, chunk_name, chunk_dir); retval = get_chunk_md5(chunk_dir, server_md5); if (retval) return retval; retval = md5_file(path, client_md5, size); if (retval) return retval; if (strcmp(client_md5, server_md5)) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] MD5 mismatch %s %s\n", client_md5, server_md5 ); } boinc_delete_file(path); } else { retval = vf.update_field("need_update=1"); if (retval) return retval; retval = ch.update_field("transfer_in_progress=0"); if (retval) return retval; } } return 0; }
int handle_add(const char* path) { char dir[MAXPATHLEN], filename[256], buf[1024]; DB_VDA_FILE vf; POLICY policy; double size; int retval; retval = file_size(path, size); if (retval) { printf("no file %s\n", path); return -1; } safe_strcpy(dir, path); char* p = strrchr(dir, '/'); *p = 0; safe_strcpy(filename, p+1); // make sure there's a valid policy file in the dir // sprintf(buf, "%s/boinc_meta.txt", dir); retval = policy.parse(buf); if (retval) { printf("Can't parse policy file.\n"); return -1; } // add a DB record and mark it for update // vf.create_time = dtime(); safe_strcpy(vf.dir, dir); safe_strcpy(vf.file_name, filename); vf.size = size; vf.chunk_size = 0; // don't know this yet; set by vdad vf.need_update = 1; vf.initialized = 0; vf.retrieving = 0; vf.retrieved = 0; retval = vf.insert(); if (retval) { printf("Can't insert DB record\n"); return -1; } return 0; }
int handle_status(const char* name) { DB_VDA_FILE dvf; char buf[1024]; sprintf(buf, "where file_name='%s'", name); int retval = dvf.lookup(buf); if (retval) return retval; VDA_FILE_AUX vf = dvf; sprintf(buf, "%s/boinc_meta.txt", vf.dir); retval = vf.policy.parse(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't parse policy file %s\n", buf); return retval; } retval = vf.get_state(); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't get file state: %d\n", retval); return retval; } printf("status for file %s:", vf.file_name); vf.meta_chunk->print_status(0); return 0; }
int handle_file(VDA_FILE_AUX& vf, DB_VDA_FILE& dvf) { int retval; char buf[1024]; log_messages.printf(MSG_NORMAL, "processing file %s\n", vf.file_name); // read the policy file // sprintf(buf, "%s/boinc_meta.txt", vf.dir); retval = vf.policy.parse(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't parse policy file %s\n", buf); return retval; } if (vf.initialized) { retval = vf.get_state(); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.get_state failed %d\n", retval); return retval; } } else { retval = vf.init(); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.init failed %d\n", retval); return retval; } sprintf(buf, "initialized=1, chunk_size=%.0f", vf.policy.chunk_size()); dvf.update_field(buf); } retval = vf.meta_chunk->recovery_plan(); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.recovery_plan failed %d\n", retval); return retval; } retval = vf.meta_chunk->recovery_action(dtime()); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.recovery_action failed %d\n", retval); return retval; } return 0; }
// issue upload and download commands // static int issue_transfer_commands(CHUNK_LIST& chunks) { char xml_buf[8192], file_name[1024]; int retval; char url[1024]; CHUNK_LIST::iterator it; for (it = chunks.begin(); it != chunks.end(); it++) { vector<const char*> urls; DB_VDA_CHUNK_HOST& ch = it->second; if (!ch.transfer_in_progress) continue; if (!ch.transfer_wait) continue; DB_VDA_FILE vf; retval = vf.lookup_id(ch.vda_file_id); if (retval) return retval; if (ch.present_on_host) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] sending upload command: %s\n", ch.name ); } // upload // sprintf(file_name, "%d_%s__%s", g_reply->host.id, ch.name, vf.name); urls.push_back(config.upload_url); R_RSA_PRIVATE_KEY key; retval = get_file_xml( file_name, urls, ch.size, dtime() + VDA_HOST_TIMEOUT, false, key, xml_buf ); } else { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] sending download command: %s\n", ch.name ); } // download // char md5[64], chunk_dir[1024]; sprintf(file_name, "%s__%s", ch.name, vf.name); get_chunk_url(vf, ch.name, url); urls.push_back(url); get_chunk_dir(vf, ch.name, chunk_dir); retval = get_chunk_md5(chunk_dir, md5); if (retval) return retval; retval = put_file_xml( file_name, urls, md5, ch.size, dtime() + VDA_HOST_TIMEOUT, xml_buf ); } g_reply->file_transfer_requests.push_back(string(xml_buf)); } return 0; }
// mark a vda_file for update by vdad // static int mark_for_update(int vda_file_id) { DB_VDA_FILE f; f.id = vda_file_id; return f.update_field("need_update=1"); }
// issue upload and download commands // static int issue_transfer_commands(CHUNK_LIST& chunks) { char xml_buf[8192], chunk_name[256], file_name[1024]; int retval; char url[1024], buf[1024]; CHUNK_LIST::iterator it; for (it = chunks.begin(); it != chunks.end(); it++) { vector<const char*> urls; DB_VDA_CHUNK_HOST& ch = it->second; if (!ch.transfer_in_progress) continue; if (!ch.transfer_wait) continue; DB_VDA_FILE vf; retval = vf.lookup_id(ch.vda_file_id); if (retval) return retval; if (ch.present_on_host) { // upload // sprintf(buf, "upload_%s", ch.physical_file_name); if (result_already_on_host(buf)) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] upload of %s already in progress\n", ch.physical_file_name ); } continue; } if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] sending command to upload %s\n", ch.physical_file_name ); } urls.push_back(config.upload_url); R_RSA_PRIVATE_KEY key; retval = get_file_xml( ch.physical_file_name, urls, vf.chunk_size, dtime() + VDA_HOST_TIMEOUT, false, key, xml_buf ); } else { // download // sprintf(buf, "download_%s", ch.physical_file_name); if (result_already_on_host(buf)) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] download of %s already in progress\n", ch.physical_file_name ); } continue; } char md5[64], chunk_dir[1024]; int hostid; if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] sending command to download %s\n", ch.physical_file_name ); } parse_physical_filename( ch.physical_file_name, hostid, chunk_name, file_name ); get_chunk_url(vf, chunk_name, url); urls.push_back(url); get_chunk_dir(vf, chunk_name, chunk_dir); retval = get_chunk_md5(chunk_dir, md5); if (retval) return retval; retval = put_file_xml( ch.physical_file_name, urls, md5, vf.chunk_size, dtime() + VDA_HOST_TIMEOUT, xml_buf ); } g_reply->file_transfer_requests.push_back(string(xml_buf)); } return 0; }
// Process a file that's present on client. // A VDA_CHUNK_HOST record may not be in the DB, // e.g. because this host hasn't communicated in a while // and we deleted the VDA_CHUNK_HOST record // So: // - create a vda_chunk_host record if needed // - set present_on_host flag in vda_chunk_host // - mark our in-memory vda_chunk_host record as "found" // - mark vda_file for update // static void process_chunk_present_on_client(FILE_INFO& fi, CHUNK_LIST& chunks) { char fname[256], chunk_name[256], buf[1024]; int hostid, retval; retval = parse_physical_filename(fi.name, hostid, chunk_name, fname); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't parse VDA filename %s\n", fi.name ); return; } DB_VDA_FILE vf; sprintf(buf, "where file_name='%s'", fname); retval = vf.lookup(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "No VDA file for %s, deleting\n", fi.name ); delete_file_xml(fi.name, buf); g_reply->file_transfer_requests.push_back(string(buf)); return; } if (fi.nbytes != vf.chunk_size) { log_messages.printf(MSG_CRITICAL, "wrong chunk size for %s: %.0f != %.0f, deleting\n", fi.name, fi.nbytes, vf.chunk_size ); delete_file_xml(fi.name, buf); g_reply->file_transfer_requests.push_back(string(buf)); return; } CHUNK_LIST::iterator cli = chunks.find(string(fi.name)); if (cli == chunks.end()) { // we don't have a record of this chunk on this host; make one // DB_VDA_CHUNK_HOST ch; ch.create_time = dtime(); ch.vda_file_id = vf.id; ch.host_id = g_reply->host.id; strcpy(ch.physical_file_name, fi.name); ch.present_on_host = true; ch.transfer_in_progress = false; ch.transfer_wait = false; ch.transfer_request_time = 0; ch.transfer_send_time = 0; retval = ch.insert(); if (retval) { log_messages.printf(MSG_CRITICAL, "ch.insert() failed\n"); return; } mark_for_update(vf.id); } else { // we already have a DB record. // If needed, update it and mark file for update // DB_VDA_CHUNK_HOST* chp = &(cli->second); chp->found = true; // if file wasn't previously on host, update the main file // if (!chp->present_on_host) { mark_for_update(vf.id); chp->transfer_in_progress = false; chp->transfer_wait = false; chp->present_on_host = true; sprintf(buf, "host_id=%d and physical_file_name='%s'", chp->host_id, chp->physical_file_name ); chp->update_fields_noid( "transfer_in_progress=0, transfer_wait=0, present_on_host=1", buf ); } } }
// process a completed upload: // if vda_chunk_host found // verify md5 of upload // move it from upload dir to vda_file dir // mark vda_file for update // clear transfer_in_progress flag in vda_chunk_host // else // delete from upload dir // static int process_completed_upload(char* phys_filename, CHUNK_LIST& chunks) { char path[1024], buf[256]; char chunk_name[1024], file_name[1024]; int retval, hostid; retval = parse_physical_filename( phys_filename, hostid, chunk_name, file_name ); if (retval) { log_messages.printf(MSG_NORMAL, "[vda] bad upload filename: %s\n", phys_filename ); return retval; } dir_hier_path( phys_filename, config.upload_dir, config.uldl_dir_fanout, path, false ); // if we don't have a DB record for this chunk, delete the file // TODO: maybe we should create a DB record instead // CHUNK_LIST::iterator i2 = chunks.find(string(phys_filename)); if (i2 == chunks.end()) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] chunk_host not found for %s\n", chunk_name ); } boinc_delete_file(path); return 0; } char client_md5[64], server_md5[64]; char chunk_dir[1024]; DB_VDA_CHUNK_HOST& ch = i2->second; DB_VDA_FILE vf; double size; retval = vf.lookup_id(ch.vda_file_id); get_chunk_dir(vf, chunk_name, chunk_dir); // if file already exists on server, delete the upload file // Otherwise move the file from upload dir to data dir // sprintf(buf, "%s/data.vda", chunk_dir); if (boinc_file_exists(buf)) { boinc_delete_file(path); } else { retval = get_chunk_md5(chunk_dir, server_md5); if (retval) return retval; retval = md5_file(path, client_md5, size); if (retval) return retval; if (strcmp(client_md5, server_md5)) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] MD5 mismatch %s %s\n", client_md5, server_md5 ); } boinc_delete_file(path); } else { char dst_path[1024]; sprintf(buf, "%s/data.vda", chunk_dir); ssize_t n = readlink(buf, dst_path, sizeof(dst_path)-1); if (n < 0) { log_messages.printf(MSG_CRITICAL, "[vda] readlink() failed\n" ); } else { dst_path[n] = 0; sprintf(buf, "mv %s %s; chmod g+rw %s", path, dst_path, dst_path); retval = system(buf); if (retval == -1 || WEXITSTATUS(retval)) { log_messages.printf(MSG_NORMAL, "[vda] command failed: %s\n", buf ); } else { log_messages.printf(MSG_NORMAL, "[vda] file move succeeded: %s\n", buf ); } } retval = vf.update_field("need_update=1"); } } sprintf(buf, "host_id=%d and physical_file_name='%s'", ch.host_id, ch.physical_file_name ); ch.transfer_in_progress = 0; retval = ch.update_fields_noid("transfer_in_progress=0", buf); if (retval) return retval; return 0; }