// handle files // bool scan_files() { DB_VDA_FILE vf; bool found = false; int retval; while (1) { retval = vf.enumerate("where need_update<>0"); if (retval == ERR_DB_NOT_FOUND) break; if (retval) { log_messages.printf(MSG_CRITICAL, "VDA_FILE enumerate failed\n"); exit(1); } VDA_FILE_AUX vfa(vf); found = true; retval = handle_file(vfa, vf); if (retval) { log_messages.printf( MSG_CRITICAL, "handle_file() failed: %d\n", retval ); exit(1); } else { retval = vf.update_field("need_update=0"); if (retval) { log_messages.printf( MSG_CRITICAL, "update_field() failed: %d\n", retval ); exit(1); } } } return found; }
// this host is declared dead; deal with the loss of data // int handle_host(DB_HOST& h) { DB_VDA_CHUNK_HOST ch; char buf[256]; int retval; log_messages.printf(MSG_NORMAL, "processing dead host %d\n", h.id); sprintf(buf, "where host_id=%d", h.id); while (1) { retval = ch.enumerate(buf); if (retval == ERR_DB_NOT_FOUND) break; if (retval) return retval; log_messages.printf(MSG_NORMAL, " updating file%d\n", ch.vda_file_id); DB_VDA_FILE vf; retval = vf.lookup_id(ch.vda_file_id); if (retval) { log_messages.printf(MSG_CRITICAL, " file lookup failed%d\n", ch.vda_file_id ); return retval; } retval = vf.update_field("need_update=1"); if (retval) { log_messages.printf(MSG_CRITICAL, " file update failed%d\n", ch.vda_file_id ); return retval; } } return 0; }
int handle_update(const char* name) { DB_VDA_FILE dvf; char buf[1024]; sprintf(buf, "where file_name='%s'", name); int retval = dvf.lookup(buf); if (retval) return retval; return dvf.update_field("need_update=1"); }
int handle_retrieve(const char* name) { DB_VDA_FILE vf; char buf[1024]; sprintf(buf, "where name='%s'", name); int retval = vf.lookup(buf); if (retval) return retval; retval = vf.update_field("retrieving=1"); return retval; }
// process a completed upload: // if vda_chunk_host found // verify md5 of upload // move it from upload dir to vda_file dir // mark vda_file for update // clear transfer_in_progress flag in vda_chunk_host // else // delete from upload dir // static int process_completed_upload(char* chunk_name, CHUNK_LIST& chunks) { char path[1024], client_filename[1024], dir[1024]; int retval; physical_file_name(g_reply->host.id, chunk_name, client_filename); dir_hier_path( client_filename, config.upload_dir, config.uldl_dir_fanout, dir, false ); sprintf(path, "%s/%s", dir, client_filename); CHUNK_LIST::iterator i2 = chunks.find(string(chunk_name)); if (i2 == chunks.end()) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] chunk_host not found for %s\n", chunk_name ); } boinc_delete_file(path); } else { char client_md5[64], server_md5[64]; char chunk_dir[1024]; DB_VDA_CHUNK_HOST& ch = i2->second; DB_VDA_FILE vf; double size; retval = vf.lookup_id(ch.vda_file_id); get_chunk_dir(vf, chunk_name, chunk_dir); retval = get_chunk_md5(chunk_dir, server_md5); if (retval) return retval; retval = md5_file(path, client_md5, size); if (retval) return retval; if (strcmp(client_md5, server_md5)) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] MD5 mismatch %s %s\n", client_md5, server_md5 ); } boinc_delete_file(path); } else { retval = vf.update_field("need_update=1"); if (retval) return retval; retval = ch.update_field("transfer_in_progress=0"); if (retval) return retval; } } return 0; }
int handle_file(VDA_FILE_AUX& vf, DB_VDA_FILE& dvf) { int retval; char buf[1024]; log_messages.printf(MSG_NORMAL, "processing file %s\n", vf.file_name); // read the policy file // sprintf(buf, "%s/boinc_meta.txt", vf.dir); retval = vf.policy.parse(buf); if (retval) { log_messages.printf(MSG_CRITICAL, "Can't parse policy file %s\n", buf); return retval; } if (vf.initialized) { retval = vf.get_state(); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.get_state failed %d\n", retval); return retval; } } else { retval = vf.init(); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.init failed %d\n", retval); return retval; } sprintf(buf, "initialized=1, chunk_size=%.0f", vf.policy.chunk_size()); dvf.update_field(buf); } retval = vf.meta_chunk->recovery_plan(); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.recovery_plan failed %d\n", retval); return retval; } retval = vf.meta_chunk->recovery_action(dtime()); if (retval) { log_messages.printf(MSG_CRITICAL, "vf.recovery_action failed %d\n", retval); return retval; } return 0; }
// mark a vda_file for update by vdad // static int mark_for_update(int vda_file_id) { DB_VDA_FILE f; f.id = vda_file_id; return f.update_field("need_update=1"); }
// process a completed upload: // if vda_chunk_host found // verify md5 of upload // move it from upload dir to vda_file dir // mark vda_file for update // clear transfer_in_progress flag in vda_chunk_host // else // delete from upload dir // static int process_completed_upload(char* phys_filename, CHUNK_LIST& chunks) { char path[1024], buf[256]; char chunk_name[1024], file_name[1024]; int retval, hostid; retval = parse_physical_filename( phys_filename, hostid, chunk_name, file_name ); if (retval) { log_messages.printf(MSG_NORMAL, "[vda] bad upload filename: %s\n", phys_filename ); return retval; } dir_hier_path( phys_filename, config.upload_dir, config.uldl_dir_fanout, path, false ); // if we don't have a DB record for this chunk, delete the file // TODO: maybe we should create a DB record instead // CHUNK_LIST::iterator i2 = chunks.find(string(phys_filename)); if (i2 == chunks.end()) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] chunk_host not found for %s\n", chunk_name ); } boinc_delete_file(path); return 0; } char client_md5[64], server_md5[64]; char chunk_dir[1024]; DB_VDA_CHUNK_HOST& ch = i2->second; DB_VDA_FILE vf; double size; retval = vf.lookup_id(ch.vda_file_id); get_chunk_dir(vf, chunk_name, chunk_dir); // if file already exists on server, delete the upload file // Otherwise move the file from upload dir to data dir // sprintf(buf, "%s/data.vda", chunk_dir); if (boinc_file_exists(buf)) { boinc_delete_file(path); } else { retval = get_chunk_md5(chunk_dir, server_md5); if (retval) return retval; retval = md5_file(path, client_md5, size); if (retval) return retval; if (strcmp(client_md5, server_md5)) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, "[vda] MD5 mismatch %s %s\n", client_md5, server_md5 ); } boinc_delete_file(path); } else { char dst_path[1024]; sprintf(buf, "%s/data.vda", chunk_dir); ssize_t n = readlink(buf, dst_path, sizeof(dst_path)-1); if (n < 0) { log_messages.printf(MSG_CRITICAL, "[vda] readlink() failed\n" ); } else { dst_path[n] = 0; sprintf(buf, "mv %s %s; chmod g+rw %s", path, dst_path, dst_path); retval = system(buf); if (retval == -1 || WEXITSTATUS(retval)) { log_messages.printf(MSG_NORMAL, "[vda] command failed: %s\n", buf ); } else { log_messages.printf(MSG_NORMAL, "[vda] file move succeeded: %s\n", buf ); } } retval = vf.update_field("need_update=1"); } } sprintf(buf, "host_id=%d and physical_file_name='%s'", ch.host_id, ch.physical_file_name ); ch.transfer_in_progress = 0; retval = ch.update_fields_noid("transfer_in_progress=0", buf); if (retval) return retval; return 0; }