int main(int argc, char** argv) {
char* src_dir, *dst_dir;
int fanout=0;
std::string filename;
char dst_path[MAXPATHLEN], src_path[MAXPATHLEN];
int retval;
if ( (argc == 1) || !strcmp(argv[1], "-h") || !strcmp(argv[1],"--help") || (argc != 4) ) {
fprintf(stderr, "%s", usage);
exit(1);
}
src_dir = argv[1];
dst_dir = argv[2];
fanout = atoi(argv[3]);
if (!fanout) {
fprintf(stderr, "%s", usage);
exit(1);
}
DirScanner scanner(src_dir);
while (scanner.scan(filename)) {
retval = dir_hier_path(filename.c_str(), dst_dir, fanout, dst_path, true);
if (retval) {
fprintf(stderr, "dir_hier_path: %s\n", boincerror(retval));
exit(1);
}
sprintf(src_path, "%s/%s", src_dir, filename.c_str());
retval = rename(src_path, dst_path);
if (retval) {
perror("rename");
exit(1);
}
}
}
// get the path a WU's input file
//
int get_file_path(WORKUNIT& wu, char* path) {
char buf[256];
bool flag;
flag = parse_str(wu.xml_doc, "<name>", buf, sizeof(buf));
if (!flag) return ERR_XML_PARSE;
dir_hier_path(buf, config.download_dir, config.uldl_dir_fanout, path);
return 0;
}
// If using locality scheduling, there are probably many result that
// use same file, so decrement available space ONLY if the host
// doesn't yet have this file. Note: this gets the file size from the
// download dir.
//
// Return value 0 means that this routine was successful in adjusting
// the available disk space in the work request. Return value <0
// means that it was not successful, and that something went wrong.
// Return values >0 mean that the host does not contain the file, and
// that no previously assigned work includes the file, and so the disk
// space in the work request should be adjusted by the calling
// routine, in the same way as if there was no scheduling locality.
//
int decrement_disk_space_locality( WORKUNIT& wu) {
char filename[256], path[MAXPATHLEN];
int filesize;
struct stat buf;
// get filename from WU name
//
if (extract_filename(wu.name, filename)) {
log_messages.printf(MSG_CRITICAL,
"No filename found in [WU#%d %s]\n", wu.id, wu.name
);
return -1;
}
// when checking to see if the host has the file, we need to
// ignore the last WU included at the end of the reply, since it
// corresponds to the one that we are (possibly) going to send!
// So make a copy and pop the current WU off the end.
if (!host_has_file(filename, true))
return 1;
// If we are here, then the host ALREADY has the file, or its size
// has already been accounted for in a previous WU. In this case,
// don't count the file size again in computing the disk
// requirements of this request.
// Get path to file, and determine its size
dir_hier_path(
filename, config.download_dir, config.uldl_dir_fanout, path, false
);
if (stat(path, &buf)) {
log_messages.printf(MSG_CRITICAL,
"Unable to find file %s at path %s\n", filename, path
);
return -1;
}
filesize=buf.st_size;
if (filesize<wu.rsc_disk_bound) {
g_wreq->disk_available -= (wu.rsc_disk_bound-filesize);
if (config.debug_locality) {
log_messages.printf(MSG_NORMAL,
"[locality] [HOST#%d] reducing disk needed for WU by %d bytes (length of %s)\n",
g_reply->host.id, filesize, filename
);
}
return 0;
}
log_messages.printf(MSG_CRITICAL,
"File %s size %d bytes > wu.rsc_disk_bound for WU#%d (%s)\n",
path, filesize, wu.id, wu.name
);
return -1;
}
// Given a filename, find its full path in the upload directory hierarchy
// Return ERR_OPENDIR if dir isn't there (possibly recoverable error),
// ERR_NOT_FOUND if dir is there but not file
//
int get_file_path(
const char *filename, char* upload_dir, int fanout, char* path
) {
dir_hier_path(filename, upload_dir, fanout, path, true);
if (boinc_file_exists(path)) {
return 0;
}
char* p = strrchr(path, '/');
*p = 0;
if (boinc_file_exists(path)) {
return ERR_NOT_FOUND;
}
return ERR_OPENDIR;
}
int handle_remove(const char* name) {
DB_VDA_FILE vf;
char buf[1024];
sprintf(buf, "where name='%s'", name);
int retval = vf.lookup(buf);
if (retval) return retval;
// delete DB records
//
DB_VDA_CHUNK_HOST ch;
sprintf(buf, "vda_file_id=%d", vf.id);
ch.delete_from_db_multi(buf);
vf.delete_from_db();
dir_hier_path(name, config.download_dir, config.uldl_dir_fanout, buf);
unlink(buf);
retval = chdir(vf.dir);
if (retval) perror("chdir");
retval = system("/bin/rm -r [0-9]* Coding data.vda");
if (retval) perror("system");
return 0;
}
int main(int argc, char** argv) {
char path[256];
int retval;
if ( (argc == 1) || !strcmp(argv[1], "-h") || !strcmp(argv[1],"--help") || (argc != 2) ) {
printf (usage);
exit(1);
}
retval = config.parse_file();
if (retval) {
fprintf(stderr, "Can't parse config.xml: %s\n", boincerror(retval));
exit(1);
}
retval = dir_hier_path(
argv[1], config.download_dir, config.uldl_dir_fanout, path, true
);
if (retval) {
fprintf(stderr, "dir_hier_path(): %d\n", retval);
exit(1);
}
printf("%s\n", path);
}
// always returns HTML reply
//
int handle_get_file_size(char* file_name) {
struct stat sbuf;
char path[MAXPATHLEN], buf[256];
int retval, pid, fd;
// TODO: check to ensure path doesn't point somewhere bad
// Use 64-bit variant
//
retval = dir_hier_path(
file_name, config.upload_dir, config.uldl_dir_fanout, path
);
if (retval) {
log_messages.printf(MSG_CRITICAL,
"Failed to find/create directory for file '%s' in '%s'.\n",
file_name, config.upload_dir
);
return return_error(ERR_TRANSIENT, "can't open file");
}
// if the volume is full, report a transient error
// to prevent the client from starting a transfer
//
if (volume_full(config.upload_dir)) {
return return_error(ERR_TRANSIENT, "Server is out of disk space");
}
fd = open(path, O_WRONLY|O_APPEND);
if (fd<0 && ENOENT==errno) {
// file does not exist: return zero length
//
log_messages.printf(MSG_NORMAL,
"handle_get_file_size(): [%s] returning zero\n", file_name
);
return return_success("<file_size>0</file_size>");
}
if (fd<0) {
// can't get file descriptor: try again later
//
log_messages.printf(MSG_CRITICAL,
"handle_get_file_size(): cannot open [%s] %s\n",
file_name, strerror(errno)
);
return return_error(ERR_TRANSIENT, "can't open file");
}
if ((pid=mylockf(fd))) {
// file locked by another file_upload_handler: try again later
//
close(fd);
log_messages.printf(MSG_CRITICAL,
"handle_get_file_size(): [%s] returning error\n", file_name
);
return return_error(ERR_TRANSIENT,
"[%s] locked by file_upload_handler PID=%d", file_name, pid
);
}
// file exists, writable, not locked by anyone else, so return length.
//
retval = stat(path, &sbuf);
close(fd);
if (retval) {
// file DOES perhaps exist, but can't stat it: try again later
//
log_messages.printf(MSG_CRITICAL,
"handle_get_file_size(): [%s] returning error %s\n",
file_name, strerror(errno)
);
return return_error(ERR_TRANSIENT, "cannot stat file" );
}
log_messages.printf(MSG_NORMAL,
"handle_get_file_size(): [%s] returning %d\n",
file_name, (int)sbuf.st_size
);
sprintf(buf, "<file_size>%d</file_size>", (int)sbuf.st_size);
return return_success(buf);
}
// ALWAYS generates an HTML reply
//
int handle_file_upload(FILE* in, R_RSA_PUBLIC_KEY& key) {
char buf[256], path[MAXPATHLEN], signed_xml[1024];
char name[256], stemp[256];
double max_nbytes=-1;
char xml_signature[1024];
int retval;
double offset=0, nbytes = -1;
bool is_valid, btemp;
strcpy(name, "");
strcpy(xml_signature, "");
bool found_data = false;
while (fgets(buf, 256, in)) {
#if 1
log_messages.printf(MSG_NORMAL, "got:%s\n", buf);
#endif
if (match_tag(buf, "<file_info>")) continue;
if (match_tag(buf, "</file_info>")) continue;
if (match_tag(buf, "<signed_xml>")) continue;
if (match_tag(buf, "</signed_xml>")) continue;
if (parse_bool(buf, "generated_locally", btemp)) continue;
if (parse_bool(buf, "upload_when_present", btemp)) continue;
if (parse_str(buf, "<url>", stemp, sizeof(stemp))) continue;
if (parse_str(buf, "<md5_cksum>", stemp, sizeof(stemp))) continue;
if (match_tag(buf, "<xml_signature>")) {
copy_element_contents(
in, "</xml_signature>", xml_signature, sizeof(xml_signature)
);
continue;
}
if (parse_str(buf, "<name>", name, sizeof(name))) {
strcpy(this_filename, name);
continue;
}
if (parse_double(buf, "<max_nbytes>", max_nbytes)) continue;
if (parse_double(buf, "<offset>", offset)) continue;
if (parse_double(buf, "<nbytes>", nbytes)) continue;
if (match_tag(buf, "<data>")) {
found_data = true;
break;
}
log_messages.printf(MSG_CRITICAL, "unrecognized: %s", buf);
}
if (strlen(name) == 0) {
return return_error(ERR_PERMANENT, "Missing name");
}
if (!found_data) {
return return_error(ERR_PERMANENT, "Missing <data> tag");
}
if (!config.ignore_upload_certificates) {
if (strlen(xml_signature) == 0) {
return return_error(ERR_PERMANENT, "missing signature");
}
if (max_nbytes < 0) {
return return_error(ERR_PERMANENT, "missing max_nbytes");
}
sprintf(signed_xml,
"<name>%s</name><max_nbytes>%.0f</max_nbytes>",
name, max_nbytes
);
retval = check_string_signature(
signed_xml, xml_signature, key, is_valid
);
if (retval || !is_valid) {
log_messages.printf(MSG_CRITICAL,
"check_string_signature() [%s] [%s] retval %d, is_valid = %d\n",
signed_xml, xml_signature,
retval, is_valid
);
log_messages.printf(MSG_NORMAL,
"signed xml: %s\n", signed_xml
);
log_messages.printf(MSG_NORMAL,
"signature: %s\n", xml_signature
);
return return_error(ERR_PERMANENT, "invalid signature");
}
}
if (nbytes < 0) {
return return_error(ERR_PERMANENT, "nbytes missing or negative");
}
// enforce limits in signed XML
//
if (!config.ignore_upload_certificates) {
if (nbytes > max_nbytes) {
sprintf(buf,
"file size (%d KB) exceeds limit (%d KB)",
(int)(nbytes/1024), (int)(max_nbytes/1024)
);
copy_socket_to_null(in);
return return_error(ERR_PERMANENT, buf);
}
}
// make sure filename is legit
//
if (strstr(name, "..")) {
return return_error(ERR_PERMANENT,
"file_upload_handler: .. found in filename: %s",
name
);
}
if (strlen(name) == 0) {
return return_error(ERR_PERMANENT,
"file_upload_handler: no filename; nbytes %f", nbytes
);
}
retval = dir_hier_path(
name, config.upload_dir, config.uldl_dir_fanout,
path, true
);
if (retval) {
log_messages.printf(MSG_CRITICAL,
"Failed to find/create directory for file '%s' in '%s'\n",
name, config.upload_dir
);
return return_error(ERR_TRANSIENT, "can't open file");
}
log_messages.printf(MSG_NORMAL,
"Starting upload of %s from %s [offset=%.0f, nbytes=%.0f]\n",
name,
get_remote_addr(),
offset, nbytes
);
#ifndef _USING_FCGI_
fflush(stderr);
#endif
if (offset >= nbytes) {
log_messages.printf(MSG_CRITICAL,
"ERROR: offset >= nbytes!!\n"
);
return return_success(0);
}
retval = copy_socket_to_file(in, path, offset, nbytes);
log_messages.printf(MSG_NORMAL,
"Ended upload of %s from %s; retval %d\n",
name,
get_remote_addr(),
retval
);
#ifndef _USING_FCGI_
fflush(stderr);
#endif
return retval;
}
int SCHED_CONFIG::download_path(const char* filename, char* path) {
return dir_hier_path(filename, download_dir, uldl_dir_fanout, path, true);
}
void rename_wu_files() {
int i, retval;
char oldname[256],newname[1024];
unsigned long sz;
DB_WORKUNIT db_wu;
const char *name[1];
char *wudir="./wu_inbox";
xml_encoding encoding=(noencode?_binary:_x_setiathome);
FILE *tmpfile;
if (boinc_db.open(boinc_config.db_name,boinc_config.db_host,boinc_config.db_user,boinc_config.db_passwd)) {
boinc_db.print_error("boinc_db.open");
log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,"Fatal error in boinc_db.open\n");
exit(1);
}
for (i=0;i<NSTRIPS;i++) {
name[0]=wuheaders[i].name;
sprintf(oldname,"%s/%s",wudir,name[0]);
//sprintf(newname,"%s%s/%s",projectdir,WU_SUBDIR,name[0]);
retval = dir_hier_path(name[0],
boinc_config.download_dir,
boinc_config.uldl_dir_fanout,
newname,
true
);
if (retval) {
char buf[1024];
log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,"[%s] dir_hier_path() failed: %d\n", name[0], retval);
exit(1);
}
struct stat sbuf;
if (!stat(oldname,&sbuf) && (tmpfile=fopen(oldname,"a"))) {
std::string tmpstr=xml_encode_string(bin_data[i],encoding);
fprintf(tmpfile,"<data length=%ld encoding=\"%s\">",tmpstr.size(),
xml_encoding_names[encoding]);
fwrite(tmpstr.c_str(),tmpstr.size(),1,tmpfile);
fprintf(tmpfile,"</data>\n");
fprintf(tmpfile,"</workunit>\n");
sz=bin_data[i].size();
fclose(tmpfile);
} else {
log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,"Header file no longer exists! splitter start script may be failing\n ");
exit(1);
}
if (!nodb) {
if (!filecopy(oldname,newname)) {
db_wu.clear();
db_wu.opaque=wuheaders[i].id;
strncpy(db_wu.name,name[0],sizeof(db_wu.name)-2);
db_wu.appid=app.id;
//db_wu.rsc_fpops_est=2.79248e+13*6;
//db_wu.rsc_fpops_bound=4.46797e+14*6;
double beam=wuheaders[i].group_info->receiver_cfg->beam_width;
double ar=wuheaders[i].group_info->data_desc.true_angle_range;
double dur=(double)(wuheaders[i].group_info->data_desc.nsamples)/wuheaders[i].subband_desc.sample_rate;
double min_slew=wuheaders[i].group_info->analysis_cfg->pot_min_slew;
double max_slew=wuheaders[i].group_info->analysis_cfg->pot_max_slew;
if ( ar <= beam ) {
db_wu.rsc_fpops_est=9.193e+13;
} else if ( ar <= ( dur*min_slew ) ) {
db_wu.rsc_fpops_est=5.962e+13+2.296e+12/ar;
} else if ( ar <= ( dur*max_slew ) ) {
db_wu.rsc_fpops_est=5.749e+13+1.476e+13/ar;
} else {
db_wu.rsc_fpops_est=3.535e+13;
}
db_wu.rsc_fpops_est*=(0.333/wuheaders[i].group_info->analysis_cfg->chirp_resolution);
db_wu.rsc_fpops_bound=db_wu.rsc_fpops_est*10;
db_wu.rsc_memory_bound=33554432;
db_wu.rsc_disk_bound=33554432;
// Our minimum is a 40 MFLOP machine
db_wu.delay_bound=std::max(86400.0*7,db_wu.rsc_fpops_est/4e+7);
db_wu.min_quorum=sah_config.min_quorum;
db_wu.target_nresults=sah_config.target_nresults;
db_wu.max_error_results=sah_config.max_error_results;
db_wu.max_total_results=sah_config.max_total_results;
db_wu.max_success_results=sah_config.max_success_results;
strncpy(db_wu.app_name,appname,sizeof(db_wu.app_name)-2);
if (create_work(db_wu,
wu_template,
result_template_filename,
result_template_filepath,
name,
1,
boinc_config,
NULL
)
) {
log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,"create work failed\n");
exit(1);
}
//unlink(oldname); // we now *always* unlink
} else {
log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,"file copy failed\n");
exit(1);
}
unlink(oldname);
}
}
boinc_db.close();
}
// ALWAYS generates an HTML reply
//
int handle_file_upload(FILE* in, R_RSA_PUBLIC_KEY& key) {
char buf[256], path[512], temp[256];
FILE_INFO file_info;
int retval;
double nbytes=-1, offset=0;
bool is_valid;
while (fgets(buf, 256, in)) {
#if 0
log_messages.printf(MSG_NORMAL, "got:%s\n", buf);
#endif
if (match_tag(buf, "<file_info>")) {
retval = file_info.parse(in);
if (retval) {
return return_error(ERR_PERMANENT, "FILE_INFO::parse");
}
#if 0
log_messages.printf(MSG_NORMAL,
"file info:\n%s\n", file_info.signed_xml
);
#endif
if (!config.ignore_upload_certificates) {
if (!file_info.signed_xml || !file_info.xml_signature) {
log_messages.printf(MSG_CRITICAL,
"file info is missing signature\n"
);
return return_error(ERR_PERMANENT, "invalid signature");
} else {
retval = verify_string(
file_info.signed_xml, file_info.xml_signature, key, is_valid
);
if (retval || !is_valid) {
log_messages.printf(MSG_CRITICAL,
"verify_string() [%s] [%s] retval %d, is_valid = %d\n",
file_info.signed_xml, file_info.xml_signature,
retval, is_valid
);
log_messages.printf(MSG_NORMAL,
"signed xml: %s", file_info.signed_xml
);
log_messages.printf(MSG_NORMAL,
"signature: %s", file_info.xml_signature
);
return return_error(ERR_PERMANENT, "invalid signature");
}
}
}
continue;
}
if (parse_double(buf, "<offset>", offset)) continue;
if (parse_double(buf, "<nbytes>", nbytes)) continue;
if (parse_str(buf, "<md5_cksum>", temp, sizeof(temp))) continue;
if (match_tag(buf, "<data>")) {
if (nbytes < 0) {
return return_error(ERR_PERMANENT, "nbytes missing or negative");
}
// enforce limits in signed XML
//
if (!config.ignore_upload_certificates) {
if (nbytes > file_info.max_nbytes) {
sprintf(buf,
"file size (%d KB) exceeds limit (%d KB)",
(int)(nbytes/1024), (int)(file_info.max_nbytes/1024)
);
copy_socket_to_null(in);
return return_error(ERR_PERMANENT, buf);
}
}
// make sure filename is legit
//
if (strstr(file_info.name, "..")) {
return return_error(ERR_PERMANENT,
"file_upload_handler: .. found in filename: %s",
file_info.name
);
}
if (strlen(file_info.name) == 0) {
return return_error(ERR_PERMANENT,
"file_upload_handler: no filename; nbytes %f", nbytes
);
}
retval = dir_hier_path(
file_info.name, config.upload_dir, config.uldl_dir_fanout,
path, true
);
if (retval) {
log_messages.printf(MSG_CRITICAL,
"Failed to find/create directory for file '%s' in '%s'\n",
file_info.name, config.upload_dir
);
return return_error(ERR_TRANSIENT, "can't open file");
}
log_messages.printf(MSG_NORMAL,
"Starting upload of %s from %s [offset=%.0f, nbytes=%.0f]\n",
file_info.name,
get_remote_addr(),
offset, nbytes
);
#ifndef _USING_FCGI_
fflush(stderr);
#endif
if (offset >= nbytes) {
log_messages.printf(MSG_CRITICAL,
"ERROR: offset >= nbytes!!\n"
);
return return_success(0);
}
retval = copy_socket_to_file(in, path, offset, nbytes);
log_messages.printf(MSG_NORMAL,
"Ended upload of %s from %s; retval %d\n",
file_info.name,
get_remote_addr(),
retval
);
#ifndef _USING_FCGI_
fflush(stderr);
#endif
return retval;
}
log_messages.printf(MSG_CRITICAL,
"unrecognized: %s", buf
);
}
return return_error(ERR_PERMANENT, "Missing <data> tag");
}
