static int write_token(void *link, void *buf, size_t size)
{
time_t stoptime = time(0) + 3600;
link_putfstring(link, "%zu\n", stoptime, size);
if(link_putlstring(link, buf, size, stoptime) == (int) size) {
return GLOBUS_SUCCESS;
} else {
return GLOBUS_GSS_ASSIST_TOKEN_EOF;
}
}
void work_queue_resources_send( struct link *master, struct work_queue_resources *r, time_t stoptime )
{
debug(D_WQ, "Sending resource description to master:");
work_queue_resource_send(master, &r->workers, "workers",stoptime);
work_queue_resource_send(master, &r->disk, "disk", stoptime);
work_queue_resource_send(master, &r->memory, "memory", stoptime);
work_queue_resource_send(master, &r->gpus, "gpus", stoptime);
work_queue_resource_send(master, &r->cores, "cores", stoptime);
/* send the tag last, the master knows when the resource update is complete */
link_putfstring(master, "resource tag %"PRId64"\n", stoptime, r->tag);
}
int dispatch_task(struct link *mpi_link, struct mpi_queue_task *t, int timeout)
{
struct mpi_queue_file *tf;
int stoptime = time(0) + timeout;
debug(D_MPI, "sending task %d\n", t->taskid);
if(t->input_files) {
list_first_item(t->input_files);
while((tf = list_next_item(t->input_files))) {
link_putfstring(mpi_link, "stat %d %s\n", stoptime, t->taskid, tf->name);
}
}
t->start_time = timestamp_get();
link_putfstring(mpi_link, "work %d %zu\n%s", stoptime, t->taskid, strlen(t->command_line), t->command_line);
t->status = MPI_QUEUE_TASK_STATUS_EXECUTING;
link_putfstring(mpi_link, "close %d\n", stoptime, t->taskid);
debug(D_MPI, "'%s' sent as task %d", t->command_line, t->taskid);
return 1;
}
INT64_T cfs_basic_getfile(const char *path, struct link * link, time_t stoptime)
{
int fd;
INT64_T result;
struct chirp_stat info;
result = cfs->stat(path, &info);
if(result < 0)
return result;
if(S_ISDIR(info.cst_mode)) {
errno = EISDIR;
return -1;
}
fd = cfs->open(path, O_RDONLY, 0);
if(fd >= 0) {
char buffer[65536];
INT64_T total = 0;
INT64_T ractual, wactual;
INT64_T length = info.cst_size;
link_putfstring(link, "%" PRId64 "\n", stoptime, length);
while(length > 0) {
INT64_T chunk = MIN((int) sizeof(buffer), length);
ractual = cfs->pread(fd, buffer, chunk, total);
if(ractual <= 0)
break;
wactual = link_putlstring(link, buffer, ractual, stoptime);
if(wactual != ractual) {
total = -1;
break;
}
total += ractual;
length -= ractual;
}
result = total;
cfs->close(fd);
} else {
result = -1;
}
return result;
}
int do_direct_query( const char *master_host, int master_port, time_t stoptime )
{
static struct nvpair_header *query_headers[4] = { queue_headers, task_headers, worker_headers, master_resource_headers };
static const char * query_strings[4] = {"queue","task","worker", "master_resource"};
struct nvpair_header *query_header = query_headers[query_mode];
const char * query_string = query_strings[query_mode];
struct link *l;
struct nvpair *nv;
char master_addr[LINK_ADDRESS_MAX];
if(!domain_name_cache_lookup(master_host,master_addr)) {
fprintf(stderr,"couldn't find address of %s\n",master_host);
return 1;
}
l = link_connect(master_addr,master_port,stoptime);
if(!l) {
fprintf(stderr,"couldn't connect to %s port %d: %s\n",master_host,master_port,strerror(errno));
return 1;
}
link_putfstring(l,"%s_status\n",stoptime,query_string);
if(format_mode==FORMAT_TABLE) {
nvpair_print_table_header(stdout, query_header);
}
while((nv = link_nvpair_read(l,stoptime))) {
if(format_mode == FORMAT_TABLE) {
nvpair_print_table(nv, stdout, query_header);
} else {
nvpair_print_text(nv, stdout);
}
nvpair_delete(nv);
}
if(format_mode == FORMAT_TABLE) {
nvpair_print_table_footer(stdout, query_header);
}
return EXIT_SUCCESS;
}
struct list* do_direct_query( const char *master_host, int master_port )
{
const char * query_string = "queue";
struct link *l;
char master_addr[LINK_ADDRESS_MAX];
time_t stoptime = time(0) + work_queue_status_timeout;
if(!domain_name_cache_lookup(master_host,master_addr)) {
fprintf(stderr,"couldn't find address of %s\n",master_host);
return 0;
}
l = link_connect(master_addr,master_port,stoptime);
if(!l) {
fprintf(stderr,"couldn't connect to %s port %d: %s\n",master_host,master_port,strerror(errno));
return 0;
}
link_putfstring(l,"%s_status\n",stoptime,query_string);
struct jx *jarray = jx_parse_link(l,stoptime);
struct jx *j = jarray->u.items->value;
link_close(l);
if(!j) {
fprintf(stderr,"couldn't read from %s port %d: %s\n",master_host,master_port,strerror(errno));
return 0;
}
j->type = JX_OBJECT;
struct list* master_list = list_create();
list_push_head(master_list, j);
return master_list;
}
static int auth_unix_accept(struct link *link, char **subject, time_t stoptime)
{
char path[AUTH_LINE_MAX];
char line[AUTH_LINE_MAX];
int success = 0;
struct stat buf;
struct passwd *p;
debug(D_AUTH, "unix: generating challenge");
make_challenge_path(path);
link_putfstring(link, "%s\n", stoptime, path);
debug(D_AUTH, "unix: waiting for response");
if(link_readline(link, line, sizeof(line), stoptime)) {
if(!strcmp(line, "yes")) {
int file_exists = 0;
int i=0;
for(i=0;i<challenge_timeout;i++) {
/*
This is an odd hack, but invoking ls -la appears to help to force
some NFS clients to refresh cached metadata.
*/
DIR *d = opendir(challenge_dir);
if(d) {
closedir(d);
}
if(stat(path,&buf)==0) {
file_exists = 1;
break;
} else {
debug(D_AUTH,"unix: client claims success, but I don't see it yet...");
sleep(1);
}
}
if(file_exists) {
debug(D_AUTH, "unix: got response");
debug(D_AUTH, "unix: client is uid %d", buf.st_uid);
p = auth_get_passwd_from_uid(buf.st_uid);
if(p) {
debug(D_AUTH, "unix: client is subject %s", p->pw_name);
link_putliteral(link, "yes\n", stoptime);
*subject = xxstrdup(p->pw_name);
success = 1;
} else {
debug(D_AUTH, "unix: there is no user corresponding to uid %d", buf.st_uid);
link_putliteral(link, "no\n", stoptime);
}
} else {
debug(D_AUTH, "unix: client failed the challenge: %s", strerror(errno));
link_putliteral(link, "no\n", stoptime);
}
} else {
debug(D_AUTH, "unix: client declined the challenge");
}
}
unlink(path);
return success;
}
int main(int argc, char *argv[])
{
struct link *link, *master;
char *subject = 0, *type = 0;
time_t stoptime;
char line[1024];
signed char c;
int portnum = 30000;
char *hostname = 0;
int timeout = 30;
debug_config(argv[0]);
static struct option long_options[] = {
{"auth", required_argument, 0, 'a'},
{"port", required_argument, 0, 'p'},
{"host", required_argument, 0, 'r'},
{"help", required_argument, 0, 'h'},
{"debug-file", required_argument, 0, 'o'},
{"debug-rotate-max", required_argument, 0, 'O'},
{"debug", required_argument, 0, 'd'},
{0,0,0,0}
};
while((c = getopt_long(argc, argv, "a:p:r:d:o:O:", long_options, NULL)) > -1) {
switch (c) {
case 'p':
portnum = atoi(optarg);
break;
case 'h':
show_help(argv[0]);
exit(0);
break;
case 'r':
hostname = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'a':
if(!auth_register_byname(optarg))
fatal("couldn't register %s authentication", optarg);
break;
default:
show_use(argv[0]);
exit(1);
break;
}
}
if(hostname) {
char addr[LINK_ADDRESS_MAX];
stoptime = time(0) + timeout;
if(!domain_name_cache_lookup(hostname, addr))
fatal("unknown host name: %s", hostname);
link = link_connect(addr, portnum, stoptime);
if(!link)
fatal("couldn't connect to %s:%d: %s", hostname, portnum, strerror(errno));
if(auth_assert(link, &type, &subject, stoptime)) {
printf("server thinks I am %s %s\n", type, subject);
if(link_readline(link, line, sizeof(line), stoptime)) {
printf("got message: %s\n", line);
} else {
printf("lost connection!\n");
}
} else {
printf("unable to authenticate.\n");
}
link_close(link);
} else {
stoptime = time(0) + timeout;
master = link_serve(portnum);
if(!master)
fatal("couldn't serve port %d: %s\n", portnum, strerror(errno));
while(time(0) < stoptime) {
link = link_accept(master, stoptime);
if(!link)
continue;
if(auth_accept(link, &type, &subject, stoptime)) {
time_t t = time(0);
link_putfstring(link, "Hello %s:%s, it is now %s", stoptime, type, subject, ctime(&t)); /* ctime ends with newline */
} else {
printf("couldn't auth accept\n");
}
link_close(link);
}
}
return 0;
}
struct mpi_queue_task *mpi_queue_wait(struct mpi_queue *q, int timeout)
{
struct mpi_queue_task *t;
time_t stoptime;
int result;
if(timeout == MPI_QUEUE_WAITFORTASK) {
stoptime = 0;
} else {
stoptime = time(0) + timeout;
}
while(1) {
// If a task is already complete, return it
t = list_pop_head(q->complete_list);
if(t)
return t;
if(list_size(q->ready_list) == 0 && itable_size(q->active_list) == 0)
break;
// Wait no longer than the caller's patience.
int msec;
int sec;
if(stoptime) {
sec = MAX(0, stoptime - time(0));
msec = sec * 1000;
} else {
sec = 5;
msec = 5000;
}
if(!q->mpi_link) {
q->mpi_link = link_accept(q->master_link, stoptime);
if(q->mpi_link) {
char working_dir[MPI_QUEUE_LINE_MAX];
link_tune(q->mpi_link, LINK_TUNE_INTERACTIVE);
link_usleep(q->mpi_link, msec, 0, 1);
getcwd(working_dir, MPI_QUEUE_LINE_MAX);
link_putfstring(q->mpi_link, "workdir %s\n", stoptime, working_dir);
result = link_usleep(q->mpi_link, msec, 1, 1);
} else {
result = 0;
}
} else {
debug(D_MPI, "Waiting for link to be ready\n");
result = link_usleep(q->mpi_link, msec, 1, 1);
}
// If nothing was awake, restart the loop or return without a task.
if(result <= 0) {
if(stoptime && time(0) >= stoptime) {
return 0;
} else {
continue;
}
}
debug(D_MPI, "sending %d tasks to the MPI master process\n", list_size(q->ready_list));
// Send all ready tasks to the MPI master process
while(list_size(q->ready_list)) {
struct mpi_queue_task *t = list_pop_head(q->ready_list);
result = dispatch_task(q->mpi_link, t, msec/1000);
if(result <= 0)
return 0;
itable_insert(q->active_list, t->taskid, t);
}
// Receive any results back
result = get_results(q->mpi_link, q->active_list, q->complete_list, msec/1000);
if(result < 0) {
return 0;
}
}
return 0;
}
static void work_queue_resource_send( struct link *master, struct work_queue_resource *r, const char *name, time_t stoptime )
{
work_queue_resource_debug(r, name);
link_putfstring(master, "resource %s %"PRId64" %"PRId64" %"PRId64"\n", stoptime, name, r->total, r->smallest, r->largest );
}
// NOT IMPLEMENTED YET
int s3_setacl(char* bucketname, char *filename, const char* owner, struct hash_table* acls, const char* access_key_id, const char* access_key) {
struct s3_message mesg;
struct link* server;
time_t stoptime = time(0)+s3_timeout;
char path[HEADER_LINE_MAX];
char response[HEADER_LINE_MAX];
//char * text;
//int length;
char *id;
struct s3_acl_object *acl;
if(!s3_endpoint) return -1;
if(filename) sprintf(path, "%s?acl", filename);
else
sprintf(path, "/?acl");
mesg.content_length = 39 + 32 + strlen(owner) + 32;
hash_table_firstkey(acls);
while(hash_table_nextkey(acls, &id, (void**)&acl)) {
int glength;
switch(acl->acl_type) {
case S3_ACL_URI:
glength = 140+strlen(id);
break;
case S3_ACL_EMAIL:
glength = 135+strlen(id);
break;
default:
glength = 107+strlen(id);
}
if(acl->perm & S3_ACL_FULL_CONTROL) mesg.content_length += 40 + glength + 12;
if(acl->perm & S3_ACL_READ) mesg.content_length += 40 + glength + 4;
if(acl->perm & S3_ACL_WRITE) mesg.content_length += 40 + glength + 5;
if(acl->perm & S3_ACL_READ_ACP) mesg.content_length += 40 + glength + 8;
if(acl->perm & S3_ACL_WRITE_ACP) mesg.content_length += 40 + glength + 9;
}
mesg.content_length += 43;
mesg.type = S3_MESG_PUT;
mesg.path = path;
mesg.bucket = bucketname;
mesg.content_type = NULL;
mesg.content_md5 = NULL;
mesg.date = time(0);
mesg.expect = 0;
mesg.amz_headers = NULL;
//server = link_connect(s3_address, 80, stoptime);
sign_message(&mesg, access_key_id, access_key);
server = s3_send_message(&mesg, NULL, stoptime);
if(!server)
return -1;
//length = s3_message_to_string(&mesg, &text);
//fprintf(stderr, "Message:\n%s\n", text);
//link_putlstring(server, text, length, stoptime);
//free(text);
link_putliteral(server, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n", stoptime);
link_putliteral(server, "<AccessControlPolicy><Owner><ID>", stoptime);
link_putstring(server, owner, stoptime);
link_putliteral(server, "</ID></Owner><AccessControlList>", stoptime);
hash_table_firstkey(acls);
while(hash_table_nextkey(acls, &id, (void**)&acl)) {
char grantee[HEADER_LINE_MAX];
switch(acl->acl_type) {
case S3_ACL_URI:
sprintf(grantee, "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"Group\"><URI>http://acs.amazonaws.com/groups/global/%s</URI></Grantee>", id);
break;
case S3_ACL_EMAIL:
sprintf(grantee, "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"AmazonCustomerByEmail\"><EmailAddress>%s</EmailAddress></Grantee>", id);
break;
default:
sprintf(grantee, "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"CanonicalUser\"><ID>%s</ID></Grantee>", id);
}
if(acl->perm & S3_ACL_FULL_CONTROL) {
link_putfstring(server, "<Grant>%s<Permission>FULL_CONTROL</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_READ) {
link_putfstring(server, "<Grant>%s<Permission>READ</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_WRITE) {
link_putfstring(server, "<Grant>%s<Permission>WRITE</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_READ_ACP) {
link_putfstring(server, "<Grant>%s<Permission>READ_ACP</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_WRITE_ACP) {
link_putfstring(server, "<Grant>%s<Permission>WRITE_ACP</Permission></Grant>", stoptime, grantee);
}
}
link_putliteral(server, "</AccessControlList></AccessControlPolicy>\n", stoptime);
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(strcmp(response, "HTTP/1.1 200 OK")) {
// Error: transfer failed; close connection and return failure
fprintf(stderr, "Error: send file failed\nResponse: %s\n", response);
link_close(server);
return -1;
}
// fprintf(stderr, "Response:\n");
do {
// fprintf(stderr, "\t%s\n", response);
if(!strcmp(response, "Server: AmazonS3")) break;
} while(link_readline(server, response, HEADER_LINE_MAX, stoptime));
link_close(server);
return 0;
}
int master_main(const char *host, int port, const char *addr) {
time_t idle_stoptime;
struct link *master = NULL;
int num_workers, i;
struct mpi_queue_job **workers;
struct itable *active_jobs = itable_create(0);
struct itable *waiting_jobs = itable_create(0);
struct list *complete_jobs = list_create();
MPI_Comm_size(MPI_COMM_WORLD, &num_workers);
workers = malloc(num_workers * sizeof(*workers));
memset(workers, 0, num_workers * sizeof(*workers));
idle_stoptime = time(0) + idle_timeout;
while(!abort_flag) {
char line[MPI_QUEUE_LINE_MAX];
if(time(0) > idle_stoptime) {
if(master) {
printf("mpi master: gave up after waiting %ds to receive a task.\n", idle_timeout);
} else {
printf("mpi master: gave up after waiting %ds to connect to %s port %d.\n", idle_timeout, host, port);
}
break;
}
if(!master) {
char working_dir[MPI_QUEUE_LINE_MAX];
master = link_connect(addr, port, idle_stoptime);
if(!master) {
sleep(5);
continue;
}
link_tune(master, LINK_TUNE_INTERACTIVE);
link_readline(master, line, sizeof(line), time(0) + active_timeout);
memset(working_dir, 0, MPI_QUEUE_LINE_MAX);
if(sscanf(line, "workdir %s", working_dir) == 1) {
MPI_Bcast(working_dir, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
master = NULL;
continue;
}
}
if(link_readline(master, line, sizeof(line), time(0) + short_timeout)) {
struct mpi_queue_operation *op;
int jobid, mode;
INT64_T length;
char path[MPI_QUEUE_LINE_MAX];
op = NULL;
debug(D_MPI, "received: %s\n", line);
if(!strcmp(line, "get results")) {
struct mpi_queue_job *job;
debug(D_MPI, "results requested: %d available\n", list_size(complete_jobs));
link_putfstring(master, "num results %d\n", time(0) + active_timeout, list_size(complete_jobs));
while(list_size(complete_jobs)) {
job = list_pop_head(complete_jobs);
link_putfstring(master, "result %d %d %d %lld\n", time(0) + active_timeout, job->jobid, job->status, job->result, job->output_length);
if(job->output_length) {
link_write(master, job->output, job->output_length, time(0)+active_timeout);
}
mpi_queue_job_delete(job);
}
} else if(sscanf(line, "work %d %lld", &jobid, &length)) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_WORK;
op->buffer_length = length+1;
op->buffer = malloc(length+1);
op->buffer[op->buffer_length] = 0;
link_read(master, op->buffer, length, time(0) + active_timeout);
op->result = -1;
} else if(sscanf(line, "stat %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_STAT;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "unlink %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_UNLINK;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "mkdir %d %s %o", &jobid, path, &mode) == 3) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_MKDIR;
sprintf(op->args, "%s %o", path, mode);
op->result = -1;
} else if(sscanf(line, "close %d", &jobid) == 1) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_CLOSE;
op->result = -1;
// } else if(sscanf(line, "symlink %d %s %s", &jobid, path, filename) == 3) {
// } else if(sscanf(line, "put %d %s %lld %o", &jobid, filename, &length, &mode) == 4) {
// } else if(sscanf(line, "rget %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "get %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "thirdget %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
// } else if(sscanf(line, "thirdput %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
} else if(!strcmp(line, "exit")) {
break;
} else {
abort_flag = 1;
continue;
}
if(op) {
struct mpi_queue_job *job;
job = itable_lookup(active_jobs, jobid);
if(!job) {
job = itable_lookup(waiting_jobs, jobid);
}
if(!job) {
job = malloc(sizeof(*job));
memset(job, 0, sizeof(*job));
job->jobid = jobid;
job->operations = list_create();
job->status = MPI_QUEUE_JOB_WAITING;
job->worker_rank = -1;
itable_insert(waiting_jobs, jobid, job);
}
list_push_tail(job->operations, op);
}
idle_stoptime = time(0) + idle_timeout;
} else {
link_close(master);
master = 0;
sleep(5);
}
int num_waiting_jobs = itable_size(waiting_jobs);
int num_unvisited_jobs = itable_size(active_jobs);
for(i = 1; i < num_workers && (num_unvisited_jobs > 0 || num_waiting_jobs > 0); i++) {
struct mpi_queue_job *job;
struct mpi_queue_operation *op;
int flag = 0;
UINT64_T jobid;
if(!workers[i]) {
if(num_waiting_jobs) {
itable_firstkey(waiting_jobs);
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
itable_insert(active_jobs, jobid, job);
workers[i] = job;
num_waiting_jobs--;
job->worker_rank = i;
job->status = MPI_QUEUE_JOB_READY;
} else {
continue;
}
} else {
num_unvisited_jobs--;
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Test(&workers[i]->request, &flag, &workers[i]->mpi_status);
if(flag) {
op = list_pop_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
workers[i]->status = MPI_QUEUE_JOB_READY;
if(op->type == MPI_QUEUE_OP_WORK || op->result < 0) {
if(workers[i]->output)
free(workers[i]->output);
workers[i]->output = op->output_buffer;
op->output_buffer = NULL;
workers[i]->output_length = op->output_length;
workers[i]->result = op->result;
if(op->result < 0) {
workers[i]->status = MPI_QUEUE_JOB_FAILED | op->type;
op->type = MPI_QUEUE_OP_CLOSE;
list_push_head(workers[i]->operations, op);
op = NULL;
}
}
if(op) {
if(op->buffer)
free(op->buffer);
if(op->output_buffer)
free(op->output_buffer);
free(op);
}
}
}
}
if( workers[i]->status != MPI_QUEUE_JOB_BUSY && list_size(workers[i]->operations)) {
op = list_peek_head(workers[i]->operations);
if(op->type == MPI_QUEUE_OP_CLOSE) {
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
if(!(workers[i]->status & MPI_QUEUE_JOB_FAILED))
workers[i]->status = MPI_QUEUE_JOB_COMPLETE;
workers[i] = NULL;
i--;
continue;
}
MPI_Send(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
if(op->buffer_length) {
MPI_Send(op->buffer, op->buffer_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
free(op->buffer);
op->buffer_length = 0;
op->buffer = NULL;
}
MPI_Irecv(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->request);
workers[i]->status = MPI_QUEUE_JOB_BUSY;
}
}
}
/** Clean up waiting & complete jobs, send Exit commands to each worker */
if(!master) {
// If the master link hasn't been set up yet
// the workers will be waiting for the working directory
char line[MPI_QUEUE_LINE_MAX];
memset(line, 0, MPI_QUEUE_LINE_MAX);
MPI_Bcast(line, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
}
for(i = 1; i < num_workers; i++) {
struct mpi_queue_operation *op, close;
memset(&close, 0, sizeof(close));
close.type = MPI_QUEUE_OP_EXIT;
if(workers[i]) {
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Wait(&workers[i]->request, &workers[i]->mpi_status);
op = list_peek_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
}
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
}
MPI_Send(&close, sizeof(close), MPI_BYTE, i, 0, MPI_COMM_WORLD);
}
itable_firstkey(waiting_jobs);
while(itable_size(waiting_jobs)) {
struct mpi_queue_job *job;
UINT64_T jobid;
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
list_push_tail(complete_jobs, job);
}
while(list_size(complete_jobs)) {
mpi_queue_job_delete(list_pop_head(complete_jobs));
}
MPI_Finalize();
return abort_flag;
}
static int search_directory(const char *subject, const char *const base, char fullpath[CHIRP_PATH_MAX], const char *pattern, int flags, struct link *l, time_t stoptime)
{
if(strlen(pattern) == 0)
return 0;
debug(D_DEBUG, "search_directory(subject = `%s', base = `%s', fullpath = `%s', pattern = `%s', flags = %d, ...)", subject, base, fullpath, pattern, flags);
int access_flags = search_to_access(flags);
int includeroot = flags & CHIRP_SEARCH_INCLUDEROOT;
int metadata = flags & CHIRP_SEARCH_METADATA;
int stopatfirst = flags & CHIRP_SEARCH_STOPATFIRST;
int result = 0;
struct chirp_dir *dirp = cfs->opendir(fullpath);
char *current = fullpath + strlen(fullpath); /* point to end to current directory */
if(dirp) {
errno = 0;
struct chirp_dirent *entry;
while((entry = cfs->readdir(dirp))) {
char *name = entry->name;
if(strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || strncmp(name, ".__", 3) == 0)
continue;
sprintf(current, "/%s", name);
if(search_match_file(pattern, base)) {
const char *matched;
if(includeroot) {
if(base - fullpath == 1 && fullpath[0] == '/') {
matched = base;
} else {
matched = fullpath;
}
} else {
matched = base;
}
result += 1;
if(access_flags == F_OK || cfs->access(fullpath, access_flags) == 0) {
if(metadata) {
/* A match was found, but the matched file couldn't be statted. Generate a result and an error. */
if(entry->lstatus == -1) {
link_putfstring(l, "0:%s::\n", stoptime, matched); // FIXME is this a bug?
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_STAT, matched);
} else {
BUFFER_STACK_ABORT(B, 4096)
chirp_stat_encode(B, &entry->info);
link_putfstring(l, "0:%s:%s:\n", stoptime, matched, buffer_tostring(B));
if(stopatfirst)
return 1;
}
} else {
link_putfstring(l, "0:%s::\n", stoptime, matched);
if(stopatfirst)
return 1;
}
} /* FIXME access failure */
}
if(cfs_isdir(fullpath) && search_should_recurse(base, pattern)) {
if(chirp_acl_check_dir(fullpath, subject, CHIRP_ACL_LIST)) {
int n = search_directory(subject, base, fullpath, pattern, flags, l, stoptime);
if(n > 0) {
result += n;
if(stopatfirst)
return result;
}
} else {
link_putfstring(l, "%d:%d:%s:\n", stoptime, EPERM, CHIRP_SEARCH_ERR_OPEN, fullpath);
}
}
*current = '\0'; /* clear current entry */
errno = 0;
}
if(errno)
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_READ, fullpath);
errno = 0;
cfs->closedir(dirp);
if(errno)
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_CLOSE, fullpath);
} else {
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_OPEN, fullpath);
}
return result;
}
struct link *http_query_size_via_proxy(const char *proxy, const char *urlin, const char *action, INT64_T * size, time_t stoptime, int cache_reload)
{
char url[HTTP_LINE_MAX];
char newurl[HTTP_LINE_MAX];
char line[HTTP_LINE_MAX];
char addr[LINK_ADDRESS_MAX];
struct link *link;
int save_errno;
int response;
char actual_host[HTTP_LINE_MAX];
int actual_port;
*size = 0;
url_encode(urlin, url, sizeof(url));
if(proxy && !strcmp(proxy, "DIRECT"))
proxy = 0;
if(proxy) {
int fields = sscanf(proxy, "http://%[^:]:%d", actual_host, &actual_port);
if(fields == 2) {
/* host and port are good */
} else if(fields == 1) {
actual_port = HTTP_PORT;
} else {
debug(D_HTTP, "invalid proxy syntax: %s", proxy);
return 0;
}
} else {
int fields = sscanf(url, "http://%[^:]:%d", actual_host, &actual_port);
if(fields != 2) {
fields = sscanf(url, "http://%[^/]", actual_host);
if(fields == 1) {
actual_port = HTTP_PORT;
} else {
debug(D_HTTP, "malformed url: %s", url);
return 0;
}
}
}
debug(D_HTTP, "connect %s port %d", actual_host, actual_port);
if(!domain_name_lookup(actual_host, addr))
return 0;
link = link_connect(addr, actual_port, stoptime);
if(!link) {
errno = ECONNRESET;
return 0;
}
if(cache_reload == 0) {
debug(D_HTTP, "%s %s HTTP/1.0\r\nHost: %s\r\nConnection: close\r\n\r\n", action, url, actual_host);
link_putfstring(link, "%s %s HTTP/1.0\r\nHost: %s\r\nConnection: close\r\n\r\n", stoptime, action, url, actual_host);
} else {
// force refresh of cache end-to-end (RFC 2616)
debug(D_HTTP, "%s %s HTTP/1.1\r\nHost: %s\r\nCache-Control: max-age=0\r\nConnection: close\r\n\r\n", action, url, actual_host);
link_putfstring(link, "%s %s HTTP/1.1\r\nHost: %s\r\nCache-Control: max-age=0\r\nConnection: close\r\n\r\n", stoptime, action, url, actual_host);
}
if(link_readline(link, line, HTTP_LINE_MAX, stoptime)) {
string_chomp(line);
debug(D_HTTP, "%s", line);
if(sscanf(line, "HTTP/%*d.%*d %d", &response) == 1) {
newurl[0] = 0;
while(link_readline(link, line, HTTP_LINE_MAX, stoptime)) {
string_chomp(line);
debug(D_HTTP, "%s", line);
sscanf(line, "Location: %s", newurl);
sscanf(line, "Content-Length: %lld", size);
if(strlen(line) <= 2) {
break;
}
}
switch (response) {
case 200:
return link;
break;
case 301:
case 302:
case 303:
case 307:
link_close(link);
if(newurl[0]) {
if(!strcmp(url, newurl)) {
debug(D_HTTP, "error: server gave %d redirect from %s back to the same url!", response, url);
errno = EIO;
return 0;
} else {
return http_query(newurl, action, stoptime);
}
} else {
errno = ENOENT;
return 0;
}
break;
default:
link_close(link);
errno = http_response_to_errno(response);
return 0;
break;
}
} else {
debug(D_HTTP, "malformed response");
save_errno = ECONNRESET;
}
} else {
debug(D_HTTP, "malformed response");
save_errno = ECONNRESET;
}
link_close(link);
errno = save_errno;
return 0;
}
static int search_directory(const char *subject, const char * const base, char *fullpath, const char *pattern, int flags, struct link *l, time_t stoptime)
{
if(strlen(pattern) == 0)
return 0;
debug(D_DEBUG, "search_directory(subject = `%s', base = `%s', fullpath = `%s', pattern = `%s', flags = %d, ...)", subject, base, fullpath, pattern, flags);
int access_flags = search_to_access(flags);
int includeroot = flags & CHIRP_SEARCH_INCLUDEROOT;
int metadata = flags & CHIRP_SEARCH_METADATA;
int stopatfirst = flags & CHIRP_SEARCH_STOPATFIRST;
int result = 0;
void *dirp = chirp_fs_local_opendir(fullpath);
char *current = fullpath + strlen(fullpath); /* point to end to current directory */
if(dirp) {
errno = 0;
struct chirp_dirent *entry;
while((entry = chirp_fs_local_readdir(dirp))) {
char *name = entry->name;
if(strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || strncmp(name, ".__", 3) == 0)
continue;
sprintf(current, "/%s", name);
if(search_match_file(pattern, base)) {
const char *matched = includeroot ? fullpath+strlen(chirp_root_path) : base; /* add strlen(chirp_root_path) to strip out */
result += 1;
if (access_flags == F_OK || chirp_fs_local_access(fullpath, access_flags) == 0) {
if(metadata) {
/* A match was found, but the matched file couldn't be statted. Generate a result and an error. */
struct chirp_stat buf;
if((chirp_fs_local_stat(fullpath, &buf)) == -1) {
link_putfstring(l, "0:%s::\n", stoptime, matched); // FIXME is this a bug?
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_STAT, matched);
} else {
link_putfstring(l, "0:%s:%s:\n", stoptime, matched, chirp_stat_string(&buf));
if(stopatfirst) return 1;
}
} else {
link_putfstring(l, "0:%s::\n", stoptime, matched);
if(stopatfirst) return 1;
}
} /* FIXME access failure */
}
if(cfs_isdir(fullpath) && search_should_recurse(base, pattern)) {
if(chirp_acl_check_dir(fullpath, subject, CHIRP_ACL_LIST)) {
int n = search_directory(subject, base, fullpath, pattern, flags, l, stoptime);
if(n > 0) {
result += n;
if(stopatfirst)
return result;
}
} else {
link_putfstring(l, "%d:%d:%s:\n", stoptime, EPERM, CHIRP_SEARCH_ERR_OPEN, fullpath);
}
}
*current = '\0'; /* clear current entry */
errno = 0;
}
if(errno)
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_READ, fullpath);
errno = 0;
chirp_alloc_closedir(dirp);
if(errno)
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_CLOSE, fullpath);
} else {
link_putfstring(l, "%d:%d:%s:\n", stoptime, errno, CHIRP_SEARCH_ERR_OPEN, fullpath);
}
return result;
}
static void work_queue_resource_send( struct link *master, struct work_queue_resource *r, const char *name, time_t stoptime )
{
link_putfstring(master, "resource %s %d %d %d %d\n", stoptime, name, r->inuse, r->total, r->smallest, r->largest );
}
static int auth_ticket_accept(struct link *link, char **subject, time_t stoptime)
{
int serrno = errno;
int status = 0;
char line[AUTH_LINE_MAX];
char ticket_subject[AUTH_LINE_MAX];
char *ticket = NULL;
errno = 0;
debug(D_AUTH, "ticket: waiting for tickets");
while(1) {
if(link_readline(link, line, sizeof(line), stoptime) > 0) {
if(strcmp(line, "==") == 0) {
debug(D_AUTH, "ticket: exhausted all ticket challenges");
break;
} else if(strlen(line) == DIGEST_LENGTH) {
char ticket_digest[DIGEST_LENGTH + 1];
strcpy(ticket_digest, line);
strcpy(ticket_subject, line);
debug(D_AUTH, "ticket: read ticket digest: %s", ticket_digest);
if(server_callback) {
free(ticket); /* free previously allocated ticket string or NULL (noop) */
ticket = server_callback(ticket_digest);
if(ticket) {
static const char command_template[] = "T1=`mktemp`\n" /* The RSA Public Key */
"T2=`mktemp`\n" /* The Challenge */
"T3=`mktemp`\n" /* The Signed Challenge */
"T4=`mktemp`\n" /* The Decrypted (verified) Signed Challenge */
"echo -n '%s' > \"$T1\"\n" "dd if=/dev/urandom of=\"$T2\" bs=%u count=1 > /dev/null 2> /dev/null\n" "cat \"$T2\"\n" /* to stdout */
"cat > \"$T3\"\n" /* from stdin */
"openssl rsautl -inkey \"$T1\" -pubin -verify < \"$T3\" > \"$T4\" 2> /dev/null\n" "cmp \"$T2\" \"$T4\" > /dev/null 2> /dev/null\n" "R=\"$?\"\n"
"rm -f \"$T1\" \"$T2\" \"$T3\" \"$T4\" > /dev/null 2> /dev/null\n" "exit \"$R\"\n";
char *command = xxmalloc(sizeof(command_template) + strlen(ticket) + 64);
sprintf(command, command_template, ticket, CHALLENGE_LENGTH);
FILE *in, *out;
pid_t pid = dpopen(command, &in, &out);
free(command);
if(pid == 0)
break;
if(!link_putfstring(link, "%zu\n", stoptime, CHALLENGE_LENGTH))
break;
if(!link_stream_from_file(link, out, CHALLENGE_LENGTH, stoptime))
break;
if(link_readline(link, line, sizeof(line), stoptime) <= 0)
break;
unsigned long length = strtoul(line, NULL, 10);
if(errno == ERANGE || errno == EINVAL)
break; /* not a number? */
if(!link_stream_to_file(link, in, length, stoptime))
break;
int result = dpclose(in, out, pid);
if(result == 0) {
debug(D_AUTH, "succeeded challenge for %s\n", ticket_digest);
link_putliteral(link, "success\n", stoptime);
status = 1;
break;
} else {
debug(D_AUTH, "failed challenge for %s\n", ticket_digest);
link_putliteral(link, "failure\n", stoptime);
break;
}
} else {
debug(D_AUTH, "declining key %s", ticket_digest);
link_putliteral(link, "declined\n", stoptime);
}
} else {
debug(D_AUTH, "declining key %s", ticket_digest);
link_putliteral(link, "declined\n", stoptime);
}
} else {
debug(D_AUTH, "ticket: bad response");
break;
}
} else {
break;
}
}
if(status) {
*subject = xxmalloc(AUTH_LINE_MAX);
strcpy(*subject, ticket_subject);
}
free(ticket); /* free previously allocated ticket string or NULL (noop) */
errno = serrno;
return status;
}
