INT64_T link_stream_from_file(struct link * link, FILE * file, INT64_T length, time_t stoptime)
{
char buffer[65536];
INT64_T total = 0;
INT64_T ractual, wactual;
while(1) {
INT64_T chunk = MIN(sizeof(buffer), length);
ractual = full_fread(file, buffer, chunk);
if(ractual <= 0)
break;
wactual = link_write(link, buffer, ractual, stoptime);
if(wactual != ractual) {
total = -1;
break;
}
total += ractual;
length -= ractual;
}
return total;
}
struct link * s3_send_message(struct s3_message *mesg, struct link *server, time_t stoptime) {
char *message_text;
char address[16];
int message_length, data_sent;
if(!server) {
if(!domain_name_cache_lookup(s3_endpoint, address))
return NULL;
server = link_connect(address, 80, stoptime);
}
if(!server)
return NULL;
message_length = s3_message_to_string(mesg, &message_text);
if(message_length <= 0) {
link_close(server);
return NULL;
}
data_sent = link_write(server, message_text, message_length, stoptime);
debug(D_TCP, "S3 Message Sent:\n%s\n", message_text);
free(message_text);
if(data_sent < message_length) {
link_close(server);
server = NULL;
}
return server;
}
void jx_print_link( struct jx *j, struct link *l, time_t stoptime )
{
buffer_t buffer;
buffer_init(&buffer);
jx_print_buffer(j,&buffer);
link_write(l,buffer.buf,buffer.len,stoptime);
buffer_free(&buffer);
}
int son_phy_write_fileno(son_phy_t * phy, int fd, const void * buffer, u32 nbyte){
#if defined __link
if( phy->driver ){
return link_write(phy->driver, fd, buffer, nbyte);
}
#endif
return -1;
}
int link_putlstring(struct link *link, const char *data, size_t count, time_t stoptime)
{
size_t total = 0;
ssize_t written = 0;
while(count > 0 && (written = link_write(link, data, count, stoptime)) > 0) {
count -= written;
total += written;
data += written;
}
return written < 0 ? written : total;
}
void jx_print_link( struct jx *j, struct link *l, time_t stoptime )
{
buffer_t buffer;
buffer_init(&buffer);
jx_print_buffer(j,&buffer);
size_t len;
const char *str = buffer_tolstring(&buffer, &len);
link_write(l,str,len,stoptime);
buffer_free(&buffer);
}
static int catalog_update_tcp( const char *host, const char *address, int port, const char *text )
{
debug(D_DEBUG, "sending update via tcp to %s(%s):%d", host, address, port);
time_t stoptime = time(0) + 15;
struct link *l = link_connect(address,port,stoptime);
if(!l) {
debug(D_DEBUG, "failed to connect to %s(%s):%d: %s",host,address,port,strerror(errno));
return 0;
}
link_write(l,text,strlen(text),stoptime);
link_close(l);
return 1;
}
int son_phy_write(son_phy_t * phy, const void * buffer, u32 nbyte){
if( phy->message ){
return phy_write_message(phy, buffer, nbyte);
}
if( phy->driver == 0 ){
//write using fwrite
return fwrite(buffer, 1, nbyte, phy->f);
} else {
#if defined __link
return link_write(phy->driver, phy->fd, buffer, nbyte);
#else
return -1;
#endif
}
}
ssize_t link_putlstring(struct link *link, const char *data, size_t count, time_t stoptime)
{
ssize_t total = 0;
if (!link)
return errno = EINVAL, -1;
/* Loop because, unlike link_write, we do not allow partial writes. */
while (count > 0) {
ssize_t w = link_write(link, data, count, stoptime);
if (w == -1)
return -1;
count -= w;
total += w;
data += w;
}
return total;
}
int64_t link_stream_from_file(struct link * link, FILE * file, int64_t length, time_t stoptime)
{
int64_t total = 0;
while(1) {
char buffer[1<<16];
size_t chunk = MIN(sizeof(buffer), (size_t)length);
ssize_t ractual = full_fread(file, buffer, chunk);
if(ractual <= 0)
break;
ssize_t wactual = link_write(link, buffer, ractual, stoptime);
if(wactual != ractual) {
total = -1;
break;
}
total += ractual;
length -= ractual;
}
return total;
}
void Link::write(unsigned addr, uint8 data) {
if(link_write) return link_write(addr, data);
}
INT64_T chirp_client_ticket_register(struct chirp_client * c, const char *name, const char *subject, time_t duration, time_t stoptime)
{
char command[PATH_MAX * 2 + 4096];
char ticket_subject[CHIRP_LINE_MAX];
FILE *shell;
int status;
if(access(name, R_OK) != 0)
return -1; /* the 'name' argument must be a client ticket filename */
ticket_translate(name, ticket_subject);
/* BEWARE: we don't bother to escape the filename, a user could
* provide a malicious filename that makes us execute code we don't want to.
*/
sprintf(command, "if [ -r '%s' ]; then sed '/^\\s*#/d' < '%s' | openssl rsa -pubout 2> /dev/null; exit 0; else exit 1; fi", name, name);
shell = popen(command, "r");
if(!shell)
return -1;
/* read the ticket file (public key) */
char *ticket = xxrealloc(NULL, 4096);
size_t read, length = 0;
while((read = fread(ticket + length, 1, 4096, shell)) > 0) {
length += read;
ticket = xxrealloc(ticket, length + 4096);
}
if(ferror(shell)) {
status = pclose(shell);
errno = ferror(shell);
return -1;
}
assert(feof(shell));
status = pclose(shell);
if(status) {
errno = EINVAL;
return -1;
}
if(subject == NULL)
subject = "self";
INT64_T result = send_command(c, stoptime, "ticket_register %s %llu %zu\n", subject, (unsigned long long) duration, length);
if(result < 0) {
free(ticket);
return result;
}
result = link_write(c->link, ticket, length, stoptime);
free(ticket);
if(result != (int) length) {
c->broken = 1;
errno = ECONNRESET;
return -1;
}
result = get_result(c, stoptime);
if(result == 0) {
time_t t;
struct tm tm;
char now[1024];
char expiration[1024];
time(&t);
localtime_r(&t, &tm);
my_strftime(now, sizeof(now) / sizeof(char), "%c", &tm);
t += duration;
localtime_r(&t, &tm);
my_strftime(expiration, sizeof(expiration) / sizeof(char), "%c", &tm);
FILE *file = fopen(name, "a");
if(file == NULL)
return -1;
fprintf(file, "# %s: Registered with %s as \"%s\". Expires on %s\n", now, c->hostport, subject, expiration);
fclose(file);
}
return result;
}
int Periph::write(const void * buf, int nbyte){
return link_write(handle, fd[port_], buf, nbyte);
}
int master_main(const char *host, int port, const char *addr) {
time_t idle_stoptime;
struct link *master = NULL;
int num_workers, i;
struct mpi_queue_job **workers;
struct itable *active_jobs = itable_create(0);
struct itable *waiting_jobs = itable_create(0);
struct list *complete_jobs = list_create();
MPI_Comm_size(MPI_COMM_WORLD, &num_workers);
workers = malloc(num_workers * sizeof(*workers));
memset(workers, 0, num_workers * sizeof(*workers));
idle_stoptime = time(0) + idle_timeout;
while(!abort_flag) {
char line[MPI_QUEUE_LINE_MAX];
if(time(0) > idle_stoptime) {
if(master) {
printf("mpi master: gave up after waiting %ds to receive a task.\n", idle_timeout);
} else {
printf("mpi master: gave up after waiting %ds to connect to %s port %d.\n", idle_timeout, host, port);
}
break;
}
if(!master) {
char working_dir[MPI_QUEUE_LINE_MAX];
master = link_connect(addr, port, idle_stoptime);
if(!master) {
sleep(5);
continue;
}
link_tune(master, LINK_TUNE_INTERACTIVE);
link_readline(master, line, sizeof(line), time(0) + active_timeout);
memset(working_dir, 0, MPI_QUEUE_LINE_MAX);
if(sscanf(line, "workdir %s", working_dir) == 1) {
MPI_Bcast(working_dir, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
master = NULL;
continue;
}
}
if(link_readline(master, line, sizeof(line), time(0) + short_timeout)) {
struct mpi_queue_operation *op;
int jobid, mode;
INT64_T length;
char path[MPI_QUEUE_LINE_MAX];
op = NULL;
debug(D_MPI, "received: %s\n", line);
if(!strcmp(line, "get results")) {
struct mpi_queue_job *job;
debug(D_MPI, "results requested: %d available\n", list_size(complete_jobs));
link_putfstring(master, "num results %d\n", time(0) + active_timeout, list_size(complete_jobs));
while(list_size(complete_jobs)) {
job = list_pop_head(complete_jobs);
link_putfstring(master, "result %d %d %d %lld\n", time(0) + active_timeout, job->jobid, job->status, job->result, job->output_length);
if(job->output_length) {
link_write(master, job->output, job->output_length, time(0)+active_timeout);
}
mpi_queue_job_delete(job);
}
} else if(sscanf(line, "work %d %lld", &jobid, &length)) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_WORK;
op->buffer_length = length+1;
op->buffer = malloc(length+1);
op->buffer[op->buffer_length] = 0;
link_read(master, op->buffer, length, time(0) + active_timeout);
op->result = -1;
} else if(sscanf(line, "stat %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_STAT;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "unlink %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_UNLINK;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "mkdir %d %s %o", &jobid, path, &mode) == 3) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_MKDIR;
sprintf(op->args, "%s %o", path, mode);
op->result = -1;
} else if(sscanf(line, "close %d", &jobid) == 1) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_CLOSE;
op->result = -1;
// } else if(sscanf(line, "symlink %d %s %s", &jobid, path, filename) == 3) {
// } else if(sscanf(line, "put %d %s %lld %o", &jobid, filename, &length, &mode) == 4) {
// } else if(sscanf(line, "rget %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "get %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "thirdget %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
// } else if(sscanf(line, "thirdput %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
} else if(!strcmp(line, "exit")) {
break;
} else {
abort_flag = 1;
continue;
}
if(op) {
struct mpi_queue_job *job;
job = itable_lookup(active_jobs, jobid);
if(!job) {
job = itable_lookup(waiting_jobs, jobid);
}
if(!job) {
job = malloc(sizeof(*job));
memset(job, 0, sizeof(*job));
job->jobid = jobid;
job->operations = list_create();
job->status = MPI_QUEUE_JOB_WAITING;
job->worker_rank = -1;
itable_insert(waiting_jobs, jobid, job);
}
list_push_tail(job->operations, op);
}
idle_stoptime = time(0) + idle_timeout;
} else {
link_close(master);
master = 0;
sleep(5);
}
int num_waiting_jobs = itable_size(waiting_jobs);
int num_unvisited_jobs = itable_size(active_jobs);
for(i = 1; i < num_workers && (num_unvisited_jobs > 0 || num_waiting_jobs > 0); i++) {
struct mpi_queue_job *job;
struct mpi_queue_operation *op;
int flag = 0;
UINT64_T jobid;
if(!workers[i]) {
if(num_waiting_jobs) {
itable_firstkey(waiting_jobs);
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
itable_insert(active_jobs, jobid, job);
workers[i] = job;
num_waiting_jobs--;
job->worker_rank = i;
job->status = MPI_QUEUE_JOB_READY;
} else {
continue;
}
} else {
num_unvisited_jobs--;
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Test(&workers[i]->request, &flag, &workers[i]->mpi_status);
if(flag) {
op = list_pop_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
workers[i]->status = MPI_QUEUE_JOB_READY;
if(op->type == MPI_QUEUE_OP_WORK || op->result < 0) {
if(workers[i]->output)
free(workers[i]->output);
workers[i]->output = op->output_buffer;
op->output_buffer = NULL;
workers[i]->output_length = op->output_length;
workers[i]->result = op->result;
if(op->result < 0) {
workers[i]->status = MPI_QUEUE_JOB_FAILED | op->type;
op->type = MPI_QUEUE_OP_CLOSE;
list_push_head(workers[i]->operations, op);
op = NULL;
}
}
if(op) {
if(op->buffer)
free(op->buffer);
if(op->output_buffer)
free(op->output_buffer);
free(op);
}
}
}
}
if( workers[i]->status != MPI_QUEUE_JOB_BUSY && list_size(workers[i]->operations)) {
op = list_peek_head(workers[i]->operations);
if(op->type == MPI_QUEUE_OP_CLOSE) {
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
if(!(workers[i]->status & MPI_QUEUE_JOB_FAILED))
workers[i]->status = MPI_QUEUE_JOB_COMPLETE;
workers[i] = NULL;
i--;
continue;
}
MPI_Send(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
if(op->buffer_length) {
MPI_Send(op->buffer, op->buffer_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
free(op->buffer);
op->buffer_length = 0;
op->buffer = NULL;
}
MPI_Irecv(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->request);
workers[i]->status = MPI_QUEUE_JOB_BUSY;
}
}
}
/** Clean up waiting & complete jobs, send Exit commands to each worker */
if(!master) {
// If the master link hasn't been set up yet
// the workers will be waiting for the working directory
char line[MPI_QUEUE_LINE_MAX];
memset(line, 0, MPI_QUEUE_LINE_MAX);
MPI_Bcast(line, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
}
for(i = 1; i < num_workers; i++) {
struct mpi_queue_operation *op, close;
memset(&close, 0, sizeof(close));
close.type = MPI_QUEUE_OP_EXIT;
if(workers[i]) {
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Wait(&workers[i]->request, &workers[i]->mpi_status);
op = list_peek_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
}
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
}
MPI_Send(&close, sizeof(close), MPI_BYTE, i, 0, MPI_COMM_WORLD);
}
itable_firstkey(waiting_jobs);
while(itable_size(waiting_jobs)) {
struct mpi_queue_job *job;
UINT64_T jobid;
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
list_push_tail(complete_jobs, job);
}
while(list_size(complete_jobs)) {
mpi_queue_job_delete(list_pop_head(complete_jobs));
}
MPI_Finalize();
return abort_flag;
}