int s3_mk_bucket(char* bucketname, enum amz_base_perm perms, const char* access_key_id, const char* access_key) {
struct link* server;
char path[] = "/";
struct s3_header_object *head;
time_t stoptime = time(0)+s3_timeout;
struct s3_message mesg;
char response[HEADER_LINE_MAX];
if(!access_key_id || !access_key || !s3_endpoint) return -1;
mesg.type = S3_MESG_PUT;
mesg.path = path;
mesg.bucket = bucketname;
mesg.content_length = 0;
mesg.content_type = NULL;
mesg.content_md5 = NULL;
mesg.date = time(0);
mesg.expect = 0;
switch(perms) {
case AMZ_PERM_PRIVATE: head = s3_new_header_object(S3_HEADER_AMZ_ACL, NULL, "private"); break;
case AMZ_PERM_PUBLIC_READ: head = s3_new_header_object(S3_HEADER_AMZ_ACL, NULL, "public-read"); break;
case AMZ_PERM_PUBLIC_WRITE: head = s3_new_header_object(S3_HEADER_AMZ_ACL, NULL, "public-read-write"); break;
case AMZ_PERM_AUTH_READ: head = s3_new_header_object(S3_HEADER_AMZ_ACL, NULL, "authenticated-read"); break;
case AMZ_PERM_BUCKET_READ: head = s3_new_header_object(S3_HEADER_AMZ_ACL, NULL, "bucket-owner-read"); break;
case AMZ_PERM_BUCKET_FULL: head = s3_new_header_object(S3_HEADER_AMZ_ACL, NULL, "bucket-owner-full-control"); break;
default: return -1;
}
mesg.amz_headers = list_create();
list_push_tail(mesg.amz_headers, head);
sign_message(&mesg, access_key_id, access_key);
server = s3_send_message(&mesg, NULL, stoptime);
list_free(mesg.amz_headers);
list_delete(mesg.amz_headers);
if(!server)
return -1;
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(strcmp(response, "HTTP/1.1 200 OK")) {
// Error: transfer failed; close connection and return failure
//fprintf(stderr, "Error: create bucket failed\nResponse: %s\n", response);
link_close(server);
return -1;
}
do {
if(!strcmp(response, "Server: AmazonS3")) break;
} while(link_readline(server, response, HEADER_LINE_MAX, stoptime));
link_close(server);
return 0;
}
static INT64_T get_stat_result(struct chirp_client *c, struct chirp_stat *info, time_t stoptime)
{
char line[CHIRP_LINE_MAX];
INT64_T fields;
memset(info, 0, sizeof(*info));
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime)) {
c->broken = 1;
errno = ECONNRESET;
return -1;
}
fields = sscanf(line, "%" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 "\n", &info->cst_dev, &info->cst_ino, &info->cst_mode, &info->cst_nlink, &info->cst_uid, &info->cst_gid, &info->cst_rdev, &info->cst_size, &info->cst_blksize,
&info->cst_blocks, &info->cst_atime, &info->cst_mtime, &info->cst_ctime);
info->cst_dev = -1;
info->cst_rdev = 0;
if(fields != 13) {
c->broken = 1;
errno = ECONNRESET;
return -1;
}
return 0;
}
static INT64_T get_result(struct chirp_client *c, time_t stoptime)
{
char line[CHIRP_LINE_MAX];
INT64_T result;
INT64_T fields;
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime)) {
c->broken = 1;
errno = ECONNRESET;
return -1;
}
fields = sscanf(line, "%" SCNd64 , &result);
if(fields != 1) {
errno = ECONNRESET;
c->broken = 1;
return -1;
}
result = convert_result(result);
if(result >= 0) {
debug(D_CHIRP, " = %lld", result);
} else {
debug(D_CHIRP, " = %lld (%s)", result, strerror(errno));
}
return result;
}
CHIRP_SEARCH *chirp_client_opensearch(struct chirp_client *c, const char *path, const char *pattern, int flags, time_t stoptime) {
INT64_T result = simple_command(c, stoptime, "search %s %s %d\n", pattern, path, flags);
char p[CHIRP_PATH_MAX];
size_t buffer_size = 2048;
char *buffer = malloc(buffer_size);
size_t l, i=0;
if (result == 0) {
while (link_readline(c->link, p, sizeof(p), stoptime)) {
if (strcmp(p, "") == 0) break;
while ((l = (size_t)snprintf(buffer+i, buffer_size-i, p)) >= buffer_size-i) {
buffer_size *= 2;
char *rbuffer = (char*) realloc(buffer, buffer_size);
if (rbuffer==NULL) return NULL;
buffer = rbuffer;
}
i += l;
}
if (i==0) *buffer = '\0';
CHIRP_SEARCH *result = malloc(sizeof(CHIRP_SEARCH));
result->entry = (struct chirp_searchent*) malloc(sizeof(struct chirp_searchent));
result->entry->info = NULL;
result->entry->path = NULL;
result->data = buffer;
result->i = 0;
return result;
} else
return NULL;
}
static int auth_unix_assert(struct link *link, time_t stoptime)
{
int success = 0;
FILE *file;
char line[AUTH_LINE_MAX];
debug(D_AUTH, "unix: waiting for challenge");
if(link_readline(link, line, sizeof(line), stoptime)) {
debug(D_AUTH, "unix: challenge is %s", line);
file = fopen(line, "w");
if(file) {
fsync(fileno(file));
fclose(file);
debug(D_AUTH, "unix: issued response");
if(auth_barrier(link, "yes\n", stoptime)) {
debug(D_AUTH, "unix: response accepted");
success = 1;
} else {
debug(D_AUTH, "unix: response rejected");
}
} else {
debug(D_AUTH, "unix: could not meet challenge: %s", strerror(errno));
link_putliteral(link, "no\n", stoptime);
}
unlink(line);
} else {
debug(D_AUTH, "unix: couldn't read challenge");
}
return success;
}
INT64_T chirp_client_getlongdir(struct chirp_client * c, const char *path, chirp_longdir_t callback, void *arg, time_t stoptime)
{
char name[CHIRP_LINE_MAX];
struct chirp_stat info;
int result;
char safepath[CHIRP_LINE_MAX];
url_encode(path, safepath, sizeof(safepath));
result = simple_command(c, stoptime, "getlongdir %s\n", safepath);
if(result < 0)
return result;
while(link_readline(c->link, name, sizeof(name), stoptime)) {
if(!name[0])
return 0;
if(get_stat_result(c, &info, stoptime) >= 0) {
callback(name, &info, arg);
} else {
break;
}
}
c->broken = 1;
errno = ECONNRESET;
return -1;
}
INT64_T chirp_client_audit(struct chirp_client * c, const char *path, struct chirp_audit ** list, time_t stoptime)
{
INT64_T result;
struct chirp_audit *entry;
int i, actual;
char line[CHIRP_LINE_MAX];
char safepath[CHIRP_LINE_MAX];
url_encode(path, safepath, sizeof(safepath));
result = simple_command(c, stoptime, "audit %s\n", safepath);
if(result <= 0)
return result;
*list = malloc(sizeof(struct chirp_audit) * result);
entry = *list;
for(i = 0; i < result; i++) {
actual = link_readline(c->link, line, sizeof(line), stoptime);
if(actual <= 0) {
free(*list);
result = -1;
errno = ECONNRESET;
break;
} else {
sscanf(line, "%s %" SCNd64 " %" SCNd64 " %" SCNd64 , entry->name, &entry->nfiles, &entry->ndirs, &entry->nbytes);
}
entry++;
}
return result;
}
int get_results(struct link *mpi_link, struct itable *active_list, struct list *complete_list, int timeout)
{
char line[MPI_QUEUE_LINE_MAX];
int num_results, n = 0;
int stoptime = time(0) + timeout;
debug(D_MPI, "Getting any results\n");
link_putliteral(mpi_link, "get results\n", stoptime);
if(link_readline(mpi_link, line, sizeof(line), stoptime)) {
debug(D_MPI, "received: %s\n", line);
sscanf(line, "num results %d", &num_results);
} else {
return 0;
}
debug(D_MPI, "%d results available\n", num_results);
while(n++ < num_results && link_readline(mpi_link, line, sizeof(line), stoptime)) {
struct mpi_queue_task *t;
int taskid, status, result, result_length;
sscanf(line, "result %d %d %d %d", &taskid, &status, &result, &result_length);
t = itable_remove(active_list, taskid);
if(!t) {
debug(D_NOTICE, "Invalid taskid (%d) returned\n", taskid);
return -1;
}
if(result_length) {
t->output = malloc(result_length+1);
link_read(mpi_link, t->output, result_length, time(0) + timeout);
t->output[result_length] = 0;
}
t->status = MPI_QUEUE_TASK_STATUS_COMPLETE;
t->return_status = result;
t->result = status;
list_push_tail(complete_list, t);
}
return num_results;
}
int s3_rm_bucket(char* bucketname, const char* access_key_id, const char* access_key) {
struct s3_message mesg;
struct link* server;
time_t stoptime = time(0)+s3_timeout;
char response[HEADER_LINE_MAX];
char path[] = "/";
if(!access_key_id || !access_key || !s3_endpoint) return -1;
mesg.type = S3_MESG_DELETE;
mesg.path = path;
mesg.bucket = bucketname;
mesg.content_type = NULL;
mesg.content_md5 = NULL;
mesg.content_length = 0;
mesg.date = time(0);
mesg.expect = 0;
mesg.amz_headers = NULL;
sign_message(&mesg, access_key_id, access_key);
server = s3_send_message(&mesg, NULL, stoptime);
if(!server) return -1;
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(strcmp(response, "HTTP/1.1 204 No Content")) {
// Error: transfer failed; close connection and return failure
//fprintf(stderr, "Error: delete bucket failed\nResponse: %s\n", response);
link_close(server);
return -1;
}
do {
if(!strcmp(response, "Server: AmazonS3")) break;
} while(link_readline(server, response, HEADER_LINE_MAX, stoptime));
link_close(server);
return 0;
}
static int auth_address_assert(struct link *link, time_t stoptime)
{
char line[AUTH_LINE_MAX];
if(!link_readline(link, line, sizeof(line), stoptime)) {
debug(D_AUTH, "address: lost connection");
return 0;
}
if(!strcmp(line, "yes")) {
debug(D_AUTH, "address: accepted");
return 1;
}
return 0;
}
INT64_T chirp_client_ticket_list(struct chirp_client * c, const char *subject, char ***list, time_t stoptime)
{
INT64_T result;
size_t size = 0;
*list = NULL;
result = simple_command(c, stoptime, "ticket_list %s\n", subject);
if(result == 0) {
while(1) {
char line[CHIRP_LINE_MAX];
size_t length;
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime))
goto failure;
if(sscanf(line, "%zu", &length) != 1)
goto failure;
if(length == 0)
break;
size++;
*list = xxrealloc(*list, sizeof(char *) * (size + 1));
(*list)[size - 1] = xxmalloc(sizeof(char) * (length + 1));
if(!link_read(c->link, (*list)[size - 1], length, stoptime))
goto failure;
(*list)[size - 1][length] = '\0';
(*list)[size] = NULL;
}
return 0;
failure:
if(*list != NULL) {
char **tmp = *list;
while(tmp[0]) {
free(tmp[0]);
}
free(*list);
}
c->broken = 1;
errno = ECONNRESET;
return -1;
}
return result;
}
const char *chirp_client_readdir(struct chirp_client *c, time_t stoptime)
{
static char name[CHIRP_PATH_MAX];
if(link_readline(c->link, name, sizeof(name), stoptime)) {
if(name[0]) {
return name;
} else {
return 0;
}
} else {
c->broken = 1;
errno = ECONNRESET;
return 0;
}
}
const char *chirp_client_readacl(struct chirp_client *c, time_t stoptime)
{
static char acl[CHIRP_PATH_MAX];
if(link_readline(c->link, acl, sizeof(acl), stoptime)) {
if(acl[0]) {
return acl;
} else {
return 0;
}
} else {
c->broken = 1;
errno = ECONNRESET;
return 0;
}
}
static int read_token(void *link, void **bufp, size_t * sizep)
{
char line[AUTH_LINE_MAX];
time_t stoptime = time(0) + 3600;
int result;
if(link_readline(link, line, sizeof(line), stoptime)) {
*sizep = atoi(line);
*bufp = malloc(*sizep);
if(*bufp) {
result = link_read(link, *bufp, *sizep, stoptime);
if(result == (int) *sizep) {
return GLOBUS_SUCCESS;
}
free(*bufp);
}
}
return GLOBUS_GSS_ASSIST_TOKEN_EOF;
}
INT64_T chirp_client_lsalloc(struct chirp_client * c, char const *path, char *allocpath, INT64_T * total, INT64_T * inuse, time_t stoptime)
{
int result;
char line[CHIRP_LINE_MAX];
char safepath[CHIRP_LINE_MAX];
url_encode(path, safepath, sizeof(safepath));
result = simple_command(c, stoptime, "lsalloc %s\n", safepath);
if(result == 0) {
if(link_readline(c->link, line, sizeof(line), stoptime)) {
sscanf(line, "%s %" SCNd64 " %" SCNd64 , allocpath, total, inuse);
} else {
c->broken = 1;
errno = ECONNRESET;
result = -1;
}
}
return result;
}
struct nvpair *catalog_query_read(struct catalog_query *q, time_t stoptime)
{
struct nvpair *nv = nvpair_create();
char line[65536];
int lines = 0;
while(link_readline(q->link, line, sizeof(line), stoptime)) {
string_chomp(line);
if(!line[0])
break;
nvpair_parse(nv, line);
lines++;
}
if(lines) {
return nv;
} else {
nvpair_delete(nv);
return 0;
}
}
static INT64_T get_statfs_result(struct chirp_client *c, struct chirp_statfs *info, time_t stoptime)
{
char line[CHIRP_LINE_MAX];
INT64_T fields;
memset(info, 0, sizeof(*info));
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime)) {
c->broken = 1;
errno = ECONNRESET;
return -1;
}
fields = sscanf(line, "%" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 " %" SCNd64 "\n", &info->f_type, &info->f_bsize, &info->f_blocks, &info->f_bfree, &info->f_bavail, &info->f_files, &info->f_ffree);
if(fields != 7) {
c->broken = 1;
errno = ECONNRESET;
return -1;
}
return 0;
}
int s3_getacl(char* bucketname, char* filename, char* owner, struct hash_table* acls, const char* access_key_id, const char* access_key) {
struct s3_message mesg;
struct link* server;
time_t stoptime = time(0)+s3_timeout;
char path[HEADER_LINE_MAX];
char response[HEADER_LINE_MAX];
char * text;
char * start;
char * temp;
int length;
if(!s3_endpoint) return -1;
if(filename) sprintf(path, "%s?acl", filename);
else sprintf(path, "/?acl");
mesg.type = S3_MESG_GET;
mesg.path = path;
mesg.bucket = bucketname;
mesg.content_type = NULL;
mesg.content_md5 = NULL;
mesg.content_length = 0;
mesg.date = time(0);
mesg.expect = 0;
mesg.amz_headers = NULL;
//server = link_connect(s3_address, 80, stoptime);
sign_message(&mesg, access_key_id, access_key);
server = s3_send_message(&mesg, NULL, stoptime);
if(!server)
return -1;
//length = s3_message_to_string(&mesg, &text);
//link_putlstring(server, text, length, stoptime);
//free(text);
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(strcmp(response, "HTTP/1.1 200 OK")) {
// Error: transfer failed; close connection and return failure
//fprintf(stderr, "Error: request file failed\nResponse: %s\n", response);
link_close(server);
return -1;
}
do {
if(!strncmp(response, "Content-Length:", 14)) sscanf(response, "Content-Length: %d", &length);
if(!strcmp(response, "Transfer-Encoding: chunked")) length = 0;
if(!strcmp(response, "Server: AmazonS3")) break;
} while(link_readline(server, response, HEADER_LINE_MAX, stoptime));
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(length) {
text = malloc(length+1);
link_read(server, text, length, stoptime);
} else {
struct list *buf;
char *temp;
unsigned int clen = 0;
buf = list_create();
do {
link_readline(server, response, HEADER_LINE_MAX, stoptime);
sscanf(response, "%x", &clen);
//link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(clen) {
text = malloc(clen+1);
link_read(server, text, clen, stoptime);
link_readline(server, response, HEADER_LINE_MAX, stoptime);
list_push_tail(buf, text);
length += clen;
}
} while(clen);
text = malloc(length+1);
text[0] = '\0';
while((temp = list_pop_head(buf))) {
sprintf(text, "%s%s", text, temp);
free(temp);
}
list_delete(buf);
}
link_close(server);
if(owner) sscanf(strstr(text, "<Owner>"), "<Owner><ID>%[^<]</ID>", owner);
temp = text;
while( (start = strstr(temp, "<Grant>")) ) {
char id[1024];
char display_name[1024];
char permission[1024];
char type;
struct s3_acl_object *acl;
char *end;
end = strstr(start, "</Grant>");
end[7] = '\0';
temp = end + 8;
memset(display_name, 0, 1024);
type = S3_ACL_ID;
if( sscanf(start, "<Grant><Grantee %*[^>]><ID>%[^<]</ID><DisplayName>%[^<]</DisplayName></Grantee><Permission>%[^<]</Permission></Grantee>", id, display_name, permission) != 3 ) {
type = S3_ACL_URI;
sscanf(start, "<Grant><Grantee %*[^>]><URI>http://acs.amazonaws.com/groups/global/%[^<]</URI></Grantee><Permission>%[^<]</Permission></Grantee>", id, permission);
}
if( !(acl = hash_table_lookup(acls, id)) ) {
acl = malloc(sizeof(*acl));
acl->acl_type = type;
if(*display_name) acl->display_name = strdup(display_name);
else acl->display_name = NULL;
acl->perm = 0;
hash_table_insert(acls, id, acl);
}
if(!strcmp(permission, "FULL_CONTROL")) {
acl->perm = acl->perm | S3_ACL_FULL_CONTROL;
} else if(!strcmp(permission, "READ")) {
acl->perm = acl->perm | S3_ACL_READ;
} else if(!strcmp(permission, "WRITE")) {
acl->perm = acl->perm | S3_ACL_WRITE;
} else if(!strcmp(permission, "READ_ACP")) {
acl->perm = acl->perm | S3_ACL_READ_ACP;
} else if(!strcmp(permission, "WRITE_ACP")) {
acl->perm = acl->perm | S3_ACL_WRITE_ACP;
}
}
free(text);
return 0;
}
// NOT IMPLEMENTED YET
int s3_setacl(char* bucketname, char *filename, const char* owner, struct hash_table* acls, const char* access_key_id, const char* access_key) {
struct s3_message mesg;
struct link* server;
time_t stoptime = time(0)+s3_timeout;
char path[HEADER_LINE_MAX];
char response[HEADER_LINE_MAX];
//char * text;
//int length;
char *id;
struct s3_acl_object *acl;
if(!s3_endpoint) return -1;
if(filename) sprintf(path, "%s?acl", filename);
else
sprintf(path, "/?acl");
mesg.content_length = 39 + 32 + strlen(owner) + 32;
hash_table_firstkey(acls);
while(hash_table_nextkey(acls, &id, (void**)&acl)) {
int glength;
switch(acl->acl_type) {
case S3_ACL_URI:
glength = 140+strlen(id);
break;
case S3_ACL_EMAIL:
glength = 135+strlen(id);
break;
default:
glength = 107+strlen(id);
}
if(acl->perm & S3_ACL_FULL_CONTROL) mesg.content_length += 40 + glength + 12;
if(acl->perm & S3_ACL_READ) mesg.content_length += 40 + glength + 4;
if(acl->perm & S3_ACL_WRITE) mesg.content_length += 40 + glength + 5;
if(acl->perm & S3_ACL_READ_ACP) mesg.content_length += 40 + glength + 8;
if(acl->perm & S3_ACL_WRITE_ACP) mesg.content_length += 40 + glength + 9;
}
mesg.content_length += 43;
mesg.type = S3_MESG_PUT;
mesg.path = path;
mesg.bucket = bucketname;
mesg.content_type = NULL;
mesg.content_md5 = NULL;
mesg.date = time(0);
mesg.expect = 0;
mesg.amz_headers = NULL;
//server = link_connect(s3_address, 80, stoptime);
sign_message(&mesg, access_key_id, access_key);
server = s3_send_message(&mesg, NULL, stoptime);
if(!server)
return -1;
//length = s3_message_to_string(&mesg, &text);
//fprintf(stderr, "Message:\n%s\n", text);
//link_putlstring(server, text, length, stoptime);
//free(text);
link_putliteral(server, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n", stoptime);
link_putliteral(server, "<AccessControlPolicy><Owner><ID>", stoptime);
link_putstring(server, owner, stoptime);
link_putliteral(server, "</ID></Owner><AccessControlList>", stoptime);
hash_table_firstkey(acls);
while(hash_table_nextkey(acls, &id, (void**)&acl)) {
char grantee[HEADER_LINE_MAX];
switch(acl->acl_type) {
case S3_ACL_URI:
sprintf(grantee, "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"Group\"><URI>http://acs.amazonaws.com/groups/global/%s</URI></Grantee>", id);
break;
case S3_ACL_EMAIL:
sprintf(grantee, "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"AmazonCustomerByEmail\"><EmailAddress>%s</EmailAddress></Grantee>", id);
break;
default:
sprintf(grantee, "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"CanonicalUser\"><ID>%s</ID></Grantee>", id);
}
if(acl->perm & S3_ACL_FULL_CONTROL) {
link_putfstring(server, "<Grant>%s<Permission>FULL_CONTROL</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_READ) {
link_putfstring(server, "<Grant>%s<Permission>READ</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_WRITE) {
link_putfstring(server, "<Grant>%s<Permission>WRITE</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_READ_ACP) {
link_putfstring(server, "<Grant>%s<Permission>READ_ACP</Permission></Grant>", stoptime, grantee);
}
if(acl->perm & S3_ACL_WRITE_ACP) {
link_putfstring(server, "<Grant>%s<Permission>WRITE_ACP</Permission></Grant>", stoptime, grantee);
}
}
link_putliteral(server, "</AccessControlList></AccessControlPolicy>\n", stoptime);
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(strcmp(response, "HTTP/1.1 200 OK")) {
// Error: transfer failed; close connection and return failure
fprintf(stderr, "Error: send file failed\nResponse: %s\n", response);
link_close(server);
return -1;
}
// fprintf(stderr, "Response:\n");
do {
// fprintf(stderr, "\t%s\n", response);
if(!strcmp(response, "Server: AmazonS3")) break;
} while(link_readline(server, response, HEADER_LINE_MAX, stoptime));
link_close(server);
return 0;
}
static void handle_query(struct link *query_link)
{
FILE *stream;
char line[LINE_MAX];
char url[LINE_MAX];
char path[LINE_MAX];
char action[LINE_MAX];
char version[LINE_MAX];
char hostport[LINE_MAX];
char addr[LINK_ADDRESS_MAX];
char key[LINE_MAX];
int port;
time_t current;
char *hkey;
struct nvpair *nv;
int i, n;
link_address_remote(query_link, addr, &port);
debug(D_DEBUG, "www query from %s:%d", addr, port);
if(link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) {
string_chomp(line);
if(sscanf(line, "%s %s %s", action, url, version) != 3) {
return;
}
// Consume the rest of the query
while(1) {
if(!link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) {
return;
}
if(line[0] == 0) {
break;
}
}
} else {
return;
}
// Output response
stream = fdopen(link_fd(query_link), "w");
if(!stream) {
return;
}
link_nonblocking(query_link, 0);
current = time(0);
fprintf(stream, "HTTP/1.1 200 OK\n");
fprintf(stream, "Date: %s", ctime(¤t));
fprintf(stream, "Server: catalog_server\n");
fprintf(stream, "Connection: close\n");
if(sscanf(url, "http://%[^/]%s", hostport, path) == 2) {
// continue on
} else {
strcpy(path, url);
}
/* load the hash table entries into one big array */
n = 0;
nvpair_database_firstkey(table);
while(nvpair_database_nextkey(table, &hkey, &nv)) {
array[n] = nv;
n++;
}
/* sort the array by name before displaying */
qsort(array, n, sizeof(struct nvpair *), compare_nvpair);
if(!strcmp(path, "/query.text")) {
fprintf(stream, "Content-type: text/plain\n\n");
for(i = 0; i < n; i++)
nvpair_print_text(array[i], stream);
} else if(!strcmp(path, "/query.json")) {
fprintf(stream, "Content-type: text/plain\n\n");
fprintf(stream,"[\n");
for(i = 0; i < n; i++) {
nvpair_print_json(array[i], stream);
fprintf(stream,",\n");
}
fprintf(stream,"]\n");
} else if(!strcmp(path, "/query.oldclassads")) {
fprintf(stream, "Content-type: text/plain\n\n");
for(i = 0; i < n; i++)
nvpair_print_old_classads(array[i], stream);
} else if(!strcmp(path, "/query.newclassads")) {
fprintf(stream, "Content-type: text/plain\n\n");
for(i = 0; i < n; i++)
nvpair_print_new_classads(array[i], stream);
} else if(!strcmp(path, "/query.xml")) {
fprintf(stream, "Content-type: text/xml\n\n");
fprintf(stream, "<?xml version=\"1.0\" standalone=\"yes\"?>\n");
fprintf(stream, "<catalog>\n");
for(i = 0; i < n; i++)
nvpair_print_xml(array[i], stream);
fprintf(stream, "</catalog>\n");
} else if(sscanf(path, "/detail/%s", key) == 1) {
struct nvpair *nv;
fprintf(stream, "Content-type: text/html\n\n");
nv = nvpair_database_lookup(table, key);
if(nv) {
const char *name = nvpair_lookup_string(nv, "name");
if(!name)
name = "unknown";
fprintf(stream, "<title>%s catalog server: %s</title>\n", preferred_hostname, name);
fprintf(stream, "<center>\n");
fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
fprintf(stream, "<h2>%s</h2>\n", name);
fprintf(stream, "<p><a href=/>return to catalog view</a><p>\n");
nvpair_print_html_solo(nv, stream);
fprintf(stream, "</center>\n");
} else {
fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname);
fprintf(stream, "<center>\n");
fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
fprintf(stream, "<h2>Unknown Item!</h2>\n");
fprintf(stream, "</center>\n");
}
} else {
char avail_line[LINE_MAX];
char total_line[LINE_MAX];
INT64_T sum_total = 0;
INT64_T sum_avail = 0;
INT64_T sum_devices = 0;
fprintf(stream, "Content-type: text/html\n\n");
fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname);
fprintf(stream, "<center>\n");
fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
fprintf(stream, "<a href=/query.text>text</a> - ");
fprintf(stream, "<a href=/query.html>html</a> - ");
fprintf(stream, "<a href=/query.xml>xml</a> - ");
fprintf(stream, "<a href=/query.json>json</a> - ");
fprintf(stream, "<a href=/query.oldclassads>oldclassads</a> - ");
fprintf(stream, "<a href=/query.newclassads>newclassads</a>");
fprintf(stream, "<p>\n");
for(i = 0; i < n; i++) {
nv = array[i];
sum_total += nvpair_lookup_integer(nv, "total");
sum_avail += nvpair_lookup_integer(nv, "avail");
sum_devices++;
}
string_metric(sum_avail, -1, avail_line);
string_metric(sum_total, -1, total_line);
fprintf(stream, "<b>%sB available out of %sB on %d devices</b><p>\n", avail_line, total_line, (int) sum_devices);
nvpair_print_html_header(stream, html_headers);
for(i = 0; i < n; i++) {
nv = array[i];
make_hash_key(nv, key);
sprintf(url, "/detail/%s", key);
nvpair_print_html_with_link(nv, stream, html_headers, "name", url);
}
nvpair_print_html_footer(stream, html_headers);
fprintf(stream, "</center>\n");
}
fclose(stream);
}
struct link *http_query_size_via_proxy(const char *proxy, const char *urlin, const char *action, INT64_T * size, time_t stoptime, int cache_reload)
{
char url[HTTP_LINE_MAX];
char newurl[HTTP_LINE_MAX];
char line[HTTP_LINE_MAX];
char addr[LINK_ADDRESS_MAX];
struct link *link;
int save_errno;
int response;
char actual_host[HTTP_LINE_MAX];
int actual_port;
*size = 0;
url_encode(urlin, url, sizeof(url));
if(proxy && !strcmp(proxy, "DIRECT"))
proxy = 0;
if(proxy) {
int fields = sscanf(proxy, "http://%[^:]:%d", actual_host, &actual_port);
if(fields == 2) {
/* host and port are good */
} else if(fields == 1) {
actual_port = HTTP_PORT;
} else {
debug(D_HTTP, "invalid proxy syntax: %s", proxy);
return 0;
}
} else {
int fields = sscanf(url, "http://%[^:]:%d", actual_host, &actual_port);
if(fields != 2) {
fields = sscanf(url, "http://%[^/]", actual_host);
if(fields == 1) {
actual_port = HTTP_PORT;
} else {
debug(D_HTTP, "malformed url: %s", url);
return 0;
}
}
}
debug(D_HTTP, "connect %s port %d", actual_host, actual_port);
if(!domain_name_lookup(actual_host, addr))
return 0;
link = link_connect(addr, actual_port, stoptime);
if(!link) {
errno = ECONNRESET;
return 0;
}
if(cache_reload == 0) {
debug(D_HTTP, "%s %s HTTP/1.0\r\nHost: %s\r\nConnection: close\r\n\r\n", action, url, actual_host);
link_putfstring(link, "%s %s HTTP/1.0\r\nHost: %s\r\nConnection: close\r\n\r\n", stoptime, action, url, actual_host);
} else {
// force refresh of cache end-to-end (RFC 2616)
debug(D_HTTP, "%s %s HTTP/1.1\r\nHost: %s\r\nCache-Control: max-age=0\r\nConnection: close\r\n\r\n", action, url, actual_host);
link_putfstring(link, "%s %s HTTP/1.1\r\nHost: %s\r\nCache-Control: max-age=0\r\nConnection: close\r\n\r\n", stoptime, action, url, actual_host);
}
if(link_readline(link, line, HTTP_LINE_MAX, stoptime)) {
string_chomp(line);
debug(D_HTTP, "%s", line);
if(sscanf(line, "HTTP/%*d.%*d %d", &response) == 1) {
newurl[0] = 0;
while(link_readline(link, line, HTTP_LINE_MAX, stoptime)) {
string_chomp(line);
debug(D_HTTP, "%s", line);
sscanf(line, "Location: %s", newurl);
sscanf(line, "Content-Length: %lld", size);
if(strlen(line) <= 2) {
break;
}
}
switch (response) {
case 200:
return link;
break;
case 301:
case 302:
case 303:
case 307:
link_close(link);
if(newurl[0]) {
if(!strcmp(url, newurl)) {
debug(D_HTTP, "error: server gave %d redirect from %s back to the same url!", response, url);
errno = EIO;
return 0;
} else {
return http_query(newurl, action, stoptime);
}
} else {
errno = ENOENT;
return 0;
}
break;
default:
link_close(link);
errno = http_response_to_errno(response);
return 0;
break;
}
} else {
debug(D_HTTP, "malformed response");
save_errno = ECONNRESET;
}
} else {
debug(D_HTTP, "malformed response");
save_errno = ECONNRESET;
}
link_close(link);
errno = save_errno;
return 0;
}
static int auth_unix_accept(struct link *link, char **subject, time_t stoptime)
{
char path[AUTH_LINE_MAX];
char line[AUTH_LINE_MAX];
int success = 0;
struct stat buf;
struct passwd *p;
debug(D_AUTH, "unix: generating challenge");
make_challenge_path(path);
link_putfstring(link, "%s\n", stoptime, path);
debug(D_AUTH, "unix: waiting for response");
if(link_readline(link, line, sizeof(line), stoptime)) {
if(!strcmp(line, "yes")) {
int file_exists = 0;
int i=0;
for(i=0;i<challenge_timeout;i++) {
/*
This is an odd hack, but invoking ls -la appears to help to force
some NFS clients to refresh cached metadata.
*/
DIR *d = opendir(challenge_dir);
if(d) {
closedir(d);
}
if(stat(path,&buf)==0) {
file_exists = 1;
break;
} else {
debug(D_AUTH,"unix: client claims success, but I don't see it yet...");
sleep(1);
}
}
if(file_exists) {
debug(D_AUTH, "unix: got response");
debug(D_AUTH, "unix: client is uid %d", buf.st_uid);
p = auth_get_passwd_from_uid(buf.st_uid);
if(p) {
debug(D_AUTH, "unix: client is subject %s", p->pw_name);
link_putliteral(link, "yes\n", stoptime);
*subject = xxstrdup(p->pw_name);
success = 1;
} else {
debug(D_AUTH, "unix: there is no user corresponding to uid %d", buf.st_uid);
link_putliteral(link, "no\n", stoptime);
}
} else {
debug(D_AUTH, "unix: client failed the challenge: %s", strerror(errno));
link_putliteral(link, "no\n", stoptime);
}
} else {
debug(D_AUTH, "unix: client declined the challenge");
}
}
unlink(path);
return success;
}
INT64_T chirp_client_ticket_get(struct chirp_client * c, const char *name, char **subject, char **ticket, time_t * duration, char ***rights, time_t stoptime)
{
INT64_T result;
char ticket_subject[CHIRP_LINE_MAX];
*subject = *ticket = NULL;
*rights = NULL;
ticket_translate(name, ticket_subject);
result = simple_command(c, stoptime, "ticket_get %s\n", ticket_subject);
if(result == 0) {
char line[CHIRP_LINE_MAX];
size_t length;
size_t nrights = 0;
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime))
goto failure;
if(sscanf(line, "%zu", &length) != 1)
goto failure;
*subject = xxmalloc((length + 1) * sizeof(char));
if(!link_read(c->link, *subject, length, stoptime))
goto failure;
(*subject)[length] = '\0';
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime))
goto failure;
if(sscanf(line, "%zu", &length) != 1)
goto failure;
*ticket = xxmalloc((length + 1) * sizeof(char));
if(!link_read(c->link, *ticket, length, stoptime))
goto failure;
(*ticket)[length] = '\0';
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime))
goto failure;
unsigned long long tmp;
if(sscanf(line, "%llu", &tmp) != 1)
goto failure;
*duration = (time_t) tmp;
while(1) {
char path[CHIRP_PATH_MAX];
char acl[CHIRP_LINE_MAX];
if(!link_readline(c->link, line, CHIRP_LINE_MAX, stoptime))
goto failure;
if(sscanf(line, "%s %s", path, acl) == 2) {
*rights = xxrealloc(*rights, sizeof(char *) * 2 * (nrights + 2));
(*rights)[nrights * 2 + 0] = xxstrdup(path);
(*rights)[nrights * 2 + 1] = xxstrdup(acl);
(*rights)[nrights * 2 + 2] = NULL;
(*rights)[nrights * 2 + 3] = NULL;
nrights++;
} else if(sscanf(line, "%" SCNd64 , &result) == 1 && result == 0) {
break;
} else
goto failure;
}
return 0;
failure:
free(*subject);
free(*ticket);
if(*rights != NULL) {
char **tmp = *rights;
while(tmp[0] && tmp[1]) {
free(tmp[0]);
free(tmp[1]);
}
free(*rights);
}
*subject = *ticket = NULL;
c->broken = 1;
errno = ECONNRESET;
return -1;
}
return result;
}
int master_main(const char *host, int port, const char *addr) {
time_t idle_stoptime;
struct link *master = NULL;
int num_workers, i;
struct mpi_queue_job **workers;
struct itable *active_jobs = itable_create(0);
struct itable *waiting_jobs = itable_create(0);
struct list *complete_jobs = list_create();
MPI_Comm_size(MPI_COMM_WORLD, &num_workers);
workers = malloc(num_workers * sizeof(*workers));
memset(workers, 0, num_workers * sizeof(*workers));
idle_stoptime = time(0) + idle_timeout;
while(!abort_flag) {
char line[MPI_QUEUE_LINE_MAX];
if(time(0) > idle_stoptime) {
if(master) {
printf("mpi master: gave up after waiting %ds to receive a task.\n", idle_timeout);
} else {
printf("mpi master: gave up after waiting %ds to connect to %s port %d.\n", idle_timeout, host, port);
}
break;
}
if(!master) {
char working_dir[MPI_QUEUE_LINE_MAX];
master = link_connect(addr, port, idle_stoptime);
if(!master) {
sleep(5);
continue;
}
link_tune(master, LINK_TUNE_INTERACTIVE);
link_readline(master, line, sizeof(line), time(0) + active_timeout);
memset(working_dir, 0, MPI_QUEUE_LINE_MAX);
if(sscanf(line, "workdir %s", working_dir) == 1) {
MPI_Bcast(working_dir, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
master = NULL;
continue;
}
}
if(link_readline(master, line, sizeof(line), time(0) + short_timeout)) {
struct mpi_queue_operation *op;
int jobid, mode;
INT64_T length;
char path[MPI_QUEUE_LINE_MAX];
op = NULL;
debug(D_MPI, "received: %s\n", line);
if(!strcmp(line, "get results")) {
struct mpi_queue_job *job;
debug(D_MPI, "results requested: %d available\n", list_size(complete_jobs));
link_putfstring(master, "num results %d\n", time(0) + active_timeout, list_size(complete_jobs));
while(list_size(complete_jobs)) {
job = list_pop_head(complete_jobs);
link_putfstring(master, "result %d %d %d %lld\n", time(0) + active_timeout, job->jobid, job->status, job->result, job->output_length);
if(job->output_length) {
link_write(master, job->output, job->output_length, time(0)+active_timeout);
}
mpi_queue_job_delete(job);
}
} else if(sscanf(line, "work %d %lld", &jobid, &length)) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_WORK;
op->buffer_length = length+1;
op->buffer = malloc(length+1);
op->buffer[op->buffer_length] = 0;
link_read(master, op->buffer, length, time(0) + active_timeout);
op->result = -1;
} else if(sscanf(line, "stat %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_STAT;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "unlink %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_UNLINK;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "mkdir %d %s %o", &jobid, path, &mode) == 3) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_MKDIR;
sprintf(op->args, "%s %o", path, mode);
op->result = -1;
} else if(sscanf(line, "close %d", &jobid) == 1) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_CLOSE;
op->result = -1;
// } else if(sscanf(line, "symlink %d %s %s", &jobid, path, filename) == 3) {
// } else if(sscanf(line, "put %d %s %lld %o", &jobid, filename, &length, &mode) == 4) {
// } else if(sscanf(line, "rget %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "get %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "thirdget %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
// } else if(sscanf(line, "thirdput %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
} else if(!strcmp(line, "exit")) {
break;
} else {
abort_flag = 1;
continue;
}
if(op) {
struct mpi_queue_job *job;
job = itable_lookup(active_jobs, jobid);
if(!job) {
job = itable_lookup(waiting_jobs, jobid);
}
if(!job) {
job = malloc(sizeof(*job));
memset(job, 0, sizeof(*job));
job->jobid = jobid;
job->operations = list_create();
job->status = MPI_QUEUE_JOB_WAITING;
job->worker_rank = -1;
itable_insert(waiting_jobs, jobid, job);
}
list_push_tail(job->operations, op);
}
idle_stoptime = time(0) + idle_timeout;
} else {
link_close(master);
master = 0;
sleep(5);
}
int num_waiting_jobs = itable_size(waiting_jobs);
int num_unvisited_jobs = itable_size(active_jobs);
for(i = 1; i < num_workers && (num_unvisited_jobs > 0 || num_waiting_jobs > 0); i++) {
struct mpi_queue_job *job;
struct mpi_queue_operation *op;
int flag = 0;
UINT64_T jobid;
if(!workers[i]) {
if(num_waiting_jobs) {
itable_firstkey(waiting_jobs);
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
itable_insert(active_jobs, jobid, job);
workers[i] = job;
num_waiting_jobs--;
job->worker_rank = i;
job->status = MPI_QUEUE_JOB_READY;
} else {
continue;
}
} else {
num_unvisited_jobs--;
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Test(&workers[i]->request, &flag, &workers[i]->mpi_status);
if(flag) {
op = list_pop_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
workers[i]->status = MPI_QUEUE_JOB_READY;
if(op->type == MPI_QUEUE_OP_WORK || op->result < 0) {
if(workers[i]->output)
free(workers[i]->output);
workers[i]->output = op->output_buffer;
op->output_buffer = NULL;
workers[i]->output_length = op->output_length;
workers[i]->result = op->result;
if(op->result < 0) {
workers[i]->status = MPI_QUEUE_JOB_FAILED | op->type;
op->type = MPI_QUEUE_OP_CLOSE;
list_push_head(workers[i]->operations, op);
op = NULL;
}
}
if(op) {
if(op->buffer)
free(op->buffer);
if(op->output_buffer)
free(op->output_buffer);
free(op);
}
}
}
}
if( workers[i]->status != MPI_QUEUE_JOB_BUSY && list_size(workers[i]->operations)) {
op = list_peek_head(workers[i]->operations);
if(op->type == MPI_QUEUE_OP_CLOSE) {
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
if(!(workers[i]->status & MPI_QUEUE_JOB_FAILED))
workers[i]->status = MPI_QUEUE_JOB_COMPLETE;
workers[i] = NULL;
i--;
continue;
}
MPI_Send(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
if(op->buffer_length) {
MPI_Send(op->buffer, op->buffer_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
free(op->buffer);
op->buffer_length = 0;
op->buffer = NULL;
}
MPI_Irecv(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->request);
workers[i]->status = MPI_QUEUE_JOB_BUSY;
}
}
}
/** Clean up waiting & complete jobs, send Exit commands to each worker */
if(!master) {
// If the master link hasn't been set up yet
// the workers will be waiting for the working directory
char line[MPI_QUEUE_LINE_MAX];
memset(line, 0, MPI_QUEUE_LINE_MAX);
MPI_Bcast(line, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
}
for(i = 1; i < num_workers; i++) {
struct mpi_queue_operation *op, close;
memset(&close, 0, sizeof(close));
close.type = MPI_QUEUE_OP_EXIT;
if(workers[i]) {
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Wait(&workers[i]->request, &workers[i]->mpi_status);
op = list_peek_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
}
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
}
MPI_Send(&close, sizeof(close), MPI_BYTE, i, 0, MPI_COMM_WORLD);
}
itable_firstkey(waiting_jobs);
while(itable_size(waiting_jobs)) {
struct mpi_queue_job *job;
UINT64_T jobid;
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
list_push_tail(complete_jobs, job);
}
while(list_size(complete_jobs)) {
mpi_queue_job_delete(list_pop_head(complete_jobs));
}
MPI_Finalize();
return abort_flag;
}
static int auth_ticket_accept(struct link *link, char **subject, time_t stoptime)
{
int serrno = errno;
int status = 0;
char line[AUTH_LINE_MAX];
char ticket_subject[AUTH_LINE_MAX];
char *ticket = NULL;
errno = 0;
debug(D_AUTH, "ticket: waiting for tickets");
while(1) {
if(link_readline(link, line, sizeof(line), stoptime) > 0) {
if(strcmp(line, "==") == 0) {
debug(D_AUTH, "ticket: exhausted all ticket challenges");
break;
} else if(strlen(line) == DIGEST_LENGTH) {
char ticket_digest[DIGEST_LENGTH + 1];
strcpy(ticket_digest, line);
strcpy(ticket_subject, line);
debug(D_AUTH, "ticket: read ticket digest: %s", ticket_digest);
if(server_callback) {
free(ticket); /* free previously allocated ticket string or NULL (noop) */
ticket = server_callback(ticket_digest);
if(ticket) {
static const char command_template[] = "T1=`mktemp`\n" /* The RSA Public Key */
"T2=`mktemp`\n" /* The Challenge */
"T3=`mktemp`\n" /* The Signed Challenge */
"T4=`mktemp`\n" /* The Decrypted (verified) Signed Challenge */
"echo -n '%s' > \"$T1\"\n" "dd if=/dev/urandom of=\"$T2\" bs=%u count=1 > /dev/null 2> /dev/null\n" "cat \"$T2\"\n" /* to stdout */
"cat > \"$T3\"\n" /* from stdin */
"openssl rsautl -inkey \"$T1\" -pubin -verify < \"$T3\" > \"$T4\" 2> /dev/null\n" "cmp \"$T2\" \"$T4\" > /dev/null 2> /dev/null\n" "R=\"$?\"\n"
"rm -f \"$T1\" \"$T2\" \"$T3\" \"$T4\" > /dev/null 2> /dev/null\n" "exit \"$R\"\n";
char *command = xxmalloc(sizeof(command_template) + strlen(ticket) + 64);
sprintf(command, command_template, ticket, CHALLENGE_LENGTH);
FILE *in, *out;
pid_t pid = dpopen(command, &in, &out);
free(command);
if(pid == 0)
break;
if(!link_putfstring(link, "%zu\n", stoptime, CHALLENGE_LENGTH))
break;
if(!link_stream_from_file(link, out, CHALLENGE_LENGTH, stoptime))
break;
if(link_readline(link, line, sizeof(line), stoptime) <= 0)
break;
unsigned long length = strtoul(line, NULL, 10);
if(errno == ERANGE || errno == EINVAL)
break; /* not a number? */
if(!link_stream_to_file(link, in, length, stoptime))
break;
int result = dpclose(in, out, pid);
if(result == 0) {
debug(D_AUTH, "succeeded challenge for %s\n", ticket_digest);
link_putliteral(link, "success\n", stoptime);
status = 1;
break;
} else {
debug(D_AUTH, "failed challenge for %s\n", ticket_digest);
link_putliteral(link, "failure\n", stoptime);
break;
}
} else {
debug(D_AUTH, "declining key %s", ticket_digest);
link_putliteral(link, "declined\n", stoptime);
}
} else {
debug(D_AUTH, "declining key %s", ticket_digest);
link_putliteral(link, "declined\n", stoptime);
}
} else {
debug(D_AUTH, "ticket: bad response");
break;
}
} else {
break;
}
}
if(status) {
*subject = xxmalloc(AUTH_LINE_MAX);
strcpy(*subject, ticket_subject);
}
free(ticket); /* free previously allocated ticket string or NULL (noop) */
errno = serrno;
return status;
}
static int auth_ticket_assert(struct link *link, time_t stoptime)
{
/* FIXME need to save errno ? */
char line[AUTH_LINE_MAX];
char **tickets = client_tickets;
if(tickets) {
char *ticket;
char digest[DIGEST_LENGTH];
for (ticket = *tickets; ticket; ticket = *(++tickets)) {
if (access(ticket, R_OK) == -1) {
debug(D_AUTH, "could not access ticket %s: %s", ticket, strerror(errno));
continue;
}
/* load the digest */
/* WARNING: openssl is *very* bad at giving sensible output. Use the last
* 32 non-space characters as the MD5 sum.
*/
char command[PATH_MAX * 2 + 4096];
sprintf(command, "openssl rsa -in '%s' -pubout 2> /dev/null | openssl md5 2> /dev/null | tr -d '[:space:]' | tail -c 32", ticket);
FILE *digestf = popen(command, "r");
if(full_fread(digestf, digest, DIGEST_LENGTH) < DIGEST_LENGTH) {
pclose(digestf);
return 0;
}
pclose(digestf);
debug(D_AUTH, "trying ticket %.*s", DIGEST_LENGTH, digest);
if(link_putlstring(link, digest, DIGEST_LENGTH, stoptime) <= 0)
return 0;
if(link_putliteral(link, "\n", stoptime) <= 0)
return 0;
if(link_readline(link, line, sizeof(line), stoptime) <= 0)
return 0;
if(strcmp(line, "declined") == 0)
continue;
unsigned long length = strtoul(line, NULL, 10);
if(errno == ERANGE || errno == EINVAL)
return 0; /* not a number? */
debug(D_AUTH, "receiving challenge of %d bytes", length);
FILE *in, *out;
static const char command_template[] = "T1=`mktemp`\n" /* signed challenge */
"T2=`mktemp`\n" /* private key without comments */
"sed '/^\\s*#/d' < '%s' > \"$T2\"\n" "openssl rsautl -inkey \"$T2\" -sign > \"$T1\" 2> /dev/null\n" "R=\"$?\"\n" "if [ \"$R\" -ne 0 ]; then\n" " rm -f \"$T1\" \"$T2\"\n" " exit \"$R\"\n" "fi\n"
"ls -l \"$T1\" | awk '{ print $5 }'\n" "cat \"$T1\"\n" "rm -f \"$T1\" \"$T2\"\n";
sprintf(command, command_template, ticket);
pid_t pid = dpopen(command, &in, &out);
if(pid == 0)
return 0;
if(link_stream_to_file(link, in, length, stoptime) <= 0) {
dpclose(in, out, pid);
debug(D_AUTH, "openssl failed, your keysize may be too small");
debug(D_AUTH, "please debug using \"dd if=/dev/urandom count=64 bs=1 | openssl rsautl -inkey <ticket file> -sign\"");
return 0;
}
fclose(in);
in = NULL;
if(link_stream_from_file(link, out, 1 << 20, stoptime) <= 0) {
dpclose(in, out, pid);
debug(D_AUTH, "openssl failed, your keysize may be too small");
debug(D_AUTH, "please debug using \"dd if=/dev/urandom count=64 bs=1 | openssl rsautl -inkey <ticket file> -sign\"");
return 0;
}
dpclose(in, out, pid);
if(link_readline(link, line, sizeof(line), stoptime) <= 0)
return 0;
if(strcmp(line, "success") == 0) {
debug(D_AUTH, "succeeded challenge for %.*s\n", DIGEST_LENGTH, digest);
return 1;
} else if(strcmp(line, "failure") == 0) {
debug(D_AUTH, "failed challenge for %.*s\n", DIGEST_LENGTH, digest);
errno = EINVAL;
return 0;
} else {
debug(D_AUTH, "received bad response: '%s'", line);
errno = EINVAL;
return 0;
}
}
}
link_putliteral(link, "==\n", stoptime);
return 0;
}
int s3_ls_bucket(char* bucketname, struct list* dirents, const char* access_key_id, const char* access_key) {
struct s3_message mesg;
struct link* server = NULL;
time_t stoptime = time(0)+s3_timeout;
char response[HEADER_LINE_MAX];
char path[HEADER_LINE_MAX];
int length;
char done = 0;
if(!access_key_id || !access_key || !s3_endpoint) return -1;
sprintf(path, "/");
mesg.type = S3_MESG_GET;
mesg.path = path;
mesg.bucket = bucketname;
mesg.content_type = NULL;
mesg.content_md5 = NULL;
mesg.content_length = 0;
mesg.date = time(0);
mesg.expect = 0;
mesg.amz_headers = NULL;
do {
char *buffer, *temp, *start, *end;
char trunc[25];
int keys;
sign_message(&mesg, access_key_id, access_key);
server = s3_send_message(&mesg, server, stoptime);
if(!server)
return -1;
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(strcmp(response, "HTTP/1.1 200 OK")) {
// Error: transfer failed; close connection and return failure
fprintf(stderr, "Error: list bucket failed\nResponse: %s\n", response);
link_close(server);
return -1;
}
length = 0;
do {
if(!strncmp(response, "Content-Length:", 14)) sscanf(response, "Content-Length: %d", &length);
if(!strcmp(response, "Transfer-Encoding: chunked")) length = 0;
if(!strcmp(response, "Server: AmazonS3")) break;
} while(link_readline(server, response, HEADER_LINE_MAX, stoptime));
link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(length) {
buffer = malloc(length+1);
link_read(server, buffer, length, stoptime);
} else {
struct list *buf;
unsigned int clen = 0;
buf = list_create();
do {
link_readline(server, response, HEADER_LINE_MAX, stoptime);
sscanf(response, "%x", &clen);
//link_readline(server, response, HEADER_LINE_MAX, stoptime);
if(clen) {
buffer = malloc(clen+1);
link_read(server, buffer, clen, stoptime);
link_readline(server, response, HEADER_LINE_MAX, stoptime);
list_push_tail(buf, buffer);
length += clen;
}
} while(clen);
buffer = malloc(length+1);
buffer[0] = '\0';
while((temp = list_pop_head(buf))) {
sprintf(buffer, "%s%s", buffer, temp);
free(temp);
}
list_delete(buf);
}
sscanf(buffer, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>%*[^<]</Name><Prefix></Prefix><Marker></Marker><MaxKeys>%d</MaxKeys><IsTruncated>%[^<]</IsTruncated>", &keys, trunc);
if(!strcmp(trunc, "false")) done = 1;
temp = buffer;
while( (start = strstr(temp, "<Contents>")) ) {
struct s3_dirent_object *dirent;
struct tm date;
char display_name[1024];
end = strstr(start, "</Contents>");
end[10] = '\0';
temp = end + 11;
dirent = malloc(sizeof(*dirent));
date.tm_isdst = -1;
sscanf(strstr(start, "<Key>"), "<Key>%[^<]</Key>", dirent->key);
sscanf(strstr(start, "<LastModified>"), "<LastModified>%d-%d-%dT%d:%d:%d.%*dZ</LastModified>", &date.tm_year, &date.tm_mon, &date.tm_mday, &date.tm_hour, &date.tm_min, &date.tm_sec);
sscanf(strstr(start, "<ETag>"), "<ETag>"%[^&]"</ETag>", dirent->digest);
sscanf(strstr(start, "<Size>"), "<Size>%d</Size>", &dirent->size);
sscanf(strstr(start, "<Owner>"), "<Owner><ID>%*[^<]</ID><DisplayName>%[^<]</DisplayName></Owner>", display_name);
if(strlen(display_name)) dirent->display_name = strdup(display_name);
date.tm_mon -= 1;
dirent->last_modified = mktime(&date);
list_push_tail(dirents, dirent);
}
free(buffer);
} while(!done);
link_close(server);
return 0;
}
struct link *http_query_size_via_proxy(const char *proxy, const char *urlin, const char *action, INT64_T * size, time_t stoptime, int cache_reload)
{
char url[HTTP_LINE_MAX];
char newurl[HTTP_LINE_MAX];
char line[HTTP_LINE_MAX];
char addr[LINK_ADDRESS_MAX];
struct link *link;
int save_errno;
int response;
char actual_host[HTTP_LINE_MAX];
int actual_port;
*size = 0;
url_encode(urlin, url, sizeof(url));
if(proxy && !strcmp(proxy, "DIRECT"))
proxy = 0;
if(proxy) {
int fields = sscanf(proxy, "http://%[^:]:%d", actual_host, &actual_port);
if(fields == 2) {
/* host and port are good */
} else if(fields == 1) {
actual_port = HTTP_PORT;
} else {
debug(D_HTTP, "invalid proxy syntax: %s", proxy);
return 0;
}
} else {
int fields = sscanf(url, "http://%[^:]:%d", actual_host, &actual_port);
if(fields != 2) {
fields = sscanf(url, "http://%[^/]", actual_host);
if(fields == 1) {
actual_port = HTTP_PORT;
} else {
debug(D_HTTP, "malformed url: %s", url);
return 0;
}
}
}
debug(D_HTTP, "connect %s port %d", actual_host, actual_port);
if(!domain_name_lookup(actual_host, addr))
return 0;
link = link_connect(addr, actual_port, stoptime);
if(!link) {
errno = ECONNRESET;
return 0;
}
{
buffer_t B;
buffer_init(&B);
buffer_abortonfailure(&B, 1);
buffer_printf(&B, "%s %s HTTP/1.1\r\n", action, url);
if(cache_reload)
buffer_putliteral(&B, "Cache-Control: max-age=0\r\n");
buffer_putliteral(&B, "Connection: close\r\n");
buffer_printf(&B, "Host: %s\r\n", actual_host);
if(getenv("HTTP_USER_AGENT"))
buffer_printf(&B, "User-Agent: Mozilla/5.0 (compatible; CCTools %d.%d.%s Parrot; http://www.nd.edu/~ccl/ %s)\r\n", CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO, getenv("HTTP_USER_AGENT"));
else
buffer_printf(&B, "User-Agent: Mozilla/5.0 (compatible; CCTools %d.%d.%s Parrot; http://www.nd.edu/~ccl/)\r\n", CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO);
buffer_putliteral(&B, "\r\n"); /* header terminator */
debug(D_HTTP, "%s", buffer_tostring(&B, NULL));
link_putstring(link, buffer_tostring(&B, NULL), stoptime);
buffer_free(&B);
}
if(link_readline(link, line, HTTP_LINE_MAX, stoptime)) {
string_chomp(line);
debug(D_HTTP, "%s", line);
if(sscanf(line, "HTTP/%*d.%*d %d", &response) == 1) {
newurl[0] = 0;
while(link_readline(link, line, HTTP_LINE_MAX, stoptime)) {
string_chomp(line);
debug(D_HTTP, "%s", line);
sscanf(line, "Location: %s", newurl);
sscanf(line, "Content-Length: %" SCNd64, size);
if(strlen(line) <= 2) {
break;
}
}
switch (response) {
case 200:
return link;
break;
case 301:
case 302:
case 303:
case 307:
link_close(link);
if(newurl[0]) {
if(!strcmp(url, newurl)) {
debug(D_HTTP, "error: server gave %d redirect from %s back to the same url!", response, url);
errno = EIO;
return 0;
} else {
return http_query_size_via_proxy(proxy,newurl,action,size,stoptime,cache_reload);
}
} else {
errno = ENOENT;
return 0;
}
break;
default:
link_close(link);
errno = http_response_to_errno(response);
return 0;
break;
}
} else {
debug(D_HTTP, "malformed response");
save_errno = ECONNRESET;
}
} else {
debug(D_HTTP, "malformed response");
save_errno = ECONNRESET;
}
link_close(link);
errno = save_errno;
return 0;
}
int main(int argc, char *argv[])
{
struct link *link, *master;
char *subject = 0, *type = 0;
time_t stoptime;
char line[1024];
signed char c;
int portnum = 30000;
char *hostname = 0;
int timeout = 30;
debug_config(argv[0]);
static struct option long_options[] = {
{"auth", required_argument, 0, 'a'},
{"port", required_argument, 0, 'p'},
{"host", required_argument, 0, 'r'},
{"help", required_argument, 0, 'h'},
{"debug-file", required_argument, 0, 'o'},
{"debug-rotate-max", required_argument, 0, 'O'},
{"debug", required_argument, 0, 'd'},
{0,0,0,0}
};
while((c = getopt_long(argc, argv, "a:p:r:d:o:O:", long_options, NULL)) > -1) {
switch (c) {
case 'p':
portnum = atoi(optarg);
break;
case 'h':
show_help(argv[0]);
exit(0);
break;
case 'r':
hostname = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'a':
if(!auth_register_byname(optarg))
fatal("couldn't register %s authentication", optarg);
break;
default:
show_use(argv[0]);
exit(1);
break;
}
}
if(hostname) {
char addr[LINK_ADDRESS_MAX];
stoptime = time(0) + timeout;
if(!domain_name_cache_lookup(hostname, addr))
fatal("unknown host name: %s", hostname);
link = link_connect(addr, portnum, stoptime);
if(!link)
fatal("couldn't connect to %s:%d: %s", hostname, portnum, strerror(errno));
if(auth_assert(link, &type, &subject, stoptime)) {
printf("server thinks I am %s %s\n", type, subject);
if(link_readline(link, line, sizeof(line), stoptime)) {
printf("got message: %s\n", line);
} else {
printf("lost connection!\n");
}
} else {
printf("unable to authenticate.\n");
}
link_close(link);
} else {
stoptime = time(0) + timeout;
master = link_serve(portnum);
if(!master)
fatal("couldn't serve port %d: %s\n", portnum, strerror(errno));
while(time(0) < stoptime) {
link = link_accept(master, stoptime);
if(!link)
continue;
if(auth_accept(link, &type, &subject, stoptime)) {
time_t t = time(0);
link_putfstring(link, "Hello %s:%s, it is now %s", stoptime, type, subject, ctime(&t)); /* ctime ends with newline */
} else {
printf("couldn't auth accept\n");
}
link_close(link);
}
}
return 0;
}
