/* fprintf and fputs should not be put into the following function. */ ssize_t __write(int filedes, const void *buf, size_t nbyte) { GFS_File gf; char *e; int n; /* * DO NOT put the following line here. This causes infinite loop! * * _gfs_hook_debug_v(fprintf(stderr, "Hooking __write(%d, , %lu)\n", * filedes, (unsigned long)nbyte)); */ if ((gf = gfs_hook_is_open(filedes)) == NULL) return (syscall(SYS_write, filedes, buf, nbyte)); if (gfs_hook_gfs_file_type(filedes) == GFS_DT_DIR) { /* * DO NOT put the following line here, which results * in infinite loop. * * _gfs_hook_debug(fprintf(stderr, * "GFS: Hooking __write(%d, , %d)\n", * filedes, nbyte)); */ e = GFARM_ERR_IS_A_DIRECTORY; goto error; } _gfs_hook_debug(fprintf(stderr, "GFS: Hooking __write(%d(%d), , %lu)\n", filedes, gfs_pio_fileno(gf), (unsigned long)nbyte)); e = gfs_pio_write(gf, buf, nbyte, &n); if (e == NULL) { _gfs_hook_debug_v(fprintf(stderr, "GFS: Hooking __write --> %d\n", n)); return (n); } error: /* * DO NOT put the following line here. * * _gfs_hook_debug(fprintf(stderr, "GFS: __write: %s\n", e)); */ errno = gfarm_error_to_errno(e); return (-1); }
static char * gfs_pio_view_global_write(GFS_File gf, const char *buffer, size_t size, size_t *lengthp) { struct gfs_file_global_context *gc = gf->view_context; char *e = gfs_pio_view_global_adjust(gf, buffer, &size); int length; /* XXX - should be size_t */ if (e != NULL) return (e); e = gfs_pio_write(gc->fragment_gf, buffer, size, &length); if (e != NULL) return (e); if (gc->fragment_index == gf->pi.status.st_nsections - 1 && gf->io_offset + length > gc->offsets[gf->pi.status.st_nsections]) gc->offsets[gf->pi.status.st_nsections] = gf->io_offset + length; *lengthp = length; return (NULL); }
static gfarm_error_t gfs_pio_view_global_write(GFS_File gf, const char *buffer, size_t size, size_t *lengthp) { struct gfs_file_global_context *gc = gf->view_context; gfarm_error_t e = gfs_pio_view_global_adjust(gf, buffer, &size); int length; /* XXX - should be size_t */ if (e != GFARM_ERR_NO_ERROR) return (e); e = gfs_pio_write(gc->fragment_gf, buffer, size, &length); if (e != GFARM_ERR_NO_ERROR) return (e); /* XXX - should notify this change to all of the parallel process. */ if (gc->fragment_index == gf->pi.status.st_nsections - 1 && gf->io_offset + length > gc->offsets[gf->pi.status.st_nsections]) gc->offsets[gf->pi.status.st_nsections] = gf->io_offset + length; *lengthp = length; return (GFARM_ERR_NO_ERROR); }
static void copy_file(int fd, GFS_File gf, char *gfarm_url, char *section) { char *e; ssize_t rv; int length; /* XXX - should be size_t */ char buffer[GFS_FILE_BUFSIZE]; for (;;) { rv = read(fd, buffer, sizeof(buffer)); if (rv <= 0) break; /* XXX - partial write case ? */ e = gfs_pio_write(gf, buffer, rv, &length); if (e != NULL) { fprintf(stderr, "%s: writing to %s:%s: %s\n", program_name, gfarm_url, section, e); error_happened = 1; break; } } }
int webgfarm_api_v1_write_to_file(request_rec *r) { // get filepath from request char *filepath = webgfarm_api_v1_getfilepath(r); if (filepath == NULL || strlen(filepath) == 0) { return HTTP_FORBIDDEN; } gfarm_error_t gerr; GFS_File gfs_file; webgfarm_config *wconfig = ap_get_module_config(r->server->module_config, &webgfarm_module); if (wconfig->write_redirect) { char *redirect_url = apr_palloc(r->pool, sizeof (char)*WEBGFARM_BUFFER_SIZE); const char *port; int available_nhosts; struct gfarm_host_sched_info *available_hosts; gerr = gfarm_schedule_hosts_domain_by_file(filepath, GFARM_FILE_RDONLY, "", &available_nhosts, &available_hosts); if (available_nhosts > 0) { if (gerr == GFARM_ERR_NO_ERROR) { int *ports; char **hosts; int nhosts = available_nhosts; int i; GFARM_MALLOC_ARRAY(hosts, available_nhosts); GFARM_MALLOC_ARRAY(ports, available_nhosts); gerr = gfarm_schedule_hosts_acyclic_to_write(filepath, available_nhosts, available_hosts, &nhosts, hosts, ports); if (nhosts > 0 && gerr == GFARM_ERR_NO_ERROR) { for (i = 0; i < nhosts; i++) { port = apr_table_get(wconfig->hosts, hosts[i]); if (port != NULL) { if (strcmp(wconfig->localhost, hosts[i]) != 0) { if (wconfig->ssl) { snprintf(redirect_url, WEBGFARM_BUFFER_SIZE, "https://%s:%s%s", hosts[i], port, r->uri); } else { snprintf(redirect_url, WEBGFARM_BUFFER_SIZE, "http://%s:%s%s", hosts[i], port, r->uri); } apr_table_set(r->headers_out, "Location", redirect_url); free(ports); free(hosts); return HTTP_TEMPORARY_REDIRECT; } else { break; } } } } free(ports); free(hosts); } else if (gerr != GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "gfarm_schedule_hosts_domain_by_file can not get the hosts of the file(%s) with error: %s", filepath, gfarm_error_string(gerr)); return HTTP_INTERNAL_SERVER_ERROR; } } } // open if (r->method_number == M_POST) { gerr = gfs_pio_open(filepath, GFARM_FILE_WRONLY | GFARM_FILE_APPEND, &gfs_file); } else if (r->method_number == M_PUT) { gerr = gfs_pio_open(filepath, GFARM_FILE_WRONLY | GFARM_FILE_TRUNC, &gfs_file); if (gerr == GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY) { gerr = gfs_pio_create(filepath, GFARM_FILE_WRONLY, 0644, &gfs_file); } } else { return HTTP_INTERNAL_SERVER_ERROR; } if (gerr != GFARM_ERR_NO_ERROR) { switch (gerr) { case GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY: return HTTP_NOT_FOUND; break; case GFARM_ERR_PERMISSION_DENIED: return HTTP_FORBIDDEN; break; //FIXEME: Support more error; default: return HTTP_INTERNAL_SERVER_ERROR; } } // get input body size; int in_size; const char *clen = apr_table_get(r->headers_in, "Content-Length"); if (clen != NULL) { in_size = strtol(clen, NULL, 0); if (in_size >= WEBGFFARM_MAX_POST_SIZE) { gfs_pio_close(gfs_file); return HTTP_BAD_REQUEST; } } else { gfs_pio_close(gfs_file); return HTTP_BAD_REQUEST; } if (in_size != 0) { apr_off_t size = 0; char *buf; if (util_read(r, &buf, &size) != OK) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[write] failed reading POST body"); gfs_pio_close(gfs_file); return HTTP_INTERNAL_SERVER_ERROR; } in_size = (int) size; if (buf == NULL) { return HTTP_BAD_REQUEST; } // write buffer int write_size; gerr = gfs_pio_write(gfs_file, buf, in_size, &write_size); if (gerr != GFARM_ERR_NO_ERROR) { if (in_size != write_size) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[write] gfs_pio_write failure..(in: %d, out: %d)", (int) in_size, (int) write_size); } gerr = gfs_pio_close(gfs_file); return HTTP_INTERNAL_SERVER_ERROR; } } // close gerr = gfs_pio_close(gfs_file); if (gerr != GFARM_ERR_NO_ERROR) { return HTTP_INTERNAL_SERVER_ERROR; } else { return OK; } }
static char * gfarm_url_fragment_register(char *gfarm_url, char *section, int nfrags, char *hostname, char *filename) { char *e, *e_save = NULL; int fd; size_t rv; int length; /* XXX - should be size_t */ GFS_File gf; struct stat s; char buffer[GFS_FILE_BUFSIZE]; /* * register the fragment */ if (strcmp(filename, "-") == 0) { fd = 0; /* stdin */ s.st_mode = 0664; /* XXX */ } else { if (stat(filename, &s) == -1) return "no such file or directory"; fd = open(filename, O_RDONLY); if (fd == -1) return "cannot open"; } e = gfs_pio_create(gfarm_url, GFARM_FILE_WRONLY, s.st_mode & GFARM_S_ALLPERM, &gf); if (e != NULL) { close(fd); return (e); } if (nfrags == GFARM_FILE_DONTCARE) e = gfs_pio_set_view_section(gf, section, hostname, 0); else e = gfs_pio_set_view_index(gf, nfrags, strtol(section, NULL, 0), hostname, 0); if (e != NULL) { char *gfarm_file; gfs_pio_close(gf); close(fd); /* try to unlink path info when there is no fragment file */ if (gfarm_url_make_path(gfarm_url, &gfarm_file) == NULL) { (void)gfarm_path_info_remove(gfarm_file); free(gfarm_file); } return (e); } for (;;) { rv = read(fd, buffer, sizeof(buffer)); if (rv <= 0) break; /* XXX - partial write case ? */ e = gfs_pio_write(gf, buffer, rv, &length); if (e != NULL) break; } e_save = e; e = gfs_pio_close(gf); close(fd); if (e_save != NULL) return (e_save); return (e); }
char * gfarm_url_program_register(char *gfarm_url, char *architecture, char *filename, int nreplicas) { char *e, *e_save = NULL, *if_hostname, *gfarm_file; int nhosts, fd, i; struct gfarm_host_info *hosts; struct sockaddr peer_addr; size_t rv; int length; /* XXX - should be size_t */ GFS_File gf; struct stat s; char buffer[GFS_FILE_BUFSIZE]; char *self_name; char **hostnames; if (stat(filename, &s) == -1) return (gfarm_errno_to_error(errno)); if (!GFARM_S_IS_PROGRAM(s.st_mode)) return (GFARM_ERR_OPERATION_NOT_PERMITTED); /* XXX - use better strategy for the replication */ e = gfarm_host_info_get_allhost_by_architecture(architecture, &nhosts, &hosts); if (e == GFARM_ERR_NO_SUCH_OBJECT) return ("gfarm_url_program_register(): no such architecture"); if (e != NULL) goto finish; hostnames = malloc(sizeof(char *) * nhosts); if (hostnames == NULL) { e = GFARM_ERR_NO_MEMORY; goto finish_host_info; } for (i = 0; i < nhosts; i++) hostnames[i] = hosts[i].hostname; if (nhosts < nreplicas) nreplicas = nhosts; if (gfarm_host_get_canonical_self_name(&self_name) != NULL) { e = gfarm_schedule_search_idle_hosts(nhosts, hostnames, nreplicas, hostnames); } else { /* give the top priority to self host */ char * tmp; for (i = 0; i < nhosts; i++) if (strcmp(hostnames[i], self_name) == 0) break; if (i != 0 && i < nhosts) { tmp = hostnames[0]; hostnames[0] = hostnames[i]; hostnames[i] = tmp; } if (nreplicas > 1) e = gfarm_schedule_search_idle_hosts(nhosts - 1, hostnames + 1, nreplicas - 1, hostnames + 1); } if (e != NULL) goto finish_hostnames; /* reflect "address_use" directive in the `hostnames[0]' */ e = gfarm_host_address_get(hostnames[0], gfarm_spool_server_port, &peer_addr, &if_hostname); if (e != NULL) goto finish_hostnames; /* * register the program */ fd = open(filename, O_RDONLY); if (fd == -1) { e = "gfarm_url_program_register(): can't open program"; goto finish_if_hostname; } /* XXX - overwrite case */ e = gfs_pio_create(gfarm_url, GFARM_FILE_WRONLY, s.st_mode & GFARM_S_ALLPERM, &gf); if (e != NULL) { close(fd); goto finish_if_hostname; } /* XXX - better strategy to select replica */ e = gfs_pio_set_view_section(gf, architecture, if_hostname, 0); if (e != NULL) { /* XXX - take care of the case where node is down */ gfs_pio_close(gf); close(fd); goto finish_if_hostname; } for (;;) { rv = read(fd, buffer, sizeof(buffer)); if (rv <= 0) break; /* XXX - partial write case ? */ e = gfs_pio_write(gf, buffer, rv, &length); if (e != NULL) break; } e_save = e; e = gfs_pio_close(gf); close(fd); if (e_save != NULL) e = e_save; if (e != NULL) goto finish_if_hostname; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) goto finish_if_hostname; /* * replicate the program */ for (i = 1; i < nreplicas; i++) { /* XXX - better strategy to select replica */ e = gfarm_file_section_replicate_from_to_internal( gfarm_file, architecture, s.st_mode & GFARM_S_ALLPERM, s.st_size, hostnames[0], if_hostname, hostnames[i]); if (e != NULL) e_save = e; } e = NULL; /* there is at least on copy available. */ /* XXX - partial error case? */ free(gfarm_file); finish_if_hostname: free(if_hostname); finish_hostnames: free(hostnames); finish_host_info: gfarm_host_info_free_all(nhosts, hosts); finish: return (e); }
gfarm_error_t gfperf_create_file_on_gfarm(const char *url, char *hostname, long long file_size) { const char *filename; char *buf; long long leftsize; int ret, s; GFS_File fp; gfarm_error_t e; struct gfs_stat sb; filename = url; GFARM_CALLOC_ARRAY(buf, GFPERF_COPY_BUF_SIZE); if (buf == NULL) { fprintf(stderr, "can not allocate memory.\n"); return (GFARM_ERR_NO_MEMORY); } e = gfs_stat(filename, &sb); if (e != GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY) { fprintf(stderr, "file exists: %s\n", filename); if (e == GFARM_ERR_NO_ERROR) gfs_stat_free(&sb); free(buf); return (GFARM_ERR_ALREADY_EXISTS); } e = gfs_pio_create(filename, GFARM_FILE_WRONLY|GFARM_FILE_TRUNC, 0644, &fp); if (e != GFARM_ERR_NO_ERROR) { fprintf(stderr, "open: %s\n", gfarm_error_string(e)); free(buf); return (e); } if (hostname != NULL) { /* XXX FIXME: INTERNAL FUNCTION SHOULD NOT BE USED */ e = gfs_pio_internal_set_view_section(fp, hostname); if (e != GFARM_ERR_NO_ERROR) { fprintf(stderr, "gfs_pio_internal_set_view_section() error! " "%s: %s\n", hostname, gfarm_error_string(e)); goto err_return; } } for (leftsize = file_size; leftsize > 0 ; leftsize -= ret) { s = (leftsize < GFPERF_COPY_BUF_SIZE) ? leftsize : GFPERF_COPY_BUF_SIZE; e = gfs_pio_write(fp, buf, s, &ret); if (e != GFARM_ERR_NO_ERROR) { fprintf(stderr, "write: %s\n", gfarm_error_string(e)); goto err_return; } } e = gfs_pio_close(fp); if (e != GFARM_ERR_NO_ERROR) { fprintf(stderr, "close: %s\n", gfarm_error_string(e)); } free(buf); return (e); err_return: gfs_pio_close(fp); free(buf); return (e); }