Beispiel #1
0
/*
 * the following function is for client,
 * server/daemon process shouldn't call it.
 * Because this function may read incorrect setting from user specified
 * $USER or $HOME.
 */
gfarm_error_t
gfarm_terminate(void)
{
#if 0 /* not yet in gfarm v2 */
	gfarm_error_t e;

	gfs_profile(gfs_display_timers());

	if (gf_stdout != NULL) {
		fflush(stdout);
		e = gfs_pio_close(gf_stdout);
		gf_stdout = NULL;
		if (e != GFARM_ERR_NO_ERROR)
			return (e);
	}
	if (gf_stderr != NULL) {
		fflush(stderr);
		e = gfs_pio_close(gf_stderr);
		gf_stderr = NULL;
		if (e != GFARM_ERR_NO_ERROR)
			return (e);
	}
	e = gfarm_metadb_terminate();
	if (e != GFARM_ERR_NO_ERROR)
		return (e);
#endif /* not yet in gfarm v2 */

	return (GFARM_ERR_NO_ERROR);
}
Beispiel #2
0
int
gfs_hook_insert_gfs_file(GFS_File gf)
{
	int fd, save_errno;

	_gfs_hook_debug(fprintf(stderr, "gfs_hook_insert_gfs_file: %p\n", gf));

	fd = dup(gfs_pio_fileno(gf));
	if (fd == -1) {
		save_errno = errno;
		gfs_pio_close(gf);
		errno = save_errno;
		return (-1);
	}
	if (fd >= MAX_GFS_FILE_BUF) {
		__syscall_close(fd);
		gfs_pio_close(gf);
		errno = EMFILE;
		return (-1);
	}
	if (_gfs_file_buf[fd] != NULL) {
		__syscall_close(fd);
		gfs_pio_close(gf);
		errno = EBADF; /* XXX - something broken */
		return (-1);
	}
	_gfs_file_buf[fd] = gf;
	return (fd);
}
Beispiel #3
0
/*
 * Instead of just calling gfs_pio_set_view_index(),
 * we use another GFS_File to move to another fragment.
 * This is because we don't want to leave this context inconsistent,
 * if failure happens at the gfs_pio_set_view_index().
 */
static char *
gfs_pio_view_global_move_to(GFS_File gf, int fragment_index)
{
	struct gfs_file_global_context *gc = gf->view_context;
	char *e;
	GFS_File new_fragment;

	if ((gf->open_flags & GFARM_FILE_CREATE) != 0)
		e = gfs_pio_create(gc->url, gf->open_flags,
		    gf->pi.status.st_mode & GFARM_S_ALLPERM,
		    &new_fragment);
	else
		e = gfs_pio_open(gc->url, gf->open_flags, &new_fragment);
	if (e != NULL)
		return (e);
	e = gfs_pio_set_view_index(new_fragment, gf->pi.status.st_nsections,
	    fragment_index, NULL, gf->view_flags);
	if (e != NULL) {
		gfs_pio_close(new_fragment);
		return (e);
	}
	if (gc->fragment_gf != NULL) {
		gfs_pio_close(gc->fragment_gf);
		/* XXX need a way to report error on here */
	}
	gc->fragment_gf = new_fragment;
	gc->fragment_index = fragment_index;
	return (NULL);
}
Beispiel #4
0
int
gfs_hook_insert_gfs_file(GFS_File gf)
{
	int fd, save_errno;
	struct stat st;

	_gfs_hook_debug(fprintf(stderr, "GFS: insert_gfs_file: %p\n", gf));

	/*
	 * A new file descriptor is needed to identify a hooked file
	 * descriptor.
	 */
	fd = gfs_pio_fileno(gf);
	if (fstat(fd, &st) == -1) {
		save_errno = errno;
		gfs_hook_delete_creating_file(gf);
		gfs_pio_close(gf);
		errno = save_errno;
		return (-1);
	}
	if (S_ISREG(st.st_mode))
		fd = dup(fd);
	else /* don't return a socket, to make select(2) work with this fd */
		fd = open("/dev/null", O_RDWR);
	if (fd == -1) {
		save_errno = errno;
		gfs_hook_delete_creating_file(gf);
		gfs_pio_close(gf);
		errno = save_errno;
		return (-1);
	}
	if (fd >= MAX_GFS_FILE_BUF) {
		__syscall_close(fd);
		gfs_hook_delete_creating_file(gf);
		gfs_pio_close(gf);
		errno = EMFILE;
		return (-1);
	}
	if (_gfs_file_buf[fd] != NULL) {
		__syscall_close(fd);
		gfs_hook_delete_creating_file(gf);
		gfs_pio_close(gf);
		errno = EBADF; /* XXX - something broken */
		return (-1);
	}
	_gfs_file_buf[fd] = malloc(sizeof(*_gfs_file_buf[fd]));
	if (_gfs_file_buf[fd] == NULL) {
		__syscall_close(fd);
		gfs_hook_delete_creating_file(gf);
		gfs_pio_close(gf);
		errno = ENOMEM;
		return (-1);
	}
	_gfs_file_buf[fd]->refcount = 1;
	_gfs_file_buf[fd]->d_type = GFS_DT_REG;
	_gfs_file_buf[fd]->u.f = gf;
	return (fd);
}
Beispiel #5
0
static void
register_fragment(int is_dir, char *gfarm_url, int index, int nfragments,
	char *hostname,
	char *filename, int use_file_mode, gfarm_mode_t file_mode)
{
	char *e;
	int fd, fd_needs_close;
	char *target_url;
	GFS_File gf;
	char section[GFARM_INT32STRLEN + 1];

	if (!open_file(filename, &fd, &fd_needs_close))
		return;
	if (!use_file_mode && !get_file_mode(fd, filename, &file_mode))
		goto finish;

	if (!is_dir)
		target_url = gfarm_url;
	else if (!concat_dir_name(gfarm_url, gfarm_path_dir_skip(filename),
	    &target_url))
		goto finish;

	if (nfragments == GFARM_FILE_DONTCARE &&
	    !get_nsections(target_url, &nfragments))
		goto finish_url;

	sprintf(section, "%d", index);
	if (opt_force || section_does_not_exists(target_url, section)) {
		e = gfs_pio_create(target_url,
		    GFARM_FILE_WRONLY|GFARM_FILE_TRUNC, file_mode, &gf);
		if (e != NULL) {
			fprintf(stderr, "%s: cannot open %s: %s\n",
			    program_name, target_url, e);
			error_happened = 1;
		} else {
			if ((e = gfs_pio_set_view_index(gf, nfragments, index,
			    hostname, 0)) != NULL) {
				fprintf(stderr, "%s: cannot open %s:%d: %s\n",
				    program_name, target_url, index, e);
				error_happened = 1;
			} else {
				copy_file(fd, gf, target_url, section);
			}
			gfs_pio_close(gf);
		}
	}
 finish_url:
	if (target_url != gfarm_url)
		free(target_url);
 finish:
	if (fd_needs_close)
		close(fd);
}
Beispiel #6
0
static char *
gfs_pio_view_global_close(GFS_File gf)
{
	struct gfs_file_global_context *gc = gf->view_context;
	char *e;

	e = gfs_pio_close(gc->fragment_gf);
	free(gc->url);
	free(gc->offsets);
	free(gc);
	gf->view_context = NULL;
	gfs_pio_set_view_default(gf);
	return (e);
}
Beispiel #7
0
static void
register_program(int is_dir, char *gfarm_url, char *section, char *hostname,
	char *filename, int use_file_mode, gfarm_mode_t file_mode)
{
	char *e;
	int fd, fd_needs_close;
	char *target_url;
	GFS_File gf;

	if (!open_file(filename, &fd, &fd_needs_close))
		return;
	if (!use_file_mode && !get_file_mode(fd, filename, &file_mode))
		goto finish;

	if (!is_dir)
		target_url = gfarm_url;
	else if (!concat_dir_name(gfarm_url, gfarm_path_dir_skip(filename),
	    &target_url))
		goto finish;

	if (opt_force || section_does_not_exists(target_url, section)) {
		e = gfs_pio_create(target_url,
		    GFARM_FILE_WRONLY|GFARM_FILE_TRUNC, file_mode, &gf);
		if (e != NULL) {
			fprintf(stderr, "%s: cannot open %s: %s\n",
			    program_name, target_url, e);
			error_happened = 1;
		} else {
			if ((e = gfs_pio_set_view_section(gf, section,
			    hostname, 0)) != NULL) {
				fprintf(stderr, "%s: cannot open %s:%s: %s\n",
				    program_name, target_url, section, e);
				error_happened = 1;
			} else {
				copy_file(fd, gf, target_url, section);
			}
			gfs_pio_close(gf);
		}
	}
	if (target_url != gfarm_url)
		free(target_url);
 finish:
	if (fd_needs_close)
		close(fd);
}
Beispiel #8
0
static gfarm_error_t
gfs_replicate_to_internal(char *file, char *dsthost, int dstport, int migrate)
{
	char *srchost;
	int srcport;
	gfarm_error_t e, e2;
	GFS_File gf;

	e = gfs_pio_open(file, GFARM_FILE_RDONLY, &gf);
	if (e != GFARM_ERR_NO_ERROR) {
		gflog_debug(GFARM_MSG_1001389,
			"gfs_pio_open(%s) failed: %s",
			file,
			gfarm_error_string(e));
		return (e);
	}

	e = gfarm_schedule_file(gf, &srchost, &srcport);
	if (e == GFARM_ERR_NO_ERROR) {

#ifndef V2_4
		e = gfs_replicate_from_to_internal(gf, srchost, srcport,
			dsthost, dstport);
#else
		e = gfs_replicate_file_from_to(file, srchost, dsthost,
		    GFS_REPLICATE_FILE_FORCE
		    /* | GFS_REPLICATE_FILE_WAIT */ /* XXX NOTYET */);
#endif
		if (e == GFARM_ERR_NO_ERROR && migrate)
			e = gfs_replica_remove_by_file(file, srchost);
		free(srchost);
	}
	e2 = gfs_pio_close(gf);

	if (e != GFARM_ERR_NO_ERROR || e2 != GFARM_ERR_NO_ERROR) {
		gflog_debug(GFARM_MSG_1001390,
			"error occurred in gfs_replicate_to_internal(): %s",
			gfarm_error_string(e != GFARM_ERR_NO_ERROR ? e : e2));
	}

	return (e != GFARM_ERR_NO_ERROR ? e : e2);
}
Beispiel #9
0
static char *
gfs_pio_view_global_close(GFS_File gf)
{
	struct gfs_file_global_context *gc = gf->view_context;
	char *e, *e_save = NULL;

	if ((gf->mode & GFS_FILE_MODE_WRITE) != 0) {
		e = gfs_pio_flush(gf);
		if (e != NULL)
			e_save = e;
	}

	e = gfs_pio_close(gc->fragment_gf);
	free(gc->url);
	free(gc->offsets);
	free(gc);
	gf->view_context = NULL;
	gfs_pio_set_view_default(gf);
	return (e_save != NULL ? e_save : e);
}
Beispiel #10
0
gfarm_error_t
gfs_replicate_from_to(char *file, char *srchost, int srcport,
	char *dsthost, int dstport)
{
#ifndef V2_4
	gfarm_error_t e, e2;
	GFS_File gf;

	if (srchost == NULL)
		return (gfs_replicate_to(file, dsthost, dstport));

	e = gfs_pio_open(file, GFARM_FILE_RDONLY, &gf);
	if (e != GFARM_ERR_NO_ERROR) {
		gflog_debug(GFARM_MSG_1001392,
			"gfs_pio_open(%s) failed: %s",
			file,
			gfarm_error_string(e));
		return (e);
	}

	e = gfs_replicate_from_to_internal(gf, srchost, srcport,
		dsthost, dstport);
	e2 = gfs_pio_close(gf);

	if (e != GFARM_ERR_NO_ERROR || e2 != GFARM_ERR_NO_ERROR) {
		gflog_debug(GFARM_MSG_1001393,
			"replication failed (%s) from (%s:%d) to (%s:%d): %s",
			file, srchost, srcport, dsthost, dstport,
			gfarm_error_string(e != GFARM_ERR_NO_ERROR ? e : e2));
	}

	return (e != GFARM_ERR_NO_ERROR ? e : e2);
#else
	return (gfs_replicate_file_from_to(file, srchost, dsthost,
	    GFS_REPLICATE_FILE_FORCE
	    /* | GFS_REPLICATE_FILE_WAIT */ /* XXX NOTYET */));

#endif
}
Beispiel #11
0
gfarm_error_t
gfs_replicate_file_to_request(
	const char *file, const char *dsthost, int flags)
{
	char *srchost;
	int srcport;
	gfarm_error_t e, e2;
	GFS_File gf;

	e = gfs_pio_open(file, GFARM_FILE_RDONLY, &gf);
	if (e != GFARM_ERR_NO_ERROR)
		return (e);

	e = gfarm_schedule_file(gf, &srchost, &srcport);
	e2 = gfs_pio_close(gf);
	if (e == GFARM_ERR_NO_ERROR) {
		e = gfs_replicate_file_from_to_request(file, srchost,
		    dsthost, flags);
		free(srchost);
	}

	return (e != GFARM_ERR_NO_ERROR ? e : e2);
}
Beispiel #12
0
char *
gfs_hook_clear_gfs_file(int fd)
{
	GFS_File gf;
	char *e = NULL;

	_gfs_hook_debug(fprintf(stderr, "GFS: clear_gfs_file: %d\n", fd));

	gf = gfs_hook_is_open(fd);
	if (gf == NULL) {
		_gfs_hook_debug(fprintf(stderr,
			"GFS: ERROR: not a Gfarm file: %d\n", fd));
		return ("not a Gfarm file");
	}

	if (--_gfs_file_buf[fd]->refcount > 0) {
	  	/* fd is duplicated, skip closing the file. */
		_gfs_hook_debug(fprintf(stderr,
					"GFS: clear_gfs_file: skipped\n"));
	} else {
		if (gfs_hook_gfs_file_type(fd) == GFS_DT_REG) {
			gfs_hook_delete_creating_file(gf);
			e = gfs_pio_close(gf);
		} else if (gfs_hook_gfs_file_type(fd) == GFS_DT_DIR) {
			_gfs_file_buf[fd]->u.d->dir = NULL;
			_gfs_file_buf[fd]->u.d->suspended = NULL;
			gfs_stat_free(&_gfs_file_buf[fd]->u.d->gst);
			free(_gfs_file_buf[fd]->u.d->canonical_path); 
			free(_gfs_file_buf[fd]->u.d);
			e = gfs_closedir((GFS_Dir)gf);
		}
		free(_gfs_file_buf[fd]);
	}
	__syscall_close(fd);
	_gfs_file_buf[fd] = NULL;
	return (e);
}
Beispiel #13
0
int webgfarm_api_v1_write_to_file(request_rec *r) {

    // get filepath from request
    char *filepath = webgfarm_api_v1_getfilepath(r);

    if (filepath == NULL || strlen(filepath) == 0) {
        return HTTP_FORBIDDEN;
    }

    gfarm_error_t gerr;
    GFS_File gfs_file;


    webgfarm_config *wconfig = ap_get_module_config(r->server->module_config, &webgfarm_module);
    if (wconfig->write_redirect) {

        char *redirect_url = apr_palloc(r->pool, sizeof (char)*WEBGFARM_BUFFER_SIZE);
        const char *port;

        int available_nhosts;
        struct gfarm_host_sched_info *available_hosts;
        gerr = gfarm_schedule_hosts_domain_by_file(filepath, GFARM_FILE_RDONLY, "", &available_nhosts, &available_hosts);

        if (available_nhosts > 0) {
            if (gerr == GFARM_ERR_NO_ERROR) {
                int *ports;
                char **hosts;
                int nhosts = available_nhosts;
                int i;
                GFARM_MALLOC_ARRAY(hosts, available_nhosts);
                GFARM_MALLOC_ARRAY(ports, available_nhosts);
                gerr = gfarm_schedule_hosts_acyclic_to_write(filepath, available_nhosts, available_hosts, &nhosts, hosts, ports);

                if (nhosts > 0 && gerr == GFARM_ERR_NO_ERROR) {
                    for (i = 0; i < nhosts; i++) {
                        port = apr_table_get(wconfig->hosts, hosts[i]);
                        if (port != NULL) {
                            if (strcmp(wconfig->localhost, hosts[i]) != 0) {
                                if (wconfig->ssl) {
                                    snprintf(redirect_url, WEBGFARM_BUFFER_SIZE, "https://%s:%s%s", hosts[i], port, r->uri);
                                } else {
                                    snprintf(redirect_url, WEBGFARM_BUFFER_SIZE, "http://%s:%s%s", hosts[i], port, r->uri);
                                }
                                apr_table_set(r->headers_out, "Location", redirect_url);
                                free(ports);
                                free(hosts);
                                return HTTP_TEMPORARY_REDIRECT;
                            } else {
                                break;
                            }
                        }
                    }
                }

                free(ports);
                free(hosts);
            } else if (gerr != GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY) {
                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "gfarm_schedule_hosts_domain_by_file can not get the hosts of the file(%s) with error: %s", filepath, gfarm_error_string(gerr));
                return HTTP_INTERNAL_SERVER_ERROR;
            }
        }
    }

    // open
    if (r->method_number == M_POST) {
        gerr = gfs_pio_open(filepath, GFARM_FILE_WRONLY | GFARM_FILE_APPEND, &gfs_file);
    } else if (r->method_number == M_PUT) {
        gerr = gfs_pio_open(filepath, GFARM_FILE_WRONLY | GFARM_FILE_TRUNC, &gfs_file);
        if (gerr == GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY) {
            gerr = gfs_pio_create(filepath, GFARM_FILE_WRONLY, 0644, &gfs_file);
        }
    } else {
        return HTTP_INTERNAL_SERVER_ERROR;
    }

    if (gerr != GFARM_ERR_NO_ERROR) {
        switch (gerr) {
            case GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY:
                return HTTP_NOT_FOUND;
                break;
            case GFARM_ERR_PERMISSION_DENIED:
                return HTTP_FORBIDDEN;
                break;
                //FIXEME: Support more error;
            default:
                return HTTP_INTERNAL_SERVER_ERROR;
        }
    }

    // get input body size;
    int in_size;
    const char *clen = apr_table_get(r->headers_in, "Content-Length");
    if (clen != NULL) {
        in_size = strtol(clen, NULL, 0);
        if (in_size >= WEBGFFARM_MAX_POST_SIZE) {
            gfs_pio_close(gfs_file);
            return HTTP_BAD_REQUEST;
        }
    } else {
        gfs_pio_close(gfs_file);
        return HTTP_BAD_REQUEST;
    }

    if (in_size != 0) {
        apr_off_t size = 0;
        char *buf;

        if (util_read(r, &buf, &size) != OK) {
            ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[write] failed reading POST body");
            gfs_pio_close(gfs_file);
            return HTTP_INTERNAL_SERVER_ERROR;
        }

        in_size = (int) size;

        if (buf == NULL) {
            return HTTP_BAD_REQUEST;
        }

        // write buffer
        int write_size;
        gerr = gfs_pio_write(gfs_file, buf, in_size, &write_size);
        if (gerr != GFARM_ERR_NO_ERROR) {
            if (in_size != write_size) {
                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[write] gfs_pio_write failure..(in: %d, out: %d)", (int) in_size, (int) write_size);
            }
            gerr = gfs_pio_close(gfs_file);
            return HTTP_INTERNAL_SERVER_ERROR;
        }
    }

    // close
    gerr = gfs_pio_close(gfs_file);
    if (gerr != GFARM_ERR_NO_ERROR) {
        return HTTP_INTERNAL_SERVER_ERROR;
    } else {
        return OK;
    }

}
Beispiel #14
0
gfarm_error_t
gfperf_create_file_on_gfarm(const char *url, char *hostname,
			    long long file_size)
{
	const char *filename;
	char *buf;
	long long leftsize;
	int ret, s;
	GFS_File fp;
	gfarm_error_t e;
	struct gfs_stat sb;

	filename = url;

	GFARM_CALLOC_ARRAY(buf, GFPERF_COPY_BUF_SIZE);
	if (buf == NULL) {
		fprintf(stderr, "can not allocate memory.\n");
		return (GFARM_ERR_NO_MEMORY);
	}

	e = gfs_stat(filename, &sb);
	if (e != GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY) {
		fprintf(stderr, "file exists: %s\n", filename);
		if (e == GFARM_ERR_NO_ERROR)
			gfs_stat_free(&sb);
		free(buf);
		return (GFARM_ERR_ALREADY_EXISTS);
	}

	e = gfs_pio_create(filename,
			   GFARM_FILE_WRONLY|GFARM_FILE_TRUNC,
			   0644,
			   &fp);
	if (e != GFARM_ERR_NO_ERROR) {
		fprintf(stderr, "open: %s\n",
			gfarm_error_string(e));
		free(buf);
		return (e);
	}
	if (hostname != NULL) {
		/* XXX FIXME: INTERNAL FUNCTION SHOULD NOT BE USED */
		e = gfs_pio_internal_set_view_section(fp, hostname);
		if (e != GFARM_ERR_NO_ERROR) {
			fprintf(stderr,
				"gfs_pio_internal_set_view_section() error! "
				"%s: %s\n", hostname,
				gfarm_error_string(e));
			goto err_return;
		}
	}

	for (leftsize = file_size; leftsize > 0 ; leftsize -= ret) {
		s = (leftsize < GFPERF_COPY_BUF_SIZE) ?
			leftsize : GFPERF_COPY_BUF_SIZE;
		e = gfs_pio_write(fp, buf, s, &ret);
		if (e != GFARM_ERR_NO_ERROR) {
			fprintf(stderr, "write: %s\n",
				gfarm_error_string(e));
			goto err_return;
		}
	}
	e = gfs_pio_close(fp);
	if (e != GFARM_ERR_NO_ERROR) {
		fprintf(stderr, "close: %s\n",
			gfarm_error_string(e));
	}
	free(buf);
	return (e);

err_return:
	gfs_pio_close(fp);
	free(buf);
	return (e);
}
Beispiel #15
0
int webgfarm_api_v1_read_file(request_rec *r) {
    // get filepath from request
    char *filepath = webgfarm_api_v1_getfilepath(r);

    if (filepath == NULL || strlen(filepath) == 0) {
        return HTTP_FORBIDDEN;
    }

    webgfarm_config *wconfig = ap_get_module_config(r->server->module_config, &webgfarm_module);

    gfarm_error_t gerr;
    GFS_File gfs_file;

    if (wconfig->read_redirect) {
        //        ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[read] read_redirect is on.");

        char *redirect_url = apr_palloc(r->pool, sizeof (char)*WEBGFARM_BUFFER_SIZE);

        const char *port;

        int available_nhosts;
        struct gfarm_host_sched_info *available_hosts;
        gerr = gfarm_schedule_hosts_domain_by_file(filepath, GFARM_FILE_RDONLY, "", &available_nhosts, &available_hosts);
        if (gerr == GFARM_ERR_NO_ERROR) {
            //            ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[read] gfarm_schedule_hosts_domain_by_file() has no error.");
            int *ports;
            char **hosts;
            int nhosts = available_nhosts;
            int i;
            GFARM_MALLOC_ARRAY(hosts, available_nhosts);
            GFARM_MALLOC_ARRAY(ports, available_nhosts);
            gerr = gfarm_schedule_hosts_acyclic(filepath, available_nhosts, available_hosts, &nhosts, hosts, ports);

            if (nhosts > 0 && gerr == GFARM_ERR_NO_ERROR) {
                //                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[read] gfarm_schedule_hosts_acyclic() has no error.");
                for (i = 0; i < nhosts; i++) {
                    port = apr_table_get(wconfig->hosts, hosts[i]);
                    if (port != NULL) {
                        if (strcmp(wconfig->localhost, hosts[i]) != 0) {
                            if (wconfig->ssl) {
                                snprintf(redirect_url, WEBGFARM_BUFFER_SIZE, "https://%s:%s%s", hosts[i], port, r->uri);
                            } else {
                                snprintf(redirect_url, WEBGFARM_BUFFER_SIZE, "http://%s:%s%s", hosts[i], port, r->uri);
                            }
                            apr_table_set(r->headers_out, "Location", redirect_url);
                            free(ports);
                            free(hosts);
                            return HTTP_TEMPORARY_REDIRECT;
                        } else {
                            break;
                        }
                    }
                }
            }

            free(ports);
            free(hosts);
        } else if (gerr != GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY) {
            ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "gfarm_schedule_hosts_domain_by_file can not get the hosts of the file(%s) with error: %s", filepath, gfarm_error_string(gerr));
            return HTTP_INTERNAL_SERVER_ERROR;
        }
    }

    // open
    gerr = gfs_pio_open(filepath, GFARM_FILE_RDONLY, &gfs_file);
    if (gerr != GFARM_ERR_NO_ERROR) {
        switch (gerr) {
            case GFARM_ERR_NO_SUCH_FILE_OR_DIRECTORY:
                return HTTP_NOT_FOUND;
                break;
            case GFARM_ERR_PERMISSION_DENIED:
                return HTTP_FORBIDDEN;
                break;
                //FIXEME: Support more error;
            default:
                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "gfs_pio_open can not open the file(%s) with error: %s", filepath, gfarm_error_string(gerr));
                return HTTP_INTERNAL_SERVER_ERROR;
        }
    }

    // get stat
    struct gfs_stat gs;
    gerr = gfs_pio_stat(gfs_file, &gs);
    if (gerr != GFARM_ERR_NO_ERROR) {
        gfs_pio_close(gfs_file);
        ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "gfs_pio_stat can not get the stat of the file(%s) with error: %s", filepath, gfarm_error_string(gerr));
        return HTTP_INTERNAL_SERVER_ERROR;
    }
    char *stat_buffer = apr_palloc(r->pool, sizeof (char)*WEBGFARM_BUFFER_SIZE);
    snprintf(stat_buffer, WEBGFARM_BUFFER_SIZE, "%lu", gs.st_ino);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-INO", stat_buffer);
    snprintf(stat_buffer, WEBGFARM_BUFFER_SIZE, "%lu", gs.st_gen);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-GEN", stat_buffer);
    snprintf(stat_buffer, WEBGFARM_BUFFER_SIZE, "%o", gs.st_mode);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-MODE", stat_buffer);
    snprintf(stat_buffer, WEBGFARM_BUFFER_SIZE, "%lu", gs.st_nlink);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-NLINK", stat_buffer);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-USER", gs.st_user);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-GROUP", gs.st_group);
    snprintf(stat_buffer, WEBGFARM_BUFFER_SIZE, "%lu", gs.st_size);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-SIZE", stat_buffer);
    snprintf(stat_buffer, WEBGFARM_BUFFER_SIZE, "%lu", gs.st_ncopy);
    apr_table_set(r->headers_out, "X-WEBGFARM-STAT-NCOPY", stat_buffer);

    //    gerr == GFARM_ERR_NO_ERROR
    //read and puts
    if (!r->header_only) {
        char *buf = apr_palloc(r->pool, sizeof (char)*WEBGFARM_BUFFER_SIZE);
        int read_size = 0;
        do {
            gerr = gfs_pio_read(gfs_file, buf, WEBGFARM_BUFFER_SIZE, &read_size);
            if (gerr != GFARM_ERR_NO_ERROR) {
                gerr = gfs_pio_close(gfs_file);
                ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "gfs_pio_read can not read the file(%s) with error: %s", filepath, gfarm_error_string(gerr));
                return HTTP_INTERNAL_SERVER_ERROR;
            } else {
                if (read_size != 0) {
                    ap_rwrite(buf, read_size, r);
                }
            }
        } while (gerr == GFARM_ERR_NO_ERROR && read_size != 0);
    }

    // close
    gerr = gfs_pio_close(gfs_file);
    if (gerr != GFARM_ERR_NO_ERROR) {
        ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "gfs_pio_close can not close the file(%s) with error: %s", filepath, gfarm_error_string(gerr));

        return HTTP_INTERNAL_SERVER_ERROR;
    } else {
        return OK;
    }
}
Beispiel #16
0
static char *
gfsck_file(char *gfarm_url)
{
	char *gfarm_file, *e, *e_save = NULL;
	int i, nsections, valid_nsections = 0;
	struct gfarm_file_section_info *sections;
	GFS_File gf;

	e = gfarm_url_make_path(gfarm_url, &gfarm_file);
	if (e != NULL)
		return (e);

	e = gfarm_file_section_info_get_all_by_file(
		gfarm_file, &nsections, &sections);
	if (e != NULL) {
		/* no section info, remove path info */
		e = path_info_remove(gfarm_url, gfarm_file);
		free(gfarm_file);
		return (e);
	}

	e = gfs_pio_open(gfarm_url, GFARM_FILE_RDONLY, &gf);
	if (e != NULL) {
		free(gfarm_file);
		return (e);
	}

	for (i = 0; i < nsections; i++) {
		int j, ncopies, valid_ncopies = 0;
		struct gfarm_file_section_copy_info *copies;
		char *section = sections[i].section;

		e = gfarm_file_section_copy_info_get_all_by_section(
			gfarm_file, section, &ncopies, &copies);
		if (e == GFARM_ERR_NO_SUCH_OBJECT) {
			/* no section copy info, remove section info */
			e = section_info_remove(gfarm_url, gfarm_file, section);
			if (e != NULL && e_save == NULL)
				e_save = e;
			continue;
		}
		else if (e != NULL) {
			fprintf(stderr, "%s (%s): %s\n",
				gfarm_url, section, e);
			if (e_save == NULL)
				e_save = e;
			continue;
		}
		for (j = 0; j < ncopies; ++j) {
			if (option_verbose)
				printf("%s (%s) on %s\n", gfarm_url, section,
				       copies[j].hostname);
			e = gfs_pio_set_view_section(gf, section,
			    copies[j].hostname,
			    GFARM_FILE_NOT_REPLICATE | GFARM_FILE_NOT_RETRY);
			if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE) {
				/* invalid section copy info removed */
				printf("%s (%s) on %s: "
				       "invalid metadata deleted\n",
				       gfarm_url, section,
				       copies[j].hostname);
				e = NULL;
			}
			else {
				++valid_ncopies;
				if (e != NULL) {
					fprintf(stderr, "%s (%s) on %s: %s\n",
						gfarm_url, section,
						copies[j].hostname, e);
					if (e_save == NULL)
						e_save = e;
				}
			}

		}
		gfarm_file_section_copy_info_free_all(ncopies, copies);
		if (valid_ncopies == 0) {
			/* no section copy info, remove section info */
			e = section_info_remove(gfarm_url, gfarm_file, section);
			if (e != NULL && e_save == NULL)
				e_save = e;
		}
		else
			++valid_nsections;
	}
	if (valid_nsections == 0) {
		/* no section info, remove path info */
		e = path_info_remove(gfarm_url, gfarm_file);
		if (e != NULL && e_save == NULL)
			e_save = e;
	}
	else if (valid_nsections < nsections) {
		printf("%s: warning: number of file sections reduced\n",
		       gfarm_url);
	}

	gfarm_file_section_info_free_all(nsections, sections);
	free(gfarm_file);

	e = gfs_pio_close(gf);
	if (e != NULL)
		return (e);

	return (e_save);
}
Beispiel #17
0
static char *
gfarm_url_fragment_register(char *gfarm_url, char *section, int nfrags,
	char *hostname, char *filename)
{
	char *e, *e_save = NULL;
	int fd;
	size_t rv;
	int length; /* XXX - should be size_t */
	GFS_File gf;
	struct stat s;
	char buffer[GFS_FILE_BUFSIZE];

	/*
	 * register the fragment
	 */
	if (strcmp(filename, "-") == 0) {
		fd = 0;			/* stdin */
		s.st_mode = 0664;	/* XXX */
	}
	else {
		if (stat(filename, &s) == -1)
			return "no such file or directory";
		fd = open(filename, O_RDONLY);
		if (fd == -1)
			return "cannot open";
	}
	e = gfs_pio_create(gfarm_url, GFARM_FILE_WRONLY,
		s.st_mode & GFARM_S_ALLPERM, &gf);
	if (e != NULL) {
		close(fd);
		return (e);
	}
	if (nfrags == GFARM_FILE_DONTCARE)
		e = gfs_pio_set_view_section(gf, section, hostname, 0);
	else
		e = gfs_pio_set_view_index(gf, nfrags,
			strtol(section, NULL, 0), hostname, 0);
	if (e != NULL) {
		char *gfarm_file;
		gfs_pio_close(gf);
		close(fd);
		/* try to unlink path info when there is no fragment file */
		if (gfarm_url_make_path(gfarm_url, &gfarm_file) == NULL) {
			(void)gfarm_path_info_remove(gfarm_file);
			free(gfarm_file);
		}
		return (e);
	}
	for (;;) {
		rv = read(fd, buffer, sizeof(buffer));
		if (rv <= 0)
			break;
		/* XXX - partial write case ? */
		e = gfs_pio_write(gf, buffer, rv, &length);
		if (e != NULL)
			break;
	}
	e_save = e;
	e = gfs_pio_close(gf);
	close(fd);

	if (e_save != NULL)
		return (e_save);

	return (e);
}
Beispiel #18
0
char *
gfarm_url_program_register(char *gfarm_url, char *architecture,
			   char *filename, int nreplicas)
{
	char *e, *e_save = NULL, *if_hostname, *gfarm_file;
	int nhosts, fd, i;
	struct gfarm_host_info *hosts;
	struct sockaddr peer_addr;
	size_t rv;
	int length; /* XXX - should be size_t */
	GFS_File gf;
	struct stat s;
	char buffer[GFS_FILE_BUFSIZE];
	char *self_name;
	char **hostnames;

	if (stat(filename, &s) == -1)
		return (gfarm_errno_to_error(errno));
	if (!GFARM_S_IS_PROGRAM(s.st_mode))
		return (GFARM_ERR_OPERATION_NOT_PERMITTED);

	/* XXX - use better strategy for the replication */

	e = gfarm_host_info_get_allhost_by_architecture(architecture,
	    &nhosts, &hosts);
	if (e == GFARM_ERR_NO_SUCH_OBJECT)
		return ("gfarm_url_program_register(): no such architecture");
	if (e != NULL)
		goto finish;


	hostnames = malloc(sizeof(char *) * nhosts);
	if (hostnames == NULL) {
		e = GFARM_ERR_NO_MEMORY;
		goto finish_host_info;
	}
	for (i = 0; i < nhosts; i++)
		hostnames[i] = hosts[i].hostname;
	if (nhosts < nreplicas)
		nreplicas = nhosts;

	if (gfarm_host_get_canonical_self_name(&self_name) != NULL) {
		e = gfarm_schedule_search_idle_hosts(nhosts, hostnames,
						     nreplicas, hostnames);
	} else {
		/* give the top priority to self host */
		char * tmp;
		for (i = 0; i < nhosts; i++)
			if (strcmp(hostnames[i], self_name) == 0)
				break;
		if (i != 0 && i < nhosts) {
			tmp = hostnames[0];
			hostnames[0] = hostnames[i];
			hostnames[i] = tmp;
		}	
		if (nreplicas > 1)
			e = gfarm_schedule_search_idle_hosts(nhosts - 1,
							     hostnames + 1,
							     nreplicas - 1,
							     hostnames + 1);
	}
	if (e != NULL)
		goto finish_hostnames;

	/* reflect "address_use" directive in the `hostnames[0]' */
	e = gfarm_host_address_get(hostnames[0], gfarm_spool_server_port,
	    &peer_addr, &if_hostname);
	if (e != NULL)
		goto finish_hostnames;

	/*
	 * register the program
	 */
	fd = open(filename, O_RDONLY);
	if (fd == -1) {
		e = "gfarm_url_program_register(): can't open program";
		goto finish_if_hostname;
	}
	/* XXX - overwrite case */
	e = gfs_pio_create(gfarm_url, GFARM_FILE_WRONLY,
	    s.st_mode & GFARM_S_ALLPERM, &gf);
	if (e != NULL) {
		close(fd);
		goto finish_if_hostname;
	}
	/* XXX - better strategy to select replica */
	e = gfs_pio_set_view_section(gf, architecture, if_hostname, 0);
	if (e != NULL) {
		/* XXX - take care of the case where node is down */
		gfs_pio_close(gf);
		close(fd);
		goto finish_if_hostname;
	}
	for (;;) {
		rv = read(fd, buffer, sizeof(buffer));
		if (rv <= 0)
			break;
		/* XXX - partial write case ? */
		e = gfs_pio_write(gf, buffer, rv, &length);
		if (e != NULL)
			break;
	}
	e_save = e;
	e = gfs_pio_close(gf);
	close(fd);
	if (e_save != NULL)
		e = e_save;
	if (e != NULL)
		goto finish_if_hostname;

	e = gfarm_url_make_path(gfarm_url, &gfarm_file);
	if (e != NULL)
		goto finish_if_hostname;

	/*
	 * replicate the program
	 */
	for (i = 1; i < nreplicas; i++) {
		/* XXX - better strategy to select replica */
		e = gfarm_file_section_replicate_from_to_internal(
		    gfarm_file, architecture,
		    s.st_mode & GFARM_S_ALLPERM, s.st_size,
		    hostnames[0], if_hostname, hostnames[i]);
		if (e != NULL)
			e_save = e;
	}
	e = NULL; /* there is at least on copy available. */
	/* XXX - partial error case? */

	free(gfarm_file);
finish_if_hostname:
	free(if_hostname);
finish_hostnames:
	free(hostnames);
finish_host_info:
	gfarm_host_info_free_all(nhosts, hosts);
finish:
	return (e);
}
Beispiel #19
0
int
main(int argc, char **argv)
{
	gfarm_error_t e;
	char *opt_domain = "";
	char *opt_mount_point = NULL;
	char *opt_file = NULL;
	int opt_metadata_only = 0;
	int opt_long_format = 0;
	int opt_nhosts = 0;
	int opt_write_mode = 0;
	int opt_create_mode = 0;
	int c, i, available_nhosts, nhosts, *ports;
	struct gfarm_host_sched_info *available_hosts;
	char *path, **hosts, *realpath = NULL;

	if (argc >= 1)
		program_name = basename(argv[0]);

	e = gfarm_initialize(&argc, &argv);
	if (e != GFARM_ERR_NO_ERROR) {
		fprintf(stderr, "%s: %s\n", program_name,
		    gfarm_error_string(e));
		exit(1);
	}

	while ((c = getopt(argc, argv, "D:LMP:cf:ln:w")) != -1) {
		switch (c) {
		case 'D':
			opt_domain = optarg;
			break;
		case 'L':
			gfarm_schedule_search_mode_use_loadavg();
			break;
		case 'M':
			opt_metadata_only = 1;
			break;
		case 'P':
			opt_mount_point = optarg;
			break;
		case 'c':
			opt_create_mode = 1;
			break;
		case 'f':
			opt_file = optarg;
			break;
		case 'l':
			opt_long_format = 1;
			break;
		case 'n':
			opt_nhosts = parse_opt_long(optarg, c, "<nhosts>");
			if (opt_nhosts <= 0) {
				fprintf(stderr, "%s: invalid value: -%c %d\n",
				    program_name, c, opt_nhosts);
				usage();
			}
			break;
		case 'w':
			opt_write_mode = 1;
			break;
		default:
			usage();
		}
	}
	argc -= optind;
	argv += optind;
	if (argc != 0)
		usage();

	if (opt_mount_point != NULL && opt_file != NULL) {
		fprintf(stderr,
		    "%s: -P and -f option cannot be specified at once.\n",
		    program_name);
		usage();
	}

	if (opt_file != NULL) {
		e = gfarm_realpath_by_gfarm2fs(opt_file, &realpath);
		if (e == GFARM_ERR_NO_ERROR)
			path = realpath;
		else
			path = opt_file;
		if (opt_create_mode) {
			GFS_File gf;

			e = gfs_pio_create(path, GFARM_FILE_WRONLY, 0666, &gf);
			if (e != GFARM_ERR_NO_ERROR) {
				fprintf(stderr, "%s: creating \"%s\": %s\n",
				    program_name, path, gfarm_error_string(e));
				exit(1);
			}
			e = gfs_pio_close(gf);
			if (e != GFARM_ERR_NO_ERROR) {
				fprintf(stderr,
				    "%s: gfs_pio_close(\"%s\"): %s\n",
				    program_name, path, gfarm_error_string(e));
				/* exit(1); */
			}
			/* NOTE: this may leave an empty file with ncopy==0 */
		}
		e = gfarm_schedule_hosts_domain_by_file(path,
		    opt_write_mode ? GFARM_FILE_RDWR : GFARM_FILE_RDONLY,
		    opt_domain,
		    &available_nhosts, &available_hosts);
	} else {
		path = opt_mount_point == NULL ? "." : opt_mount_point;
		e = gfarm_realpath_by_gfarm2fs(path, &realpath);
		if (e == GFARM_ERR_NO_ERROR)
			path = realpath;
		e = gfarm_schedule_hosts_domain_all(path, opt_domain,
		    &available_nhosts, &available_hosts);
	}
	if (e != GFARM_ERR_NO_ERROR) {
		fprintf(stderr, "%s: metadata scheduling: %s\n",
		    program_name, gfarm_error_string(e));
		exit(1);
	}

	nhosts = opt_nhosts > 0 ? opt_nhosts : available_nhosts;
	GFARM_MALLOC_ARRAY(hosts, nhosts);
	GFARM_MALLOC_ARRAY(ports, nhosts);
	if (hosts == NULL || ports == NULL) {
		fprintf(stderr, "%s: cannot allocate memory for %d hosts.\n",
		    program_name, nhosts);
		exit(1);
	}

	if (opt_metadata_only) {
		if (nhosts > available_nhosts)
			nhosts = available_nhosts;
		for (i = 0; i < nhosts; i++) {
			hosts[i] = available_hosts[i].host;
			ports[i] = available_hosts[i].port;
		}
	} else if (opt_write_mode) {
		e = gfarm_schedule_hosts_acyclic_to_write(path,
		    available_nhosts, available_hosts,
		    &nhosts, hosts, ports);
	} else {
		e = gfarm_schedule_hosts_acyclic(path,
		    available_nhosts, available_hosts,
		    &nhosts, hosts, ports);
	}
	free(realpath);
	if (e != GFARM_ERR_NO_ERROR) {
		fprintf(stderr, "%s: client side scheduling: %s\n",
		    program_name, gfarm_error_string(e));
		exit(1);
	}

	for (i = 0; i < nhosts; i++) {
		printf("%s", hosts[i]);
		if (opt_long_format)
			printf("\t%d", ports[i]);
		putchar('\n');
	}
	free(hosts);
	free(ports);

	e = gfarm_terminate();
	if (e != GFARM_ERR_NO_ERROR) {
		fprintf(stderr, "%s: %s\n",
		    program_name, gfarm_error_string(e));
		exit(1);
	}

	exit(0);
}
Beispiel #20
0
int
FUNC___OPEN(const char *path, int oflag, ...)
{
	GFS_File gf;
	const char *e;
	char *url, *sec;
	va_list ap;
	mode_t mode;
	int filedes;

	va_start(ap, oflag);
	mode = va_arg(ap, mode_t);
	va_end(ap);

	_gfs_hook_debug_v(fprintf(stderr,
	    "Hooking " S(FUNC___OPEN) "(%s, 0x%x)\n", path, oflag));

	if (!gfs_hook_is_url(path, &url, &sec))
		return (SYSCALL_OPEN(path, oflag, mode));

	/* XXX - ROOT I/O creates a new file with O_CREAT|O_RDWR mode. */
	/* XXX - FIXME */
	if ((oflag & O_CREAT) != 0 || (oflag & O_TRUNC) != 0) {
		_gfs_hook_debug(fprintf(stderr,
		    "GFS: Hooking " S(FUNC___OPEN) "(%s:%s, 0x%x, 0%o)\n",
		    url, sec != NULL ? sec : "(null)", oflag, mode));
		if (oflag & O_TRUNC) {
			/*
			 * Hooking open syscall does not mean to open
			 * an entire file but a file fragment in local and
			 * index file views.  gfs_unlink() should not be
			 * called in both views.
			 */
			if (_gfs_hook_default_view == global_view)
				gfs_unlink(url); /* XXX - FIXME */
			e = gfs_pio_create(url, oflag, mode, &gf);
		} else {
			e = gfs_pio_open(url, oflag, &gf);
			if (e == GFARM_ERR_NO_SUCH_OBJECT) /* XXX - FIXME */
				e = gfs_pio_create(url, oflag, mode, &gf);
		}
	} else {
		_gfs_hook_debug(fprintf(stderr,
		    "GFS: Hooking " S(FUNC___OPEN) "(%s:%s, 0x%x)\n",
		    url, sec != NULL ? sec : "(null)", oflag));
		e = gfs_pio_open(url, oflag, &gf);
	}
	free(url);
	if (e != NULL) {
		_gfs_hook_debug(fprintf(stderr,
		    "GFS: Hooking " S(FUNC___OPEN) ": %s\n", e));
		errno = gfarm_error_to_errno(e);
		return (-1);
	}

	if (sec != NULL || _gfs_hook_default_view == index_view) {
		if (sec != NULL) {
			_gfs_hook_debug(fprintf(stderr,
				"GFS: set_view_section(%s, %s)\n", url, sec));
			e = gfs_pio_set_view_section(gf, sec, NULL, 0);
			free(sec);
		} else {
			_gfs_hook_debug(fprintf(stderr,
				"GFS: set_view_index(%s, %d, %d)\n", url,
				_gfs_hook_num_fragments, _gfs_hook_index));
			e = gfs_pio_set_view_index(gf, _gfs_hook_num_fragments,
				_gfs_hook_index, NULL, 0);
		}
		if (e != NULL) {
			_gfs_hook_debug(fprintf(stderr,
			    "GFS: set_view_section: %s\n", e));
			gfs_pio_close(gf);
			errno = gfarm_error_to_errno(e);
			return (-1);
		}
	} else if (_gfs_hook_default_view == local_view) {
		int nf = -1, np;
		/*
		 * If the number of fragments is not the same as the
		 * number of parallel processes, or the file is not
		 * fragmented, do not change to the local file view.
		 */
		if (gfs_pio_get_nfragment(gf, &nf) ==
			GFARM_ERR_FRAGMENT_INDEX_NOT_AVAILABLE ||
		    (gfs_pio_get_node_size(&np) == NULL && nf == np)) {
			_gfs_hook_debug(fprintf(stderr,
				"GFS: set_view_local(%s (%d, %d))\n",
					url, gfarm_node, gfarm_nnode));
			if ((e = gfs_pio_set_view_local(gf, 0)) != NULL) {
				_gfs_hook_debug(fprintf(stderr,
					"GFS: set_view_local: %s\n", e));
				gfs_pio_close(gf);
				errno = gfarm_error_to_errno(e);
				return (-1);
			}
		}
	}
	filedes = gfs_hook_insert_gfs_file(gf);
	_gfs_hook_debug(
		if (filedes != -1) {
			fprintf(stderr,
			    "GFS: Hooking " S(FUNC___OPEN) " --> %d(%d)\n",
			    filedes, gfs_pio_fileno(gf));
		}
	);