Beispiel #1
0
char *
gfs_pio_close(GFS_File gf)
{
	char *e, *e_save;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e_save = gfs_pio_check_view_default(gf);
	if (e_save == NULL) {
		if ((gf->mode & GFS_FILE_MODE_WRITE) != 0)
			e_save = gfs_pio_flush(gf);

		e = (*gf->ops->view_close)(gf);
		if (e_save == NULL)
			e_save = e;
	}
	gfarm_path_info_free(&gf->pi);
	free(gf->buffer);
	free(gf);

	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_close_time += gfarm_timerval_sub(&t2, &t1));

	return (e_save);
}
Beispiel #2
0
char *
gfs_pio_truncate(GFS_File gf, file_offset_t length)
{
	char *e;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e = gfs_pio_check_view_default(gf);
	if (e != NULL)
		goto finish;

	CHECK_WRITABLE(gf);

	gf->mode &= ~GFS_FILE_MODE_CALC_DIGEST;

	if (gf->mode & GFS_FILE_MODE_BUFFER_DIRTY) {
		e = gfs_pio_flush(gf);
		if (e != NULL)
			goto finish;
	}

	gf->error = NULL; /* purge EOF/error state */
	gfs_pio_purge(gf);

	e = (*gf->ops->view_ftruncate)(gf, length);
	if (e != NULL)
		gf->error = e;
finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_truncate_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #3
0
char *
gfs_pio_close(GFS_File gf)
{
	char *e, *e_save;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e_save = gfs_pio_check_view_default(gf);
	if (e_save == NULL) {
		if ((gf->mode & GFS_FILE_MODE_WRITE) != 0)
			e_save = gfs_pio_flush(gf);

		e = (*gf->ops->view_close)(gf);
		if (e_save == NULL)
			e_save = e;
	}
	/*
	 * When there is inconsistency, do not update/overwrite the
	 * metadata. This inconsistency may come from the update by
	 * other process or oneself such as 'nvi'.
	 */
	if (e_save == NULL)
		e_save = gfs_pio_update_times(gf);

	gfarm_path_info_free(&gf->pi);
	gfs_file_free(gf);

	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_close_time += gfarm_timerval_sub(&t2, &t1));

	return (e_save);
}
Beispiel #4
0
char *
gfs_pio_open(const char *url, int flags, GFS_File *gfp)
{
	char *e, *pathname;
	GFS_File gf;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	if ((flags & GFARM_FILE_ACCMODE) != GFARM_FILE_RDONLY) {
#if 0 /*  XXX - ROOT I/O opens a new file with O_CREAT|O_RDRW mode. */
		e = GFARM_ERR_OPERATION_NOT_SUPPORTED; /* XXX */
		goto finish;
#else
		flags |= GFARM_FILE_CREATE;
#endif
	}
	e = gfarm_url_make_path(url, &pathname);
	if (e != NULL)
		goto finish;
	e = gfs_file_alloc(&gf);
	if (e != NULL) {
		free(pathname);
		goto finish;
	}
	gf->open_flags = flags;
	gf->mode |= GFS_FILE_MODE_READ;
	if (((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_RDWR)
	    || ((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_WRONLY))
		gf->mode |= GFS_FILE_MODE_WRITE;
	e = gfarm_path_info_get(pathname, &gf->pi);
	free(pathname);
	if (e != NULL)
		goto finish;
	e = gfarm_path_info_access(&gf->pi, GFS_R_OK);
	if (e != NULL) {
		gfarm_path_info_free(&gf->pi);
		gfs_file_free(gf);
		goto finish;
	}
	if (!GFARM_S_ISREG(gf->pi.status.st_mode)) {
		if (GFARM_S_ISDIR(gf->pi.status.st_mode))
			e = GFARM_ERR_IS_A_DIRECTORY;
		else
			e = GFARM_ERR_OPERATION_NOT_SUPPORTED;
		gfarm_path_info_free(&gf->pi);
		gfs_file_free(gf);
		goto finish;
	}
	gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED;
	*gfp = gf;

	e = NULL;
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_open_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #5
0
char *
gfs_pio_write(GFS_File gf, const void *buffer, int size, int *np)
{
	char *e;
	size_t written;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e = gfs_pio_check_view_default(gf);
	if (e != NULL)
		return (e);

	CHECK_WRITABLE(gf);

	if (size + gf->p > GFS_FILE_BUFSIZE) {
		/*
		 * gf->buffer[gf->p .. GFS_FILE_BUFSIZE-1] will be overridden
		 * by buffer.
		 */
		gf->length = gf->p;
		e = gfs_pio_flush(gf);
		if (e != NULL)
			goto finish;
	}
	if (size >= GFS_FILE_BUFSIZE) {
		/* shortcut to avoid unnecessary memory copy */
		assert(gf->p == 0); /* gfs_pio_flush() was called above */
		gf->length = 0;
		gf->mode &= ~GFS_FILE_MODE_BUFFER_DIRTY;

		e = do_write(gf, buffer, size, &written);
		if (e != NULL && written == 0)
			goto finish;
		gf->offset += written;
		*np = written; /* XXX - size_t vs int */

		e = NULL;
		goto finish;
	}
	gf->mode |= GFS_FILE_MODE_BUFFER_DIRTY;
	memcpy(gf->buffer + gf->p, buffer, size);
	gf->p += size;
	if (gf->p > gf->length)
		gf->length = gf->p;
	*np = size;
	e = NULL;
	if (gf->p >= GFS_FILE_BUFSIZE)
		e = gfs_pio_flush(gf);
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_write_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #6
0
char *
gfs_pio_open(const char *url, int flags, GFS_File *gfp)
{
	char *e, *pathname;
	GFS_File gf;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	if (flags & GFARM_FILE_CREATE) {
		e = GFARM_ERR_OPERATION_NOT_SUPPORTED;
		goto finish;
	}
	e = gfarm_url_make_path(url, &pathname);
	if (e != NULL)
		goto finish;
	e = gfs_file_alloc(&gf);
	if (e != NULL) {
		free(pathname);
		goto finish;
	}

	/* GFARM_FILE_EXCLUSIVE is a NOP with gfs_pio_open(). */
	flags &= ~GFARM_FILE_EXCLUSIVE;

	gfs_pio_open_initialize_mode_flags(gf, flags);

	e = gfarm_path_info_get(pathname, &gf->pi);
	free(pathname);
	if (e != NULL)
		goto free_gf;
	e = gfs_pio_open_check_perm(gf);
	if (e != NULL)
		goto free_gf_pi;
	if ((flags & GFARM_FILE_TRUNC) == 0)
		gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED;
	*gfp = gf;

	e = NULL;
	goto finish;
free_gf_pi:
	gfarm_path_info_free(&gf->pi);
free_gf:
	gfs_file_free(gf);
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_open_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #7
0
char *
gfs_stat(const char *path, struct gfs_stat *s)
{
	char *e, *p;
	gfarm_timerval_t t1, t2;
	long ino;

	gfs_profile(gfarm_gettimerval(&t1));

	path = gfarm_url_prefix_skip(path);
	e = gfarm_canonical_path(path, &p);
	if (e != NULL)
		goto finish;
	e = gfs_stat_canonical_path(p, s);
	if (e == NULL)
		goto finish_free_p;
	if (e != GFARM_ERR_NO_SUCH_OBJECT)
		goto finish_free_p;

	/*
	 * XXX - assume that it's a directory that does not have the
	 * path info.
	 */
	e = gfs_get_ino(p, &ino);
	if (e != NULL)
		goto finish_free_p;
	s->st_ino = ino;
	s->st_mode = GFARM_S_IFDIR | 0777;
	s->st_user = strdup("root");
	s->st_group = strdup("gfarm");
	s->st_atimespec.tv_sec = 0;
	s->st_atimespec.tv_nsec = 0;
	s->st_mtimespec.tv_sec = 0;
	s->st_mtimespec.tv_nsec = 0;
	s->st_ctimespec.tv_sec = 0;
	s->st_ctimespec.tv_nsec = 0;
	s->st_size = 0;
	s->st_nsections = 0;

	e = NULL;
 finish_free_p:
	free(p);
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_stat_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #8
0
char *
gfs_pio_read(GFS_File gf, void *buffer, int size, int *np)
{
	char *e;
	char *p = buffer;
	int n = 0;
	int length;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e = gfs_pio_check_view_default(gf);
	if (e != NULL)
		return (e);

	CHECK_READABLE(gf);

	while (size > 0) {
		if ((e = gfs_pio_fillbuf(gf)) != NULL)
			break;
		if (gf->error != NULL) /* EOF or error */
			break;
		length = gf->length - gf->p;
		if (length > size)
			length = size;
		memcpy(p, gf->buffer + gf->p, length);
		p += length;
		n += length;
		size -= length;
		gf->p += length;
	}
	if (e != NULL && n == 0)
		goto finish;
	*np = n;

	e = NULL;
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_read_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #9
0
char *
gfs_pio_getline(GFS_File gf, char *s, size_t size, int *eofp)
{
	char *e = gfs_pio_check_view_default(gf);
	char *p = s;
	int c;
	gfarm_timerval_t t1, t2;

	if (e != NULL)
		return (e);

	gfs_profile(gfarm_gettimerval(&t1));

#ifdef __GNUC__ /* workaround gcc warning: unused variable */
	c = EOF;
#endif
	CHECK_READABLE(gf);

	if (size <= 1) {
		gf->error = GFARM_ERR_INVALID_ARGUMENT;
		return (gf->error);
	}
	--size; /* for '\0' */
	for (; size > 0 && (c = gfs_pio_getc(gf)) != EOF; --size) {
		if (c == '\n')
			break;
		*p++ = c;
	}
	*p++ = '\0';
	if (p == s + 1 && c == EOF) {
		*eofp = 1;
		return (GFS_PIO_ERROR(gf));
	}
	*eofp = 0;

	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_getline_time += gfarm_timerval_sub(&t2, &t1));

	return (NULL);
}
Beispiel #10
0
/*
 * mostly compatible with getdelim(3) in glibc,
 * but there are the following differences:
 * 1. on EOF, *lenp == 0
 * 2. on error, *lenp isn't touched.
 */
char *
gfs_pio_readdelim(GFS_File gf, char **bufp, size_t *sizep, size_t *lenp,
	const char *delim, size_t delimlen)
{
	char *e = gfs_pio_check_view_default(gf);
	char *buf = *bufp, *p;
	size_t size = *sizep, len = 0;
	int c, delimtail;
	static const char empty_line[] = "\n\n";
	gfarm_timerval_t t1, t2;

	if (e != NULL)
		return (e);

	gfs_profile(gfarm_gettimerval(&t1));

#ifdef __GNUC__ /* workaround gcc warning: unused variable */
	c = EOF;
#endif
	CHECK_READABLE(gf);

	if (delim == NULL) { /* special case 1 */
		delimtail = 0; /* workaround gcc warning */
	} else {
		if (delimlen == 0) { /* special case 2 */
			delim = empty_line;
			delimlen = 2;
		}
		delimtail = delim[delimlen - 1];
	}
	if (buf == NULL || size <= 1) {
		if (size <= 1)
			size = ALLOC_SIZE_INIT;
		buf = realloc(buf, size);
		if (buf == NULL)
			return (GFARM_ERR_NO_MEMORY);
	}
	for (;;) {
		c = gfs_pio_getc(gf);
		if (c == EOF)
			break;
		if (size <= len) {
			p = realloc(buf, size + size);
			if (p == NULL) {
				*bufp = buf;
				*sizep = size;
				return (GFARM_ERR_NO_MEMORY);
			}
			buf = p;
			size += size;
		}
		buf[len++] = c;
		if (delim == NULL) /* special case 1: no delimiter */
			continue;
		if (len >= delimlen && c == delimtail &&
		    memcmp(&buf[len - delimlen], delim, delimlen) == 0) {
			if (delim == empty_line) { /* special case 2 */
				for (;;) {
					c = gfs_pio_getc(gf);
					if (c == EOF)
						break;
					if (c != '\n') {
						gfs_pio_ungetc(gf, c);
						break;
					}
					if (size <= len) {
						p = realloc(buf, size + size);
						if (p == NULL) {
							*bufp = buf;
							*sizep = size;
							return (
							  GFARM_ERR_NO_MEMORY);
						}
						buf = p;
						size += size;
					}
					buf[len++] = c;
				}
			}
			break;
		}
	}
	if (size <= len) {
		p = realloc(buf, size + size);
		if (p == NULL) {
			*bufp = buf;
			*sizep = size;
			return (GFARM_ERR_NO_MEMORY);
		}
		buf = p;
		size += size;
	}
	buf[len] = '\0';

	gfs_profile(gfarm_gettimerval(&t2));
	/* XXX should introduce gfs_pio_readdelim_time??? */
	gfs_profile(gfs_pio_getline_time += gfarm_timerval_sub(&t2, &t1));

	*bufp = buf;
	*sizep = size;
	*lenp = len;

	return (NULL);
}
Beispiel #11
0
/*
 * mostly compatible with getline(3) in glibc,
 * but there are the following differences:
 * 1. on EOF, *lenp == 0
 * 2. on error, *lenp isn't touched.
 */
char *
gfs_pio_readline(GFS_File gf, char **bufp, size_t *sizep, size_t *lenp)
{
	char *e = gfs_pio_check_view_default(gf);
	char *buf = *bufp, *p;
	size_t size = *sizep, len = 0;
	int c;
	gfarm_timerval_t t1, t2;

	if (e != NULL)
		return (e);

	gfs_profile(gfarm_gettimerval(&t1));

#ifdef __GNUC__ /* workaround gcc warning: unused variable */
	c = EOF;
#endif
	CHECK_READABLE(gf);

	if (buf == NULL || size <= 1) {
		if (size <= 1)
			size = ALLOC_SIZE_INIT;
		buf = realloc(buf, size);
		if (buf == NULL)
			return (GFARM_ERR_NO_MEMORY);
	}
	for (;;) {
		c = gfs_pio_getc(gf);
		if (c == EOF)
			break;
		if (size <= len) {
			p = realloc(buf, size + size);
			if (p == NULL) {
				*bufp = buf;
				*sizep = size;
				return (GFARM_ERR_NO_MEMORY);
			}
			buf = p;
			size += size;
		}
		buf[len++] = c;
		if (c == '\n')
			break;
	}
	if (size <= len) {
		p = realloc(buf, size + size);
		if (p == NULL) {
			*bufp = buf;
			*sizep = size;
			return (GFARM_ERR_NO_MEMORY);
		}
		buf = p;
		size += size;
	}
	buf[len] = '\0';

	gfs_profile(gfarm_gettimerval(&t2));
	/* XXX should introduce gfs_pio_readline_time??? */
	gfs_profile(gfs_pio_getline_time += gfarm_timerval_sub(&t2, &t1));

	*bufp = buf;
	*sizep = size;
	*lenp = len;
	

	return (NULL);
}
Beispiel #12
0
char *
gfs_pio_create(const char *url, int flags, gfarm_mode_t mode, GFS_File *gfp)
{
	char *e, *pathname;
	GFS_File gf;
	char *user;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	user = gfarm_get_global_username();
	if (user == NULL) {
		e = "gfs_pio_create(): programming error, "
		    "gfarm library isn't properly initialized";
		goto finish;
	}

	e = gfarm_url_make_path_for_creation(url, &pathname);
	if (e != NULL)
		goto finish;
	e = gfs_file_alloc(&gf);
	if (e != NULL) {
		free(pathname);
		goto finish;
	}

	flags |= GFARM_FILE_CREATE;

	gfs_pio_open_initialize_mode_flags(gf, flags);

	e = gfarm_path_info_get(pathname, &gf->pi);
	if (e != NULL && e != GFARM_ERR_NO_SUCH_OBJECT) {
		free(pathname);
		goto free_gf;
	}
	if (e == NULL) {
		free(pathname);
		if ((flags & GFARM_FILE_EXCLUSIVE) != 0) {
			e = GFARM_ERR_ALREADY_EXISTS;
			goto free_gf_pi;
		}
		e = gfs_pio_open_check_perm(gf);
		if (e != NULL)
			goto free_gf_pi;
		/*
		 * XXX should check the follows:
		 * - the mode is consistent among same job
		 * - creator of the metainfo has same job id
		 * - O_TRUNC / !O_TRUNC case
		 */
		if ((flags & GFARM_FILE_TRUNC) == 0)
			gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED;
	} else {
		mode_t mask;
		struct timeval now;

		mask = umask(0);
		umask(mask);

		gettimeofday(&now, NULL);
		gf->pi.pathname = pathname;
		gf->pi.status.st_mode = (GFARM_S_IFREG | (mode & ~mask));
		gf->pi.status.st_user = strdup(user); /* XXX NULL check */
		gf->pi.status.st_group = strdup("*"); /* XXX for now */
		gf->pi.status.st_atimespec.tv_sec =
		gf->pi.status.st_mtimespec.tv_sec =
		gf->pi.status.st_ctimespec.tv_sec = now.tv_sec;
		gf->pi.status.st_atimespec.tv_nsec =
		gf->pi.status.st_mtimespec.tv_nsec =
		gf->pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000;
		gf->pi.status.st_size = 0;
		gf->pi.status.st_nsections = 0;
		gf->mode |= GFS_FILE_MODE_FILE_CREATED;
	}
	*gfp = gf;

	e = NULL;
	goto finish;
free_gf_pi:
	gfarm_path_info_free(&gf->pi);
free_gf:
	gfs_file_free(gf);
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_create_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #13
0
char *
gfs_pio_create(const char *url, int flags, gfarm_mode_t mode, GFS_File *gfp)
{
	char *e, *pathname;
	GFS_File gf;
	int pi_available = 0;
	mode_t mask;
	char *user;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	user = gfarm_get_global_username();
	if (user == NULL) {
		e = "gfarm_pio_create(): programming error, "
		    "gfarm library isn't properly initialized";
		goto finish;
	}
#if 0 /*  XXX - ROOT I/O opens a new file with O_CREAT|O_RDRW mode. */
	if ((flags & GFARM_FILE_ACCMODE) != GFARM_FILE_WRONLY) {
		e = GFARM_ERR_OPERATION_NOT_SUPPORTED; /* XXX */
		goto finish;
	}
#endif

	mask = umask(0);
	umask(mask);
	mode &= ~mask;

	e = gfarm_url_make_path_for_creation(url, &pathname);
	if (e != NULL)
		goto finish;
	e = gfs_file_alloc(&gf);
	if (e != NULL) {
		free(pathname);
		goto finish;
	}

	/* gfs_pio_create() always assumes CREATE, TRUNC */
	flags |= GFARM_FILE_CREATE | GFARM_FILE_TRUNC;

	if ((flags & (GFARM_FILE_TRUNC|GFARM_FILE_SEQUENTIAL)) !=
	    (GFARM_FILE_TRUNC|GFARM_FILE_SEQUENTIAL)) {
		/* MODE_READ is needed to re-calculate checksum. */
		flags = (flags & ~GFARM_FILE_ACCMODE) | GFARM_FILE_RDWR;
	} else if ((flags & ~GFARM_FILE_ACCMODE) == GFARM_FILE_RDONLY) {
		flags = (flags & ~GFARM_FILE_ACCMODE) | GFARM_FILE_WRONLY;
	} 
	gf->open_flags = flags;

	gf->mode = GFS_FILE_MODE_WRITE;
	if ((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_RDWR)
		gf->mode |= GFS_FILE_MODE_READ;
	e = gfarm_path_info_get(pathname, &gf->pi);
	if (e != NULL && e != GFARM_ERR_NO_SUCH_OBJECT) {
		free(pathname);
		gfs_file_free(gf);
		goto finish;
	}
	if (e == NULL) {
		/* XXX unlink and re-create the file? */
		free(pathname);
		e = gfarm_path_info_access(&gf->pi, GFS_W_OK);
		if (e != NULL) {
			gfarm_path_info_free(&gf->pi);
			gfs_file_free(gf);
			goto finish;
		}
		if (!GFARM_S_ISREG(gf->pi.status.st_mode)) {
			if (GFARM_S_ISDIR(gf->pi.status.st_mode))
				e = GFARM_ERR_IS_A_DIRECTORY;
			else
				e = GFARM_ERR_OPERATION_NOT_SUPPORTED;
			gfarm_path_info_free(&gf->pi);
			gfs_file_free(gf);
			goto finish;
		}
		/*
		 * XXX should check the follows:
		 * - the mode is consistent among same job
		 * - creator of the metainfo has same job id
		 * - O_TRUNC / !O_TRUNC case
		 */
		mode |= GFARM_S_IFREG;
		if (GFARM_S_IS_PROGRAM(mode) != GFS_FILE_IS_PROGRAM(gf)) {
			gfarm_path_info_free(&gf->pi);
			gfs_file_free(gf);
			e = GFARM_ERR_OPERATION_NOT_PERMITTED;
			goto finish;
		}
		pi_available = 1;
	}
	if (!pi_available) {
		struct timeval now;

		gettimeofday(&now, NULL);
		gf->pi.pathname = pathname;
		gf->pi.status.st_mode = (GFARM_S_IFREG | mode);
		gf->pi.status.st_user = strdup(user); /* XXX NULL check */
		gf->pi.status.st_group = strdup("*"); /* XXX for now */
		gf->pi.status.st_atimespec.tv_sec =
		gf->pi.status.st_mtimespec.tv_sec =
		gf->pi.status.st_ctimespec.tv_sec = now.tv_sec;
		gf->pi.status.st_atimespec.tv_nsec =
		gf->pi.status.st_mtimespec.tv_nsec =
		gf->pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000;
		gf->pi.status.st_size = 0;
		gf->pi.status.st_nsections = 0;
	}
	*gfp = gf;
	gfs_uncachedir();

	e = NULL;
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_create_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #14
0
static const char *
pfor(struct action *act, int nfinfo, struct file_info **finfo,
	  int nthreads, struct gfrep_arg *arg)
{
	int i, pi, tnum, nth, nerr = 0;
	gfarm_error_t e;
	int ndst = arg->ndst, nsrc = arg->nsrc;
	const char *errmsg;

	if (ndst <= 0)
		return ("no destination node");
	if (nsrc <= 0)
		return ("no source node");
	if (ndst < opt_nrep)
		return ("not enough number of destination nodes");
	if (nfinfo <= 0)
		return (NULL); /* no file */
	/* sort 'sinfo' in descending order wrt file size */
	qsort(finfo, nfinfo, sizeof(*finfo), filesizecmp_inv);

	if (nthreads <= 0) {
		nthreads = nfinfo;
		if (ndst < nthreads)
			nthreads = ndst;
		if (nsrc < nthreads)
			nthreads = nsrc;
	}
	if (opt_verbose) {
		print_gfrep_arg(arg);
		printf("files: %d\n", nfinfo);
#ifdef _OPENMP
		printf("parallel %s in %d threads\n", act->msg, nthreads);
#endif
	}
	omp_set_num_threads(nthreads);

#pragma omp parallel reduction(+:nerr) private(pi, tnum, nth)
	{
	pi = 0;
	tnum = omp_get_thread_num();
	nth = omp_get_num_threads();

#pragma omp for schedule(dynamic)
	for (i = 0; i < nfinfo; ++i) {
		struct timeval t1, t2;
		double t;
		struct file_info *fi = finfo[i];
#ifdef LIBGFARM_NOT_MT_SAFE
		pid_t pid;
		int s, rv;
#endif
		if (opt_noexec || !opt_quiet)
			printf("%s\n", fi->pathname);
		if (opt_verbose)
			print_file_info(fi);
		if (tnum + pi * nth >= ndst)
			pi = 0;

#ifdef LIBGFARM_NOT_MT_SAFE
		pid = fork();
		if (pid == 0) {
			e = gfarm_terminate();
			if (e == GFARM_ERR_NO_ERROR)
				e = gfarm_initialize(NULL, NULL);
			if (e != GFARM_ERR_NO_ERROR) {
				errmsg = gfarm_error_string(e);
				goto skip_replication;
			}
#endif
			if (opt_verbose)
				gettimeofday(&t1, NULL);
			if (!opt_noexec) {
				e = action(act, tnum, nth, pi, fi, arg);
				errmsg = gfarm_error_string(e);
			} else {
				e = GFARM_ERR_NO_ERROR;
				errmsg = gfarm_error_string(e);
			}
			if (opt_verbose && e == GFARM_ERR_NO_ERROR) {
				gettimeofday(&t2, NULL);
				t = gfarm_timerval_sub(&t2, &t1);
				printf("%s: %f sec  %f MB/sec\n",
				       fi->pathname, t,
				       fi->filesize / t / 1024 / 1024);
			}
#ifdef LIBGFARM_NOT_MT_SAFE
 skip_replication:
#endif
			if (e != GFARM_ERR_NO_ERROR) {
#ifndef LIBGFARM_NOT_MT_SAFE
				++nerr;
#endif
				fprintf(stderr, "%s: %s\n",
					fi->pathname, errmsg);
			}
#ifdef LIBGFARM_NOT_MT_SAFE
			fflush(stdout);
			sync();
			_exit(e == GFARM_ERR_NO_ERROR ? 0 : 1);
		}
		while ((rv = waitpid(pid, &s, 0)) == -1 && errno == EINTR)
			;
		if (rv == -1 || (WIFEXITED(s) && WEXITSTATUS(s) != 0))
			++nerr;
#endif
		++pi;
	}
	}
	return (nerr == 0 ? NULL : "error happens during operations");
}
Beispiel #15
0
char *
gfs_pio_seek(GFS_File gf, file_offset_t offset, int whence,
	     file_offset_t *resultp)
{
	char *e;
	file_offset_t where;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e = gfs_pio_check_view_default(gf);
	if (e != NULL)
		return (e);

	if (whence == SEEK_SET || whence == SEEK_CUR) {
		file_offset_t tmp_offset = offset;
		/*
		 * This is the case that the file offset will be
		 * repositioned within the current io buffer.
		 */
		if (whence == SEEK_CUR)
			tmp_offset += gf->offset + gf->p;

		if (gf->offset <= tmp_offset
		    && tmp_offset <= gf->offset + gf->length){
			/*
			 * We don't have to clear
			 * GFS_FILE_MODE_CALC_DIGEST bit here, because
			 * this is no problem to calculate checksum
			 * for write-only or read-only case.
			 * This is also ok on switching from writing
			 * to reading.
			 * This is not ok on switching from reading to
			 * writing, but gfs_pio_flush() clears the bit
			 * at that case.
			 */
			gf->p = tmp_offset - gf->offset;
			if (resultp != NULL)
				*resultp = tmp_offset;

			e = NULL;
			goto finish;
		}
	} else if (whence != SEEK_END) {
		e = gf->error = GFARM_ERR_INVALID_ARGUMENT;
		goto finish;
	}

	gf->mode &= ~GFS_FILE_MODE_CALC_DIGEST;
	    
	if (gf->mode & GFS_FILE_MODE_BUFFER_DIRTY) {
		e = gfs_pio_flush(gf);
		if (e != NULL)
			goto finish;
	}
	gf->error = NULL; /* purge EOF/error state */
	gfs_pio_purge(gf);
	e = (*gf->ops->view_seek)(gf, offset, whence, &where);
	if (e != NULL) {
		gf->error = e;
		goto finish;
	}
	gf->offset = gf->io_offset = where;
	if (resultp != NULL)
		*resultp = where;

	e = NULL;
 finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_seek_time += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #16
0
char *
gfs_pio_set_view_section(GFS_File gf, const char *section,
			 char *if_hostname, int flags)
{
	struct gfs_file_section_context *vc;
	char *e;
	int is_local_host;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e = gfs_pio_set_view_default(gf);
	if (e != NULL)
		goto profile_finish;

	vc = malloc(sizeof(struct gfs_file_section_context));
	if (vc == NULL) {
		e = gf->error = GFARM_ERR_NO_MEMORY;
		goto profile_finish;
	}

	vc->section = strdup(section);
	if (vc->section == NULL) {
		free(vc);
		e = gf->error = GFARM_ERR_NO_MEMORY;
		goto profile_finish;
	}

 retry:
	if (if_hostname != NULL) {
		e = gfarm_host_get_canonical_name(if_hostname,
		    &vc->canonical_hostname);
		if (e == GFARM_ERR_UNKNOWN_HOST) {
			/* FT - invalid hostname, delete section copy info */
			if (gfarm_file_section_copy_info_remove(
				    gf->pi.pathname, vc->section, if_hostname)
			    == NULL)
				e = GFARM_ERR_INCONSISTENT_RECOVERABLE;

			if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE
			    && (flags & GFARM_FILE_NOT_RETRY) == 0
			    && (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) {
				if_hostname = NULL;
				goto retry;
			}
			goto finish;
		}
		else if (e != NULL)
			goto finish;
	} else if ((gf->open_flags & GFARM_FILE_CREATE) != 0) {
		e = gfarm_host_get_canonical_self_name(&if_hostname);
		if (e == NULL)
			e = gfarm_schedule_search_idle_hosts(
				1, &if_hostname, 1, &vc->canonical_hostname);
		if (e == NULL)
			e = gfarm_fixedstrings_dup(
				1, &vc->canonical_hostname,
				   &vc->canonical_hostname);
		if (e != NULL) {
			/*
			 * local host is not a file system node, or
			 * 'gfsd' on a local host is not running.
			 */
			e = gfarm_schedule_search_idle_by_all(1, &if_hostname);
			if (e != NULL)
				goto finish;
			vc->canonical_hostname = if_hostname;
		}
	} else {
		e = gfarm_file_section_host_schedule_with_priority_to_local(
		    gf->pi.pathname, vc->section, &if_hostname);
		if (e != NULL)
			goto finish;
		vc->canonical_hostname = if_hostname; /* must be already
							 canonical */
	}
	is_local_host = gfarm_canonical_hostname_is_local(
					vc->canonical_hostname);

	if ((gf->open_flags & GFARM_FILE_CREATE) != 0) {
		struct gfarm_path_info pi;

		e = gfarm_path_info_set(gf->pi.pathname, &gf->pi);
		if (e == GFARM_ERR_ALREADY_EXISTS &&
		    (e = gfarm_path_info_get(gf->pi.pathname, &pi)) == NULL) {
			if (GFS_FILE_IS_PROGRAM(gf) !=
			    GFARM_S_IS_PROGRAM(pi.status.st_mode))
				e = GFARM_ERR_OPERATION_NOT_PERMITTED;
			if (e == NULL && !GFS_FILE_IS_PROGRAM(gf)) {
				if (gf->pi.status.st_nsections !=
				    pi.status.st_nsections) {
					e = GFARM_ERR_FRAGMENT_NUMBER_DOES_NOT_MATCH;
				} else {
#if 0
					 assert(gf->pi.status.st_mode &
					     GFS_FILE_MODE_NSEGMENTS_FIXED);
#endif
				}
			}
			if (e != NULL) {
				gfarm_path_info_free(&pi);
			} else {
				gfarm_path_info_free(&gf->pi);
				gf->pi = pi;
			}
			/*
			 * XXX should check the follows:
			 * - creator of the metainfo has same job id
			 * - mode is consistent among same job
			 * - nfragments is consistent among same job
			 */
		}
		if (e != NULL)
			goto free_host;
	}

	/* delete the file section when opening with a truncation flag. */
	if ((gf->open_flags & GFARM_FILE_TRUNC) != 0)
		(void)gfs_unlink_section(gf->pi.pathname, vc->section);

	gf->ops = &gfs_pio_view_section_ops;
	gf->view_context = vc;
	gf->view_flags = flags;
	gf->p = gf->length = 0;
	gf->io_offset = gf->offset = 0;

	gf->mode |= GFS_FILE_MODE_CALC_DIGEST;
	EVP_DigestInit(&vc->md_ctx, GFS_DEFAULT_DIGEST_MODE);

	if (!is_local_host && 
	    ((((gf->open_flags & GFARM_FILE_REPLICATE) != 0
	       || gf_on_demand_replication ) &&  
	      (flags & GFARM_FILE_NOT_REPLICATE) == 0) ||
	     (flags & GFARM_FILE_REPLICATE) != 0)) {
		e = replicate_section_to_local(gf, vc->section, if_hostname);
		/* FT - inconsistent metadata has been fixed.  try again. */
		if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE
		    && (flags & GFARM_FILE_NOT_RETRY) == 0
		    && (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) {
			if_hostname = NULL;
			free(vc->canonical_hostname);
			goto retry;
		}
		if (e != NULL)
			goto free_host;
		free(vc->canonical_hostname);
		e = gfarm_host_get_canonical_self_name(
		    &vc->canonical_hostname); 
		if (e != NULL)
			goto finish;
		vc->canonical_hostname = strdup(vc->canonical_hostname);
		if (vc->canonical_hostname == NULL) {
			e = GFARM_ERR_NO_MEMORY;
			goto finish;
		}
		is_local_host = 1;
	}

	if (is_local_host)
		e = gfs_pio_open_local_section(gf, flags);
	else
		e = gfs_pio_open_remote_section(gf, if_hostname, flags);

	/* FT - inconsistent metadata has been fixed.  try again. */
	if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE
	    && (flags & GFARM_FILE_NOT_RETRY) == 0
	    && (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) {
		if_hostname = NULL;
		free(vc->canonical_hostname);
		goto retry;
	}

free_host:
	if (e != NULL)
		free(vc->canonical_hostname);

finish:
	if (e != NULL) {
		free(vc->section);
		free(vc);
		gf->view_context = NULL;
		gfs_pio_set_view_default(gf);
	}
	gf->error = e;

profile_finish:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_pio_set_view_section_time
		    += gfarm_timerval_sub(&t2, &t1));

	return (e);
}
Beispiel #17
0
char *
gfs_unlink(const char *gfarm_url)
{
	char *gfarm_file, *e, *e_save = NULL;
	int i, j, nsections;
	struct gfarm_path_info pi;
	struct gfarm_file_section_info *sections;
	gfarm_timerval_t t1, t2;

	gfs_profile(gfarm_gettimerval(&t1));

	e = gfarm_url_make_path_for_creation(gfarm_url, &gfarm_file);
	if (e != NULL) {
		e_save = e;
		goto finish_unlink;
	}
	e = gfarm_path_info_get(gfarm_file, &pi);
	if (e != NULL) {
		e_save = e;
		goto finish_free_gfarm_file;
	}
	if (GFARM_S_ISDIR(pi.status.st_mode)) {
		gfarm_path_info_free(&pi);
		e_save = GFARM_ERR_IS_A_DIRECTORY;
		goto finish_free_gfarm_file;
	}
	gfarm_path_info_free(&pi);
	e = gfarm_file_section_info_get_all_by_file(gfarm_file,
	    &nsections, &sections);
	if (e != NULL) {
		e_save = e;
		nsections = 0;
		sections = NULL;
	}
	/* XXX - should unlink in parallel. */
	for (i = 0; i < nsections; i++) {
		int ncopies;
		struct gfarm_file_section_copy_info *copies;

		e = gfarm_file_section_copy_info_get_all_by_section(
		    gfarm_file, sections[i].section, &ncopies, &copies);
		if (e != NULL) {
			if (e_save == NULL)
				e_save = e;
			continue;
		}
		for (j = 0; j < ncopies; j++) {
			e = gfs_unlink_replica_internal(gfarm_file,
				sections[i].section, copies[j].hostname);
			if (e != NULL) {
				if (e_save == NULL)
					e_save = e;
				continue;
			}
		}
		gfarm_file_section_copy_info_free_all(ncopies, copies);
	}
	if (sections != NULL)
		gfarm_file_section_info_free_all(nsections, sections);
	e = gfarm_file_section_info_remove_all_by_file(gfarm_file);
	if (e != NULL)
		e_save = e;
	e = gfarm_path_info_remove(gfarm_file);
	if (e != NULL)
		e_save = e;

finish_free_gfarm_file:
	free(gfarm_file);

finish_unlink:
	gfs_profile(gfarm_gettimerval(&t2));
	gfs_profile(gfs_unlink_time += gfarm_timerval_sub(&t2, &t1));

	return (e_save);	
}