char *
gfs_pio_close(GFS_File gf)
{
char *e, *e_save;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e_save = gfs_pio_check_view_default(gf);
if (e_save == NULL) {
if ((gf->mode & GFS_FILE_MODE_WRITE) != 0)
e_save = gfs_pio_flush(gf);
e = (*gf->ops->view_close)(gf);
if (e_save == NULL)
e_save = e;
}
gfarm_path_info_free(&gf->pi);
free(gf->buffer);
free(gf);
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_close_time += gfarm_timerval_sub(&t2, &t1));
return (e_save);
}
char *
gfs_pio_truncate(GFS_File gf, file_offset_t length)
{
char *e;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e = gfs_pio_check_view_default(gf);
if (e != NULL)
goto finish;
CHECK_WRITABLE(gf);
gf->mode &= ~GFS_FILE_MODE_CALC_DIGEST;
if (gf->mode & GFS_FILE_MODE_BUFFER_DIRTY) {
e = gfs_pio_flush(gf);
if (e != NULL)
goto finish;
}
gf->error = NULL; /* purge EOF/error state */
gfs_pio_purge(gf);
e = (*gf->ops->view_ftruncate)(gf, length);
if (e != NULL)
gf->error = e;
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_truncate_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_pio_close(GFS_File gf)
{
char *e, *e_save;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e_save = gfs_pio_check_view_default(gf);
if (e_save == NULL) {
if ((gf->mode & GFS_FILE_MODE_WRITE) != 0)
e_save = gfs_pio_flush(gf);
e = (*gf->ops->view_close)(gf);
if (e_save == NULL)
e_save = e;
}
/*
* When there is inconsistency, do not update/overwrite the
* metadata. This inconsistency may come from the update by
* other process or oneself such as 'nvi'.
*/
if (e_save == NULL)
e_save = gfs_pio_update_times(gf);
gfarm_path_info_free(&gf->pi);
gfs_file_free(gf);
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_close_time += gfarm_timerval_sub(&t2, &t1));
return (e_save);
}
char *
gfs_pio_open(const char *url, int flags, GFS_File *gfp)
{
char *e, *pathname;
GFS_File gf;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
if ((flags & GFARM_FILE_ACCMODE) != GFARM_FILE_RDONLY) {
#if 0 /* XXX - ROOT I/O opens a new file with O_CREAT|O_RDRW mode. */
e = GFARM_ERR_OPERATION_NOT_SUPPORTED; /* XXX */
goto finish;
#else
flags |= GFARM_FILE_CREATE;
#endif
}
e = gfarm_url_make_path(url, &pathname);
if (e != NULL)
goto finish;
e = gfs_file_alloc(&gf);
if (e != NULL) {
free(pathname);
goto finish;
}
gf->open_flags = flags;
gf->mode |= GFS_FILE_MODE_READ;
if (((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_RDWR)
|| ((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_WRONLY))
gf->mode |= GFS_FILE_MODE_WRITE;
e = gfarm_path_info_get(pathname, &gf->pi);
free(pathname);
if (e != NULL)
goto finish;
e = gfarm_path_info_access(&gf->pi, GFS_R_OK);
if (e != NULL) {
gfarm_path_info_free(&gf->pi);
gfs_file_free(gf);
goto finish;
}
if (!GFARM_S_ISREG(gf->pi.status.st_mode)) {
if (GFARM_S_ISDIR(gf->pi.status.st_mode))
e = GFARM_ERR_IS_A_DIRECTORY;
else
e = GFARM_ERR_OPERATION_NOT_SUPPORTED;
gfarm_path_info_free(&gf->pi);
gfs_file_free(gf);
goto finish;
}
gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED;
*gfp = gf;
e = NULL;
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_open_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_pio_write(GFS_File gf, const void *buffer, int size, int *np)
{
char *e;
size_t written;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e = gfs_pio_check_view_default(gf);
if (e != NULL)
return (e);
CHECK_WRITABLE(gf);
if (size + gf->p > GFS_FILE_BUFSIZE) {
/*
* gf->buffer[gf->p .. GFS_FILE_BUFSIZE-1] will be overridden
* by buffer.
*/
gf->length = gf->p;
e = gfs_pio_flush(gf);
if (e != NULL)
goto finish;
}
if (size >= GFS_FILE_BUFSIZE) {
/* shortcut to avoid unnecessary memory copy */
assert(gf->p == 0); /* gfs_pio_flush() was called above */
gf->length = 0;
gf->mode &= ~GFS_FILE_MODE_BUFFER_DIRTY;
e = do_write(gf, buffer, size, &written);
if (e != NULL && written == 0)
goto finish;
gf->offset += written;
*np = written; /* XXX - size_t vs int */
e = NULL;
goto finish;
}
gf->mode |= GFS_FILE_MODE_BUFFER_DIRTY;
memcpy(gf->buffer + gf->p, buffer, size);
gf->p += size;
if (gf->p > gf->length)
gf->length = gf->p;
*np = size;
e = NULL;
if (gf->p >= GFS_FILE_BUFSIZE)
e = gfs_pio_flush(gf);
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_write_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_pio_open(const char *url, int flags, GFS_File *gfp)
{
char *e, *pathname;
GFS_File gf;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
if (flags & GFARM_FILE_CREATE) {
e = GFARM_ERR_OPERATION_NOT_SUPPORTED;
goto finish;
}
e = gfarm_url_make_path(url, &pathname);
if (e != NULL)
goto finish;
e = gfs_file_alloc(&gf);
if (e != NULL) {
free(pathname);
goto finish;
}
/* GFARM_FILE_EXCLUSIVE is a NOP with gfs_pio_open(). */
flags &= ~GFARM_FILE_EXCLUSIVE;
gfs_pio_open_initialize_mode_flags(gf, flags);
e = gfarm_path_info_get(pathname, &gf->pi);
free(pathname);
if (e != NULL)
goto free_gf;
e = gfs_pio_open_check_perm(gf);
if (e != NULL)
goto free_gf_pi;
if ((flags & GFARM_FILE_TRUNC) == 0)
gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED;
*gfp = gf;
e = NULL;
goto finish;
free_gf_pi:
gfarm_path_info_free(&gf->pi);
free_gf:
gfs_file_free(gf);
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_open_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_stat(const char *path, struct gfs_stat *s)
{
char *e, *p;
gfarm_timerval_t t1, t2;
long ino;
gfs_profile(gfarm_gettimerval(&t1));
path = gfarm_url_prefix_skip(path);
e = gfarm_canonical_path(path, &p);
if (e != NULL)
goto finish;
e = gfs_stat_canonical_path(p, s);
if (e == NULL)
goto finish_free_p;
if (e != GFARM_ERR_NO_SUCH_OBJECT)
goto finish_free_p;
/*
* XXX - assume that it's a directory that does not have the
* path info.
*/
e = gfs_get_ino(p, &ino);
if (e != NULL)
goto finish_free_p;
s->st_ino = ino;
s->st_mode = GFARM_S_IFDIR | 0777;
s->st_user = strdup("root");
s->st_group = strdup("gfarm");
s->st_atimespec.tv_sec = 0;
s->st_atimespec.tv_nsec = 0;
s->st_mtimespec.tv_sec = 0;
s->st_mtimespec.tv_nsec = 0;
s->st_ctimespec.tv_sec = 0;
s->st_ctimespec.tv_nsec = 0;
s->st_size = 0;
s->st_nsections = 0;
e = NULL;
finish_free_p:
free(p);
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_stat_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_pio_read(GFS_File gf, void *buffer, int size, int *np)
{
char *e;
char *p = buffer;
int n = 0;
int length;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e = gfs_pio_check_view_default(gf);
if (e != NULL)
return (e);
CHECK_READABLE(gf);
while (size > 0) {
if ((e = gfs_pio_fillbuf(gf)) != NULL)
break;
if (gf->error != NULL) /* EOF or error */
break;
length = gf->length - gf->p;
if (length > size)
length = size;
memcpy(p, gf->buffer + gf->p, length);
p += length;
n += length;
size -= length;
gf->p += length;
}
if (e != NULL && n == 0)
goto finish;
*np = n;
e = NULL;
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_read_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_pio_getline(GFS_File gf, char *s, size_t size, int *eofp)
{
char *e = gfs_pio_check_view_default(gf);
char *p = s;
int c;
gfarm_timerval_t t1, t2;
if (e != NULL)
return (e);
gfs_profile(gfarm_gettimerval(&t1));
#ifdef __GNUC__ /* workaround gcc warning: unused variable */
c = EOF;
#endif
CHECK_READABLE(gf);
if (size <= 1) {
gf->error = GFARM_ERR_INVALID_ARGUMENT;
return (gf->error);
}
--size; /* for '\0' */
for (; size > 0 && (c = gfs_pio_getc(gf)) != EOF; --size) {
if (c == '\n')
break;
*p++ = c;
}
*p++ = '\0';
if (p == s + 1 && c == EOF) {
*eofp = 1;
return (GFS_PIO_ERROR(gf));
}
*eofp = 0;
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_getline_time += gfarm_timerval_sub(&t2, &t1));
return (NULL);
}
/*
* mostly compatible with getdelim(3) in glibc,
* but there are the following differences:
* 1. on EOF, *lenp == 0
* 2. on error, *lenp isn't touched.
*/
char *
gfs_pio_readdelim(GFS_File gf, char **bufp, size_t *sizep, size_t *lenp,
const char *delim, size_t delimlen)
{
char *e = gfs_pio_check_view_default(gf);
char *buf = *bufp, *p;
size_t size = *sizep, len = 0;
int c, delimtail;
static const char empty_line[] = "\n\n";
gfarm_timerval_t t1, t2;
if (e != NULL)
return (e);
gfs_profile(gfarm_gettimerval(&t1));
#ifdef __GNUC__ /* workaround gcc warning: unused variable */
c = EOF;
#endif
CHECK_READABLE(gf);
if (delim == NULL) { /* special case 1 */
delimtail = 0; /* workaround gcc warning */
} else {
if (delimlen == 0) { /* special case 2 */
delim = empty_line;
delimlen = 2;
}
delimtail = delim[delimlen - 1];
}
if (buf == NULL || size <= 1) {
if (size <= 1)
size = ALLOC_SIZE_INIT;
buf = realloc(buf, size);
if (buf == NULL)
return (GFARM_ERR_NO_MEMORY);
}
for (;;) {
c = gfs_pio_getc(gf);
if (c == EOF)
break;
if (size <= len) {
p = realloc(buf, size + size);
if (p == NULL) {
*bufp = buf;
*sizep = size;
return (GFARM_ERR_NO_MEMORY);
}
buf = p;
size += size;
}
buf[len++] = c;
if (delim == NULL) /* special case 1: no delimiter */
continue;
if (len >= delimlen && c == delimtail &&
memcmp(&buf[len - delimlen], delim, delimlen) == 0) {
if (delim == empty_line) { /* special case 2 */
for (;;) {
c = gfs_pio_getc(gf);
if (c == EOF)
break;
if (c != '\n') {
gfs_pio_ungetc(gf, c);
break;
}
if (size <= len) {
p = realloc(buf, size + size);
if (p == NULL) {
*bufp = buf;
*sizep = size;
return (
GFARM_ERR_NO_MEMORY);
}
buf = p;
size += size;
}
buf[len++] = c;
}
}
break;
}
}
if (size <= len) {
p = realloc(buf, size + size);
if (p == NULL) {
*bufp = buf;
*sizep = size;
return (GFARM_ERR_NO_MEMORY);
}
buf = p;
size += size;
}
buf[len] = '\0';
gfs_profile(gfarm_gettimerval(&t2));
/* XXX should introduce gfs_pio_readdelim_time??? */
gfs_profile(gfs_pio_getline_time += gfarm_timerval_sub(&t2, &t1));
*bufp = buf;
*sizep = size;
*lenp = len;
return (NULL);
}
/*
* mostly compatible with getline(3) in glibc,
* but there are the following differences:
* 1. on EOF, *lenp == 0
* 2. on error, *lenp isn't touched.
*/
char *
gfs_pio_readline(GFS_File gf, char **bufp, size_t *sizep, size_t *lenp)
{
char *e = gfs_pio_check_view_default(gf);
char *buf = *bufp, *p;
size_t size = *sizep, len = 0;
int c;
gfarm_timerval_t t1, t2;
if (e != NULL)
return (e);
gfs_profile(gfarm_gettimerval(&t1));
#ifdef __GNUC__ /* workaround gcc warning: unused variable */
c = EOF;
#endif
CHECK_READABLE(gf);
if (buf == NULL || size <= 1) {
if (size <= 1)
size = ALLOC_SIZE_INIT;
buf = realloc(buf, size);
if (buf == NULL)
return (GFARM_ERR_NO_MEMORY);
}
for (;;) {
c = gfs_pio_getc(gf);
if (c == EOF)
break;
if (size <= len) {
p = realloc(buf, size + size);
if (p == NULL) {
*bufp = buf;
*sizep = size;
return (GFARM_ERR_NO_MEMORY);
}
buf = p;
size += size;
}
buf[len++] = c;
if (c == '\n')
break;
}
if (size <= len) {
p = realloc(buf, size + size);
if (p == NULL) {
*bufp = buf;
*sizep = size;
return (GFARM_ERR_NO_MEMORY);
}
buf = p;
size += size;
}
buf[len] = '\0';
gfs_profile(gfarm_gettimerval(&t2));
/* XXX should introduce gfs_pio_readline_time??? */
gfs_profile(gfs_pio_getline_time += gfarm_timerval_sub(&t2, &t1));
*bufp = buf;
*sizep = size;
*lenp = len;
return (NULL);
}
char *
gfs_pio_create(const char *url, int flags, gfarm_mode_t mode, GFS_File *gfp)
{
char *e, *pathname;
GFS_File gf;
char *user;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
user = gfarm_get_global_username();
if (user == NULL) {
e = "gfs_pio_create(): programming error, "
"gfarm library isn't properly initialized";
goto finish;
}
e = gfarm_url_make_path_for_creation(url, &pathname);
if (e != NULL)
goto finish;
e = gfs_file_alloc(&gf);
if (e != NULL) {
free(pathname);
goto finish;
}
flags |= GFARM_FILE_CREATE;
gfs_pio_open_initialize_mode_flags(gf, flags);
e = gfarm_path_info_get(pathname, &gf->pi);
if (e != NULL && e != GFARM_ERR_NO_SUCH_OBJECT) {
free(pathname);
goto free_gf;
}
if (e == NULL) {
free(pathname);
if ((flags & GFARM_FILE_EXCLUSIVE) != 0) {
e = GFARM_ERR_ALREADY_EXISTS;
goto free_gf_pi;
}
e = gfs_pio_open_check_perm(gf);
if (e != NULL)
goto free_gf_pi;
/*
* XXX should check the follows:
* - the mode is consistent among same job
* - creator of the metainfo has same job id
* - O_TRUNC / !O_TRUNC case
*/
if ((flags & GFARM_FILE_TRUNC) == 0)
gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED;
} else {
mode_t mask;
struct timeval now;
mask = umask(0);
umask(mask);
gettimeofday(&now, NULL);
gf->pi.pathname = pathname;
gf->pi.status.st_mode = (GFARM_S_IFREG | (mode & ~mask));
gf->pi.status.st_user = strdup(user); /* XXX NULL check */
gf->pi.status.st_group = strdup("*"); /* XXX for now */
gf->pi.status.st_atimespec.tv_sec =
gf->pi.status.st_mtimespec.tv_sec =
gf->pi.status.st_ctimespec.tv_sec = now.tv_sec;
gf->pi.status.st_atimespec.tv_nsec =
gf->pi.status.st_mtimespec.tv_nsec =
gf->pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000;
gf->pi.status.st_size = 0;
gf->pi.status.st_nsections = 0;
gf->mode |= GFS_FILE_MODE_FILE_CREATED;
}
*gfp = gf;
e = NULL;
goto finish;
free_gf_pi:
gfarm_path_info_free(&gf->pi);
free_gf:
gfs_file_free(gf);
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_create_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_pio_create(const char *url, int flags, gfarm_mode_t mode, GFS_File *gfp)
{
char *e, *pathname;
GFS_File gf;
int pi_available = 0;
mode_t mask;
char *user;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
user = gfarm_get_global_username();
if (user == NULL) {
e = "gfarm_pio_create(): programming error, "
"gfarm library isn't properly initialized";
goto finish;
}
#if 0 /* XXX - ROOT I/O opens a new file with O_CREAT|O_RDRW mode. */
if ((flags & GFARM_FILE_ACCMODE) != GFARM_FILE_WRONLY) {
e = GFARM_ERR_OPERATION_NOT_SUPPORTED; /* XXX */
goto finish;
}
#endif
mask = umask(0);
umask(mask);
mode &= ~mask;
e = gfarm_url_make_path_for_creation(url, &pathname);
if (e != NULL)
goto finish;
e = gfs_file_alloc(&gf);
if (e != NULL) {
free(pathname);
goto finish;
}
/* gfs_pio_create() always assumes CREATE, TRUNC */
flags |= GFARM_FILE_CREATE | GFARM_FILE_TRUNC;
if ((flags & (GFARM_FILE_TRUNC|GFARM_FILE_SEQUENTIAL)) !=
(GFARM_FILE_TRUNC|GFARM_FILE_SEQUENTIAL)) {
/* MODE_READ is needed to re-calculate checksum. */
flags = (flags & ~GFARM_FILE_ACCMODE) | GFARM_FILE_RDWR;
} else if ((flags & ~GFARM_FILE_ACCMODE) == GFARM_FILE_RDONLY) {
flags = (flags & ~GFARM_FILE_ACCMODE) | GFARM_FILE_WRONLY;
}
gf->open_flags = flags;
gf->mode = GFS_FILE_MODE_WRITE;
if ((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_RDWR)
gf->mode |= GFS_FILE_MODE_READ;
e = gfarm_path_info_get(pathname, &gf->pi);
if (e != NULL && e != GFARM_ERR_NO_SUCH_OBJECT) {
free(pathname);
gfs_file_free(gf);
goto finish;
}
if (e == NULL) {
/* XXX unlink and re-create the file? */
free(pathname);
e = gfarm_path_info_access(&gf->pi, GFS_W_OK);
if (e != NULL) {
gfarm_path_info_free(&gf->pi);
gfs_file_free(gf);
goto finish;
}
if (!GFARM_S_ISREG(gf->pi.status.st_mode)) {
if (GFARM_S_ISDIR(gf->pi.status.st_mode))
e = GFARM_ERR_IS_A_DIRECTORY;
else
e = GFARM_ERR_OPERATION_NOT_SUPPORTED;
gfarm_path_info_free(&gf->pi);
gfs_file_free(gf);
goto finish;
}
/*
* XXX should check the follows:
* - the mode is consistent among same job
* - creator of the metainfo has same job id
* - O_TRUNC / !O_TRUNC case
*/
mode |= GFARM_S_IFREG;
if (GFARM_S_IS_PROGRAM(mode) != GFS_FILE_IS_PROGRAM(gf)) {
gfarm_path_info_free(&gf->pi);
gfs_file_free(gf);
e = GFARM_ERR_OPERATION_NOT_PERMITTED;
goto finish;
}
pi_available = 1;
}
if (!pi_available) {
struct timeval now;
gettimeofday(&now, NULL);
gf->pi.pathname = pathname;
gf->pi.status.st_mode = (GFARM_S_IFREG | mode);
gf->pi.status.st_user = strdup(user); /* XXX NULL check */
gf->pi.status.st_group = strdup("*"); /* XXX for now */
gf->pi.status.st_atimespec.tv_sec =
gf->pi.status.st_mtimespec.tv_sec =
gf->pi.status.st_ctimespec.tv_sec = now.tv_sec;
gf->pi.status.st_atimespec.tv_nsec =
gf->pi.status.st_mtimespec.tv_nsec =
gf->pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000;
gf->pi.status.st_size = 0;
gf->pi.status.st_nsections = 0;
}
*gfp = gf;
gfs_uncachedir();
e = NULL;
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_create_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
static const char *
pfor(struct action *act, int nfinfo, struct file_info **finfo,
int nthreads, struct gfrep_arg *arg)
{
int i, pi, tnum, nth, nerr = 0;
gfarm_error_t e;
int ndst = arg->ndst, nsrc = arg->nsrc;
const char *errmsg;
if (ndst <= 0)
return ("no destination node");
if (nsrc <= 0)
return ("no source node");
if (ndst < opt_nrep)
return ("not enough number of destination nodes");
if (nfinfo <= 0)
return (NULL); /* no file */
/* sort 'sinfo' in descending order wrt file size */
qsort(finfo, nfinfo, sizeof(*finfo), filesizecmp_inv);
if (nthreads <= 0) {
nthreads = nfinfo;
if (ndst < nthreads)
nthreads = ndst;
if (nsrc < nthreads)
nthreads = nsrc;
}
if (opt_verbose) {
print_gfrep_arg(arg);
printf("files: %d\n", nfinfo);
#ifdef _OPENMP
printf("parallel %s in %d threads\n", act->msg, nthreads);
#endif
}
omp_set_num_threads(nthreads);
#pragma omp parallel reduction(+:nerr) private(pi, tnum, nth)
{
pi = 0;
tnum = omp_get_thread_num();
nth = omp_get_num_threads();
#pragma omp for schedule(dynamic)
for (i = 0; i < nfinfo; ++i) {
struct timeval t1, t2;
double t;
struct file_info *fi = finfo[i];
#ifdef LIBGFARM_NOT_MT_SAFE
pid_t pid;
int s, rv;
#endif
if (opt_noexec || !opt_quiet)
printf("%s\n", fi->pathname);
if (opt_verbose)
print_file_info(fi);
if (tnum + pi * nth >= ndst)
pi = 0;
#ifdef LIBGFARM_NOT_MT_SAFE
pid = fork();
if (pid == 0) {
e = gfarm_terminate();
if (e == GFARM_ERR_NO_ERROR)
e = gfarm_initialize(NULL, NULL);
if (e != GFARM_ERR_NO_ERROR) {
errmsg = gfarm_error_string(e);
goto skip_replication;
}
#endif
if (opt_verbose)
gettimeofday(&t1, NULL);
if (!opt_noexec) {
e = action(act, tnum, nth, pi, fi, arg);
errmsg = gfarm_error_string(e);
} else {
e = GFARM_ERR_NO_ERROR;
errmsg = gfarm_error_string(e);
}
if (opt_verbose && e == GFARM_ERR_NO_ERROR) {
gettimeofday(&t2, NULL);
t = gfarm_timerval_sub(&t2, &t1);
printf("%s: %f sec %f MB/sec\n",
fi->pathname, t,
fi->filesize / t / 1024 / 1024);
}
#ifdef LIBGFARM_NOT_MT_SAFE
skip_replication:
#endif
if (e != GFARM_ERR_NO_ERROR) {
#ifndef LIBGFARM_NOT_MT_SAFE
++nerr;
#endif
fprintf(stderr, "%s: %s\n",
fi->pathname, errmsg);
}
#ifdef LIBGFARM_NOT_MT_SAFE
fflush(stdout);
sync();
_exit(e == GFARM_ERR_NO_ERROR ? 0 : 1);
}
while ((rv = waitpid(pid, &s, 0)) == -1 && errno == EINTR)
;
if (rv == -1 || (WIFEXITED(s) && WEXITSTATUS(s) != 0))
++nerr;
#endif
++pi;
}
}
return (nerr == 0 ? NULL : "error happens during operations");
}
char *
gfs_pio_seek(GFS_File gf, file_offset_t offset, int whence,
file_offset_t *resultp)
{
char *e;
file_offset_t where;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e = gfs_pio_check_view_default(gf);
if (e != NULL)
return (e);
if (whence == SEEK_SET || whence == SEEK_CUR) {
file_offset_t tmp_offset = offset;
/*
* This is the case that the file offset will be
* repositioned within the current io buffer.
*/
if (whence == SEEK_CUR)
tmp_offset += gf->offset + gf->p;
if (gf->offset <= tmp_offset
&& tmp_offset <= gf->offset + gf->length){
/*
* We don't have to clear
* GFS_FILE_MODE_CALC_DIGEST bit here, because
* this is no problem to calculate checksum
* for write-only or read-only case.
* This is also ok on switching from writing
* to reading.
* This is not ok on switching from reading to
* writing, but gfs_pio_flush() clears the bit
* at that case.
*/
gf->p = tmp_offset - gf->offset;
if (resultp != NULL)
*resultp = tmp_offset;
e = NULL;
goto finish;
}
} else if (whence != SEEK_END) {
e = gf->error = GFARM_ERR_INVALID_ARGUMENT;
goto finish;
}
gf->mode &= ~GFS_FILE_MODE_CALC_DIGEST;
if (gf->mode & GFS_FILE_MODE_BUFFER_DIRTY) {
e = gfs_pio_flush(gf);
if (e != NULL)
goto finish;
}
gf->error = NULL; /* purge EOF/error state */
gfs_pio_purge(gf);
e = (*gf->ops->view_seek)(gf, offset, whence, &where);
if (e != NULL) {
gf->error = e;
goto finish;
}
gf->offset = gf->io_offset = where;
if (resultp != NULL)
*resultp = where;
e = NULL;
finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_seek_time += gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_pio_set_view_section(GFS_File gf, const char *section,
char *if_hostname, int flags)
{
struct gfs_file_section_context *vc;
char *e;
int is_local_host;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e = gfs_pio_set_view_default(gf);
if (e != NULL)
goto profile_finish;
vc = malloc(sizeof(struct gfs_file_section_context));
if (vc == NULL) {
e = gf->error = GFARM_ERR_NO_MEMORY;
goto profile_finish;
}
vc->section = strdup(section);
if (vc->section == NULL) {
free(vc);
e = gf->error = GFARM_ERR_NO_MEMORY;
goto profile_finish;
}
retry:
if (if_hostname != NULL) {
e = gfarm_host_get_canonical_name(if_hostname,
&vc->canonical_hostname);
if (e == GFARM_ERR_UNKNOWN_HOST) {
/* FT - invalid hostname, delete section copy info */
if (gfarm_file_section_copy_info_remove(
gf->pi.pathname, vc->section, if_hostname)
== NULL)
e = GFARM_ERR_INCONSISTENT_RECOVERABLE;
if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE
&& (flags & GFARM_FILE_NOT_RETRY) == 0
&& (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) {
if_hostname = NULL;
goto retry;
}
goto finish;
}
else if (e != NULL)
goto finish;
} else if ((gf->open_flags & GFARM_FILE_CREATE) != 0) {
e = gfarm_host_get_canonical_self_name(&if_hostname);
if (e == NULL)
e = gfarm_schedule_search_idle_hosts(
1, &if_hostname, 1, &vc->canonical_hostname);
if (e == NULL)
e = gfarm_fixedstrings_dup(
1, &vc->canonical_hostname,
&vc->canonical_hostname);
if (e != NULL) {
/*
* local host is not a file system node, or
* 'gfsd' on a local host is not running.
*/
e = gfarm_schedule_search_idle_by_all(1, &if_hostname);
if (e != NULL)
goto finish;
vc->canonical_hostname = if_hostname;
}
} else {
e = gfarm_file_section_host_schedule_with_priority_to_local(
gf->pi.pathname, vc->section, &if_hostname);
if (e != NULL)
goto finish;
vc->canonical_hostname = if_hostname; /* must be already
canonical */
}
is_local_host = gfarm_canonical_hostname_is_local(
vc->canonical_hostname);
if ((gf->open_flags & GFARM_FILE_CREATE) != 0) {
struct gfarm_path_info pi;
e = gfarm_path_info_set(gf->pi.pathname, &gf->pi);
if (e == GFARM_ERR_ALREADY_EXISTS &&
(e = gfarm_path_info_get(gf->pi.pathname, &pi)) == NULL) {
if (GFS_FILE_IS_PROGRAM(gf) !=
GFARM_S_IS_PROGRAM(pi.status.st_mode))
e = GFARM_ERR_OPERATION_NOT_PERMITTED;
if (e == NULL && !GFS_FILE_IS_PROGRAM(gf)) {
if (gf->pi.status.st_nsections !=
pi.status.st_nsections) {
e = GFARM_ERR_FRAGMENT_NUMBER_DOES_NOT_MATCH;
} else {
#if 0
assert(gf->pi.status.st_mode &
GFS_FILE_MODE_NSEGMENTS_FIXED);
#endif
}
}
if (e != NULL) {
gfarm_path_info_free(&pi);
} else {
gfarm_path_info_free(&gf->pi);
gf->pi = pi;
}
/*
* XXX should check the follows:
* - creator of the metainfo has same job id
* - mode is consistent among same job
* - nfragments is consistent among same job
*/
}
if (e != NULL)
goto free_host;
}
/* delete the file section when opening with a truncation flag. */
if ((gf->open_flags & GFARM_FILE_TRUNC) != 0)
(void)gfs_unlink_section(gf->pi.pathname, vc->section);
gf->ops = &gfs_pio_view_section_ops;
gf->view_context = vc;
gf->view_flags = flags;
gf->p = gf->length = 0;
gf->io_offset = gf->offset = 0;
gf->mode |= GFS_FILE_MODE_CALC_DIGEST;
EVP_DigestInit(&vc->md_ctx, GFS_DEFAULT_DIGEST_MODE);
if (!is_local_host &&
((((gf->open_flags & GFARM_FILE_REPLICATE) != 0
|| gf_on_demand_replication ) &&
(flags & GFARM_FILE_NOT_REPLICATE) == 0) ||
(flags & GFARM_FILE_REPLICATE) != 0)) {
e = replicate_section_to_local(gf, vc->section, if_hostname);
/* FT - inconsistent metadata has been fixed. try again. */
if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE
&& (flags & GFARM_FILE_NOT_RETRY) == 0
&& (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) {
if_hostname = NULL;
free(vc->canonical_hostname);
goto retry;
}
if (e != NULL)
goto free_host;
free(vc->canonical_hostname);
e = gfarm_host_get_canonical_self_name(
&vc->canonical_hostname);
if (e != NULL)
goto finish;
vc->canonical_hostname = strdup(vc->canonical_hostname);
if (vc->canonical_hostname == NULL) {
e = GFARM_ERR_NO_MEMORY;
goto finish;
}
is_local_host = 1;
}
if (is_local_host)
e = gfs_pio_open_local_section(gf, flags);
else
e = gfs_pio_open_remote_section(gf, if_hostname, flags);
/* FT - inconsistent metadata has been fixed. try again. */
if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE
&& (flags & GFARM_FILE_NOT_RETRY) == 0
&& (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) {
if_hostname = NULL;
free(vc->canonical_hostname);
goto retry;
}
free_host:
if (e != NULL)
free(vc->canonical_hostname);
finish:
if (e != NULL) {
free(vc->section);
free(vc);
gf->view_context = NULL;
gfs_pio_set_view_default(gf);
}
gf->error = e;
profile_finish:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_pio_set_view_section_time
+= gfarm_timerval_sub(&t2, &t1));
return (e);
}
char *
gfs_unlink(const char *gfarm_url)
{
char *gfarm_file, *e, *e_save = NULL;
int i, j, nsections;
struct gfarm_path_info pi;
struct gfarm_file_section_info *sections;
gfarm_timerval_t t1, t2;
gfs_profile(gfarm_gettimerval(&t1));
e = gfarm_url_make_path_for_creation(gfarm_url, &gfarm_file);
if (e != NULL) {
e_save = e;
goto finish_unlink;
}
e = gfarm_path_info_get(gfarm_file, &pi);
if (e != NULL) {
e_save = e;
goto finish_free_gfarm_file;
}
if (GFARM_S_ISDIR(pi.status.st_mode)) {
gfarm_path_info_free(&pi);
e_save = GFARM_ERR_IS_A_DIRECTORY;
goto finish_free_gfarm_file;
}
gfarm_path_info_free(&pi);
e = gfarm_file_section_info_get_all_by_file(gfarm_file,
&nsections, §ions);
if (e != NULL) {
e_save = e;
nsections = 0;
sections = NULL;
}
/* XXX - should unlink in parallel. */
for (i = 0; i < nsections; i++) {
int ncopies;
struct gfarm_file_section_copy_info *copies;
e = gfarm_file_section_copy_info_get_all_by_section(
gfarm_file, sections[i].section, &ncopies, &copies);
if (e != NULL) {
if (e_save == NULL)
e_save = e;
continue;
}
for (j = 0; j < ncopies; j++) {
e = gfs_unlink_replica_internal(gfarm_file,
sections[i].section, copies[j].hostname);
if (e != NULL) {
if (e_save == NULL)
e_save = e;
continue;
}
}
gfarm_file_section_copy_info_free_all(ncopies, copies);
}
if (sections != NULL)
gfarm_file_section_info_free_all(nsections, sections);
e = gfarm_file_section_info_remove_all_by_file(gfarm_file);
if (e != NULL)
e_save = e;
e = gfarm_path_info_remove(gfarm_file);
if (e != NULL)
e_save = e;
finish_free_gfarm_file:
free(gfarm_file);
finish_unlink:
gfs_profile(gfarm_gettimerval(&t2));
gfs_profile(gfs_unlink_time += gfarm_timerval_sub(&t2, &t1));
return (e_save);
}