static int get_nsections(char *gfarm_url, int *nsectionsp) { char *e, *gfarm_file; struct gfarm_path_info pi; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) { fprintf(stderr, "%s: gfarm_url_make_path(%s): %s\n", program_name, gfarm_url, e); error_happened = 1; return (0); } e = gfarm_path_info_get(gfarm_file, &pi); free(gfarm_file); if (e != NULL) { if (e == GFARM_ERR_NO_SUCH_OBJECT) fprintf(stderr, "%s: missing -N <total number of fragments>\n", program_name); else fprintf(stderr, "%s: gfarm_get_path_info(%s): %s\n", program_name, gfarm_url, e); error_happened = 1; return (0); } *nsectionsp = pi.status.st_nsections; gfarm_path_info_free(&pi); return (1); }
char * gfs_utimes(const char *gfarm_url, const struct gfarm_timespec *tsp) { char *e, *gfarm_file; struct gfarm_path_info pi; struct timeval now; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); free(gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_access(&pi, GFS_W_OK); if (e != NULL) return (e); gettimeofday(&now, NULL); if (tsp == NULL) { pi.status.st_atimespec.tv_sec = pi.status.st_mtimespec.tv_sec = now.tv_sec; pi.status.st_atimespec.tv_nsec = pi.status.st_mtimespec.tv_nsec = now.tv_usec * 1000; } else { pi.status.st_atimespec = tsp[0]; pi.status.st_mtimespec = tsp[1]; } pi.status.st_ctimespec.tv_sec = now.tv_sec; pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000; e = gfarm_path_info_replace(pi.pathname, &pi); gfarm_path_info_free(&pi); return (e); }
char * gfs_pio_open(const char *url, int flags, GFS_File *gfp) { char *e, *pathname; GFS_File gf; gfarm_timerval_t t1, t2; gfs_profile(gfarm_gettimerval(&t1)); if ((flags & GFARM_FILE_ACCMODE) != GFARM_FILE_RDONLY) { #if 0 /* XXX - ROOT I/O opens a new file with O_CREAT|O_RDRW mode. */ e = GFARM_ERR_OPERATION_NOT_SUPPORTED; /* XXX */ goto finish; #else flags |= GFARM_FILE_CREATE; #endif } e = gfarm_url_make_path(url, &pathname); if (e != NULL) goto finish; e = gfs_file_alloc(&gf); if (e != NULL) { free(pathname); goto finish; } gf->open_flags = flags; gf->mode |= GFS_FILE_MODE_READ; if (((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_RDWR) || ((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_WRONLY)) gf->mode |= GFS_FILE_MODE_WRITE; e = gfarm_path_info_get(pathname, &gf->pi); free(pathname); if (e != NULL) goto finish; e = gfarm_path_info_access(&gf->pi, GFS_R_OK); if (e != NULL) { gfarm_path_info_free(&gf->pi); gfs_file_free(gf); goto finish; } if (!GFARM_S_ISREG(gf->pi.status.st_mode)) { if (GFARM_S_ISDIR(gf->pi.status.st_mode)) e = GFARM_ERR_IS_A_DIRECTORY; else e = GFARM_ERR_OPERATION_NOT_SUPPORTED; gfarm_path_info_free(&gf->pi); gfs_file_free(gf); goto finish; } gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED; *gfp = gf; e = NULL; finish: gfs_profile(gfarm_gettimerval(&t2)); gfs_profile(gfs_pio_open_time += gfarm_timerval_sub(&t2, &t1)); return (e); }
static gfarm_error_t gfs_pio_view_global_ftruncate(GFS_File gf, gfarm_off_t length) { struct gfs_file_global_context *gc = gf->view_context; gfarm_error_t e; int i, fragment, nsections; gfarm_off_t section_length; struct gfarm_file_section_info *sections; char section_string[GFARM_INT32STRLEN + 1]; if (length < 0) return (GFARM_ERR_INVALID_ARGUMENT); if (length >= gc->offsets[gf->pi.status.st_nsections - 1]) fragment = gf->pi.status.st_nsections - 1; else fragment = gfs_pio_view_global_bsearch( length, gc->offsets, gf->pi.status.st_nsections - 1); section_length = length - gc->offsets[fragment]; e = gfs_pio_view_global_move_to(gf, fragment); if (e != GFARM_ERR_NO_ERROR) return (e); e = gfs_pio_truncate(gc->fragment_gf, section_length); if (e != GFARM_ERR_NO_ERROR) return (e); /* * Before updating path_info, try to update most recent information, * because the file mode may be updated by e.g. gfs_chmod(). */ if (gfarm_path_info_get(gf->pi.pathname, &pi) == NULL) { gfarm_path_info_free(&gf->pi); gf->pi = pi; } #if 0 /* We don't store file size in gfarm_path_info, this is just ignored */ gf->pi.status.st_size = length; #endif gf->pi.status.st_nsections = fragment + 1; e = gfarm_path_info_replace(gf->pi.pathname, &gf->pi); if (e != GFARM_ERR_NO_ERROR) return (e); e = gfarm_file_section_info_get_sorted_all_serial_by_file( gf->pi.pathname, &nsections, §ions); if (e != GFARM_ERR_NO_ERROR) return (e); sections[fragment].filesize = section_length; sprintf(section_string, "%d", fragment); e = gfarm_file_section_info_replace(gf->pi.pathname, section_string, §ions[fragment]); for (i = fragment + 1; i < nsections; i++) (void)gfs_unlink_section_internal(gf->pi.pathname, sections[i].section); gfarm_file_section_info_free_all(nsections, sections); return (e); }
char * gfarm_url_section_replicate_from_to(char *gfarm_url, char *section, char *srchost, char *dsthost) { char *e, *gfarm_file, *canonical_hostname, *if_hostname; struct sockaddr peer_addr; struct gfarm_path_info pi; struct gfarm_file_section_info si; gfarm_mode_t mode_allowed = 0; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) goto finish; e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) goto finish_gfarm_file; e = gfarm_file_section_info_get(gfarm_file, section, &si); if (e != NULL) goto finish_path_info; e = gfarm_host_get_canonical_name(srchost, &canonical_hostname); if (e != NULL) goto finish_canonical_hostname; /* reflect "address_use" directive in the `srchost' */ e = gfarm_host_address_get(srchost, gfarm_spool_server_port, &peer_addr, &if_hostname); if (e != NULL) goto finish_section_info; /* * XXX - if the owner of a file is not the same, permit a * group/other write access - This should be fixed in the next * major release. */ if (strcmp(pi.status.st_user, gfarm_get_global_username()) != 0) { e = gfarm_path_info_access(&pi, GFS_R_OK); if (e != NULL) goto finish_path_info; mode_allowed = 022; } e = gfarm_file_section_replicate_from_to_internal( gfarm_file, section, (pi.status.st_mode | mode_allowed) & GFARM_S_ALLPERM, si.filesize, canonical_hostname, if_hostname, dsthost); free(if_hostname); finish_canonical_hostname: free(canonical_hostname); finish_section_info: gfarm_file_section_info_free(&si); finish_path_info: gfarm_path_info_free(&pi); finish_gfarm_file: free(gfarm_file); finish: return (e); }
char * gfs_rmdir(const char *pathname) { char *canonic_path, *e, *e_tmp; struct gfarm_path_info pi; GFS_Dir dir; struct gfs_dirent *entry; e = gfarm_url_make_path_for_creation(pathname, &canonic_path); /* We permit missing gfarm: prefix here as a special case */ if (e == GFARM_ERR_GFARM_URL_PREFIX_IS_MISSING) e = gfarm_canonical_path_for_creation(pathname, &canonic_path); if (e != NULL) return (e); e = gfarm_path_info_get(canonic_path, &pi); if (e != NULL) goto error_free_canonic_path; if (!GFARM_S_ISDIR(pi.status.st_mode)) { gfarm_path_info_free(&pi); e = GFARM_ERR_NOT_A_DIRECTORY; goto error_free_canonic_path; } gfarm_path_info_free(&pi); e = gfs_opendir(pathname, &dir); if (e == NULL) { while ((e = gfs_readdir(dir, &entry)) == NULL) { if (entry == NULL) { /* OK, remove the directory */ e = gfarm_path_info_remove(canonic_path); break; } if ((entry->d_namlen == 1 && entry->d_name[0] == '.') || (entry->d_namlen == 2 && entry->d_name[0] == '.' && entry->d_name[1] == '.')) continue; /* "." or ".." */ /* Not OK */ e = GFARM_ERR_DIRECTORY_NOT_EMPTY; break; } e_tmp = gfs_closedir(dir); if (e == NULL) e = e_tmp; } error_free_canonic_path: free(canonic_path); return (e); }
char * gfs_stat_canonical_path(char *gfarm_file, struct gfs_stat *s) { char *e; int i, nsections; struct gfarm_file_section_info *sections; struct gfarm_path_info pi; long ino; e = gfs_get_ino(gfarm_file, &ino); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) return (e); *s = pi.status; s->st_ino = ino; s->st_user = strdup(s->st_user); s->st_group = strdup(s->st_group); gfarm_path_info_free(&pi); if (s->st_user == NULL || s->st_group == NULL) { gfs_stat_free(s); return (GFARM_ERR_NO_MEMORY); } if (!GFARM_S_ISREG(s->st_mode)) return (NULL); /* regular file */ e = gfarm_file_section_info_get_all_by_file(gfarm_file, &nsections, §ions); if (e != NULL) { gfs_stat_free(s); /* * If GFARM_ERR_NO_SUCH_OBJECT is returned here, * gfs_stat() incorrectly assumes that this is a directory, * and reports GFARM_ERR_NOT_A_DIRECTORY. */ return (GFARM_ERR_NO_FRAGMENT_INFORMATION); } s->st_size = 0; for (i = 0; i < nsections; i++) s->st_size += sections[i].filesize; s->st_nsections = nsections; gfarm_file_section_info_free_all(nsections, sections); return (NULL); }
/* Create a set of architectures that the program is registered for */ static char * program_arch_set(char *program, struct gfarm_hash_table **arch_setp) { char *e, *gfarm_file; struct gfarm_path_info pi; struct gfarm_file_section_info *sections; struct gfarm_hash_table *arch_set; int i, nsections, created; e = gfarm_url_make_path(program, &gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) { if (e == GFARM_ERR_NO_SUCH_OBJECT) e = "such program isn't registered"; free(gfarm_file); return (e); } if (!GFARM_S_IS_PROGRAM(pi.status.st_mode)) { gfarm_path_info_free(&pi); free(gfarm_file); return ("specified command is not an executable"); } e = gfarm_file_section_info_get_all_by_file(gfarm_file, &nsections, §ions); gfarm_path_info_free(&pi); free(gfarm_file); if (e != NULL) return ("no binary is registered as the specified command"); arch_set = gfarm_hash_table_alloc(ARCH_SET_HASHTAB_SIZE, gfarm_hash_default, gfarm_hash_key_equal_default); if (arch_set == NULL) { gfarm_file_section_info_free_all(nsections, sections); return (GFARM_ERR_NO_MEMORY); } /* register architectures of the program to `arch_set' */ for (i = 0; i < nsections; i++) { if (gfarm_hash_enter(arch_set, sections[i].section, strlen(sections[i].section) + 1, sizeof(int), &created) == NULL) { free_arch_set(arch_set); gfarm_file_section_info_free_all(nsections, sections); return (GFARM_ERR_NO_MEMORY); } } gfarm_file_section_info_free_all(nsections, sections); *arch_setp = arch_set; return (NULL); }
static char * check_path_info(char *gfarm_file) { struct gfarm_path_info p_info; char *e; /* check whether the path info is already registered. */ e = gfarm_path_info_get(gfarm_file, &p_info); if (e != NULL) return (e); gfarm_path_info_free(&p_info); return (NULL); }
char * gfs_pio_open(const char *url, int flags, GFS_File *gfp) { char *e, *pathname; GFS_File gf; gfarm_timerval_t t1, t2; gfs_profile(gfarm_gettimerval(&t1)); if (flags & GFARM_FILE_CREATE) { e = GFARM_ERR_OPERATION_NOT_SUPPORTED; goto finish; } e = gfarm_url_make_path(url, &pathname); if (e != NULL) goto finish; e = gfs_file_alloc(&gf); if (e != NULL) { free(pathname); goto finish; } /* GFARM_FILE_EXCLUSIVE is a NOP with gfs_pio_open(). */ flags &= ~GFARM_FILE_EXCLUSIVE; gfs_pio_open_initialize_mode_flags(gf, flags); e = gfarm_path_info_get(pathname, &gf->pi); free(pathname); if (e != NULL) goto free_gf; e = gfs_pio_open_check_perm(gf); if (e != NULL) goto free_gf_pi; if ((flags & GFARM_FILE_TRUNC) == 0) gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED; *gfp = gf; e = NULL; goto finish; free_gf_pi: gfarm_path_info_free(&gf->pi); free_gf: gfs_file_free(gf); finish: gfs_profile(gfarm_gettimerval(&t2)); gfs_profile(gfs_pio_open_time += gfarm_timerval_sub(&t2, &t1)); return (e); }
char * gfs_mkdir(const char *pathname, gfarm_mode_t mode) { char *user, *canonic_path, *e; struct gfarm_path_info pi; struct timeval now; mode_t mask; user = gfarm_get_global_username(); if (user == NULL) return ("unknown user"); e = gfarm_url_make_path_for_creation(pathname, &canonic_path); /* We permit missing gfarm: prefix here as a special case */ if (e == GFARM_ERR_GFARM_URL_PREFIX_IS_MISSING) e = gfarm_canonical_path_for_creation(pathname, &canonic_path); if (e != NULL) return (e); if (gfarm_path_info_get(canonic_path, &pi) == NULL) { gfarm_path_info_free(&pi); free(canonic_path); return (GFARM_ERR_ALREADY_EXISTS); } mask = umask(0); umask(mask); mode &= ~mask; gettimeofday(&now, NULL); pi.pathname = canonic_path; pi.status.st_mode = (GFARM_S_IFDIR | mode); pi.status.st_user = user; pi.status.st_group = "*"; /* XXX for now */ pi.status.st_atimespec.tv_sec = pi.status.st_mtimespec.tv_sec = pi.status.st_ctimespec.tv_sec = now.tv_sec; pi.status.st_atimespec.tv_nsec = pi.status.st_mtimespec.tv_nsec = pi.status.st_ctimespec.tv_nsec = now.tv_usec * GFARM_MILLISEC_BY_MICROSEC; pi.status.st_size = 0; pi.status.st_nsections = 0; e = gfarm_path_info_set(canonic_path, &pi); free(canonic_path); return (e); }
char * gfs_stat_section(const char *gfarm_url, const char *section, struct gfs_stat *s) { char *e, *gfarm_file; struct gfarm_file_section_info sinfo; struct gfarm_path_info pi; long ino; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) return (e); e = gfs_get_ino(gfarm_file, &ino); if (e != NULL) { free(gfarm_file); return (e); } e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) { free(gfarm_file); return (e); } *s = pi.status; s->st_ino = ino; s->st_user = strdup(s->st_user); s->st_group = strdup(s->st_group); gfarm_path_info_free(&pi); if (!GFARM_S_ISREG(s->st_mode)) { free(gfarm_file); return (NULL); } e = gfarm_file_section_info_get(gfarm_file, section, &sinfo); free(gfarm_file); if (e != NULL) return (e); s->st_size = sinfo.filesize; s->st_nsections = 1; gfarm_file_section_info_free(&sinfo); return (NULL); }
char * gfs_access(const char *gfarm_url, int mode) { char *e, *gfarm_file; struct gfarm_path_info pi; gfarm_mode_t stat_mode; int stat_nsections, nsections; struct gfarm_file_section_info *sections; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) { free(gfarm_file); return (e); } stat_mode = pi.status.st_mode; stat_nsections = pi.status.st_nsections; e = gfarm_path_info_access(&pi, mode); gfarm_path_info_free(&pi); if (e != NULL) { free(gfarm_file); return (e); } if (GFARM_S_ISDIR(stat_mode)) { free(gfarm_file); return (NULL); } /* * Check all fragments are ready or not. * XXX - is this check necessary? */ e = gfarm_file_section_info_get_all_by_file( gfarm_file, &nsections, §ions); free(gfarm_file); if (e != NULL) return (e); gfarm_file_section_info_free_all(nsections, sections); if (!GFARM_S_IS_PROGRAM(stat_mode) && nsections != stat_nsections) e = GFARM_ERR_FRAGMENT_NUMBER_DOES_NOT_MATCH; return (e); }
char * gfarm_url_fragment_number(char *gfarm_url, int *np) { char *e, *gfarm_file; struct gfarm_path_info pi; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); free(gfarm_file); if (e != NULL) return (e); if (!GFARM_S_IS_FRAGMENTED_FILE(pi.status.st_mode)) return (GFARM_ERR_OPERATION_NOT_PERMITTED); *np = pi.status.st_nsections; gfarm_path_info_free(&pi); return (NULL); }
char * gfarm_url_section_replicate_to(char *gfarm_url, char *section, char *dsthost) { char *e, *gfarm_file; struct gfarm_path_info pi; struct gfarm_file_section_info si; gfarm_mode_t mode_allowed = 0; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) goto finish; e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) goto finish_gfarm_file; e = gfarm_file_section_info_get(gfarm_file, section, &si); if (e != NULL) goto finish_path_info; /* * XXX - if the owner of a file is not the same, permit a * group/other write access - This should be fixed in the next * major release. */ if (strcmp(pi.status.st_user, gfarm_get_global_username()) != 0) { e = gfarm_path_info_access(&pi, GFS_R_OK); if (e != NULL) goto finish_path_info; mode_allowed = 022; } e = gfarm_file_section_replicate_to_internal( gfarm_file, section, (pi.status.st_mode | mode_allowed) & GFARM_S_ALLPERM, si.filesize, dsthost); gfarm_file_section_info_free(&si); finish_path_info: gfarm_path_info_free(&pi); finish_gfarm_file: free(gfarm_file); finish: return (e); }
char * gfs_access(char *gfarm_url, int mode) { char *e, *gfarm_file; struct gfarm_path_info pi; int nsections; struct gfarm_file_section_info *sections; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) { free(gfarm_file); return (e); } /* * Check whether the gfarm_url can be accessible by other * processes or not. * * XXX - temporal solution until file locking will be implemented. */ e = gfarm_file_section_info_get_all_by_file(gfarm_file, &nsections, §ions); free(gfarm_file); if (e != NULL) { return (e); } gfarm_file_section_info_free_all(nsections, sections); if (!GFARM_S_IS_PROGRAM(pi.status.st_mode) && nsections != pi.status.st_nsections) return (GFARM_ERR_FRAGMENT_NUMBER_DOES_NOT_MATCH); e = gfarm_path_info_access(&pi, mode); gfarm_path_info_free(&pi); return (e); }
char * gfarm_canonical_path_for_creation(const char *gfarm_file, char **canonic_pathp) { const char *basename, *p0; char *e, *p1, *dir, *dir_canonic, *lastc, cwd[PATH_MAX + 1]; *canonic_pathp = NULL; /* cause SEGV, if return value is ignored */ /* '' or 'gfarm:' case */ if (gfarm_file[0] == '\0') { e = gfs_getcwd(cwd, sizeof(cwd)); if (e != NULL) return (e); p0 = cwd; } else p0 = gfarm_file; /* Expand '~'. */ e = gfarm_path_expand_home(p0, &p1); if (e != NULL) return (e); /* Eliminate unnecessary '/'s following the basename. */ lastc = &p1[strlen(p1) - 1]; if (*lastc == '/') { while (p1 < lastc && *lastc == '/') --lastc; if (p1 == lastc) { /* * In this case, given gfarm_file is '/' or contains * only several '/'s. This means to attempt to create * the root directory. Because the root directory * should exist, the attempt will fail with the error * of 'already exist'. However, this case such that * the canonical name is "" causes many problems. * That is why the error of 'already exist' is * returned here. */ free(p1); return (GFARM_ERR_ALREADY_EXISTS); } else { *(lastc + 1) = '\0'; } } basename = gfarm_path_dir_skip(p1); if (basename == p1) /* "filename" */ dir = "."; else if (basename == p1 + 1) /* "/filename" */ dir = "/"; else { /* /.../.../filename */ p1[basename - 1 - p1] = '\0'; dir = p1; } /* Check the existence of the parent directory. */ e = gfarm_canonical_path(dir, &dir_canonic); if (e != NULL) goto free_p1; /* * check whether parent directory is writable or not. * XXX this isn't enough yet, due to missing X-bits check. */ if (dir_canonic[0] != '\0') { /* XXX "/" is always OK for now */ struct gfarm_path_info pi; e = gfarm_path_info_get(dir_canonic, &pi); if (e != NULL) goto free_dir_canonic; e = gfarm_path_info_access(&pi, GFS_W_OK); gfarm_path_info_free(&pi); if (e != NULL) goto free_dir_canonic; } *canonic_pathp = malloc(strlen(dir_canonic) + 1 + strlen(basename) + 1); if (*canonic_pathp == NULL) { e = GFARM_ERR_NO_MEMORY; goto free_dir_canonic; } /* * When the 'dir_canonic' is a null string, *canonic_pathp * will start with '/' incorrectly. */ if (dir_canonic[0] == '\0') strcpy(*canonic_pathp, basename); else sprintf(*canonic_pathp, "%s/%s", dir_canonic, basename); e = NULL; free_dir_canonic: free(dir_canonic); free_p1: free(p1); return (e); }
int main(int argc, char **argv) { gfarm_int32_t interleave_factor = 0; gfarm_int32_t ndivisions = 1; int recv_stripe_sync = 0; int send_stripe_sync = 0; int file_sync_stripe = 0; struct xxx_connection *from_frontend, *to_frontend; struct xxx_connection *from_server, *to_server; char *e, *src_host, *my_canonical_name; char **gfrcmd_paths, **gfrep_backend_server_paths; int c, i, n, replication_method_save; gfarm_stringlist files, sections; struct gfarm_path_info *path_infos; gfarm_int32_t *results; struct sockaddr server_addr; e = gfarm_initialize(&argc, &argv); if (e != NULL) { fprintf(stderr, "%s: gfarm_initialize on %s: %s\n", program_name, my_name, e); fatal(); } while ((c = getopt(argc, argv, "i:n:rsS:")) != -1) { switch (c) { case 'i': interleave_factor = atoi(optarg); break; case 'n': ndivisions = atoi(optarg); break; case 'r': recv_stripe_sync = 1; break; case 's': send_stripe_sync = 1; break; case 'S': file_sync_stripe = atoi(optarg); break; default: fprintf(stderr, "%s: unknown option -%c\n", program_name, c); usage(); /*NOTREACHED*/ } } argc -= optind; argv += optind; my_name = gfarm_host_get_self_name(); if (argc != 1) { fprintf(stderr, "%s: missing server name argument on %s\n", program_name, my_name); usage(); /*NOTREACHED*/ } src_host = argv[0]; e = gfarm_host_get_canonical_self_name(&my_canonical_name); if (e != NULL) { fprintf(stderr, "%s: host %s isn't a filesystem node: %s\n", program_name, my_name, e); fatal(); } replication_method_save = gfarm_replication_get_method(); gfarm_replication_set_method(GFARM_REPLICATION_BOOTSTRAP_METHOD); e = gfarm_url_program_deliver(gfrcmd_url, 1, &my_canonical_name, &gfrcmd_paths); if (e != NULL) { fprintf(stderr, "%s: cannot deliver %s to host %s on %s: %s\n", program_name, gfrcmd_url, my_canonical_name, my_name, e); fatal(); } e = gfarm_url_program_deliver(gfrep_backend_server, 1, &src_host, &gfrep_backend_server_paths); if (e != NULL) { fprintf(stderr, "%s: cannot deliver %s to host %s on %s: %s\n", program_name, gfrep_backend_server, src_host, my_name, e); fatal(); } gfarm_replication_set_method(replication_method_save); e = gfs_client_rep_backend_invoke( src_host, gfrcmd_paths[0], gfrep_backend_server_paths[0], NULL, GFS_CLIENT_REP_ALGORITHM_LATEST, ndivisions, interleave_factor, 0, send_stripe_sync, 0, program_name, &from_server, &to_server, &backend); if (e != NULL) fatal(); e = gfarm_host_address_get(src_host, 0, &server_addr, NULL); if (e != NULL) { fprintf(stderr, "%s: %s: %s\n", program_name, src_host, e); fatal(); } e = gfarm_netparam_config_get_long(&gfarm_netparam_file_sync_rate, src_host, (struct sockaddr *)&server_addr, &file_sync_rate); if (e != NULL) {/* shouldn't happen */ fprintf(stderr, "%s: get netparam file_sync_rate on %s (%s): %s\n", program_name, my_name, src_host, e); fatal(); } /* XXX read-only connection */ e = xxx_fd_connection_new(STDIN_FILENO, &from_frontend); if (e != NULL) { fprintf(stderr, "%s: %s for stdin\n", program_name, e); fatal(); } /* XXX write-only connection */ e = xxx_fd_connection_new(STDOUT_FILENO, &to_frontend); if (e != NULL) { fprintf(stderr, "%s: %s for stdout\n", program_name, e); fatal(); } e = gfs_client_rep_filelist_receive(from_frontend, &n, &files, §ions, program_name); if (e != NULL) fatal(); e = gfs_client_rep_filelist_send(src_host, to_server, program_name, n, &files, §ions); if (e != NULL) fatal(); results = malloc(sizeof(*results) * n); if (results == NULL) { fprintf(stderr, "%s: no memory for %d ints on %s\n", program_name, n, my_name); fatal(); } /* make current directory == spool_root */ if (chdir(gfarm_spool_root) == -1) { fprintf(stderr, "%s: chdir(%s) on %s: %s\n", program_name, gfarm_spool_root, my_name, e); fatal(); } path_infos = malloc(sizeof(*path_infos) * n); if (results == NULL) { fprintf(stderr, "%s: no memory for %d path_info on %s\n", program_name, n, my_name); fatal(); } for (i = 0; i < n; i++) { e = gfarm_path_info_get(gfarm_stringlist_elem(&files, i), &path_infos[i]); results[i] = e == NULL ? GFS_ERROR_NOERROR : gfs_string_to_proto_error(e); } umask(0); /* don't mask, just use the 3rd parameter of open(2) */ session(src_host, &server_addr, from_server, to_server, my_canonical_name, ndivisions, interleave_factor, file_sync_stripe, recv_stripe_sync, n, &files, §ions, path_infos, results); #ifdef HACK_FOR_BWC /* * If this program fails with fatal(), or is killed by a signal, * this metadata update isn't executed. (So, #undef HACK_FOR_BWC) */ for (i = 0; i < n; i++) { char *file = gfarm_stringlist_elem(&files, i); char *section = gfarm_stringlist_elem(§ions, i); struct gfarm_file_section_copy_info fci; if (results[i] == GFS_ERROR_NOERROR) { e = gfarm_file_section_copy_info_set(file, section, my_canonical_name, &fci); if (e != NULL) results[i] = gfs_string_to_proto_error(e); } } #endif for (i = 0; i < n; i++) xxx_proto_send(to_frontend, "i", results[i]); xxx_proto_flush(to_frontend); e = gfarm_terminate(); if (e != NULL) { fprintf(stderr, "%s: gfarm_terminate on %s: %s\n", program_name, my_name, e); fatal(); } return (0); }
int try_to_make_parent_dir(char *pathname) { char *e, *p, *parent, *dirbuf; int rv, dirlen; gfarm_mode_t mode; struct stat st; struct gfarm_path_info pi; parent = strdup(pathname); if (parent == NULL) /* give up */ return (0); /* get parent directory */ p = (char *)gfarm_path_dir_skip(parent); while (p > parent && p[-1] == '/') *--p = '\0'; if (p == parent) { /* no directory part */ free(parent); return (0); } if (stat(parent, &st) == 0) { /* parent directory already exists */ free(parent); return (0); } dirbuf = strdup(parent); if (dirbuf == NULL) { free(parent); return (0); } for (dirlen = 0;;) { dirlen += strcspn(parent + dirlen, "/"); memcpy(dirbuf, parent, dirlen); dirbuf[dirlen] = '\0'; if (stat(dirbuf, &st) == 0) { if (!S_ISDIR(st.st_mode)) { free(dirbuf); free(parent); return (0); } } else { e = gfarm_path_info_get(dirbuf, &pi); mode = pi.status.st_mode; if (e != NULL || !GFARM_S_ISDIR(mode)) { if (e == NULL) gfarm_path_info_free(&pi); free(dirbuf); free(parent); return (0); } mode &= GFARM_S_ALLPERM; if (strcmp(pi.status.st_user, gfarm_get_global_username()) != 0) { mode |= 0777; } gfarm_path_info_free(&pi); rv = mkdir(dirbuf, mode); if (rv == -1) { free(dirbuf); free(parent); return (0); } } dirlen += strspn(parent + dirlen, "/"); if (parent[dirlen] == '\0') { /* OK, made */ free(dirbuf); free(parent); return (1); } } }
int main(int argc, char **argv) { char *e, *canonic_path; int i, c; extern int optind; char *user; if (argc <= 1) usage(); program_name = basename(argv[0]); while ((c = getopt(argc, argv, "h")) != EOF) { switch (c) { case 'h': default: usage(); } } argc -= optind; argv += optind; e = gfarm_initialize(&argc, &argv); if (e != NULL) { fprintf(stderr, "%s: %s\n", program_name, e); exit(1); } user = gfarm_get_global_username(); if (user == NULL) { fprintf(stderr, "%s:%s%s\n", program_name, "programming error, gfarm library isn't properly ", "initialized"); exit(1); } for (i = 0; i < argc; i++) { struct gfarm_path_info pi; struct timeval now; e = gfarm_url_make_path_for_creation(argv[i], &canonic_path); /* We permit missing gfarm: prefix here as a special case */ if (e == GFARM_ERR_GFARM_URL_PREFIX_IS_MISSING) e = gfarm_canonical_path_for_creation(argv[i], &canonic_path); if (e != NULL) { fprintf(stderr, "%s: %s: %s\n", program_name, argv[i], e); exit(1); } if (gfarm_path_info_get(canonic_path, &pi) == 0) { fprintf(stderr, "%s: %s: %s\n", program_name, argv[i], GFARM_ERR_ALREADY_EXISTS); gfarm_path_info_free(&pi); continue; } gettimeofday(&now, NULL); pi.pathname = canonic_path; pi.status.st_mode = (GFARM_S_IFDIR | 0755); pi.status.st_user = user; pi.status.st_group = "*"; /* XXX for now */ pi.status.st_atimespec.tv_sec = pi.status.st_mtimespec.tv_sec = pi.status.st_ctimespec.tv_sec = now.tv_sec; pi.status.st_atimespec.tv_nsec = pi.status.st_mtimespec.tv_nsec = pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000; pi.status.st_size = 0; pi.status.st_nsections = 0; e = gfarm_path_info_set(canonic_path, &pi); if (e != NULL) { fprintf(stderr, "%s: %s: %s\n", program_name, argv[i], e); exit(1); } free(canonic_path); } e = gfarm_terminate(); return (0); }
char * gfarm_url_fragments_replicate(char *gfarm_url, int ndsthosts, char **dsthosts) { char *e, *gfarm_file, **srchosts, **edsthosts; int nsrchosts; gfarm_mode_t mode; int i, pid, *pids; struct gfarm_path_info pi; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) goto finish_gfarm_file; mode = pi.status.st_mode; gfarm_path_info_free(&pi); if (!GFARM_S_IS_FRAGMENTED_FILE(mode)) { e = GFARM_ERR_OPERATION_NOT_PERMITTED; goto finish_gfarm_file; } /* * XXX - if the owner of a file is not the same, permit a * group/other write access - This should be fixed in the next * major release. */ if (strcmp(pi.status.st_user, gfarm_get_global_username()) != 0) { e = gfarm_path_info_access(&pi, GFS_R_OK); if (e != NULL) { gfarm_path_info_free(&pi); free(gfarm_file); return (e); } mode |= 022; } e = gfarm_url_hosts_schedule(gfarm_url, "", &nsrchosts, &srchosts); if (e != NULL) goto finish_gfarm_file; edsthosts = malloc(sizeof(*edsthosts) * nsrchosts); if (edsthosts == NULL) { e = GFARM_ERR_NO_MEMORY; goto finish_srchosts; } gfarm_strings_expand_cyclic(ndsthosts, dsthosts, nsrchosts, edsthosts); pids = malloc(sizeof(int) * nsrchosts); if (pids == NULL) { e = GFARM_ERR_NO_MEMORY; goto finish_edsthosts; } /* * To use different connection for each metadb access. * * XXX: FIXME layering violation */ e = gfarm_metadb_terminate(); if (e != NULL) goto finish_pids; for (i = 0; i < nsrchosts; i++) { struct sockaddr peer_addr; char *if_hostname; char section_string[GFARM_INT32STRLEN + 1]; struct gfarm_file_section_info si; pid = fork(); if (pid < 0) break; if (pid) { /* parent */ pids[i] = pid; continue; } /* child */ /* * use different connection for each metadb access. * * XXX: FIXME layering violation */ e = gfarm_metadb_initialize(); if (e != NULL) _exit(1); /* reflect "address_use" directive in the `srchosts[i]' */ e = gfarm_host_address_get(srchosts[i], gfarm_spool_server_port, &peer_addr, &if_hostname); if (e != NULL) _exit(2); sprintf(section_string, "%d", i); e = gfarm_file_section_info_get(gfarm_file, section_string, &si); if (e != NULL) _exit(3); e = gfarm_file_section_replicate_from_to_internal( gfarm_file, section_string, mode & GFARM_S_ALLPERM, si.filesize, srchosts[i], if_hostname, edsthosts[i]); if (e != NULL) _exit(1); _exit(0); } while (--i >= 0) { int rv, s; while ((rv = waitpid(pids[i], &s, 0)) == -1 && errno == EINTR) ; if (rv == -1) { if (e == NULL) e = gfarm_errno_to_error(errno); } else if (WIFEXITED(s) && WEXITSTATUS(s) != 0) { e = "error happens on replication"; } } /* * recover temporary closed metadb connection * * XXX: FIXME layering violation */ if (e != NULL) { gfarm_metadb_initialize(); } else { e = gfarm_metadb_initialize(); } finish_pids: free(pids); finish_edsthosts: free(edsthosts); finish_srchosts: gfarm_strings_free_deeply(nsrchosts, srchosts); finish_gfarm_file: free(gfarm_file); return (e); }
char * gfs_pio_create(const char *url, int flags, gfarm_mode_t mode, GFS_File *gfp) { char *e, *pathname; GFS_File gf; int pi_available = 0; mode_t mask; char *user; gfarm_timerval_t t1, t2; gfs_profile(gfarm_gettimerval(&t1)); user = gfarm_get_global_username(); if (user == NULL) { e = "gfarm_pio_create(): programming error, " "gfarm library isn't properly initialized"; goto finish; } #if 0 /* XXX - ROOT I/O opens a new file with O_CREAT|O_RDRW mode. */ if ((flags & GFARM_FILE_ACCMODE) != GFARM_FILE_WRONLY) { e = GFARM_ERR_OPERATION_NOT_SUPPORTED; /* XXX */ goto finish; } #endif mask = umask(0); umask(mask); mode &= ~mask; e = gfarm_url_make_path_for_creation(url, &pathname); if (e != NULL) goto finish; e = gfs_file_alloc(&gf); if (e != NULL) { free(pathname); goto finish; } /* gfs_pio_create() always assumes CREATE, TRUNC */ flags |= GFARM_FILE_CREATE | GFARM_FILE_TRUNC; if ((flags & (GFARM_FILE_TRUNC|GFARM_FILE_SEQUENTIAL)) != (GFARM_FILE_TRUNC|GFARM_FILE_SEQUENTIAL)) { /* MODE_READ is needed to re-calculate checksum. */ flags = (flags & ~GFARM_FILE_ACCMODE) | GFARM_FILE_RDWR; } else if ((flags & ~GFARM_FILE_ACCMODE) == GFARM_FILE_RDONLY) { flags = (flags & ~GFARM_FILE_ACCMODE) | GFARM_FILE_WRONLY; } gf->open_flags = flags; gf->mode = GFS_FILE_MODE_WRITE; if ((flags & GFARM_FILE_ACCMODE) == GFARM_FILE_RDWR) gf->mode |= GFS_FILE_MODE_READ; e = gfarm_path_info_get(pathname, &gf->pi); if (e != NULL && e != GFARM_ERR_NO_SUCH_OBJECT) { free(pathname); gfs_file_free(gf); goto finish; } if (e == NULL) { /* XXX unlink and re-create the file? */ free(pathname); e = gfarm_path_info_access(&gf->pi, GFS_W_OK); if (e != NULL) { gfarm_path_info_free(&gf->pi); gfs_file_free(gf); goto finish; } if (!GFARM_S_ISREG(gf->pi.status.st_mode)) { if (GFARM_S_ISDIR(gf->pi.status.st_mode)) e = GFARM_ERR_IS_A_DIRECTORY; else e = GFARM_ERR_OPERATION_NOT_SUPPORTED; gfarm_path_info_free(&gf->pi); gfs_file_free(gf); goto finish; } /* * XXX should check the follows: * - the mode is consistent among same job * - creator of the metainfo has same job id * - O_TRUNC / !O_TRUNC case */ mode |= GFARM_S_IFREG; if (GFARM_S_IS_PROGRAM(mode) != GFS_FILE_IS_PROGRAM(gf)) { gfarm_path_info_free(&gf->pi); gfs_file_free(gf); e = GFARM_ERR_OPERATION_NOT_PERMITTED; goto finish; } pi_available = 1; } if (!pi_available) { struct timeval now; gettimeofday(&now, NULL); gf->pi.pathname = pathname; gf->pi.status.st_mode = (GFARM_S_IFREG | mode); gf->pi.status.st_user = strdup(user); /* XXX NULL check */ gf->pi.status.st_group = strdup("*"); /* XXX for now */ gf->pi.status.st_atimespec.tv_sec = gf->pi.status.st_mtimespec.tv_sec = gf->pi.status.st_ctimespec.tv_sec = now.tv_sec; gf->pi.status.st_atimespec.tv_nsec = gf->pi.status.st_mtimespec.tv_nsec = gf->pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000; gf->pi.status.st_size = 0; gf->pi.status.st_nsections = 0; } *gfp = gf; gfs_uncachedir(); e = NULL; finish: gfs_profile(gfarm_gettimerval(&t2)); gfs_profile(gfs_pio_create_time += gfarm_timerval_sub(&t2, &t1)); return (e); }
char * gfs_pio_set_view_section(GFS_File gf, char *section, char *if_hostname, int flags) { struct gfs_file_section_context *vc; char *e; int is_local_host; e = gfs_pio_set_view_default(gf); if (e != NULL) return (e); vc = malloc(sizeof(struct gfs_file_section_context)); if (vc == NULL) { gf->error = GFARM_ERR_NO_MEMORY; return (gf->error); } vc->section = strdup(section); if (vc->section == NULL) { free(vc); gf->error = GFARM_ERR_NO_MEMORY; return (gf->error); } if (if_hostname != NULL) { e = gfarm_host_get_canonical_name(if_hostname, &vc->canonical_hostname); if (e != NULL) goto finish; } else if ((gf->open_flags & GFARM_FILE_CREATE) != 0) { e = gfarm_host_get_canonical_self_name(&if_hostname); if (e == NULL) { vc->canonical_hostname = strdup(if_hostname); if (vc->canonical_hostname == NULL) { e = GFARM_ERR_NO_MEMORY; goto finish; } } else { char **hostnames; hostnames = malloc(sizeof(char *)); if (hostnames == NULL) { e = GFARM_ERR_NO_MEMORY; goto finish; } e = gfarm_schedule_search_idle_by_all(1, hostnames); if (e != NULL) goto finish; vc->canonical_hostname = if_hostname = hostnames[0]; free(hostnames); } } else { e = gfarm_file_section_host_schedule_with_priority_to_local( gf->pi.pathname, vc->section, &if_hostname); if (e != NULL) goto finish; vc->canonical_hostname = if_hostname; /* must be already canonical */ } is_local_host = gfarm_canonical_hostname_is_local( vc->canonical_hostname); if ((gf->open_flags & GFARM_FILE_CREATE) != 0) { struct gfarm_path_info pi; e = gfarm_path_info_set(gf->pi.pathname, &gf->pi); if (e == GFARM_ERR_ALREADY_EXISTS && (e = gfarm_path_info_get(gf->pi.pathname, &pi)) == NULL) { if (GFS_FILE_IS_PROGRAM(gf) != GFARM_S_IS_PROGRAM(pi.status.st_mode)) e = GFARM_ERR_OPERATION_NOT_PERMITTED; if (e == NULL && !GFS_FILE_IS_PROGRAM(gf)) { if (gf->pi.status.st_nsections != pi.status.st_nsections) { e = GFARM_ERR_FRAGMENT_NUMBER_DOES_NOT_MATCH; } else { #if 0 assert(gf->pi.status.st_mode & GFS_FILE_MODE_NSEGMENTS_FIXED); #endif } } if (e != NULL) { gfarm_path_info_free(&pi); } else { gfarm_path_info_free(&gf->pi); gf->pi = pi; } /* * XXX should check the follows: * - creator of the metainfo has same job id * - mode is consistent among same job * - nfragments is consistent among same job */ } if (e != NULL) goto free_host; } /* delete section info when opening with a trancation flag. */ if ((gf->open_flags & GFARM_FILE_TRUNC) != 0) (void)gfs_unlink_section(gf->pi.pathname, vc->section); gf->ops = &gfs_pio_view_section_ops; gf->view_context = vc; gf->view_flags = flags; gf->p = gf->length = 0; gf->io_offset = gf->offset = 0; gf->mode |= GFS_FILE_MODE_CALC_DIGEST; EVP_DigestInit(&vc->md_ctx, GFS_DEFAULT_DIGEST_MODE); if (!is_local_host && (((gf->open_flags & GFARM_FILE_REPLICATE) != 0 && (flags & GFARM_FILE_NOT_REPLICATE) == 0) || (flags & GFARM_FILE_REPLICATE) != 0)) { e = replicate_section_to_local(gf, vc->section, if_hostname); if (e != NULL) goto free_host; free(vc->canonical_hostname); e = gfarm_host_get_canonical_self_name( &vc->canonical_hostname); if (e != NULL) goto finish; vc->canonical_hostname = strdup(vc->canonical_hostname); if (vc->canonical_hostname == NULL) { e = GFARM_ERR_NO_MEMORY; goto finish; } is_local_host = 1; } if (is_local_host) e = gfs_pio_open_local_section(gf, flags); else e = gfs_pio_open_remote_section(gf, if_hostname, flags); free_host: /* XXX - if failed, try another replica */ if (e != NULL) free(vc->canonical_hostname); finish: if (e != NULL) { free(vc->section); free(vc); gf->view_context = NULL; gfs_pio_set_view_default(gf); } gf->error = e; return (e); }
char * gfs_pio_create(const char *url, int flags, gfarm_mode_t mode, GFS_File *gfp) { char *e, *pathname; GFS_File gf; char *user; gfarm_timerval_t t1, t2; gfs_profile(gfarm_gettimerval(&t1)); user = gfarm_get_global_username(); if (user == NULL) { e = "gfs_pio_create(): programming error, " "gfarm library isn't properly initialized"; goto finish; } e = gfarm_url_make_path_for_creation(url, &pathname); if (e != NULL) goto finish; e = gfs_file_alloc(&gf); if (e != NULL) { free(pathname); goto finish; } flags |= GFARM_FILE_CREATE; gfs_pio_open_initialize_mode_flags(gf, flags); e = gfarm_path_info_get(pathname, &gf->pi); if (e != NULL && e != GFARM_ERR_NO_SUCH_OBJECT) { free(pathname); goto free_gf; } if (e == NULL) { free(pathname); if ((flags & GFARM_FILE_EXCLUSIVE) != 0) { e = GFARM_ERR_ALREADY_EXISTS; goto free_gf_pi; } e = gfs_pio_open_check_perm(gf); if (e != NULL) goto free_gf_pi; /* * XXX should check the follows: * - the mode is consistent among same job * - creator of the metainfo has same job id * - O_TRUNC / !O_TRUNC case */ if ((flags & GFARM_FILE_TRUNC) == 0) gf->mode |= GFS_FILE_MODE_NSEGMENTS_FIXED; } else { mode_t mask; struct timeval now; mask = umask(0); umask(mask); gettimeofday(&now, NULL); gf->pi.pathname = pathname; gf->pi.status.st_mode = (GFARM_S_IFREG | (mode & ~mask)); gf->pi.status.st_user = strdup(user); /* XXX NULL check */ gf->pi.status.st_group = strdup("*"); /* XXX for now */ gf->pi.status.st_atimespec.tv_sec = gf->pi.status.st_mtimespec.tv_sec = gf->pi.status.st_ctimespec.tv_sec = now.tv_sec; gf->pi.status.st_atimespec.tv_nsec = gf->pi.status.st_mtimespec.tv_nsec = gf->pi.status.st_ctimespec.tv_nsec = now.tv_usec * 1000; gf->pi.status.st_size = 0; gf->pi.status.st_nsections = 0; gf->mode |= GFS_FILE_MODE_FILE_CREATED; } *gfp = gf; e = NULL; goto finish; free_gf_pi: gfarm_path_info_free(&gf->pi); free_gf: gfs_file_free(gf); finish: gfs_profile(gfarm_gettimerval(&t2)); gfs_profile(gfs_pio_create_time += gfarm_timerval_sub(&t2, &t1)); return (e); }
char * gfs_pio_set_view_section(GFS_File gf, const char *section, char *if_hostname, int flags) { struct gfs_file_section_context *vc; char *e; int is_local_host; gfarm_timerval_t t1, t2; gfs_profile(gfarm_gettimerval(&t1)); e = gfs_pio_set_view_default(gf); if (e != NULL) goto profile_finish; vc = malloc(sizeof(struct gfs_file_section_context)); if (vc == NULL) { e = gf->error = GFARM_ERR_NO_MEMORY; goto profile_finish; } vc->section = strdup(section); if (vc->section == NULL) { free(vc); e = gf->error = GFARM_ERR_NO_MEMORY; goto profile_finish; } retry: if (if_hostname != NULL) { e = gfarm_host_get_canonical_name(if_hostname, &vc->canonical_hostname); if (e == GFARM_ERR_UNKNOWN_HOST) { /* FT - invalid hostname, delete section copy info */ if (gfarm_file_section_copy_info_remove( gf->pi.pathname, vc->section, if_hostname) == NULL) e = GFARM_ERR_INCONSISTENT_RECOVERABLE; if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE && (flags & GFARM_FILE_NOT_RETRY) == 0 && (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) { if_hostname = NULL; goto retry; } goto finish; } else if (e != NULL) goto finish; } else if ((gf->open_flags & GFARM_FILE_CREATE) != 0) { e = gfarm_host_get_canonical_self_name(&if_hostname); if (e == NULL) e = gfarm_schedule_search_idle_hosts( 1, &if_hostname, 1, &vc->canonical_hostname); if (e == NULL) e = gfarm_fixedstrings_dup( 1, &vc->canonical_hostname, &vc->canonical_hostname); if (e != NULL) { /* * local host is not a file system node, or * 'gfsd' on a local host is not running. */ e = gfarm_schedule_search_idle_by_all(1, &if_hostname); if (e != NULL) goto finish; vc->canonical_hostname = if_hostname; } } else { e = gfarm_file_section_host_schedule_with_priority_to_local( gf->pi.pathname, vc->section, &if_hostname); if (e != NULL) goto finish; vc->canonical_hostname = if_hostname; /* must be already canonical */ } is_local_host = gfarm_canonical_hostname_is_local( vc->canonical_hostname); if ((gf->open_flags & GFARM_FILE_CREATE) != 0) { struct gfarm_path_info pi; e = gfarm_path_info_set(gf->pi.pathname, &gf->pi); if (e == GFARM_ERR_ALREADY_EXISTS && (e = gfarm_path_info_get(gf->pi.pathname, &pi)) == NULL) { if (GFS_FILE_IS_PROGRAM(gf) != GFARM_S_IS_PROGRAM(pi.status.st_mode)) e = GFARM_ERR_OPERATION_NOT_PERMITTED; if (e == NULL && !GFS_FILE_IS_PROGRAM(gf)) { if (gf->pi.status.st_nsections != pi.status.st_nsections) { e = GFARM_ERR_FRAGMENT_NUMBER_DOES_NOT_MATCH; } else { #if 0 assert(gf->pi.status.st_mode & GFS_FILE_MODE_NSEGMENTS_FIXED); #endif } } if (e != NULL) { gfarm_path_info_free(&pi); } else { gfarm_path_info_free(&gf->pi); gf->pi = pi; } /* * XXX should check the follows: * - creator of the metainfo has same job id * - mode is consistent among same job * - nfragments is consistent among same job */ } if (e != NULL) goto free_host; } /* delete the file section when opening with a truncation flag. */ if ((gf->open_flags & GFARM_FILE_TRUNC) != 0) (void)gfs_unlink_section(gf->pi.pathname, vc->section); gf->ops = &gfs_pio_view_section_ops; gf->view_context = vc; gf->view_flags = flags; gf->p = gf->length = 0; gf->io_offset = gf->offset = 0; gf->mode |= GFS_FILE_MODE_CALC_DIGEST; EVP_DigestInit(&vc->md_ctx, GFS_DEFAULT_DIGEST_MODE); if (!is_local_host && ((((gf->open_flags & GFARM_FILE_REPLICATE) != 0 || gf_on_demand_replication ) && (flags & GFARM_FILE_NOT_REPLICATE) == 0) || (flags & GFARM_FILE_REPLICATE) != 0)) { e = replicate_section_to_local(gf, vc->section, if_hostname); /* FT - inconsistent metadata has been fixed. try again. */ if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE && (flags & GFARM_FILE_NOT_RETRY) == 0 && (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) { if_hostname = NULL; free(vc->canonical_hostname); goto retry; } if (e != NULL) goto free_host; free(vc->canonical_hostname); e = gfarm_host_get_canonical_self_name( &vc->canonical_hostname); if (e != NULL) goto finish; vc->canonical_hostname = strdup(vc->canonical_hostname); if (vc->canonical_hostname == NULL) { e = GFARM_ERR_NO_MEMORY; goto finish; } is_local_host = 1; } if (is_local_host) e = gfs_pio_open_local_section(gf, flags); else e = gfs_pio_open_remote_section(gf, if_hostname, flags); /* FT - inconsistent metadata has been fixed. try again. */ if (e == GFARM_ERR_INCONSISTENT_RECOVERABLE && (flags & GFARM_FILE_NOT_RETRY) == 0 && (gf->open_flags & GFARM_FILE_NOT_RETRY) == 0) { if_hostname = NULL; free(vc->canonical_hostname); goto retry; } free_host: if (e != NULL) free(vc->canonical_hostname); finish: if (e != NULL) { free(vc->section); free(vc); gf->view_context = NULL; gfs_pio_set_view_default(gf); } gf->error = e; profile_finish: gfs_profile(gfarm_gettimerval(&t2)); gfs_profile(gfs_pio_set_view_section_time += gfarm_timerval_sub(&t2, &t1)); return (e); }
char * gfs_unlink(const char *gfarm_url) { char *gfarm_file, *e, *e_save = NULL; int i, j, nsections; struct gfarm_path_info pi; struct gfarm_file_section_info *sections; gfarm_timerval_t t1, t2; gfs_profile(gfarm_gettimerval(&t1)); e = gfarm_url_make_path_for_creation(gfarm_url, &gfarm_file); if (e != NULL) { e_save = e; goto finish_unlink; } e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) { e_save = e; goto finish_free_gfarm_file; } if (GFARM_S_ISDIR(pi.status.st_mode)) { gfarm_path_info_free(&pi); e_save = GFARM_ERR_IS_A_DIRECTORY; goto finish_free_gfarm_file; } gfarm_path_info_free(&pi); e = gfarm_file_section_info_get_all_by_file(gfarm_file, &nsections, §ions); if (e != NULL) { e_save = e; nsections = 0; sections = NULL; } /* XXX - should unlink in parallel. */ for (i = 0; i < nsections; i++) { int ncopies; struct gfarm_file_section_copy_info *copies; e = gfarm_file_section_copy_info_get_all_by_section( gfarm_file, sections[i].section, &ncopies, &copies); if (e != NULL) { if (e_save == NULL) e_save = e; continue; } for (j = 0; j < ncopies; j++) { e = gfs_unlink_replica_internal(gfarm_file, sections[i].section, copies[j].hostname); if (e != NULL) { if (e_save == NULL) e_save = e; continue; } } gfarm_file_section_copy_info_free_all(ncopies, copies); } if (sections != NULL) gfarm_file_section_info_free_all(nsections, sections); e = gfarm_file_section_info_remove_all_by_file(gfarm_file); if (e != NULL) e_save = e; e = gfarm_path_info_remove(gfarm_file); if (e != NULL) e_save = e; finish_free_gfarm_file: free(gfarm_file); finish_unlink: gfs_profile(gfarm_gettimerval(&t2)); gfs_profile(gfs_unlink_time += gfarm_timerval_sub(&t2, &t1)); return (e_save); }
int main(int argc, char **argv) { char *e, *canonic_path; int i; if (argc <= 1) usage(); program_name = basename(argv[0]); e = gfarm_initialize(&argc, &argv); if (e != NULL) { fprintf(stderr, "%s: %s\n", program_name, e); exit(1); } for (i = 1; i < argc; i++) { char *e; struct args a; struct gfarm_path_info pi; GFS_Dir dir; struct gfs_dirent *entry; int nhosts_succeed; e = gfarm_url_make_path(argv[i], &canonic_path); /* We permit missing gfarm: prefix here as a special case */ if (e == GFARM_ERR_GFARM_URL_PREFIX_IS_MISSING) e = gfarm_canonical_path(argv[i], &canonic_path); if (e != NULL) { fprintf(stderr, "%s: %s\n", program_name, e); continue; } e = gfarm_path_info_get(canonic_path, &pi); if (e != NULL) { fprintf(stderr, "%s: %s\n", program_name, e); exit(1); } if (!GFARM_S_ISDIR(pi.status.st_mode)) { fprintf(stderr, "%s: %s: %s\n", program_name, argv[i], GFARM_ERR_NOT_A_DIRECTORY); free(canonic_path); gfarm_path_info_free(&pi); continue; } gfarm_path_info_free(&pi); e = gfs_opendir(argv[i], &dir); if (e != NULL) { fprintf(stderr, "%s: %s: %s\n", program_name, argv[i], e); exit(1); } e = gfs_readdir(dir, &entry); if (e != NULL) { fprintf(stderr, "%s: %s: %s\n", program_name, argv[i], e); exit(1); } if (entry != NULL) { fprintf(stderr, "%s: %s: %s\n", program_name, argv[i], "directory not empty"); free(canonic_path); continue; } a.path = canonic_path; e = gfs_client_apply_all_hosts(gfrmdir, &a, program_name, &nhosts_succeed); if (e != NULL) { fprintf(stderr, "%s: %s\n", program_name, e); exit(1); } e = gfarm_path_info_remove(canonic_path); if (e != NULL) { fprintf(stderr, "%s: %s\n", program_name, e); exit(1); } free(canonic_path); } e = gfarm_terminate(); return (0); }
gfarm_error_t gfs_chdir_canonical(const char *canonic_dir) { static int cwd_len = 0; static char env_name[] = "GFS_PWD="; static char *env = NULL; static int env_len = 0; int len, old_len; char *e, *tmp, *old_env; struct gfarm_path_info pi; e = gfarm_path_info_get(canonic_dir, &pi); if (e == NULL) { e = gfarm_path_info_access(&pi, X_OK); gfarm_path_info_free(&pi); } if (e != NULL) return (e); len = 1 + strlen(canonic_dir) + 1; if (cwd_len < len) { GFARM_REALLOC_ARRAY(tmp, gfarm_current_working_directory, len); if (tmp == NULL) return (GFARM_ERR_NO_MEMORY); gfarm_current_working_directory = tmp; cwd_len = len; } sprintf(gfarm_current_working_directory, "/%s", canonic_dir); len += sizeof(env_name) - 1 + GFARM_URL_PREFIX_LENGTH; tmp = getenv("GFS_PWD"); if (tmp == NULL || tmp != env + sizeof(env_name) - 1) { /* * changed by an application instead of this function, and * probably it's already free()ed. In this case, realloc() * does not work well at least using bash. allocate it again. */ env = NULL; env_len = 0; } old_env = env; old_len = env_len; if (env_len < len) { /* * We cannot use realloc(env, ...) here, because `env' may be * still pointed by environ[somewhere] (at least with glibc), * and realloc() may break the memory. So, allocate different * memory. */ GFARM_MALLOC_ARRAY(tmp, len); if (tmp == NULL) return (GFARM_ERR_NO_MEMORY); env = tmp; env_len = len; } sprintf(env, "%s%s%s", env_name, GFARM_URL_PREFIX, gfarm_current_working_directory); if (putenv(env) != 0) { if (env != old_env && env != NULL) free(env); env = old_env; env_len = old_len; return (gfarm_errno_to_error(errno)); } if (old_env != env && old_env != NULL) free(old_env); return (NULL); }
char * gfarm_url_program_deliver(const char *gfarm_url, int nhosts, char **hosts, char ***delivered_paths) { char *e, **dp, *gfarm_file, *root, *arch, **canonical_hostnames; gfarm_mode_t mode; int i; struct gfarm_path_info pi; e = gfarm_url_make_path(gfarm_url, &gfarm_file); if (e != NULL) return (e); e = gfarm_path_info_get(gfarm_file, &pi); if (e != NULL) { free(gfarm_file); return (e); } mode = pi.status.st_mode; if (!GFARM_S_IS_PROGRAM(mode)) { gfarm_path_info_free(&pi); free(gfarm_file); return ("gfarm_url_program_deliver(): not a program"); } /* * XXX - if the owner of a file is not the same, permit a * group/other write access - This should be fixed in the next * major release. */ if (strcmp(pi.status.st_user, gfarm_get_global_username()) != 0) { e = gfarm_path_info_access(&pi, GFS_X_OK); if (e != NULL) { gfarm_path_info_free(&pi); free(gfarm_file); return (e); } mode |= 022; } gfarm_path_info_free(&pi); dp = malloc(sizeof(char *) * (nhosts + 1)); if (dp == NULL) { free(gfarm_file); return (GFARM_ERR_NO_MEMORY); } dp[nhosts] = NULL; e = gfarm_host_get_canonical_names(nhosts, hosts, &canonical_hostnames); if (e != NULL) { free(dp); free(gfarm_file); return (e); } /* XXX - this is too slow */ for (i = 0; i < nhosts; i++) { struct sockaddr peer_addr; struct gfs_connection *gfs_server; struct gfarm_file_section_info si; dp[i] = NULL; /* for error handling */ arch = gfarm_host_info_get_architecture_by_host( canonical_hostnames[i]); if (arch == NULL) { /* architecture of the hosts[i] is not registered */ e = "cannot deliver program to an unregistered host"; goto error; } /* XXX - which to use? `hosts[i]' vs `copies[j].hostname' */ e = gfarm_host_address_get(hosts[i], gfarm_spool_server_port, &peer_addr, NULL); if (e != NULL) { free(arch); goto error; } e = gfs_client_connection(canonical_hostnames[i], &peer_addr, &gfs_server); if (e != NULL) { free(arch); goto error; } e = gfs_client_get_spool_root(gfs_server, &root); if (e != NULL) { free(arch); goto error; } e = gfarm_full_path_file_section(root, gfarm_file, arch, &dp[i]); free(root); if (e != NULL) { free(arch); goto error; } e = gfarm_file_section_info_get(gfarm_file, arch, &si); if (e != NULL) { free(arch); goto error; } /* * replicate the program */ e = gfarm_file_section_replicate_to_internal(gfarm_file, arch, mode & GFARM_S_ALLPERM, si.filesize, hosts[i]); gfarm_file_section_info_free(&si); free(arch); if (e != NULL) goto error; } gfarm_strings_free_deeply(nhosts, canonical_hostnames); free(gfarm_file); *delivered_paths = dp; return (NULL); error: gfarm_strings_free_deeply(nhosts, canonical_hostnames); free(gfarm_file); gfarm_strings_free_deeply(i + 1, dp); *delivered_paths = NULL; return (e); }