void
gfarm_errlist_hashtab_initialize(void)
{
int i, created;
struct gfarm_hash_entry *p;
gfarm_errlist_hashtab = gfarm_hash_table_alloc(ERRLIST_HASHTAB_SIZE,
gfarm_hash_default, gfarm_hash_key_equal_default);
if (gfarm_errlist_hashtab == NULL) {
fprintf(stderr, "gfarm_errlist_hashtab_initialize(): "
"no memory\n");
exit(1);
}
for (i = 1; IN_ERRNO(i); i++) {
if (STRERROR(i) == NULL)
continue;
p = gfarm_hash_enter(gfarm_errlist_hashtab,
STRERROR(i), strlen(STRERROR(i)) + 1,
sizeof(int), &created);
if (p == NULL) {
fprintf(stderr, "gfarm_errlist_hashtab_initialize(): "
"no memory for errno %d\n", i);
exit(1);
}
if (created)
*(int *)gfarm_hash_entry_data(p) = i;
}
}
static void
change_file_node_to_dir(struct node *n)
{
n->flags |= NODE_FLAG_IS_DIR;
if (n->u.d.children == NULL) {
n->u.d.children = gfarm_hash_table_alloc(NODE_HASH_SIZE,
gfarm_hash_default, gfarm_hash_key_equal_default);
n->u.d.mtime.tv_sec = n->u.d.mtime.tv_usec = 0;
}
}
static struct node *
init_dir_node(struct node *n, const char *name, int len)
{
if (init_node_name(n, name, len) == NULL)
return (NULL);
n->flags = NODE_FLAG_IS_DIR;
n->u.d.children = gfarm_hash_table_alloc(NODE_HASH_SIZE,
gfarm_hash_default, gfarm_hash_key_equal_default);
n->u.d.mtime.tv_sec = n->u.d.mtime.tv_usec = 0;
return (n);
}
gfarm_error_t
gfp_conn_hash_table_init(
struct gfarm_hash_table **hashtabp, int hashtabsize)
{
struct gfarm_hash_table *hashtab;
hashtab = gfarm_hash_table_alloc(hashtabsize,
gfp_conn_hash_index, gfp_conn_hash_equal);
if (hashtab == NULL)
return (GFARM_ERR_NO_MEMORY);
*hashtabp = hashtab;
return (GFARM_ERR_NO_ERROR);
}
/* Create a set of architectures that the program is registered for */
static char *
program_arch_set(char *program, struct gfarm_hash_table **arch_setp)
{
char *e, *gfarm_file;
struct gfarm_path_info pi;
struct gfarm_file_section_info *sections;
struct gfarm_hash_table *arch_set;
int i, nsections, created;
e = gfarm_url_make_path(program, &gfarm_file);
if (e != NULL)
return (e);
e = gfarm_path_info_get(gfarm_file, &pi);
if (e != NULL) {
if (e == GFARM_ERR_NO_SUCH_OBJECT)
e = "such program isn't registered";
free(gfarm_file);
return (e);
}
if (!GFARM_S_IS_PROGRAM(pi.status.st_mode)) {
gfarm_path_info_free(&pi);
free(gfarm_file);
return ("specified command is not an executable");
}
e = gfarm_file_section_info_get_all_by_file(gfarm_file,
&nsections, §ions);
gfarm_path_info_free(&pi);
free(gfarm_file);
if (e != NULL)
return ("no binary is registered as the specified command");
arch_set = gfarm_hash_table_alloc(ARCH_SET_HASHTAB_SIZE,
gfarm_hash_default, gfarm_hash_key_equal_default);
if (arch_set == NULL) {
gfarm_file_section_info_free_all(nsections, sections);
return (GFARM_ERR_NO_MEMORY);
}
/* register architectures of the program to `arch_set' */
for (i = 0; i < nsections; i++) {
if (gfarm_hash_enter(arch_set,
sections[i].section, strlen(sections[i].section) + 1,
sizeof(int), &created) == NULL) {
free_arch_set(arch_set);
gfarm_file_section_info_free_all(nsections, sections);
return (GFARM_ERR_NO_MEMORY);
}
}
gfarm_file_section_info_free_all(nsections, sections);
*arch_setp = arch_set;
return (NULL);
}
static gfarm_error_t
create_hash_table_from_string_list(int array_length, char **array,
int hashsize, struct gfarm_hash_table **hashp)
{
struct gfarm_hash_table *hash;
int i;
hash = gfarm_hash_table_alloc(hashsize,
gfarm_hash_casefold, gfarm_hash_key_equal_casefold);
if (hash == NULL)
return (GFARM_ERR_NO_MEMORY);
for (i = 0; i < array_length; ++i)
gfarm_hash_enter(hash, array[i], strlen(array[i])+1, 0, NULL);
*hashp = hash;
return (GFARM_ERR_NO_ERROR);
}
static int
cache_path_info_init()
{
static int env_init = 0;
if (env_init == 0) {
char *envval;
envval = getenv("GFARM_PATH_INFO_HASH_SIZE");
if (envval != NULL)
cache_hash_size_set(atoi(envval));
/* millisecond */
envval = getenv("GFARM_PATH_INFO_TIMEOUT");
if (envval != NULL)
cache_timeout_set(atoi(envval));
/* millisecond */
envval = getenv("GFARM_UPDATE_TIME_INTERVAL");
if (envval != NULL)
cache_update_time_interval_set(atoi(envval));
env_init = 1;
}
if (cache_timeout.tv_sec == 0 && cache_timeout.tv_usec == 0)
return (0); /* disable */
if (prepare_cache_table == 1)
return (1); /* enable */
cache_table = gfarm_hash_table_alloc(hash_size,
gfarm_hash_default,
gfarm_hash_key_equal_default);
prepare_cache_table = 1;
current_cache_num = 0;
_debug("! cache_path_info_init: hash_size=%d\n", hash_size);
_debug("! path_info_timeout=%u.%u(sec.microsec)\n",
(unsigned int) cache_timeout.tv_sec,
(unsigned int) cache_timeout.tv_usec);
_debug("! update_time_interval=%u.%u(sec.nanosec)\n",
(unsigned int) update_time_interval.tv_sec,
(unsigned int) update_time_interval.tv_nsec);
return (1); /* enable */
}
static char *
alloc_hosts_state(int *n_all_hostsp, struct gfarm_host_info **all_hostsp,
struct gfarm_hash_table **hosts_statep)
{
char *e;
int i, created, n_all_hosts;
struct gfarm_host_info *all_hosts;
struct gfarm_hash_table *hosts_state;
struct gfarm_hash_entry *entry;
struct search_idle_host_state *h;
e = gfarm_host_info_get_all(&n_all_hosts, &all_hosts);
if (e != NULL)
return (e);
if (n_all_hosts == 0) {
gfarm_host_info_free_all(n_all_hosts, all_hosts);
return (GFARM_ERR_NO_HOST);
}
hosts_state = gfarm_hash_table_alloc(HOSTS_HASHTAB_SIZE,
gfarm_hash_casefold, gfarm_hash_key_equal_casefold);
if (hosts_state == NULL) {
gfarm_host_info_free_all(n_all_hosts, all_hosts);
return (GFARM_ERR_NO_MEMORY);
}
for (i = 0; i < n_all_hosts; i++) {
entry = gfarm_hash_enter(hosts_state,
all_hosts[i].hostname, strlen(all_hosts[i].hostname) + 1,
sizeof(struct search_idle_host_state), &created);
if (entry == NULL) {
free_hosts_state(n_all_hosts, all_hosts, hosts_state);
return (GFARM_ERR_NO_MEMORY);
}
/* `created' must be always true. */
h = gfarm_hash_entry_data(entry);
h->host_info = &all_hosts[i];
h->flags = 0;
}
*n_all_hostsp = n_all_hosts;
*all_hostsp = all_hosts;
*hosts_statep = hosts_state;
return (NULL);
}
gfarm_error_t
gfs_stat_cache_init(void)
{
if (!stat_cache_lifespan_is_set) {
/* always reflect gfarm_attr_cache_timeout */
stat_cache_lifespan.tv_sec = gfarm_attr_cache_timeout /
(GFARM_SECOND_BY_MICROSEC / GFARM_MILLISEC_BY_MICROSEC);
stat_cache_lifespan.tv_usec = (gfarm_attr_cache_timeout -
stat_cache_lifespan.tv_sec *
(GFARM_SECOND_BY_MICROSEC / GFARM_MILLISEC_BY_MICROSEC)) *
GFARM_MILLISEC_BY_MICROSEC;
}
if (stat_cache != NULL) /* already initialized */
return (GFARM_ERR_NO_ERROR);
stat_cache = gfarm_hash_table_alloc(
STAT_HASH_SIZE, gfarm_hash_default, gfarm_hash_key_equal_default);
if (stat_cache == NULL)
return (GFARM_ERR_NO_MEMORY);
return (GFARM_ERR_NO_ERROR);
}
int
main(int argc, char *argv[])
{
char *src_hostfile = NULL, *dst_hostfile = NULL;
gfarm_stringlist paths;
gfs_glob_t types;
int mode_src_ch = 0, mode_dst_ch = 0, parallel = -1;
int i, ch;
gfarm_error_t e;
const char *errmsg, *errmsg2 = NULL;
struct gfrep_arg gfrep_arg;
struct flist flist;
if (argc >= 1)
program_name = basename(argv[0]);
memset(&gfrep_arg, 0, sizeof(gfrep_arg));
memset(&flist, 0, sizeof(flist));
flist.src_domain = "";
flist.dst_domain = "";
e = gfarm_initialize(&argc, &argv);
error_check(e);
#ifdef _OPENMP
while ((ch = getopt(argc, argv, "h:j:mnqvxS:D:H:N:?")) != -1) {
#else
while ((ch = getopt(argc, argv, "h:mnqvxS:D:H:N:?")) != -1) {
#endif
switch (ch) {
case 'h':
src_hostfile = optarg;
conflict_check(&mode_src_ch, ch);
break;
#ifdef _OPENMP
case 'j':
parallel = strtol(optarg, NULL, 0);
break;
#endif
case 'm':
act = &migrate_mode;
break;
case 'n':
opt_noexec = 1;
break;
case 'q':
opt_quiet = 1;
break;
case 'v':
opt_verbose = 1;
break;
case 'x':
opt_remove = 1;
break;
case 'S':
flist.src_domain = optarg;
conflict_check(&mode_src_ch, ch);
break;
case 'D':
flist.dst_domain = optarg;
conflict_check(&mode_dst_ch, ch);
break;
case 'H':
dst_hostfile = optarg;
conflict_check(&mode_dst_ch, ch);
break;
case 'N':
opt_nrep = strtol(optarg, NULL, 0);
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
/* make writing-to-stderr atomic, for GfarmFS-FUSE log output */
setvbuf(stderr, NULL, _IOLBF, 0);
if (!opt_quiet) {
printf("constructing file list...");
fflush(stdout);
}
e = gfarm_stringlist_init(&paths);
if (e == GFARM_ERR_NO_ERROR) {
e = gfs_glob_init(&types);
if (e == GFARM_ERR_NO_ERROR) {
for (i = 0; i < argc; i++)
gfs_glob(argv[i], &paths, &types);
gfs_glob_free(&types);
}
}
error_check(e);
e = gfarm_list_init(&flist.slist);
error_check(e);
e = gfarm_list_init(&flist.dlist);
error_check(e);
flist.srchash = gfarm_hash_table_alloc(HOSTHASH_SIZE,
gfarm_hash_casefold, gfarm_hash_key_equal_casefold);
if (flist.srchash == NULL)
error_check(GFARM_ERR_NO_MEMORY);
e = create_hosthash_from_file(src_hostfile,
HOSTHASH_SIZE, &flist.src_hosthash);
error_check(e);
e = create_hosthash_from_file(dst_hostfile,
HOSTHASH_SIZE, &flist.dst_hosthash);
error_check(e);
for (i = 0; i < gfarm_stringlist_length(&paths); i++) {
char *file = gfarm_stringlist_elem(&paths, i), *realpath = NULL;
e = gfarm_realpath_by_gfarm2fs(file, &realpath);
if (e == GFARM_ERR_NO_ERROR)
file = realpath;
e = gfarm_foreach_directory_hierarchy(
create_filelist, NULL, NULL, file, &flist);
free(realpath);
if (e != GFARM_ERR_NO_ERROR)
break;
}
gfarm_stringlist_free_deeply(&paths);
error_check(e);
if (!opt_quiet)
printf(" done\n");
if (opt_verbose) {
printf("files to be replicated\n");
print_file_list(&flist.slist);
}
if (opt_verbose && opt_remove) {
printf("files having too many replicas\n");
print_file_list(&flist.dlist);
}
if (gfarm_list_length(&flist.slist) <= 0
&& (!opt_remove || gfarm_list_length(&flist.dlist) <= 0))
exit(0); /* no file */
/* replicate files */
e = gfarm_hash_to_string_array(
flist.srchash, &gfrep_arg.nsrc, &gfrep_arg.src);
error_check(e);
gfarm_hash_table_free(flist.srchash);
if (!opt_quiet) {
printf("investigating hosts...");
fflush(stdout);
}
e = create_hostlist_by_domain_and_hash(
gfarm_list_length(&flist.slist) > 0 ?
gfarm_list_elem(&flist.slist, 0) :
gfarm_list_elem(&flist.dlist, 0),
flist.dst_domain, flist.dst_hosthash,
&gfrep_arg.ndst, &gfrep_arg.dst, &gfrep_arg.dst_port);
error_check(e);
if (!opt_quiet)
printf(" done\n");
errmsg = pfor_list(act, &flist.slist, parallel, &gfrep_arg);
gfarm_list_free(&flist.slist);
/* remove file replicas */
if (opt_remove)
errmsg2 = pfor_list(
&remove_mode, &flist.dlist, parallel, &gfrep_arg);
gfarm_list_free(&flist.dlist);
if (errmsg == NULL)
errmsg = errmsg2;
if (errmsg != NULL)
fprintf(stderr, "%s\n", errmsg), exit(EXIT_FAILURE);
gfarm_strings_free_deeply(gfrep_arg.nsrc, gfrep_arg.src);
gfarm_strings_free_deeply(gfrep_arg.ndst, gfrep_arg.dst);
free(gfrep_arg.dst_port);
e = gfarm_terminate();
error_check(e);
return (0);
}
