/* searches for name and returns a character pointer to its value if set, * returns NULL if not found */ char* scr_param_get(char* name) { char* value = NULL; /* see if this parameter is one which is restricted from user */ scr_hash* no_user = scr_hash_get(scr_no_user_hash, name); /* if parameter is set in environment, return that value */ if (no_user == NULL && getenv(name) != NULL) { /* we don't just return the getenv value directly because that causes * segfaults on some systems, so instead we add it to a hash and return * the pointer into the hash */ /* try to lookup the value for this name in case we've already cached it */ if (scr_hash_util_get_str(scr_env_hash, name, &value) != SCR_SUCCESS) { /* it's not in the hash yet, so add it */ char* tmp_value = strdup(getenv(name)); scr_hash_util_set_str(scr_env_hash, name, tmp_value); scr_free(&tmp_value); /* now issue our lookup again */ if (scr_hash_util_get_str(scr_env_hash, name, &value) != SCR_SUCCESS) { /* it's an error if we don't find it this time */ scr_abort(-1, "Failed to find value for %s in env hash @ %s:%d", name, __FILE__, __LINE__ ); } } return value; } /* otherwise, if parameter is set in user configuration file, * return that value */ value = scr_hash_elem_get_first_val(scr_user_hash, name); if (no_user == NULL && value != NULL) { return value; } /* otherwise, if parameter is set in system configuration file, * return that value */ value = scr_hash_elem_get_first_val(scr_system_hash, name); if (value != NULL) { return value; } /* parameter not found, return NULL */ return NULL; }
/* given a hash of files and a file name, check whether the named * file needs data transfered, if so, strdup its destination name * and set its position and filesize */ int need_transfer(scr_hash* files, char* src, char** dst, off_t* position, off_t* filesize) { /* check that we got a hash of files and a file name */ if (files == NULL || src == NULL) { return SCR_FAILURE; } /* lookup the specified file in the hash */ scr_hash* file_hash = scr_hash_get(files, src); if (file_hash == NULL) { return SCR_FAILURE; } /* extract the values for file size, bytes written, and destination */ unsigned long size, written; char* dest; if (scr_hash_util_get_bytecount(file_hash, SCR_TRANSFER_KEY_SIZE, &size) == SCR_SUCCESS && scr_hash_util_get_bytecount(file_hash, SCR_TRANSFER_KEY_WRITTEN, &written) == SCR_SUCCESS && scr_hash_util_get_str(file_hash, SCR_TRANSFER_KEY_DESTINATION, &dest) == SCR_SUCCESS) { /* if the bytes written value is less than the file size, * we've got a valid file */ if (written < size) { /* got our file, fill in output parameters */ *dst = strdup(dest); /* TODO: check for error */ *position = (off_t) written; *filesize = (off_t) size; return SCR_SUCCESS; } } return SCR_FAILURE; }
/* extract container name, size, offset, and length values * for container that holds the specified segment */ int scr_container_get_name_offset_length( const scr_hash* segment, char** name, unsigned long* offset, unsigned long* length) { /* check that our parameters are valid */ if (segment == NULL || name == NULL || offset == NULL || length == NULL) { return SCR_FAILURE; } /* get name of container */ if (scr_hash_util_get_str(segment, SCR_SUMMARY_6_KEY_FILE, name) != SCR_SUCCESS) { return SCR_FAILURE; } /* lookup the offset value */ if (scr_hash_util_get_bytecount(segment, SCR_SUMMARY_6_KEY_OFFSET, offset) != SCR_SUCCESS) { return SCR_FAILURE; } /* lookup the segment length */ if (scr_hash_util_get_bytecount(segment, SCR_SUMMARY_6_KEY_LENGTH, length) != SCR_SUCCESS) { return SCR_FAILURE; } return SCR_SUCCESS; }
/* flush files specified in list, and record corresponding entries for summary file */ static int scr_flush_files_list(scr_hash* file_list, scr_hash* summary) { /* assume we will succeed in this flush */ int rc = SCR_SUCCESS; /* flush each of my files and fill in summary data structure */ scr_hash_elem* elem = NULL; scr_hash* files = scr_hash_get(file_list, SCR_KEY_FILE); for (elem = scr_hash_elem_first(files); elem != NULL; elem = scr_hash_elem_next(elem)) { /* get the filename */ char* file = scr_hash_elem_key(elem); /* convert file to path and extract name of file */ scr_path* path_name = scr_path_from_str(file); scr_path_basename(path_name); /* get the hash for this element */ scr_hash* hash = scr_hash_elem_hash(elem); /* get meta data for this file */ scr_meta* meta = scr_hash_get(hash, SCR_KEY_META); /* if segments are defined, we flush the file to its containers, * otherwise we copy the file out as is */ scr_hash* segments = scr_hash_get(hash, SCR_SUMMARY_6_KEY_SEGMENT); if (segments != NULL) { /* TODO: PRESERVE get original filename here */ /* add this file to the summary file */ char* name = scr_path_strdup(path_name); scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name); scr_free(&name); // USERDEF fixme! /* flush the file to the containers listed in its segmenets */ if (scr_flush_file_to_containers(file, meta, segments, scr_prefix) == SCR_SUCCESS) { /* successfully flushed this file, record the filesize */ unsigned long filesize = 0; if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) { scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize); } /* record the crc32 if one was computed */ uLong crc = 0; if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) { scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc); } /* record segment information in summary file */ scr_hash* segments_copy = scr_hash_new(); scr_hash_merge(segments_copy, segments); scr_hash_set(file_hash, SCR_SUMMARY_6_KEY_SEGMENT, segments_copy); } else { /* the flush failed */ rc = SCR_FAILURE; /* explicitly mark file as incomplete */ scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0); } } else { /* get directory to flush file to */ char* dir; if (scr_hash_util_get_str(hash, SCR_KEY_PATH, &dir) == SCR_SUCCESS) { /* create full path of destination file */ scr_path* path_full = scr_path_from_str(dir); scr_path_append(path_full, path_name); /* get relative path to flushed file from SCR_PREFIX directory */ scr_path* path_relative = scr_path_relative(scr_prefix_path, path_full); if (! scr_path_is_null(path_relative)) { /* record the name of the file in the summary hash, and get reference to a hash for this file */ char* name = scr_path_strdup(path_relative); scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name); scr_free(&name); /* flush the file and fill in the meta data for this file */ if (scr_flush_a_file(file, dir, meta) == SCR_SUCCESS) { /* successfully flushed this file, record the filesize */ unsigned long filesize = 0; if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) { scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize); } /* record the crc32 if one was computed */ uLong crc = 0; if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) { scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc); } } else { /* the flush failed */ rc = SCR_FAILURE; /* explicitly mark incomplete files */ scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0); } } else { scr_abort(-1, "Failed to get relative path to directory %s from %s @ %s:%d", dir, scr_prefix, __FILE__, __LINE__ ); } /* free relative and full paths */ scr_path_delete(&path_relative); scr_path_delete(&path_full); } else { scr_abort(-1, "Failed to read directory to flush file to @ %s:%d", __FILE__, __LINE__ ); } } /* free the file name path */ scr_path_delete(&path_name); } return rc; }
static int scr_fetch_rank2file_map( const scr_path* dataset_path, int depth, int* ptr_valid, char** ptr_file, unsigned long* ptr_offset) { int rc = SCR_SUCCESS; /* get local variables so we don't have to deference everything */ int valid = *ptr_valid; char* file = *ptr_file; unsigned long offset = *ptr_offset; /* create a hash to hold section of file */ scr_hash* hash = scr_hash_new(); /* if we can read from file do it */ if (valid) { /* open file if we haven't already */ int fd = scr_open(file, O_RDONLY); if (fd >= 0) { /* read our segment from the file */ scr_lseek(file, fd, offset, SEEK_SET); ssize_t read_rc = scr_hash_read_fd(file, fd, hash); if (read_rc < 0) { scr_err("Failed to read from %s @ %s:%d", file, __FILE__, __LINE__ ); rc = SCR_FAILURE; } /* close the file */ scr_close(file, fd); } else { scr_err("Failed to open rank2file map %s @ %s:%d", file, __FILE__, __LINE__ ); rc = SCR_FAILURE; } } /* check for read errors */ if (! scr_alltrue(rc == SCR_SUCCESS)) { rc = SCR_FAILURE; goto cleanup; } /* create hashes to exchange data */ scr_hash* send = scr_hash_new(); scr_hash* recv = scr_hash_new(); /* copy rank data into send hash */ if (valid) { scr_hash* rank_hash = scr_hash_get(hash, SCR_SUMMARY_6_KEY_RANK); scr_hash_merge(send, rank_hash); } /* exchange hashes */ scr_hash_exchange_direction(send, recv, scr_comm_world, SCR_HASH_EXCHANGE_RIGHT); /* see if anyone sent us anything */ int newvalid = 0; char* newfile = NULL; unsigned long newoffset = 0; scr_hash_elem* elem = scr_hash_elem_first(recv); if (elem != NULL) { /* got something, so now we'll read in the next step */ newvalid = 1; /* get file name we should read */ scr_hash* elem_hash = scr_hash_elem_hash(elem); char* value; if (scr_hash_util_get_str(elem_hash, SCR_SUMMARY_6_KEY_FILE, &value) == SCR_SUCCESS) { /* return string of full path to file to caller */ scr_path* newpath = scr_path_dup(dataset_path); scr_path_append_str(newpath, value); newfile = scr_path_strdup(newpath); scr_path_delete(&newpath); } else { rc = SCR_FAILURE; } /* get offset we should start reading from */ if (scr_hash_util_get_bytecount(elem_hash, SCR_SUMMARY_6_KEY_OFFSET, &newoffset) != SCR_SUCCESS) { rc = SCR_FAILURE; } } /* free the send and receive hashes */ scr_hash_delete(&recv); scr_hash_delete(&send); /* get level id, and broadcast it from rank 0, * which we assume to be a reader in all steps */ int level_id = -1; if (valid) { if (scr_hash_util_get_int(hash, SCR_SUMMARY_6_KEY_LEVEL, &level_id) != SCR_SUCCESS) { rc = SCR_FAILURE; } } MPI_Bcast(&level_id, 1, MPI_INT, 0, scr_comm_world); /* check for read errors */ if (! scr_alltrue(rc == SCR_SUCCESS)) { rc = SCR_FAILURE; goto cleanup; } /* set parameters for output or next iteration, * we already took care of updating ptr_fd earlier */ if (valid) { scr_free(ptr_file); } *ptr_valid = newvalid; *ptr_file = newfile; *ptr_offset = newoffset; /* recurse if we still have levels to read */ if (level_id > 1) { rc = scr_fetch_rank2file_map(dataset_path, depth+1, ptr_valid, ptr_file, ptr_offset); } cleanup: /* free the hash */ scr_hash_delete(&hash); return rc; }
/* fetch files listed in hash into specified cache directory, * update filemap and fill in total number of bytes fetched, * returns SCR_SUCCESS if successful */ static int scr_fetch_files_list( const scr_hash* file_list, const char* dir, scr_filemap* map) { /* assume we'll succeed in fetching our files */ int rc = SCR_SUCCESS; /* assume we don't have any files to fetch */ int my_num_files = 0; /* get dataset id */ int id; scr_dataset* dataset = scr_hash_get(file_list, SCR_KEY_DATASET); scr_dataset_get_id(dataset, &id); /* now iterate through the file list and fetch each file */ scr_hash_elem* file_elem = NULL; scr_hash* files = scr_hash_get(file_list, SCR_KEY_FILE); for (file_elem = scr_hash_elem_first(files); file_elem != NULL; file_elem = scr_hash_elem_next(file_elem)) { /* get the filename */ char* file = scr_hash_elem_key(file_elem); /* get a pointer to the hash for this file */ scr_hash* hash = scr_hash_elem_hash(file_elem); /* check whether we are supposed to fetch this file */ /* TODO: this is a hacky way to avoid reading a redundancy file * back in under the assumption that it's an original file, which * breaks our redundancy computation due to a name conflict on * the file names */ scr_hash_elem* no_fetch_hash = scr_hash_elem_get(hash, SCR_SUMMARY_6_KEY_NOFETCH); if (no_fetch_hash != NULL) { continue; } /* increment our file count */ my_num_files++; /* build the destination file name */ scr_path* path_newfile = scr_path_from_str(file); scr_path_basename(path_newfile); scr_path_prepend_str(path_newfile, dir); char* newfile = scr_path_strdup(path_newfile); /* add the file to our filemap and write it to disk before creating * the file, this way we have a record that it may exist before we * actually start to fetch it */ scr_filemap_add_file(map, id, scr_my_rank_world, newfile); scr_filemap_write(scr_map_file, map); /* get the file size */ unsigned long filesize = 0; if (scr_hash_util_get_unsigned_long(hash, SCR_KEY_SIZE, &filesize) != SCR_SUCCESS) { scr_err("Failed to read file size from summary data @ %s:%d", __FILE__, __LINE__ ); rc = SCR_FAILURE; /* free path and string */ scr_free(&newfile); scr_path_delete(&path_newfile); break; } /* check for a complete flag */ int complete = 1; if (scr_hash_util_get_int(hash, SCR_KEY_COMPLETE, &complete) != SCR_SUCCESS) { /* in summary file, the absence of a complete flag on a file * implies the file is complete */ complete = 1; } /* create a new meta data object for this file */ scr_meta* meta = scr_meta_new(); /* set the meta data */ scr_meta_set_filename(meta, newfile); scr_meta_set_filetype(meta, SCR_META_FILE_USER); scr_meta_set_filesize(meta, filesize); scr_meta_set_complete(meta, 1); /* TODODSET: move the ranks field elsewhere, for now it's needed * by scr_index.c */ scr_meta_set_ranks(meta, scr_ranks_world); /* get the crc, if set, and add it to the meta data */ uLong crc; if (scr_hash_util_get_crc32(hash, SCR_KEY_CRC, &crc) == SCR_SUCCESS) { scr_meta_set_crc32(meta, crc); } /* fetch file from containers if they are defined, otherwise fetch * the native file */ scr_hash* segments = scr_hash_get(hash, SCR_SUMMARY_6_KEY_SEGMENT); if (segments != NULL) { /* get source path */ char* from_dir; if (scr_hash_util_get_str(file_list, SCR_KEY_PATH, &from_dir) == SCR_SUCCESS) { /* fetch file from containers */ if (scr_fetch_file_from_containers(newfile, meta, segments, from_dir) != SCR_SUCCESS) { /* failed to fetch file, mark it as incomplete */ scr_meta_set_complete(meta, 0); rc = SCR_FAILURE; } } else { /* failed to find base dataset directory in file list */ rc = SCR_FAILURE; } } else { /* fetch native file, lookup directory for this file */ char* from_dir; if (scr_hash_util_get_str(hash, SCR_KEY_PATH, &from_dir) == SCR_SUCCESS) { if (scr_fetch_file(newfile, from_dir, meta) != SCR_SUCCESS) { /* failed to fetch file, mark it as incomplete */ scr_meta_set_complete(meta, 0); rc = SCR_FAILURE; } } else { /* failed to read source directory, mark file as incomplete */ scr_meta_set_complete(meta, 0); rc = SCR_FAILURE; } } /* TODODSET: want to write out filemap before we start to fetch * each file? */ /* mark the file as complete */ scr_filemap_set_meta(map, id, scr_my_rank_world, newfile, meta); /* free the meta data object */ scr_meta_delete(&meta); /* free path and string */ scr_free(&newfile); scr_path_delete(&path_newfile); } /* set the expected number of files for this dataset */ scr_filemap_set_expected_files(map, id, scr_my_rank_world, my_num_files); scr_filemap_write(scr_map_file, map); return rc; }