/* searches for name and returns a character pointer to its value if set,
* returns NULL if not found */
char* scr_param_get(char* name)
{
char* value = NULL;
/* see if this parameter is one which is restricted from user */
scr_hash* no_user = scr_hash_get(scr_no_user_hash, name);
/* if parameter is set in environment, return that value */
if (no_user == NULL && getenv(name) != NULL) {
/* we don't just return the getenv value directly because that causes
* segfaults on some systems, so instead we add it to a hash and return
* the pointer into the hash */
/* try to lookup the value for this name in case we've already cached it */
if (scr_hash_util_get_str(scr_env_hash, name, &value) != SCR_SUCCESS) {
/* it's not in the hash yet, so add it */
char* tmp_value = strdup(getenv(name));
scr_hash_util_set_str(scr_env_hash, name, tmp_value);
scr_free(&tmp_value);
/* now issue our lookup again */
if (scr_hash_util_get_str(scr_env_hash, name, &value) != SCR_SUCCESS) {
/* it's an error if we don't find it this time */
scr_abort(-1, "Failed to find value for %s in env hash @ %s:%d",
name, __FILE__, __LINE__
);
}
}
return value;
}
/* otherwise, if parameter is set in user configuration file,
* return that value */
value = scr_hash_elem_get_first_val(scr_user_hash, name);
if (no_user == NULL && value != NULL) {
return value;
}
/* otherwise, if parameter is set in system configuration file,
* return that value */
value = scr_hash_elem_get_first_val(scr_system_hash, name);
if (value != NULL) {
return value;
}
/* parameter not found, return NULL */
return NULL;
}
/* given a hash of files and a file name, check whether the named
* file needs data transfered, if so, strdup its destination name
* and set its position and filesize */
int need_transfer(scr_hash* files, char* src, char** dst, off_t* position, off_t* filesize)
{
/* check that we got a hash of files and a file name */
if (files == NULL || src == NULL) {
return SCR_FAILURE;
}
/* lookup the specified file in the hash */
scr_hash* file_hash = scr_hash_get(files, src);
if (file_hash == NULL) {
return SCR_FAILURE;
}
/* extract the values for file size, bytes written, and destination */
unsigned long size, written;
char* dest;
if (scr_hash_util_get_bytecount(file_hash, SCR_TRANSFER_KEY_SIZE, &size) == SCR_SUCCESS &&
scr_hash_util_get_bytecount(file_hash, SCR_TRANSFER_KEY_WRITTEN, &written) == SCR_SUCCESS &&
scr_hash_util_get_str(file_hash, SCR_TRANSFER_KEY_DESTINATION, &dest) == SCR_SUCCESS)
{
/* if the bytes written value is less than the file size,
* we've got a valid file */
if (written < size) {
/* got our file, fill in output parameters */
*dst = strdup(dest);
/* TODO: check for error */
*position = (off_t) written;
*filesize = (off_t) size;
return SCR_SUCCESS;
}
}
return SCR_FAILURE;
}
/* extract container name, size, offset, and length values
* for container that holds the specified segment */
int scr_container_get_name_offset_length(
const scr_hash* segment,
char** name,
unsigned long* offset,
unsigned long* length)
{
/* check that our parameters are valid */
if (segment == NULL || name == NULL || offset == NULL || length == NULL) {
return SCR_FAILURE;
}
/* get name of container */
if (scr_hash_util_get_str(segment, SCR_SUMMARY_6_KEY_FILE, name) != SCR_SUCCESS) {
return SCR_FAILURE;
}
/* lookup the offset value */
if (scr_hash_util_get_bytecount(segment, SCR_SUMMARY_6_KEY_OFFSET, offset) != SCR_SUCCESS) {
return SCR_FAILURE;
}
/* lookup the segment length */
if (scr_hash_util_get_bytecount(segment, SCR_SUMMARY_6_KEY_LENGTH, length) != SCR_SUCCESS) {
return SCR_FAILURE;
}
return SCR_SUCCESS;
}
/* flush files specified in list, and record corresponding entries for summary file */
static int scr_flush_files_list(scr_hash* file_list, scr_hash* summary)
{
/* assume we will succeed in this flush */
int rc = SCR_SUCCESS;
/* flush each of my files and fill in summary data structure */
scr_hash_elem* elem = NULL;
scr_hash* files = scr_hash_get(file_list, SCR_KEY_FILE);
for (elem = scr_hash_elem_first(files);
elem != NULL;
elem = scr_hash_elem_next(elem))
{
/* get the filename */
char* file = scr_hash_elem_key(elem);
/* convert file to path and extract name of file */
scr_path* path_name = scr_path_from_str(file);
scr_path_basename(path_name);
/* get the hash for this element */
scr_hash* hash = scr_hash_elem_hash(elem);
/* get meta data for this file */
scr_meta* meta = scr_hash_get(hash, SCR_KEY_META);
/* if segments are defined, we flush the file to its containers,
* otherwise we copy the file out as is */
scr_hash* segments = scr_hash_get(hash, SCR_SUMMARY_6_KEY_SEGMENT);
if (segments != NULL) {
/* TODO: PRESERVE get original filename here */
/* add this file to the summary file */
char* name = scr_path_strdup(path_name);
scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name);
scr_free(&name);
// USERDEF fixme!
/* flush the file to the containers listed in its segmenets */
if (scr_flush_file_to_containers(file, meta, segments, scr_prefix) == SCR_SUCCESS) {
/* successfully flushed this file, record the filesize */
unsigned long filesize = 0;
if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) {
scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize);
}
/* record the crc32 if one was computed */
uLong crc = 0;
if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) {
scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc);
}
/* record segment information in summary file */
scr_hash* segments_copy = scr_hash_new();
scr_hash_merge(segments_copy, segments);
scr_hash_set(file_hash, SCR_SUMMARY_6_KEY_SEGMENT, segments_copy);
} else {
/* the flush failed */
rc = SCR_FAILURE;
/* explicitly mark file as incomplete */
scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0);
}
} else {
/* get directory to flush file to */
char* dir;
if (scr_hash_util_get_str(hash, SCR_KEY_PATH, &dir) == SCR_SUCCESS) {
/* create full path of destination file */
scr_path* path_full = scr_path_from_str(dir);
scr_path_append(path_full, path_name);
/* get relative path to flushed file from SCR_PREFIX directory */
scr_path* path_relative = scr_path_relative(scr_prefix_path, path_full);
if (! scr_path_is_null(path_relative)) {
/* record the name of the file in the summary hash, and get reference to a hash for this file */
char* name = scr_path_strdup(path_relative);
scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name);
scr_free(&name);
/* flush the file and fill in the meta data for this file */
if (scr_flush_a_file(file, dir, meta) == SCR_SUCCESS) {
/* successfully flushed this file, record the filesize */
unsigned long filesize = 0;
if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) {
scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize);
}
/* record the crc32 if one was computed */
uLong crc = 0;
if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) {
scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc);
}
} else {
/* the flush failed */
rc = SCR_FAILURE;
/* explicitly mark incomplete files */
scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0);
}
} else {
scr_abort(-1, "Failed to get relative path to directory %s from %s @ %s:%d",
dir, scr_prefix, __FILE__, __LINE__
);
}
/* free relative and full paths */
scr_path_delete(&path_relative);
scr_path_delete(&path_full);
} else {
scr_abort(-1, "Failed to read directory to flush file to @ %s:%d",
__FILE__, __LINE__
);
}
}
/* free the file name path */
scr_path_delete(&path_name);
}
return rc;
}
static int scr_fetch_rank2file_map(
const scr_path* dataset_path,
int depth,
int* ptr_valid,
char** ptr_file,
unsigned long* ptr_offset)
{
int rc = SCR_SUCCESS;
/* get local variables so we don't have to deference everything */
int valid = *ptr_valid;
char* file = *ptr_file;
unsigned long offset = *ptr_offset;
/* create a hash to hold section of file */
scr_hash* hash = scr_hash_new();
/* if we can read from file do it */
if (valid) {
/* open file if we haven't already */
int fd = scr_open(file, O_RDONLY);
if (fd >= 0) {
/* read our segment from the file */
scr_lseek(file, fd, offset, SEEK_SET);
ssize_t read_rc = scr_hash_read_fd(file, fd, hash);
if (read_rc < 0) {
scr_err("Failed to read from %s @ %s:%d",
file, __FILE__, __LINE__
);
rc = SCR_FAILURE;
}
/* close the file */
scr_close(file, fd);
} else {
scr_err("Failed to open rank2file map %s @ %s:%d",
file, __FILE__, __LINE__
);
rc = SCR_FAILURE;
}
}
/* check for read errors */
if (! scr_alltrue(rc == SCR_SUCCESS)) {
rc = SCR_FAILURE;
goto cleanup;
}
/* create hashes to exchange data */
scr_hash* send = scr_hash_new();
scr_hash* recv = scr_hash_new();
/* copy rank data into send hash */
if (valid) {
scr_hash* rank_hash = scr_hash_get(hash, SCR_SUMMARY_6_KEY_RANK);
scr_hash_merge(send, rank_hash);
}
/* exchange hashes */
scr_hash_exchange_direction(send, recv, scr_comm_world, SCR_HASH_EXCHANGE_RIGHT);
/* see if anyone sent us anything */
int newvalid = 0;
char* newfile = NULL;
unsigned long newoffset = 0;
scr_hash_elem* elem = scr_hash_elem_first(recv);
if (elem != NULL) {
/* got something, so now we'll read in the next step */
newvalid = 1;
/* get file name we should read */
scr_hash* elem_hash = scr_hash_elem_hash(elem);
char* value;
if (scr_hash_util_get_str(elem_hash, SCR_SUMMARY_6_KEY_FILE, &value)
== SCR_SUCCESS)
{
/* return string of full path to file to caller */
scr_path* newpath = scr_path_dup(dataset_path);
scr_path_append_str(newpath, value);
newfile = scr_path_strdup(newpath);
scr_path_delete(&newpath);
} else {
rc = SCR_FAILURE;
}
/* get offset we should start reading from */
if (scr_hash_util_get_bytecount(elem_hash, SCR_SUMMARY_6_KEY_OFFSET, &newoffset)
!= SCR_SUCCESS)
{
rc = SCR_FAILURE;
}
}
/* free the send and receive hashes */
scr_hash_delete(&recv);
scr_hash_delete(&send);
/* get level id, and broadcast it from rank 0,
* which we assume to be a reader in all steps */
int level_id = -1;
if (valid) {
if (scr_hash_util_get_int(hash, SCR_SUMMARY_6_KEY_LEVEL, &level_id)
!= SCR_SUCCESS)
{
rc = SCR_FAILURE;
}
}
MPI_Bcast(&level_id, 1, MPI_INT, 0, scr_comm_world);
/* check for read errors */
if (! scr_alltrue(rc == SCR_SUCCESS)) {
rc = SCR_FAILURE;
goto cleanup;
}
/* set parameters for output or next iteration,
* we already took care of updating ptr_fd earlier */
if (valid) {
scr_free(ptr_file);
}
*ptr_valid = newvalid;
*ptr_file = newfile;
*ptr_offset = newoffset;
/* recurse if we still have levels to read */
if (level_id > 1) {
rc = scr_fetch_rank2file_map(dataset_path, depth+1, ptr_valid, ptr_file, ptr_offset);
}
cleanup:
/* free the hash */
scr_hash_delete(&hash);
return rc;
}
/* fetch files listed in hash into specified cache directory,
* update filemap and fill in total number of bytes fetched,
* returns SCR_SUCCESS if successful */
static int scr_fetch_files_list(
const scr_hash* file_list,
const char* dir,
scr_filemap* map)
{
/* assume we'll succeed in fetching our files */
int rc = SCR_SUCCESS;
/* assume we don't have any files to fetch */
int my_num_files = 0;
/* get dataset id */
int id;
scr_dataset* dataset = scr_hash_get(file_list, SCR_KEY_DATASET);
scr_dataset_get_id(dataset, &id);
/* now iterate through the file list and fetch each file */
scr_hash_elem* file_elem = NULL;
scr_hash* files = scr_hash_get(file_list, SCR_KEY_FILE);
for (file_elem = scr_hash_elem_first(files);
file_elem != NULL;
file_elem = scr_hash_elem_next(file_elem))
{
/* get the filename */
char* file = scr_hash_elem_key(file_elem);
/* get a pointer to the hash for this file */
scr_hash* hash = scr_hash_elem_hash(file_elem);
/* check whether we are supposed to fetch this file */
/* TODO: this is a hacky way to avoid reading a redundancy file
* back in under the assumption that it's an original file, which
* breaks our redundancy computation due to a name conflict on
* the file names */
scr_hash_elem* no_fetch_hash = scr_hash_elem_get(hash, SCR_SUMMARY_6_KEY_NOFETCH);
if (no_fetch_hash != NULL) {
continue;
}
/* increment our file count */
my_num_files++;
/* build the destination file name */
scr_path* path_newfile = scr_path_from_str(file);
scr_path_basename(path_newfile);
scr_path_prepend_str(path_newfile, dir);
char* newfile = scr_path_strdup(path_newfile);
/* add the file to our filemap and write it to disk before creating
* the file, this way we have a record that it may exist before we
* actually start to fetch it */
scr_filemap_add_file(map, id, scr_my_rank_world, newfile);
scr_filemap_write(scr_map_file, map);
/* get the file size */
unsigned long filesize = 0;
if (scr_hash_util_get_unsigned_long(hash, SCR_KEY_SIZE, &filesize) != SCR_SUCCESS) {
scr_err("Failed to read file size from summary data @ %s:%d",
__FILE__, __LINE__
);
rc = SCR_FAILURE;
/* free path and string */
scr_free(&newfile);
scr_path_delete(&path_newfile);
break;
}
/* check for a complete flag */
int complete = 1;
if (scr_hash_util_get_int(hash, SCR_KEY_COMPLETE, &complete) != SCR_SUCCESS) {
/* in summary file, the absence of a complete flag on a file
* implies the file is complete */
complete = 1;
}
/* create a new meta data object for this file */
scr_meta* meta = scr_meta_new();
/* set the meta data */
scr_meta_set_filename(meta, newfile);
scr_meta_set_filetype(meta, SCR_META_FILE_USER);
scr_meta_set_filesize(meta, filesize);
scr_meta_set_complete(meta, 1);
/* TODODSET: move the ranks field elsewhere, for now it's needed
* by scr_index.c */
scr_meta_set_ranks(meta, scr_ranks_world);
/* get the crc, if set, and add it to the meta data */
uLong crc;
if (scr_hash_util_get_crc32(hash, SCR_KEY_CRC, &crc) == SCR_SUCCESS) {
scr_meta_set_crc32(meta, crc);
}
/* fetch file from containers if they are defined, otherwise fetch
* the native file */
scr_hash* segments = scr_hash_get(hash, SCR_SUMMARY_6_KEY_SEGMENT);
if (segments != NULL) {
/* get source path */
char* from_dir;
if (scr_hash_util_get_str(file_list, SCR_KEY_PATH, &from_dir) == SCR_SUCCESS) {
/* fetch file from containers */
if (scr_fetch_file_from_containers(newfile, meta, segments, from_dir) != SCR_SUCCESS) {
/* failed to fetch file, mark it as incomplete */
scr_meta_set_complete(meta, 0);
rc = SCR_FAILURE;
}
} else {
/* failed to find base dataset directory in file list */
rc = SCR_FAILURE;
}
} else {
/* fetch native file, lookup directory for this file */
char* from_dir;
if (scr_hash_util_get_str(hash, SCR_KEY_PATH, &from_dir) == SCR_SUCCESS) {
if (scr_fetch_file(newfile, from_dir, meta) != SCR_SUCCESS) {
/* failed to fetch file, mark it as incomplete */
scr_meta_set_complete(meta, 0);
rc = SCR_FAILURE;
}
} else {
/* failed to read source directory, mark file as incomplete */
scr_meta_set_complete(meta, 0);
rc = SCR_FAILURE;
}
}
/* TODODSET: want to write out filemap before we start to fetch
* each file? */
/* mark the file as complete */
scr_filemap_set_meta(map, id, scr_my_rank_world, newfile, meta);
/* free the meta data object */
scr_meta_delete(&meta);
/* free path and string */
scr_free(&newfile);
scr_path_delete(&path_newfile);
}
/* set the expected number of files for this dataset */
scr_filemap_set_expected_files(map, id, scr_my_rank_world, my_num_files);
scr_filemap_write(scr_map_file, map);
return rc;
}
