void jag_store::setup(
data_reader_jag_conduit_hdf5 *reader,
bool num_stores,
int my_rank) {
double tm1 = get_time();
m_master = m_comm->am_world_master();
options *opts = options::get();
m_reader = reader;
m_max_samples = INT_MAX;
if (opts->has_int("max_samples")) {
m_max_samples = (size_t)opts->get_int("max_samples");
}
bool has_conduit_filenames = false;
if (opts->has_string("conduit_filelist")) {
std::string f = opts->get_string("conduit_filelist");
std::ifstream in(f.c_str());
if (!in) {
throw lbann_exception(std::string{} + __FILE__ + " " + std::to_string(__LINE__) + " :: failed to open " + f + " for reading");
}
std::string line;
while (getline(in, line)) {
m_conduit_filenames.push_back(line);
}
in.close();
if (m_max_samples < m_conduit_filenames.size()) {
m_conduit_filenames.resize(m_max_samples);
}
has_conduit_filenames = true;
}
if (m_image_size == 0) {
throw lbann_exception(std::string{} + __FILE__ + " " + std::to_string(__LINE__) + " :: image_size = 0; probably set_image_size() has not been called");
}
// optionally build an index file, then exit. Each line of the file will
// contain a conduit filename, followed by the valid sample_ids in
// the conduit file
if (opts->has_string("build_conduit_index")) {
if (! has_conduit_filenames) {
throw lbann_exception(std::string{} + __FILE__ + " " + std::to_string(__LINE__) + " :: you must pass --conduit_filenames=<string> on the cmd line when building a conduit index");
}
build_conduit_index(m_conduit_filenames);
exit(0);
}
load_variable_names();
build_data_sizes();
report_linearized_sizes();
allocate_memory();
load_normalization_values();
if (!opts->has_int("mode")) {
throw lbann_exception(std::string{} + __FILE__ + " " + std::to_string(__LINE__) + " :: you must pass --mode=<int> on cmd line, where <int> is 1 (to use conduit files) or 2 or 3 (for testing) (to use binary files)");
}
m_mode = opts->get_int("mode");
if (! (m_mode == 1 || m_mode == 2 || m_mode == 3)) {
throw lbann_exception(std::string{} + __FILE__ + " " + std::to_string(__LINE__) + " :: you must pass --mode=<int> on cmd line, where <int> is 1 (to use conduit files) or 2 (to use binary files); or 4 (for testing) you passed: " + std::to_string(m_mode));
}
if (m_master) std::cerr << "Running in mode: " << m_mode << "\n";
// optionally convert conduit files to our binary format, then exit
if (opts->has_string("convert_conduit")) {
if (! has_conduit_filenames) {
throw lbann_exception(std::string{} + __FILE__ + " " + std::to_string(__LINE__) + " :: you must pass --conduit_filenames=<string> on the cmd line when converting conduit filenames to binary");
}
setup_conduit();
convert_conduit_to_binary(m_conduit_filenames);
exit(0);
}
if (m_mode == 1) {
setup_conduit();
} else if (m_mode == 2) {
setup_binary();
} else {
setup_testing();
}
if (m_master) {
std::cerr << "jag_store::setup time: " << get_time() - tm1 << "; num samples: " << m_num_samples << std::endl;
}
if (m_mode == 3) {
test_converted_files();
m_comm->global_barrier();
exit(0);
}
// optionally compute min/max values, then exit.
// This is only needed for one-time computation of normalization values
if (opts->has_string("compute_min_max")) {
compute_min_max();
exit(0);
}
// optionally check bandwidth (sort of), then exit
if (opts->has_int("bandwidth")) {
if (m_mode == 0) {
compute_bandwidth();
} else {
compute_bandwidth_binary();
}
exit(0);
}
}
static inline ssize_t writeMem(struct file *filp, const char *buf, size_t count, loff_t *f_pos)
{
unsigned short int transfer_size;
ssize_t transferred = 0;
unsigned long src_addr, trgt_addr;
struct drvr_mem * mem_to_write = &(((struct drvr_device *) filp->private_data)->data);
#ifdef USE_WORD_ADDRESSING
if (count % 2 != 0) {
DBG_LOG("write: Transfer must be 16bits aligned\n");
return -EFAULT;
}
#endif
if (count < MAX_DMA_TRANSFER_IN_BYTES) {
transfer_size = count;
} else {
transfer_size = MAX_DMA_TRANSFER_IN_BYTES;
}
#ifdef USE_WORD_ADDRESSING
trgt_addr = (unsigned long) &(mem_to_write->base_addr[(*f_pos) / 2]);
#else
trgt_addr = (unsigned long) &(mem_to_write->base_addr[(*f_pos)]);
#endif
src_addr = (unsigned long) dmaphysbuf;
if (copy_from_user(mem_to_write->dma.buf, buf, transfer_size)) {
return -EFAULT;
}
if(transfer_size <= 256){
memcpy((void *) &(mem_to_write->virt_addr[(*f_pos)]), mem_to_write->dma.buf,transfer_size);
return transfer_size ;
}
while (transferred < count) {
int res;
#ifdef PROFILE
DBG_LOG("Write\n");
start_profile();
#endif
res = logi_dma_copy(mem_to_write, trgt_addr, src_addr,
transfer_size);
if (res < 0) {
DBG_LOG("write: Failed to trigger EDMA transfer\n");
return res;
}
#ifdef PROFILE
stop_profile();
compute_bandwidth(transfer_size);
#endif
trgt_addr += transfer_size;
transferred += transfer_size;
if ((count - transferred) < MAX_DMA_TRANSFER_IN_BYTES) {
transfer_size = count - transferred;
} else {
transfer_size = MAX_DMA_TRANSFER_IN_BYTES;
}
if (copy_from_user(mem_to_write->dma.buf, &buf[transferred], transfer_size)) {
return -EFAULT;
}
}
return transferred;
}
ssize_t writeMem(struct file *filp, const char *buf, size_t count, loff_t *f_pos)
{
unsigned short int transfer_size;
ssize_t transferred = 0;
unsigned long src_addr, trgt_addr;
struct drvr_mem * mem_to_write = &(((struct drvr_device *) filp->private_data)->data.mem);
if (count % 2 != 0) {
printk("%s: write: Transfer must be 16bits aligned.\n", DEVICE_NAME);
return -1;
}
if (count < MAX_DMA_TRANSFER_IN_BYTES) {
transfer_size = count;
} else {
transfer_size = MAX_DMA_TRANSFER_IN_BYTES;
}
if (mem_to_write->dma_buf == NULL) {
printk("failed to allocate DMA buffer \n");
return -1;
}
trgt_addr = (unsigned long) &(mem_to_write->base_addr[(*f_pos) / 2]);
src_addr = (unsigned long) dmaphysbuf;
if (copy_from_user(mem_to_write->dma_buf, buf, transfer_size)) {
return -1;
}
while (transferred < count) {
#ifdef PROFILE
printk("Write \n");
start_profile();
#endif
if (edma_memtomemcpy(transfer_size, src_addr, trgt_addr, mem_to_write->dma_chan) < 0) {
printk("%s: write: Failed to trigger EDMA transfer.\n", DEVICE_NAME);
return -1;
}
#ifdef PROFILE
stop_profile();
compute_bandwidth(transfer_size);
#endif
trgt_addr += transfer_size;
transferred += transfer_size;
if ((count - transferred) < MAX_DMA_TRANSFER_IN_BYTES) {
transfer_size = count - transferred;
} else {
transfer_size = MAX_DMA_TRANSFER_IN_BYTES;
}
if (copy_from_user(mem_to_write->dma_buf, &buf[transferred], transfer_size)) {
return -1;
}
}
return transferred;
}
