void BatchLogger::writeToMRDPLOT2(std::string prefix)
{
if (!_inited || _nPoints == 0)
{
std::cout << "writeToMRDPLOT2 not init or no data!" << std::endl;
return;
}
_saving = true;
std::string file_name = generate_file_name(prefix);
std::cout << file_name << " SAVING DATA ....." << std::endl;
std::vector<std::string> names;
std::vector<std::string> units;
float *data = _data;
names.resize(_nChannels);
units.resize(_nChannels);
for (int i = 0; i < _nChannels; i++) {
names[i] = _datapoints[i].names;
units[i] = _datapoints[i].units;
}
write_mrdplot_file( file_name, _nChannels*_nPoints,
_nChannels, _nPoints, _frequency,
data, names, units);
std::cout << file_name << " SAVED DATA." << std::endl;
_saving = false;
}
iow_t *bgpcorsaro_io_prepare_file_full(bgpcorsaro_t *bgpcorsaro,
const char *plugin_name,
bgpcorsaro_interval_t *interval,
int compress_type, int compress_level,
int flags)
{
iow_t *f = NULL;
char *outfileuri;
/* generate a file name based on the plugin name */
if ((outfileuri = generate_file_name(bgpcorsaro, plugin_name, interval,
compress_type)) == NULL) {
bgpcorsaro_log(__func__, bgpcorsaro, "could not generate file name for %s",
plugin_name);
return NULL;
}
if ((f = wandio_wcreate(outfileuri, compress_type, compress_level, flags)) ==
NULL) {
bgpcorsaro_log(__func__, bgpcorsaro, "could not open %s for writing",
outfileuri);
return NULL;
}
free(outfileuri);
return f;
}
static void test_generate_file_name(void)
{
int32_t rtrn = 0;
char *name = NULL;
rtrn = generate_file_name(&name, ".txt");
TEST_ASSERT(rtrn == 0);
TEST_ASSERT_NOT_NULL(name);
return;
}
void BatchLogger::writeToMRDPLOT(const char *prefix)
{
if (!_inited || _nPoints == 0)
return;
// mtx.lock();
_saving = true;
MRDPLOT_DATA *d;
d = malloc_mrdplot_data( 0, 0 );
d->filename = generate_file_name(prefix);
d->n_channels = _nChannels;
d->n_points = _nPoints;
d->total_n_numbers = d->n_channels*d->n_points;
d->frequency = _frequency;
d->data = _data;
// mtx.unlock();
_saving = false;
fprintf(stdout, "%s SAVING DATA ..... \n", d->filename);
d->names = new char*[d->n_channels];
d->units = new char*[d->n_channels];
for (int i = 0; i < d->n_channels; i++) {
d->names[i] = new char[LOGGER_MAX_CHARS];
d->units[i] = new char[LOGGER_MAX_CHARS];
strcpy(d->names[i], _datapoints[i].names);
strcpy(d->units[i], _datapoints[i].units);
}
write_mrdplot_file( d );
for (int i = 0; i < d->n_channels; i++) {
delete []d->names[i];
delete []d->units[i];
}
delete []d->names;
delete []d->units;
fprintf(stdout, "%s SAVED DATA\n", d->filename);
free(d);
}
PIDX_return_code PIDX_header_io_file_write(PIDX_header_io_id header_io_id, PIDX_block_layout block_layout, int mode)
{
int i = 0, rank = 0, ret;
char bin_file[PATH_MAX];
#if PIDX_HAVE_MPI
int nprocs = 1;
if (header_io_id->idx_d->parallel_mode == 1)
{
MPI_Comm_rank(header_io_id->comm, &rank);
MPI_Comm_size(header_io_id->comm, &nprocs);
}
#endif
for (i = 0; i < header_io_id->idx_d->max_file_count; i++)
{
#if PIDX_HAVE_MPI
if (i % nprocs == rank && block_layout->file_bitmap[i] == 1)
#else
if (rank == 0 && block_layout->file_bitmap[i] == 1)
#endif
{
int adjusted_file_index = 0;
int l = pow(2, ((int)log2(i * header_io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (header_io_id->idx_d->idx_count[0] * header_io_id->idx_d->idx_count[1] * header_io_id->idx_d->idx_count[2]) + ((i * header_io_id->idx->blocks_per_file) - l) + (header_io_id->idx_d->color * l)) / header_io_id->idx->blocks_per_file;
ret = generate_file_name(header_io_id->idx->blocks_per_file, header_io_id->idx->filename_template, adjusted_file_index/*i*/, bin_file, PATH_MAX);
if (ret == 1)
{
fprintf(stderr, "[%s] [%d] generate_file_name() failed.\n", __FILE__, __LINE__);
return 1;
}
ret = populate_meta_data(header_io_id, block_layout, i, bin_file, mode);
if (ret != PIDX_success) return PIDX_err_header;
}
}
return PIDX_success;
}
void BatchLogger::writeToMRDPLOT()
{
if (!_inited)
return;
recorded = true;
fprintf(stderr, "LOGGER SAVING DATA ..... \n");
MRDPLOT_DATA *d;
d = malloc_mrdplot_data( 0, 0 );
d->filename = generate_file_name();
d->n_channels = n_channels;
d->n_points = n_points;
d->total_n_numbers = d->n_channels*d->n_points;
d->frequency = frequency;
d->data = data;
d->names = new char*[n_channels];
d->units = new char*[n_channels];
for (int i = 0; i < n_channels; i++) {
d->names[i] = new char[LOGGER_MAX_CHARS];
d->units[i] = new char[LOGGER_MAX_CHARS];
strcpy(d->names[i], datapoints[i].names);
strcpy(d->units[i], datapoints[i].units);
}
write_mrdplot_file( d );
for (int i = 0; i < n_channels; i++) {
delete []d->names[i];
delete []d->units[i];
}
delete []d->names;
delete []d->units;
fprintf(stderr, "LOGGER SAVED DATA\n");
free(d);
}
int PIDX_header_io_file_create(PIDX_header_io_id header_io_id, PIDX_block_layout block_layout)
{
int i = 0, rank = 0, j, ret;
char bin_file[PATH_MAX];
char last_path[PATH_MAX] = {0};
char this_path[PATH_MAX] = {0};
char tmp_path[PATH_MAX] = {0};
char* pos;
#if PIDX_HAVE_MPI
int nprocs = 1;
if (header_io_id->idx_d->parallel_mode == 1)
{
MPI_Comm_rank(header_io_id->comm, &rank);
MPI_Comm_size(header_io_id->comm, &nprocs);
}
#endif
for (i = 0; i < header_io_id->idx_d->max_file_count; i++)
{
#if PIDX_HAVE_MPI
if (i % nprocs == rank && block_layout->file_bitmap[i] == 1)
#else
if (rank == 0 && block_layout->file_bitmap[i] == 1)
#endif
{
/*
int adjusted_file_index = i;
if (layout_type != 0)
{
int l = pow(2, ((int)log2(i * header_io_id->idx->blocks_per_file)));
adjusted_file_index = l * (header_io_id->idx_d->idx_count[0] * header_io_id->idx_d->idx_count[1] * header_io_id->idx_d->idx_count[2]) + (i - l) + (header_io_id->idx_d->color * l);
}
*/
int adjusted_file_index = 0;
int l = pow(2, ((int)log2(i * header_io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (header_io_id->idx_d->idx_count[0] * header_io_id->idx_d->idx_count[1] * header_io_id->idx_d->idx_count[2]) + ((i * header_io_id->idx->blocks_per_file) - l) + (header_io_id->idx_d->color * l)) / header_io_id->idx->blocks_per_file;
//if (nprocs == 2)
ret = generate_file_name(header_io_id->idx->blocks_per_file, header_io_id->idx->filename_template, adjusted_file_index, bin_file, PATH_MAX);
if (ret == 1)
{
fprintf(stderr, "[%s] [%d] generate_file_name() failed.\n", __FILE__, __LINE__);
return 1;
}
//printf("%d -> MAP [%d %d %d] -> %d [%s (%s)]\n", i, header_io_id->idx_d->idx_count[0], header_io_id->idx_d->idx_count[1], header_io_id->idx_d->idx_count[2], adjusted_file_index, bin_file, header_io_id->idx->filename_template);
//TODO: the logic for creating the subdirectory hierarchy could
//be made to be more efficient than this. This implementation
//walks up the tree attempting to mkdir() each segment every
//time we switch to a new directory when creating binary files.
// see if we need to make parent directory
strcpy(this_path, bin_file);
if ((pos = strrchr(this_path, '/')))
//if ((pos = rindex(this_path, '/')))
{
pos[1] = '\0';
if (!strcmp(this_path, last_path) == 0)
{
//this file is in a previous directory than the last
//one; we need to make sure that it exists and create
//it if not.
strcpy(last_path, this_path);
memset(tmp_path, 0, PATH_MAX * sizeof (char));
//walk up path and mkdir each segment
for (j = 0; j < (int)strlen(this_path); j++)
{
if (j > 0 && this_path[j] == '/')
{
//printf("path = %s %s [T %s]\n", tmp_path, bin_file, header_io_id->idx->filename_template);
ret = mkdir(tmp_path, S_IRWXU | S_IRWXG | S_IRWXO);
if (ret != 0 && errno != EEXIST)
{
perror("mkdir");
fprintf(stderr, "Error: failed to mkdir %s\n", tmp_path);
return 1;
}
}
tmp_path[j] = this_path[j];
}
}
}
#if PIDX_HAVE_MPI
if (header_io_id->idx_d->parallel_mode == 1)
{
MPI_File fh = 0;
MPI_File_open(MPI_COMM_SELF, bin_file, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &fh);
MPI_File_close(&fh);
}
else
{
int fp = 0;
fp = open(bin_file, O_CREAT, 0664);
close(fp);
}
#else
int fp = 0;
fp = open(bin_file, O_CREAT, 0664);
close(fp);
#endif
}
}
#if PIDX_HAVE_MPI
if (header_io_id->idx_d->parallel_mode == 1)
MPI_Barrier(header_io_id->comm);
#endif
return PIDX_success;
}
int PIDX_aggregated_io(PIDX_file_io_id io_id, Agg_buffer agg_buf, PIDX_block_layout block_layout, int MODE)
{
int64_t data_offset = 0;
char file_name[PATH_MAX];
int i = 0, k = 0;
uint32_t *headers;
int total_header_size = 0;
#ifdef PIDX_RECORD_TIME
double t1, t2, t3, t4, t5;
#endif
#if PIDX_HAVE_MPI
int mpi_ret;
MPI_File fh;
MPI_Status status;
#else
int fh;
#endif
int total_chunk_size = (io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2] * io_id->idx->chunk_size[3] * io_id->idx->chunk_size[4]);
if (enable_caching == 1 && agg_buf->var_number == io_id->init_index && agg_buf->sample_number == 0)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) /*agg_buf->file_number*/adjusted_file_index, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#else
fh = open(file_name, O_WRONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = (uint32_t*)malloc(total_header_size);
memset(headers, 0, total_header_size);
#if !SIMULATE_IO
if (enable_caching == 1)
memcpy (headers, cached_header_copy, total_header_size);
else
{
//TODO
}
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
uint64_t header_size = (io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size);
#if !SIMULATE_IO
unsigned char* temp_buffer = (unsigned char*)realloc(agg_buf->buffer, agg_buf->buffer_size + header_size);
if (temp_buffer == NULL)
{
fprintf(stderr, "[%s] [%d] realloc() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
else
{
agg_buf->buffer = temp_buffer;
memmove(agg_buf->buffer + header_size, agg_buf->buffer, agg_buf->buffer_size);
memcpy(agg_buf->buffer, headers, total_header_size);
memset(agg_buf->buffer + total_header_size, 0, (header_size - total_header_size));
}
#endif
free(headers);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_write_at(fh, 0, agg_buf->buffer, agg_buf->buffer_size + header_size, MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed for filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size + header_size, 0);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t5 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V0. [R %d] [O 0 C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long)agg_buf->buffer_size + header_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t3-t2), (t4-t3), (t5-t4));
#else
#endif
}
else if (agg_buf->var_number != -1 && agg_buf->sample_number != -1 && agg_buf->file_number != -1)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) adjusted_file_index/*agg_buf->file_number*/, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
else
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
#else
if (MODE == PIDX_WRITE)
fh = open(file_name, O_WRONLY);
else
fh = open(file_name, O_RDONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
data_offset += io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size;
if (MODE == PIDX_WRITE)
{
for (k = 0; k < agg_buf->var_number; k++)
{
PIDX_variable vark = io_id->idx->variable[k];
int bytes_per_datatype = ((vark->bits_per_value/8) * total_chunk_size) / (io_id->idx->compression_factor);
int64_t prev_var_sample = (int64_t) block_layout->block_count_per_file[agg_buf->file_number] * io_id->idx_d->samples_per_block * bytes_per_datatype * io_id->idx->variable[k]->values_per_sample;
data_offset = (int64_t) data_offset + prev_var_sample;
}
for (i = 0; i < agg_buf->sample_number; i++)
data_offset = (int64_t) data_offset + agg_buf->buffer_size;
}
else
{
int total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = malloc(total_header_size);
memset(headers, 0, total_header_size);
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_read_at(fh, 0, headers, total_header_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#endif
}
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
//int rank;
//MPI_Comm_rank(io_id->comm, &rank);
//printf("W [%d] [%d %d %d] size = %d and offset = %d\n", rank, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, agg_buf->buffer_size, data_offset);
mpi_ret = MPI_File_write_at(fh, data_offset, agg_buf->buffer, agg_buf->buffer_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count = 0;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
int data_size = 0;
int block_count = 0;
for (i = 0; i < io_id->idx->blocks_per_file; i++)
{
if (PIDX_blocks_is_block_present(agg_buf->file_number * io_id->idx->blocks_per_file + i, block_layout))
{
data_offset = htonl(headers[12 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
data_size = htonl(headers[14 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
mpi_ret = MPI_File_read_at(fh, data_offset, agg_buf->buffer + (block_count * io_id->idx_d->samples_per_block * (io_id->idx->variable[agg_buf->var_number]->bits_per_value/8) * io_id->idx->variable[agg_buf->var_number]->values_per_sample * io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2]) / io_id->idx->compression_factor, /*agg_buf->buffer_size*/data_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int read_count = 0;
MPI_Get_count(&status, MPI_BYTE, &read_count);
if (read_count != /*agg_buf->buffer_size*/data_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed. %d != %lldd\n", __FILE__, __LINE__, read_count, (long long)agg_buf->buffer_size);
return PIDX_err_io;
}
block_count++;
}
}
free(headers);
}
#else
if (MODE == PIDX_WRITE)
{
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
ssize_t read_count = pread(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (read_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pread() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V. [R %d] [O %lld C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long) data_offset, (long long)agg_buf->buffer_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t2-t2), (t3-t2), (t4-t3));
#endif
}
return PIDX_success;
}
static int write_read_samples(PIDX_file_io_id io_id, int variable_index, uint64_t hz_start_index, uint64_t hz_count, unsigned char* hz_buffer, int64_t buffer_offset, PIDX_block_layout layout, int MODE)
{
int samples_per_file, block_number, file_index, file_count, ret = 0, block_negative_offset = 0, file_number;
int bytes_per_sample, bytes_per_datatype;
int i = 0;
char file_name[PATH_MAX];
off_t data_offset = 0;
samples_per_file = io_id->idx_d->samples_per_block * io_id->idx->blocks_per_file;
bytes_per_datatype = (io_id->idx->variable[variable_index]->bits_per_value / 8) * (io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2] * io_id->idx->chunk_size[3] * io_id->idx->chunk_size[4]) / (io_id->idx->compression_factor);
#if !SIMULATE_IO
hz_buffer = hz_buffer + buffer_offset * bytes_per_datatype * io_id->idx->variable[variable_index]->values_per_sample;
#endif
while (hz_count)
{
block_number = hz_start_index / io_id->idx_d->samples_per_block;
file_number = hz_start_index / samples_per_file;
file_index = hz_start_index % samples_per_file;
file_count = samples_per_file - file_index;
if ((int64_t)file_count > hz_count)
file_count = hz_count;
// build file name
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
ret = generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, /*file_number*/adjusted_file_index, file_name, PATH_MAX);
if (ret == 1)
{
fprintf(stderr, "[%s] [%d] generate_file_name() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
data_offset = 0;
bytes_per_sample = io_id->idx->variable[variable_index]->bits_per_value / 8;
data_offset = file_index * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample;
data_offset += io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size;
block_negative_offset = PIDX_blocks_find_negative_offset(io_id->idx->blocks_per_file, block_number, layout);
data_offset -= block_negative_offset * io_id->idx_d->samples_per_block * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample;
for (l = 0; l < variable_index; l++)
{
bytes_per_sample = io_id->idx->variable[l]->bits_per_value / 8;
for (i = 0; i < io_id->idx->blocks_per_file; i++)
if (PIDX_blocks_is_block_present((i + (io_id->idx->blocks_per_file * file_number)), layout))
data_offset = data_offset + (io_id->idx->variable[l]->values_per_sample * bytes_per_sample * io_id->idx_d->samples_per_block);
}
if(MODE == PIDX_WRITE)
{
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
#ifdef PIDX_DUMP_IO
if (io_id->idx_d->dump_io_info == 1 && io_id->idx->current_time_step == 0)
{
fprintf(io_dump_fp, "[A] Count %lld Target Disp %d (%d %d)\n", (long long)file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), (file_index * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample - block_negative_offset * io_id->idx_d->samples_per_block * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample)/8, (int)io_id->idx_d->start_fs_block, (int)io_id->idx_d->fs_block_size);
fflush(io_dump_fp);
}
#endif
if (io_id->idx_d->parallel_mode == 1)
{
int rank = 0;
MPI_Comm_rank(io_id->comm, &rank);
MPI_File fh;
MPI_Status status;
int mpi_ret;
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed. (%s) [%d]\n", __FILE__, __LINE__, file_name, file_number);
return PIDX_err_io;
}
/*
printf("[%d] Data Offset %d Count %d\n", rank, data_offset, (file_count));
int x = 0;
for (x = 0; x < file_count; x++)
{
double x1;
memcpy(&x1, hz_buffer + x * sizeof(double), sizeof(double));
printf("Values %d %f\n", x, x1);
}
*/
mpi_ret = MPI_File_write_at(fh, data_offset, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
int write_count;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
MPI_File_close(&fh);
}
else
{
int fh;
fh = open(file_name, O_WRONLY);
ssize_t write_count = pwrite(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
}
#else
int fh;
fh = open(file_name, O_WRONLY);
/*
double x1, x2, x3, x4;
memcpy(&x1, hz_buffer, sizeof(double));
memcpy(&x2, hz_buffer + sizeof(double), sizeof(double));
memcpy(&x3, hz_buffer + 2*sizeof(double), sizeof(double));
memcpy(&x4, hz_buffer + 3*sizeof(double), sizeof(double));
printf("[%d] [%d %d] Values %f %f %f %f\n", variable_index, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset, x1, x2, x3, x4);
*/
ssize_t write_count = pwrite(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
#endif
#endif
}
if(MODE == PIDX_READ)
{
#if PIDX_HAVE_MPI
if (io_id->idx_d->parallel_mode == 1)
{
MPI_File fh;
MPI_Status status;
int mpi_ret;
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
mpi_ret = MPI_File_read_at(fh, data_offset, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
MPI_File_close(&fh);
}
else
{
int fh;
fh = open(file_name, O_RDONLY);
ssize_t read_count = pread(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (read_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
}
#else
int fh;
fh = open(file_name, O_RDONLY);
ssize_t read_count = pread(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (read_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
#endif
}
hz_count -= file_count;
hz_start_index += file_count;
hz_buffer += file_count * io_id->idx->variable[variable_index]->values_per_sample * bytes_per_datatype;
#if PIDX_HAVE_MPI
#else
#endif
}
return PIDX_success;
}
int main(int argc, char **argv)
{
double start_time, end_time;
start_time = get_time();
int ret = 0;
init_mpi(argc, argv);
parse_args(argc, argv);
rank_0_print("Merge Program\n");
#if 0
comm = MPI_COMM_WORLD;
#endif
ret = IDX_file_open(output_file_name);
if (ret != 0) terminate_with_error_msg("PIDX_file_create");
maxh = strlen(bitSequence);
fprintf(stderr, "Partition size :: and count %d %d %d :: %d %d %d\n", idx_count[0], idx_count[1], idx_count[2], idx_size[0], idx_size[1], idx_size[2]);
fprintf(stderr, "bitstring %s maxh = %d\n", bitSequence, maxh);
// shared_block_level is the level upto which the idx blocks are shared
int shared_block_level = (int)log2(idx_count[0] * idx_count[1] * idx_count[2]) + bits_per_block + 1;
if (shared_block_level >= maxh)
shared_block_level = maxh;
int shared_block_count = pow(2, shared_block_level - 1) / samples_per_block;
fprintf(stderr, "Shared block level = %d Shared block count = %d\n", shared_block_level, shared_block_count);
int level = 0;
int ts = 0;
// Iteration through all the timesteps
for (ts = start_time_step; ts <= end_time_step; ts++)
{
// Iteration through all the shared blocks
//for (level = 0; level < shared_block_level; level = level + 1)
{
int hz_index = (int)pow(2, level - 1);
int file_no = hz_index / (blocks_per_file * samples_per_block);
int file_count;
char existing_file_name[PIDX_FILE_PATH_LENGTH];
char new_file_name[PIDX_FILE_PATH_LENGTH];
int ic = 0;
if (level <= bits_per_block + log2(blocks_per_file) + 1)
file_count = 1;
else
file_count = (int)pow(2, level - (bits_per_block + log2(blocks_per_file) + 1));
// file_no is the index of the file that needs to be opened to read from all the partitions
// they contain the shared blocks
fprintf(stderr, "Opening file %d\n", file_no);
#if 1
// iterate throuh all the files that contains the shared blocks
// most likely this will be only the first file of all the partitions
// so fc = 1
int fc = 0;
for (fc = file_no; fc < file_no + file_count; fc++)
{
// malloc for the header for the outpur blocks, i.e. the merged blocks
uint32_t* write_binheader;
int write_binheader_count;
write_binheader_count = 10 + 10 * blocks_per_file * variable_count;
int write_binheader_length = write_binheader_count * sizeof (*write_binheader);
write_binheader = malloc(write_binheader_length);
memset(write_binheader, 0, write_binheader_length);
//iterate through all the variables/fields
int var = 0;
off_t var_offset = 0;
for (var = 0; var < 1; var++)
{
unsigned char *write_data_buffer = malloc(samples_per_block * shared_block_count * bpv[var]/8);
memset(write_data_buffer, 0, samples_per_block * shared_block_count * bpv[var]/8);
//fprintf(stderr, "Write bufer size = %d [%d x %d x %d]\n", samples_per_block * shared_block_count * bpv[var]/8, (int)pow(2, bits_per_block), shared_block_count, bpv[var]/8);
// shared block data
// doube pointer (number o fpartitions x number of shared blocks)
unsigned char **read_data_buffer = malloc(idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_data_buffer));
memset(read_data_buffer, 0, idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_data_buffer));
// shared block header info
uint32_t** read_binheader = malloc(idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_binheader));
memset(read_binheader, 0, idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_binheader));
file_initialize_time_step(ts, output_file_name, output_file_template);
generate_file_name(blocks_per_file, output_file_template, fc, new_file_name, PATH_MAX);
//fprintf(stderr, "Merged blocks to be written in %s\n", new_file_name);
// iterate through all the parttions
for (ic = 0; ic < idx_count[0] * idx_count[1] * idx_count[2]; ic++)
{
char file_name_skeleton[PIDX_FILE_PATH_LENGTH];
strncpy(file_name_skeleton, output_file_name, strlen(output_file_name) - 4);
file_name_skeleton[strlen(output_file_name) - 4] = '\0';
if (idx_count[0] != 1 || idx_count[1] != 1 || idx_count[2] != 1)
sprintf(partition_file_name, "%s_%d.idx", file_name_skeleton, ic);
else
strcpy(partition_file_name, output_file_name);
file_initialize_time_step(ts, partition_file_name, partition_file_template);
generate_file_name(blocks_per_file, partition_file_template, fc, existing_file_name, PATH_MAX);
int read_binheader_count;
read_binheader_count = 10 + 10 * blocks_per_file * variable_count;
read_binheader[ic] = (uint32_t*) malloc(sizeof (*read_binheader[ic])*(read_binheader_count));
memset(read_binheader[ic], 0, sizeof (*(read_binheader[ic]))*(read_binheader_count));
fprintf(stderr, "[%d] Partition File name %s\n", ic, existing_file_name);
// file exists
if ( access( partition_file_name, F_OK ) != -1 )
{
// contins data from the shared blocks
read_data_buffer[ic] = malloc(samples_per_block * shared_block_count * bpv[var]/8);
memset(read_data_buffer[ic], 0, samples_per_block * shared_block_count * bpv[var]/8);
int fd;
fd = open(existing_file_name, O_RDONLY | O_BINARY);
if (fd < 0)
{
fprintf(stderr, "[File : %s] [Line : %d] open\n", __FILE__, __LINE__);
continue;
return 0;
}
// reads the header infor from binary file of the partitions
ret = read(fd, read_binheader[ic], (sizeof (*(read_binheader[ic])) * read_binheader_count));
if (ret < 0)
{
fprintf(stderr, "[File : %s] [Line : %d] read\n", __FILE__, __LINE__);
return 0;
}
//assert(ret == (sizeof (*(read_binheader[ic])) * read_binheader_count));
// copy the header from the partition file to the merged output file
// do it only for first partition (this gets all info other than block offset nd count)
if (ic == 0)
memcpy(write_binheader, read_binheader[ic], 10 * sizeof (*write_binheader));
int bpf = 0;
size_t data_size = 0;
off_t data_offset = 0;
for (bpf = 0; bpf < shared_block_count; bpf++)
{
data_offset = ntohl(read_binheader[ic][(bpf + var * blocks_per_file)*10 + 12]);
data_size = ntohl(read_binheader[ic][(bpf + var * blocks_per_file)*10 + 14]);
fprintf(stderr, "[%s] [Partition %d Block %d Variable %d] --> Offset %d Count %d\n", partition_file_name, ic, bpf, var, (int)data_offset, (int)data_size);
if (data_offset != 0 && data_size != 0)
{
pread(fd, read_data_buffer[ic] + (bpf * samples_per_block * (bpv[var] / 8)), data_size, data_offset);
write_binheader[((bpf + var * blocks_per_file)*10 + 12)] = htonl(write_binheader_length + (bpf * data_size) + var * shared_block_count);
write_binheader[((bpf + var * blocks_per_file)*10 + 14)] = htonl(data_size);
// Merge happening while the shared block is being read
// Hardcoded stupid merge
// checks if value is not zero then copies to the write block
int m = 0;
for (m = 0; m < data_size / (bpv[var] / 8) ; m++)
{
double temp;
memcpy(&temp, read_data_buffer[ic] + (bpf * samples_per_block + m) * sizeof(double), sizeof(double));
if (temp != 0)
memcpy(write_data_buffer + ((bpf * samples_per_block) + m) * sizeof(double), &temp, sizeof(double));
}
}
}
close(fd);
}
else
continue;
}
//Merge after all the reads
for (ic = 0; ic < idx_count[0] * idx_count[1] * idx_count[2]; ic++)
{
//input is read_data_buffer**
//output is write_data_buffer*
}
if ( access( new_file_name, F_OK ) != -1 )
{
// file exists
int fd;
fd = open(new_file_name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
{
}
close(fd);
}
else
{
// file doesn't exist
/*
int r;
for (r = 0; r < (shared_block_count * samples_per_block * bpv[var]/8) / sizeof(double); r++)
{
double dval;
memcpy(&dval, write_data_buffer + r * sizeof(double), sizeof(double));
fprintf(stderr, "value at %d = %f\n", r, dval);
}
*/
int fd;
fd = open(new_file_name, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
pwrite(fd, write_binheader, sizeof (*write_binheader)*(write_binheader_count), 0);
pwrite(fd, write_data_buffer, shared_block_count * samples_per_block * bpv[var]/8, sizeof (*write_binheader)*(write_binheader_count));
close(fd);
}
}
}
#endif
}
}
shutdown_mpi();
end_time = get_time();
fprintf(stderr, "Total time taken = %f %f\n", end_time, start_time);
return 0;
}
int32_t create_random_file(char *root, char *ext, char **path, uint64_t *size)
{
int32_t rtrn = 0;
int32_t no_period = 0;
char *name auto_free = NULL;
char *junk auto_free = NULL;
char *extension auto_free = NULL;
/* Check for a period in the extension string passed by the user. */
char *pointer = strrchr(ext, '.');
if(pointer == NULL)
{
/* There's no period so let's create one. */
rtrn = asprintf(&extension, ".%s", ext);
if(rtrn < 0)
{
output->write(ERROR, "Can't create extension string\n");
return (-1);
}
no_period = 1;
}
if(no_period == 1)
{
/* Generate random file name. */
rtrn = generate_file_name(&name, extension);
if(rtrn < 0)
{
output->write(ERROR, "Can't generate random file name\n");
return (-1);
}
}
else
{
/* Generate random file name. */
rtrn = generate_file_name(&name, ext);
if(rtrn < 0)
{
output->write(ERROR, "Can't generate random file name\n");
return (-1);
}
}
/* Join paths together. */
rtrn = asprintf(path, "%s/%s", root, name);
if(rtrn < 0)
{
output->write(ERROR, "Can't join paths: %s\n", strerror(errno));
return (-1);
}
/* Pick a random size between zero and 4 kilobytes. */
rtrn = random_gen->range(4095, (uint32_t *)size);
if(rtrn < 0)
{
output->write(ERROR, "Can't choose random number\n");
return (-1);
}
/* Add one to size so it's not zero. */
(*size) = (*size) + 1;
/* Allocate a buffer to put junk in. */
junk = allocator->alloc((*size) + 1);
if(junk == NULL)
{
output->write(ERROR, "Can't alloc junk buf: %s\n", strerror(errno));
return (-1);
}
/* Put some junk in a buffer. */
rtrn = random_gen->bytes(&junk, (uint32_t)(*size));
if(rtrn < 0)
{
output->write(ERROR, "Can't alloc junk buf: %s\n", strerror(errno));
return (-1);
}
/* Write that junk to the file so that the file is not just blank. */
rtrn = map_file_out((*path), junk, (*size));
if(rtrn < 0)
{
output->write(ERROR, "Can't write junk to disk\n");
return (-1);
}
return (0);
}
int poweron_self_check()
{
int ret = -1;
int timeout = 0;
char set_system_time[50];
flying_attitude_s *p;
//--------------spi initial------------------
ret=spi_open();
printf("CPLD logic version:%d\n",get_fpga_version());
printf("----------------------------------------------------------------\n");
#ifndef debug
if(ret<0)
fault_status_return(ret);
#endif
/*
ret=adc_init();
#ifndef debug
if(ret<0)
fault_status_return(ret);
#endif
*/
//---------------imu initial-----------------
ret = sensor_open();
#ifndef debug
if (ret < 0) {
fault_status_return(ret);
}
#endif
pressure_sensor_init();
set_flying_status(AIRCRAFT_PREPARING);
if(command==0){
//UAV is working in normal mode
// reset CPLD ,this should be check later ,if CPLD should be reset when CPU recovers from failure.
reset_control_register(CTRL_REG_MASK_MANUAL);
// wait for flying attitude sensor data
while (timeout <= 600) {
sleep(1);
if(flying_attitude_sensor_is_active())
break;
timeout++;
}
if (timeout >600) {
print_err("flying attitude sensor is inactive, please check serial port is connected\n");
goto exit;
}
print_debug("Flying attitude sensor is active\n");
p=get_flying_attitude();
//printf("system beijing time %d-%d-%d,%d:%d:%d\n",p->year,p->month,p->day,p->hour+8,p->min,p->sec);
sprintf(set_system_time,"date -s \"%d-%d-%d %d:%d:%d\"",p->year+2000,p->month,p->day,p->hour+8,p->min,p->sec);
//printf("%s\n",set_system_time);
system(set_system_time);
}else if(command==1){
//UAV is working in test mode
reset_control_register(CTRL_REG_MASK_MANUAL);
printf("uav is working in test mode,IMU is by passed,using test data instead\n");
}else if(command==2){
//UAV is in mannual mode ,for pwm data capture
set_flying_status(AIRCRAFT_MANUAL_MODE);
set_control_register(CTRL_REG_MASK_MANUAL);
set_system_status(SYS_PREPARE_SETTING);
printf("UAV is now working in data capturing mode\n");
}else{
printf(" error command,please check the usage:\n");
printf(" usage: uav [command] [gap] ]\n");
printf(" [command]: 0--normal mode,used when run in the air\n");
printf(" [command]: 1--test mode,used for platform test.use this mode \n");
printf(" when no IMU connected ,\n");
printf(" [command]: 2--manual mode,used when capturing PWM data\n");
printf(" [frequency]: the frequency (ms/frame) in which UAV send fly status data\n");
}
generate_file_name(log_file_name);
if((fp_fly_status=fopen(log_file_name,"wb+"))==NULL){
printf("can not open file:%s\n",log_file_name);
}
heli_configuration_init();
control_parameter_init();
servo_test_enable=0;
while (1) {
if(get_system_status() < SYS_PREPARE_STEERING_TEST){
flying_status_return(1);
usleep(500000);
}else if(get_system_status() == SYS_PREPARE_STEERING_TEST){
steering_test();
usleep(20000);
}else if(get_system_status() == SYS_PREPARE_TAKEOFF){
return 0;
}
/*
if (get_system_status() >= SYS_PREPARE_SETTING) {
print_debug("Link is ready\n");
return 0;
}
link_testing_send();
usleep(20000); //20ms
*/
}
exit:
sensor_close();
return ret;
}
