void RegionTrackerRecorder::WriteFlowInfo()
{
vector<hsize_t> flowindex_dims(2);
flowindex_dims[0] = 2;
flowindex_dims[1] = numFlows;
// where should this really be handled??
vector<int> flowBlockIndex(2*numFlows);
for (int i=0; i<numFlows; i++){
int j = i+numFlows;
flowBlockIndex[i] = i % NUMFB; // within block index indexed by flow
flowBlockIndex[j] = i/NUMFB; // block id indexed by flow
}
for (int i=0; i<numFlows; i++){
flowToIndex[i] = flowBlockIndex[i];
flowToBlockId[i] = flowBlockIndex[i+numFlows];
}
H5ReplayRecorder recorder = H5ReplayRecorder(clo, FLOWINDEX);
recorder.CreateDataset(flowindex_dims);
// hyperslab layout on disk
// 2 x numFlows
// 0, 1, 2, ... NUMFB-1, 0, 1, 2, ...
// 0, 0, 0, ... 0, 1, 1, 1, ...
// start position [0,0]
vector<hsize_t> offset(2);
offset[0] = 0;
offset[1] = 0;
vector<hsize_t> count(2);
count[0] = 2;
count[1] = numFlows;
// memory layout of slab we write from is 2 x numFlows
// start position in memory slab is [0,0]
vector<hsize_t> offset_in(2);
offset_in[0] = 0;
offset_in[1] = 0;
vector<hsize_t> count_in(2);
count_in[0] = 2;
count_in[1] = numFlows;
// fprintf(stdout, "H5 buffer, flow %d, region %d: %f, %f, ...\n", flow, regionIndex, newbuff[0], newbuff[1]);
recorder.Write(offset, count, offset_in, count_in, &flowBlockIndex[0]);
}
void *do_subset(void* data)
{
pthread_detach( pthread_self() );
struct thread_args *args = (struct thread_args *)data;
hsize_t row = args->row;
hsize_t col = args->col;
vector<hsize_t>chunks( * (args->chunks));
vector<hsize_t>dims(* (args->dims));
string h5file_in = string(* (args->h5file_in));
string source = string( * (args->source));
string h5file_out = string(* (args->h5file_out));
string destination = string(* (args->destination));
hsize_t offset_out_0 = args->offset_out;
hsize_t count_out_0 = args->count_out;
vector<size_t>input_positions(* (args->input_positions));
unsigned int thread_id = args->thread_id;
unsigned int flowlimit = args->flowlimit;
H5ReplayReader reader = H5ReplayReader(h5file_in, &source[0]);
H5ReplayRecorder recorder = H5ReplayRecorder(h5file_out, &destination[0]);
// read a chunk in
vector<hsize_t> offset(3);
offset.At(0) = row;
offset.At(1) = col;
offset.At(2) = 0;
vector<hsize_t> count(3);
count.At(0) = ( row+chunks.At(0) < dims.At(0) ) ? chunks.At(0) : dims.At(0)-row;
count.At(1) = ( col+chunks.At(1) < dims.At(1) ) ? chunks.At(1) : dims.At(1)-col;
count.At(2) = dims.At(2);
vector<float> signal(count.At(0) * count.At(1) * count.At(2));
vector<float> subset(count_out_0*count.At(2));
vector<hsize_t> offset_in(1);
offset_in.At(0) = 0;
vector<hsize_t> count_in(1, signal.size());
// fprintf(stdout, "thread_id %d reading %d from row=%d (max %d), %d from column=%d (max %d)\n", thread_id, (int)count.At(0), (int)row, (int)dims.At(0), (int)count.At(1), (int)col, (int)dims.At(1));
reader.Read(offset, count, offset_in, count_in, &signal[0]);
// now subset well at a time
size_t ix = 0;
size_t incr = 0;
vector<size_t> subset_input_positions(count_out_0);
for (size_t rr=row; rr<(row+count.At(0)) && ix < input_positions.size(); rr++) {
for (size_t cc=col; cc<(col+count.At(1)) && ix < input_positions.size(); cc++) {
size_t pos = input_positions.At(ix);
size_t chp_indx = RowColToIndex1(rr,cc, dims.At(0), dims.At(1));
if ( chp_indx < pos)
continue;
while ( chp_indx > pos){
ix++;
if (ix == input_positions.size())
break;
pos = input_positions.At(ix);
}
if( chp_indx == pos) {
size_t array_indx = RowColToIndex1(rr-row,cc-col,count.At(0),count.At(1))*dims.At(2);
// fprintf(stdout, "thread_id=%d: Position match %lu at chip rr=%lu, rr-row=%d, cc=%lu, cc-col=%d, (array_indx=%lu)/(dims[2]=%llu)=%llu\n", thread_id, pos, rr, (int)(rr-row), cc, (int)(cc-col), array_indx, dims.At(2), array_indx/dims.At(2));
for(size_t flow=0; flow<flowlimit; flow++)
subset.At(incr*dims.At(2) + flow) = signal.At( array_indx + flow);
assert (incr < count_out_0 );
subset_input_positions.At(incr) = input_positions.At(ix);
incr++;
ix++;
}
assert (ix <= input_positions.size() );
}
}
assert (incr == count_out_0 );
// write the chunk back out
vector<hsize_t> count_out(2);
count_out.At(0) = count_out_0;
count_out.At(1) = dims.At(2);
vector<hsize_t> offset_out(2);
offset_out.At(0) = offset_out_0;
offset_out.At(1) = 0;
vector<hsize_t> offset_in_subset(1,0);
vector<hsize_t> count_in_subset(1,subset.size());
recorder.Write(offset_out, count_out, offset_in_subset, count_in_subset, &subset[0]);
{
string position = "position";
H5ReplayRecorder recorder_pos = H5ReplayRecorder(h5file_out, &position[0]);
vector<hsize_t> offset_pos(1, offset_out.At(0));
vector<hsize_t> count_pos(1, count_out.At(0));
recorder_pos.Write(offset_pos, count_pos, offset_in, count_pos, &subset_input_positions[0]);
}
thread_flags.at(thread_id) = 0;
int ret = 0;
pthread_exit(&ret);
return(NULL);
}
int main(int argc, const char *argv[]) {
OptArgs opts;
string position_file;
string h5file_in;
string source;
string h5file_out;
string destination;
string positions_file;
bool help;
string flowlimit_arg;
unsigned int flowlimit;
vector<string>otherArgs;
DumpStartingStateOfExtractWells (argc,argv);
opts.ParseCmdLine(argc, argv);
opts.GetOption(h5file_in, "", 'i', "input");
opts.GetOption(source, "", 's', "source");
opts.GetOption(h5file_out, "", 'o', "output");
opts.GetOption(destination, "", 'd', "destination");
opts.GetOption(flowlimit_arg, "", 'f', "flowlimit");
opts.GetOption(positions_file, "", 'p', "positions");
opts.GetOption(help, "false", 'h', "help");
opts.GetLeftoverArguments(otherArgs);
// input data processing
string line;
vector<size_t> row_val;
vector<size_t> col_val;
ifstream filestream;
if ( ! positions_file.empty() )
filestream.open(&positions_file.At(0));
istream &input = ( filestream.is_open() ) ? filestream : cin;
while ( getline(input, line) )
{
int num = -1;
vector<size_t> ints;
istringstream ss(line);
while ( ss >> num && ints.size() < 2 ) {
if (num < 0) {
fprintf(stderr, "Found negative integer %d\n", num);
exit(-1);
}
else
ints.push_back((size_t)num);
}
if (ints.size() != 2) {
fprintf(stderr, "Found %d integers in %s, expected 2\n", (int)ints.size(), &line[0]);
continue;
}
row_val.push_back(ints.at(0));
col_val.push_back(ints.at(1));
}
if (row_val.size() == 0 ) {
fprintf(stdout, "No positions to extract, check input\n");
exit(0);
}
vector<size_t>input_positions(row_val.size(), 0);
int numCPU = (int)sysconf( _SC_NPROCESSORS_ONLN );
int numThreads = MAXTHREADS < numCPU ? MAXTHREADS : numCPU;
fprintf(stdout, "Using %d threads of %d cores\n", numThreads, numCPU);
if (source.empty())
source = source + SIGNAL_IN;
H5ReplayReader reader = H5ReplayReader(h5file_in, &source[0]);
if ( h5file_out.empty() )
h5file_out = h5file_out + H5FILE_OUT;
if ( destination.empty() )
destination = destination + SIGNAL_OUT;
reader.Open();
int rank = reader.GetRank();
vector<hsize_t>dims(rank);
vector<hsize_t>chunks(rank);
reader.GetDims(dims);
reader.GetChunkSize(chunks);
reader.Close();
// convert input row, col positions to indices
for (hsize_t i=0; i<input_positions.size(); i++)
input_positions.At(i) = RowColToIndex(row_val.At(i), col_val.At(i), dims.At(0), dims.At(1));
sort(input_positions.begin(), input_positions.end());
fprintf(stdout, "Opened for read %s:%s with rank %d, row x col x flow dims=[ ", &h5file_in[0], &source[0], rank);
for (int i=0; i<rank; i++)
fprintf(stdout, "%d ", (int)dims.At(i));
fprintf(stdout, "], chunksize=[ ");
for (int i=0; i<rank; i++)
fprintf(stdout, "%d ", (int)chunks.At(i));
fprintf(stdout, "]\n");
H5ReplayRecorder recorder = H5ReplayRecorder(h5file_out, &destination[0],reader.GetType(),2);
recorder.CreateFile();
{
vector<hsize_t> dims_pos(1, input_positions.size());
string pos_name = "position";
H5ReplayRecorder recorder_pos = H5ReplayRecorder(h5file_out, &pos_name[0],H5T_NATIVE_ULONG,1);
recorder_pos.CreateDataset(dims_pos);
}
{
string chip_dims = "chip_dims";
H5ReplayRecorder recorder_chip_dims = H5ReplayRecorder(h5file_out, &chip_dims[0],H5T_NATIVE_ULLONG,1);
vector<hsize_t> offset_dims(1,0);
vector<hsize_t> count_dims(1,3);
recorder_chip_dims.CreateDataset(count_dims);
recorder_chip_dims.Write(offset_dims, count_dims, offset_dims, count_dims, &dims[0]);
}
if (flowlimit_arg.empty())
flowlimit = dims.At(2);
else
flowlimit = atoi(flowlimit_arg.c_str());
flowlimit = (flowlimit < dims.At(2)) ? flowlimit : dims.At(2);
fprintf(stdout, "Using %u flows\n", flowlimit);
// chunks no bigger than 100000
vector<hsize_t>chunks_out(2);
chunks_out.At(0) = (input_positions.size() < 10000) ? input_positions.size() : 100000;
chunks_out.At(1) = chunks.At(2);
recorder.CreateDataset(chunks_out);
vector<hsize_t> extension(2);
extension.At(0) = input_positions.size();
extension.At(1) = dims.At(2);
recorder.ExtendDataSet(extension); // extend if necessary
fprintf(stdout, "Opening for write %s:%s with rank %d, position x flow chunks=[ ", &h5file_out[0], &destination[0], (int)chunks_out.size());
for (int i=0; i<(int)chunks_out.size(); i++)
fprintf(stdout, "%d ", (int)chunks_out.At(i));
fprintf(stdout, "]\n");
int max_threads_ever = (dims.At(0)/chunks.At(0) +1)*(dims.At(1)/chunks.At(1) +1);
thread_flags.resize (max_threads_ever, 0);
// fprintf(stdout, "max_threads_ever = %d\n", max_threads_ever);
unsigned int thread_id = 0;
vector<thread_args> my_args( max_threads_ever );
size_t runningCount = 0;
// layout is rows x cols x flows
for (size_t row=0; row<dims.At(0); ) {
for (size_t col=0; col<dims.At(1); ) {
size_t ix = 0;
hsize_t offset_out = 0;
hsize_t count_out = 0;
vector<size_t> limit(2);
limit.At(0) = ( row+chunks.At(0) < dims.At(0) ) ? row+chunks.At(0) : dims.At(0);
limit.At(1) = ( col+chunks.At(1) < dims.At(1) ) ? col+chunks.At(1) : dims.At(1);
// fprintf(stdout, "Block row=%lu, col=%lu, count=[%lu %lu]\n", row, col, limit.At(0), limit.At(1));
// bool first_time=true;
for (size_t rr=row; rr<limit.At(0) && ix < input_positions.size(); rr++) {
for (size_t cc=col; cc<limit.At(1) && ix < input_positions.size(); cc++) {
size_t pos = input_positions.At(ix);
size_t chp_indx = RowColToIndex(rr,cc, dims.At(0), dims.At(1));
// if (first_time)
// fprintf(stdout, "Entering loop with pos=%lu, ix=%lu, chp_indx=%lu\n", pos, ix, chp_indx);
// first_time = false;
if ( chp_indx < pos)
continue;
while ( chp_indx > pos){
// fprintf(stdout, "chp_indx=%lu > pos=%lu, incrementing ix=%lu\n", chp_indx, pos, ix);
ix++;
if (ix == input_positions.size()){
break;
}
pos = input_positions.At(ix);
// first_time = true;
}
if( chp_indx == pos){
if ( count_out == 0)
offset_out = runningCount;
count_out++;
runningCount++;
// fprintf(stdout, "found: rr=%d, cc=%d, pos=%d, index=%d, ix=%lu, runningCount=%lu\n", (int)rr, (int)cc, (int)pos, (int)chp_indx, ix, runningCount);
ix++;
continue;
}
}
}
assert (ix <= input_positions.size() );
assert (runningCount <= input_positions.size() );
if (count_out > 0) {
pthread_t thread;
int thread_status = 0;
assert( thread_id < thread_flags.size() );
my_args.at(thread_id).row = row;
my_args.at(thread_id).col = col;
my_args.at(thread_id).chunks = &chunks;
my_args.at(thread_id).chunks_out = &chunks_out;
my_args.at(thread_id).dims = &dims;
my_args.at(thread_id).h5file_in = &h5file_in;
my_args.at(thread_id).source = &source;
my_args.at(thread_id).h5file_out = &h5file_out;
my_args.at(thread_id).destination = &destination;
my_args.at(thread_id).offset_out = offset_out;
my_args.at(thread_id).count_out = count_out;
my_args.at(thread_id).input_positions = &input_positions;
my_args.at(thread_id).thread_id = thread_id;
my_args.at(thread_id).flowlimit = flowlimit;
// fprintf(stdout, "creating thread %d from row=%d (max %d), column=%d (max %d), offset_out=%llu, count_out=%llu\n", thread_id, (int)row, (int)dims.At(0), (int)col, (int)dims.At(1), offset_out, count_out);
while (accumulate(thread_flags.begin(), thread_flags.end(), 0) > numThreads) {
// only have to be approximate, don't worry about races
fprintf(stdout, "Sleeping before creating thread %d from row=%d (max %d), column=%d (max %d), offset_out=%llu, count_out=%llu ...\n", thread_id, (int)row, (int)dims.At(0), (int)col, (int)dims.At(1), offset_out, count_out);
sleep(1);
}
thread_flags.At(thread_id) = 1;
thread_status = pthread_create(&thread, NULL, do_subset, (void *)&my_args[thread_id]);
// do_subset((void *)&my_args[thread_id]);
assert (thread_status >= 0);
thread_id++;
}
col += chunks.At(1);
//fflush(stdout);
}
row += chunks.At(0);
}
while (accumulate(thread_flags.begin(), thread_flags.end(), 0) > 0) {
// wait for the threads to finish
// fprintf(stdout, "Waiting ...\n");
sleep(1);
}
assert (runningCount == input_positions.size() );
cout << "Done." << endl;
pthread_exit(NULL);
}
int main(int argc, const char *argv[]) {
OptArgs opts;
string h5file;
string source;
string destination;
vector<string> infiles;
bool help;
string flowlimit_arg;
unsigned int flowlimit;
DumpStartingStateOfNormWells (argc,argv);
opts.ParseCmdLine(argc, argv);
opts.GetOption(h5file, "", '-', "h5file");
opts.GetOption(source, "", 's', "source");
opts.GetOption(destination, "", 'd', "destination");
opts.GetOption(flowlimit_arg, "", 'f', "flowlimit");
opts.GetOption(help, "false", 'h', "help");
opts.GetLeftoverArguments(infiles);
if(help || infiles.empty() || (infiles.size() > 1) ) {
usage();
}
h5file = infiles.front();
int numCPU = (int)sysconf( _SC_NPROCESSORS_ONLN );
int numThreads = MAXTHREADS < numCPU ? MAXTHREADS : numCPU;
fprintf(stdout, "Using %d threads of %d cores\n", numThreads, numCPU);
if (source.empty())
source = source + SIGNAL_IN;
H5ReplayReader reader = H5ReplayReader(h5file, &source[0]);
if ( destination.empty() )
destination = destination + SIGNAL_OUT;
H5ReplayRecorder recorder = (source.compare(destination)==0)
? H5ReplayRecorder(h5file, &destination[0])
: H5ReplayRecorder(h5file, &destination[0],reader.GetType(),reader.GetRank());
reader.Open();
int rank = reader.GetRank();
vector<hsize_t>dims(rank,0);
vector<hsize_t>chunks(rank,0);
reader.GetDims(dims);
reader.GetChunkSize(chunks);
reader.Close();
fprintf(stdout, "Opening for read %s:%s with rank %d, row x col x flow dims=[ ", &h5file[0], &source[0], rank);
for (int i=0; i<rank; i++)
fprintf(stdout, "%d ", (int)dims[i]);
fprintf(stdout, "], chunksize=[ ");
for (int i=0; i<rank; i++)
fprintf(stdout, "%d ", (int)chunks[i]);
fprintf(stdout, "]\n");
if (flowlimit_arg.empty())
flowlimit = dims[2];
else
flowlimit = atoi(flowlimit_arg.c_str());
flowlimit = (flowlimit < dims[2]) ? flowlimit : dims[2];
fprintf(stdout, "Using %u flows\n", flowlimit);
// hard code region size to be at least 100x100
chunks[0] = (chunks[0] < 100) ? 100 : chunks[0];
chunks[1] = (chunks[1] < 100) ? 100 : chunks[1];
recorder.CreateDataset(chunks);
int max_threads_ever = (dims[0]/chunks[0] +1)*(dims[1]/chunks[1] +1);
thread_flags.resize (max_threads_ever, 0);
// fprintf(stdout, "max_threads_ever = %d\n", max_threads_ever);
unsigned int thread_id = 0;
vector<compute_norm_args> my_args( max_threads_ever );
// layout is rows x cols x flows
for (hsize_t row=0; row<dims[0]; ) {
for (hsize_t col=0; col<dims[1]; ) {
pthread_t thread;
int thread_status;
assert( thread_id < thread_flags.size() );
my_args.at(thread_id).row = row;
my_args.at(thread_id).col = col;
my_args.at(thread_id).chunks = &chunks;
my_args.at(thread_id).dims = &dims;
my_args.at(thread_id).h5file = &h5file;
my_args.at(thread_id).source = &source;
my_args.at(thread_id).destination = &destination;
my_args.at(thread_id).thread_id = thread_id;
my_args.at(thread_id).flowlimit = flowlimit;
fprintf(stdout, "creating thread %d from row=%d (max %d), column=%d (max %d)\n", thread_id, (int)row, (int)dims[0], (int)col, (int)dims[1]);
while (accumulate(thread_flags.begin(), thread_flags.end(), 0) > numThreads) {
// only have to be approximate, don't worry about races
// fprintf(stdout, "Sleeping ...\n");
sleep(1);
}
thread_flags[thread_id] = 1;
thread_status = pthread_create(&thread, NULL, compute_norm, (void *)&my_args[thread_id]);
// compute_norm((void *)&my_args[thread_id]);
assert (thread_status >= 0);
thread_id++;
col += chunks[1];
//fflush(stdout);
}
row += chunks[0];
}
while (accumulate(thread_flags.begin(), thread_flags.end(), 0) > 0) {
// wait for the threads to finish
// fprintf(stdout, "Waiting ...\n");
sleep(1);
}
cout << "Done." << endl;
pthread_exit(NULL);
}
void *compute_norm(void* data)
{
// fprintf(stdout, "Thread count %d\n", thread_count);
pthread_detach( pthread_self() );
struct compute_norm_args *args = (struct compute_norm_args *)data;
hsize_t row = args->row;
hsize_t col = args->col;
vector<hsize_t>chunks( * (args->chunks));
vector<hsize_t>dims(* (args->dims));
string h5file = string(* (args->h5file));
string source = string( * (args->source));
string destination = string(* (args->destination));
unsigned int thread_id = args->thread_id;
unsigned int flowlimit = args->flowlimit;
H5ReplayReader reader = H5ReplayReader(h5file, &source[0]);
H5ReplayRecorder recorder = H5ReplayRecorder(h5file, &destination[0]);
// read a chunk in
vector<hsize_t> offset(3);
offset[0] = row;
offset[1] = col;
offset[2] = 0;
vector<hsize_t> count(3);
count[0] = ( row+chunks[0] < dims[0] ) ? chunks[0] : dims[0]-row;
count[1] = ( col+chunks[1] < dims[1] ) ? chunks[1] : dims[1]-col;
count[2] = dims[2];
vector<float> signal(count[0] * count[1] * count[2], 0);
vector<float> normed_signal(signal.size(), 0);
vector<hsize_t> offset_in(1);
offset_in[0] = 0;
vector<hsize_t> count_in(1);
count_in[0] = signal.size();
// fprintf(stdout, "thread_id %d reading %d from row=%d (max %d), %d from column=%d (max %d)\n", thread_id, (int)count[0], (int)row, (int)dims[0], (int)count[1], (int)col, (int)dims[1]);
reader.Read(offset, count, offset_in, count_in, &signal[0]);
// now normalize a well at a time
size_t ix = 0;
for (size_t rr=0; rr<count[0]; rr++) {
for (size_t cc=0; cc<count[1]; cc++) {
size_t wellindex = ix*count[2];
ix++;
vector<double> peaks;
assert (flowlimit <= dims[2] );
vector<float>dat(signal.begin()+wellindex, signal.begin()+wellindex+flowlimit);
vector<float>normalized(dat.size(), 0);
NoKeyCall nkc;
nkc.SetBeadLocation(cc+col,rr+row);
nkc.GetPeaks(dat, peaks);
float failure_to_normalize_value = -1.0f;
nkc.NormalizeBeadSignal(dat, peaks, normalized, failure_to_normalize_value);
// vector<float>normalized(dat.size(), 1);
// stick it back in signal
double v = 0;
for (size_t fnum = 0; fnum < count[2]; fnum++){
if (fnum < flowlimit) {
*(normed_signal.begin()+wellindex+fnum) = normalized[fnum];
v += fabs(normalized[fnum]);
}
else {
*(normed_signal.begin()+wellindex+fnum) = 0;
}
}
assert(v>0);
}
}
// write the chunk back out
vector<hsize_t> extension(3);
extension[0] = row+count[0];
extension[1] = col+count[1];
extension[2] = count[2];
recorder.ExtendDataSet(extension); // extend if necessary
recorder.Write(offset, count, offset_in, count_in, &normed_signal[0]);
thread_flags.at(thread_id) = 0;
int ret = 0;
pthread_exit(&ret);
return(NULL);
}
