wells_mask_t *
wells_mask_read(FILE *fp)
{
int32_t i;
wells_mask_t *m;
m = ion_calloc(1, sizeof(wells_mask_t), __func__, "m");
if(fread(&m->num_rows, sizeof(int32_t), 1, fp) != 1
|| fread(&m->num_cols, sizeof(int32_t), 1, fp) != 1) {
free(m);
return NULL;
}
m->masks = ion_malloc(m->num_rows * sizeof(uint16_t*), __func__, "m->mask");
for(i=0;i<m->num_rows;i++) {
m->masks[i] = ion_malloc(m->num_cols * sizeof(uint16_t), __func__, "m->masks[i]");
if(fread(m->masks[i], sizeof(uint16_t), m->num_cols, fp) != m->num_cols) {
// free
while(0 <= i) {
free(m->masks[i]);
i--;
}
free(m);
return NULL;
}
}
return m;
}
wells_data_t *
wells_data_read(FILE *fp, wells_header_t *header)
{
wells_data_t *data;
data = ion_malloc(sizeof(wells_data_t), __func__, "data");
data->flow_values = ion_malloc(sizeof(float)*header->num_flows, __func__, "data->flow_values");
if(NULL == wells_data_read1(fp, header, data)) {
wells_data_destroy(data);
return NULL;
}
return data;
}
void
sff_sort(sff_file_t *fp_in, sff_file_t *fp_out)
{
int32_t i, row, col;
sff_t *sff;
int32_t requires_sort = 0;
sff_sort_t *sffs = NULL;
int32_t sffs_mem = 0, sffs_len = 0;
// initialize memory
sffs_mem = 1024;
sffs = ion_malloc(sizeof(sff_sort_t) * sffs_mem, __func__, "sffs");
// go through the input file
while(NULL != (sff = sff_read(fp_in))) {
// get the row/col co-ordinates
if(0 == ion_readname_to_rowcol(sff->rheader->name->s, &row, &col)) {
ion_error(__func__, "could not understand the read name", Exit, OutOfRange);
}
// copy over
while(sffs_mem <= sffs_len) {
sffs_mem <<= 1; // double
sffs = ion_realloc(sffs, sizeof(sff_sort_t) * sffs_mem, __func__, "sffs");
}
sffs[sffs_len].row = row;
sffs[sffs_len].col = col;
sffs[sffs_len].sff = sff;
sff = NULL;
// check if we need to sort, for later
if(0 < sffs_len && __sff_sort_lt(sffs[sffs_len], sffs[sffs_len-1])) {
requires_sort = 1;
}
sffs_len++;
}
// resize
sffs_mem = sffs_len;
sffs = ion_realloc(sffs, sizeof(sff_sort_t) * sffs_mem, __func__, "sffs");
if(1 == requires_sort) {
// sort
ion_sort_introsort(sff_sort, sffs_len, sffs);
}
// write
for(i=0;i<sffs_len;i++) {
if(0 == sff_write(fp_out, sffs[i].sff)) {
ion_error(__func__, "sff_write", Exit, WriteFileError);
}
}
// destroy
for(i=0;i<sffs_len;i++) {
sff_destroy(sffs[i].sff);
}
free(sffs);
}
dat_frame_t *
dat_frame_read(FILE *fp, dat_frame_t *prev, dat_header_t *header)
{
dat_frame_t *cur = NULL;
// malloc
cur = ion_malloc(sizeof(dat_frame_t), __func__, "cur");
cur->data = ion_malloc(sizeof(uint16_t)*header->rows*header->cols, __func__, "cur->data");
// read in the data
if(NULL == dat_frame_read1(fp, cur, prev, header)) {
free(cur->data);
free(cur);
}
return cur;
}
sff_read_t *
sff_read_read(FILE *fp, sff_header_t *gh, sff_read_header_t *rh)
{
sff_read_t *r = NULL;
uint32_t i, n = 0;
r = sff_read_init();
r->flowgram = ion_malloc(sizeof(uint16_t)*gh->flow_length, __func__, "r->flowgram");
r->flow_index = ion_malloc(sizeof(uint8_t)*rh->n_bases, __func__, "r->flow_index");
r->bases = ion_string_init(rh->n_bases+1);
r->quality = ion_string_init(rh->n_bases+1);
if(gh->flow_length != fread(r->flowgram, sizeof(uint16_t), gh->flow_length, fp)
|| rh->n_bases != fread(r->flow_index, sizeof(uint8_t), rh->n_bases, fp)
|| rh->n_bases != fread(r->bases->s, sizeof(char), rh->n_bases, fp)
|| rh->n_bases != fread(r->quality->s, sizeof(char), rh->n_bases, fp)) {
// truncated file, error
ion_error(__func__, "fread", Exit, ReadFileError);
}
n += sizeof(uint16_t)*gh->flow_length + 3*sizeof(uint8_t)*rh->n_bases;
// set length and null-terminators
r->bases->l = rh->n_bases;
r->quality->l = rh->n_bases;
r->bases->s[r->bases->l]='\0';
r->quality->s[r->quality->l]='\0';
// convert flowgram to host order
for(i=0;i<gh->flow_length;i++) {
r->flowgram[i] = ntohs(r->flowgram[i]);
}
n += ion_read_padding(fp, n);
#ifdef ION_SFF_DEBUG
sff_read_print(stderr, r, gh, rh);
#endif
return r;
}
sff_index_t *
sff_index_read(FILE *fp)
{
int32_t i;
uint32_t n = 0;
uint64_t len = 0;
sff_index_t *idx;
idx = sff_index_init();
// index header
if(1 != fread(&idx->index_magic_number, sizeof(uint32_t), 1, fp)
|| 1 != fread(&idx->index_version, sizeof(uint32_t), 1, fp)
|| 1 != fread(&idx->num_rows, sizeof(int32_t), 1, fp)
|| 1 != fread(&idx->num_cols, sizeof(int32_t), 1, fp)
|| 1 != fread(&idx->type, sizeof(int32_t), 1, fp)) {
ion_error(__func__, "fread", Exit, WriteFileError);
}
n += sizeof(uint32_t)*2 + sizeof(int32_t)*3;
// convert values from big-endian
sff_index_ntoh(idx);
// offsets
if(SFF_INDEX_ROW_ONLY == idx->type) {
len = 1 + idx->num_rows;
}
else if(SFF_INDEX_ALL == idx->type) {
len = 1 + (idx->num_rows * idx->num_cols);
}
else {
ion_error(__func__, "could not understand index type", Exit, OutOfRange);
}
// alloc
idx->offset = ion_malloc(sizeof(uint64_t) * len, __func__, "idx->offset");
// read
if(len != fread(idx->offset, sizeof(uint64_t), len, fp)) {
ion_error(__func__, "fread", Exit, WriteFileError);
}
// convert values from big-endian
for(i=0;i<len;i++) {
idx->offset[i] = ntohll(idx->offset[i]);
}
n += sizeof(uint64_t) * len;
// padding
n += ion_read_padding(fp, n);
return idx;
}
static sff_read_t *
sff_read_clone(sff_read_t *r, sff_header_t *gh, sff_read_header_t *rh)
{
sff_read_t *ret = NULL;
int32_t i;
ret = ion_calloc(1, sizeof(sff_read_t), __func__, "r");
ret->flowgram = ion_malloc(sizeof(uint16_t)*gh->flow_length, __func__, "ret->flowgram");
for(i=0;i<gh->flow_length;i++) {
ret->flowgram[i] = r->flowgram[i];
}
ret->flow_index = ion_malloc(sizeof(uint8_t)*rh->n_bases, __func__, "ret->flow_index");
for(i=0;i<rh->n_bases;i++) {
ret->flow_index[i] = r->flow_index[i];
}
ret->bases = ion_string_clone(r->bases);
ret->quality = ion_string_clone(r->quality);
return ret;
}
dat_header_t *
dat_header_read(FILE *fp)
{
dat_header_t *h=NULL;
h = ion_malloc(sizeof(dat_header_t), __func__, "h");
if(fread_big_endian_uint32_t(fp, &h->signature) != 1
|| fread_big_endian_uint32_t(fp, &h->version) != 1
|| fread_big_endian_uint32_t(fp, &h->header_size) != 1
|| fread_big_endian_uint32_t(fp, &h->data_size) != 1
|| fread_big_endian_uint32_t(fp, &h->wall_time) != 1
|| fread_big_endian_uint16_t(fp, &h->rows) != 1
|| fread_big_endian_uint16_t(fp, &h->cols) != 1
|| fread_big_endian_uint16_t(fp, &h->channels) != 1
|| fread_big_endian_uint16_t(fp, &h->interlace_type) != 1
|| fread_big_endian_uint16_t(fp, &h->frames_in_file) != 1
|| fread_big_endian_uint16_t(fp, &h->reserved) != 1
|| fread_big_endian_uint32_t(fp, &h->sample_rate) != 1
|| fread_big_endian_uint16_t(fp, &h->full_scale_voltage[0]) != 1
|| fread_big_endian_uint16_t(fp, &h->full_scale_voltage[1]) != 1
|| fread_big_endian_uint16_t(fp, &h->full_scale_voltage[2]) != 1
|| fread_big_endian_uint16_t(fp, &h->full_scale_voltage[3]) != 1
|| fread_big_endian_uint16_t(fp, &h->channel_offsets[0]) != 1
|| fread_big_endian_uint16_t(fp, &h->channel_offsets[1]) != 1
|| fread_big_endian_uint16_t(fp, &h->channel_offsets[2]) != 1
|| fread_big_endian_uint16_t(fp, &h->channel_offsets[3]) != 1
|| fread_big_endian_uint16_t(fp, &h->ref_electrode_offset) != 1
|| fread_big_endian_uint16_t(fp, &h->frame_interval) != 1) {
free(h);
return NULL;
}
if(3 != h->version) { // from Image.cpp
ion_error(__func__, "h->version", Exit, OutOfRange);
}
// Check signature
if(h->signature != DAT_HEADER_SIGNATURE) {
ion_error(__func__, "h->signature", Exit, OutOfRange);
}
return h;
}
wells_header_t *
wells_header_read(FILE *fp)
{
wells_header_t *h;
h = ion_calloc(1, sizeof(wells_header_t), __func__, "h");
if(fread(&h->num_wells, sizeof(uint32_t), 1, fp) != 1
|| fread(&h->num_flows, sizeof(uint16_t), 1, fp) != 1) {
free(h);
return NULL;
}
h->flow_order = ion_malloc(sizeof(char)*(h->num_flows+1), __func__, "h");
if(fread(h->flow_order, sizeof(char), h->num_flows, fp) != h->num_flows) {
free(h);
return NULL;
}
h->flow_order[h->num_flows]='\0';
return h;
}
dat_frame_t *
dat_frame_read1(FILE *fp, dat_frame_t *cur, dat_frame_t *prev, dat_header_t *header)
{
int32_t i, mode;
uint32_t ctr, compressed;
int8_t *tmp_data8=NULL;
//FILE *fp_debug = stdout; // for debuggin
// read the timestamp and compression flag for this frame
if(fread_big_endian_uint32_t(fp, &cur->timestamp) != 1
|| fread_big_endian_uint32_t(fp, &compressed) != 1) {
return NULL;
}
// the first frame is always uncompressed...
if(NULL == prev || DAT_HEADER_UNINTERLACED == header->interlace_type) {
if(0 != compressed) {
ion_error(__func__, "compressed", Exit, OutOfRange);
}
// read in the data
if(fread(cur->data, sizeof(uint16_t), header->rows*header->cols, fp) != header->rows*header->cols) { // image data
return NULL;
}
// manually convert from big-endian
for(i=0;i<header->rows*header->cols;i++) {
cur->data[i] = ntohs(cur->data[i]) & DAT_FRAME_DATA_MASK; // unmask data
}
}
else {
uint32_t len, transitions, total, sentinel;
uint32_t observed_transitions = 0;
// read in the length of the frame, the # of transitions, the
// total sum of hte pixel values, and hte sentinel
if(fread_big_endian_uint32_t(fp, &len) != 1
|| fread_big_endian_uint32_t(fp, &transitions) != 1
|| fread_big_endian_uint32_t(fp, &total) != 1
|| fread_big_endian_uint32_t(fp, &sentinel) != 1) {
return NULL;
}
// check the sentinel
if(sentinel != DAT_HEADER_SIGNATURE) {
ion_error(__func__, "cur->sentinel", Exit, OutOfRange);
}
// subtract len, transitions, total, sentinel
len -= sizeof(uint32_t)*4;
assert(0 < len);
// initialize memory
cur->data = ion_calloc(header->rows*header->cols, sizeof(uint16_t), __func__, "cur->data");
tmp_data8 = ion_malloc(len*sizeof(int8_t), __func__, "tmp_data8");
// read in the whole frame
// NOTE: could read in byte-by-byte and process to reduce memory overhead
if(len != fread(tmp_data8, sizeof(int8_t), len, fp)) {
free(tmp_data8);
return NULL;
}
// de-interlace the data
i=mode=ctr=0;
while(ctr < header->rows*header->cols) {
if(len <= i) { // not enough bytes read
ion_error(__func__, "len <= i", Exit, OutOfRange);
}
// switch to 8-bit mode, or 16-bit mode where appropriate
if(i < len-1 && DAT_FRAME_KEY_0 == (uint8_t)tmp_data8[i]) {
if(DAT_FRAME_KEY_8_1 == (uint8_t)tmp_data8[i+1]) {
// 16-bit to 8-bit
observed_transitions++;
/*
fprintf(stderr, "[%d-%d] from %d-bit mode to 8-bit mode #%d/%d ctr=%d\n", i, i+1, mode,
observed_transitions, transitions, ctr);
*/
mode = 8;
i+=2;
}
else if(DAT_FRAME_KEY_16_1 == (uint8_t)tmp_data8[i+1]) {
// 8-bit to 16-bit
observed_transitions++;
/*
fprintf(stderr, "[%d-%d] from %d-bit mode to 16-bit mode #%d/%d ctr=%d\n", i, i+1, mode,
observed_transitions, transitions, ctr);
*/
mode = 16;
i+=2;
}
}
// Note: assumes we must have data read between mode switches
// read in data
switch(mode) {
case 8:
// 8-bit mode
cur->data[ctr] = tmp_data8[i] + prev->data[ctr];
ctr++;
i++;
break;
case 16:
// 16-bit mode
cur->data[ctr] = (ntohs((int16_t)((tmp_data8[i] << 8) | tmp_data8[i+1])) & DAT_FRAME_DATA_MASK) + prev->data[ctr];
ctr++;
i+=2;
break;
default:
// mode?
ion_error(__func__, "mode", Exit, OutOfRange);
break;
}
}
if(((i+3) & ~0x3) != len) { // check that the data was quad-word aligned
ion_error(__func__, "quad word alignment", Exit, OutOfRange);
}
// free tmp_data8
free(tmp_data8);
// check that the observed # of transitions equals the state # of
// transitions
if(transitions != observed_transitions) {
ion_error(__func__, "transitions != observed_transitions", Exit, OutOfRange);
}
}
return cur;
}
// TODO: should we change the header:
// - must trake index_length
// - assumes row-major order
sff_index_t*
sff_index_create(sff_file_t *fp_in, sff_header_t *fp_out_header, int32_t num_rows, int32_t num_cols, int32_t type)
{
int64_t len = 0;
int32_t i, prev_row, prev_col, row, col;
sff_index_t *idx;
sff_t *sff;
uint64_t fp_in_start, prev_pos;
idx = sff_index_init();
idx->num_rows = num_rows;
idx->num_cols = num_cols;
idx->type = type;
// alloc
switch(type) {
case SFF_INDEX_ROW_ONLY:
len = 1 + idx->num_rows;
idx->offset = ion_malloc(len * sizeof(uint64_t), __func__, "idx->offset");
break;
case SFF_INDEX_ALL:
len = 1 + (idx->num_rows * idx->num_cols);
idx->offset = ion_malloc(len * sizeof(uint64_t), __func__, "idx->offset");
break;
default:
ion_error(__func__, "this index type is currently not supported", Exit, OutOfRange);
}
// save where the sff entries started
prev_pos = fp_in_start = ftell(fp_in->fp);
if(-1L == fp_in_start) {
ion_error(__func__, "ftell", Exit, ReadFileError);
}
// go through the input file
i = 0;
prev_row = prev_col = 0;
while(NULL != (sff = sff_read(fp_in))) {
// out of range
if(len-1 <= i) {
ion_error(__func__, "bug encountered", Exit, OutOfRange);
}
// get the row/col co-ordinates
if(0 == ion_readname_to_rowcol(sff->rheader->name->s, &row, &col)) {
ion_error(__func__, "could not understand the read name", Exit, OutOfRange);
}
// assumes row-major order, skips over reads that are not present
if(row < prev_row || (row == prev_row && col < prev_col)) {
ion_error(__func__, "SFF file was not sorted in row-major order", Exit, OutOfRange);
}
while(row != prev_row || col != prev_col) {
// add in empty entry
switch(type) {
case SFF_INDEX_ROW_ONLY:
if(0 == prev_col) { // first column
idx->offset[i] = UINT64_MAX;
// do not increment i, since we only do this when moving to a new row
}
break;
case SFF_INDEX_ALL:
// all rows and columns
idx->offset[i] = UINT64_MAX;
i++;
break;
default:
ion_error(__func__, "this index type is currently not supported", Exit, OutOfRange);
}
if(len-1 <= i) {
ion_error(__func__, "x/y was out of range", Exit, OutOfRange);
}
prev_col++;
if(prev_col == idx->num_cols) {
// new row
prev_col = 0;
prev_row++;
if(SFF_INDEX_ROW_ONLY == type) {
i++;
}
}
}
// add to the index
switch(type) {
case SFF_INDEX_ROW_ONLY:
if(0 == col) { // first column
idx->offset[i] = prev_pos;
}
else if(0 < col && UINT64_MAX == idx->offset[i]) {
idx->offset[i] = prev_pos;
// do not move onto the next
}
break;
case SFF_INDEX_ALL:
// all rows and columns
idx->offset[i] = prev_pos;
i++;
break;
default:
ion_error(__func__, "this index type is currently not supported", Exit, OutOfRange);
}
prev_row = row;
prev_col = col;
// destroy
sff_destroy(sff);
// next
prev_col++;
if(prev_col == idx->num_cols) {
// new row
prev_col = 0;
prev_row++;
if(SFF_INDEX_ROW_ONLY == type) {
i++;
}
}
prev_pos = ftell(fp_in->fp);
if(-1L == prev_pos) {
ion_error(__func__, "ftell", Exit, ReadFileError);
}
}
// get the last offset
idx->offset[len-1] = prev_pos;
// update the index offset in the header
fp_out_header->index_offset = fp_in_start; // insert between the header and sff entries
// update the index length in the header
fp_out_header->index_length = sff_index_length(idx);
// update the offsets based on the index length
for(i=0;i<len;i++) {
if(UINT64_MAX != idx->offset[i]) {
idx->offset[i] += fp_out_header->index_length;
}
}
return idx;
}