void
sff_sort(sff_file_t *fp_in, sff_file_t *fp_out)
{
int32_t i, row, col;
sff_t *sff;
int32_t requires_sort = 0;
sff_sort_t *sffs = NULL;
int32_t sffs_mem = 0, sffs_len = 0;
// initialize memory
sffs_mem = 1024;
sffs = ion_malloc(sizeof(sff_sort_t) * sffs_mem, __func__, "sffs");
// go through the input file
while(NULL != (sff = sff_read(fp_in))) {
// get the row/col co-ordinates
if(0 == ion_readname_to_rowcol(sff->rheader->name->s, &row, &col)) {
ion_error(__func__, "could not understand the read name", Exit, OutOfRange);
}
// copy over
while(sffs_mem <= sffs_len) {
sffs_mem <<= 1; // double
sffs = ion_realloc(sffs, sizeof(sff_sort_t) * sffs_mem, __func__, "sffs");
}
sffs[sffs_len].row = row;
sffs[sffs_len].col = col;
sffs[sffs_len].sff = sff;
sff = NULL;
// check if we need to sort, for later
if(0 < sffs_len && __sff_sort_lt(sffs[sffs_len], sffs[sffs_len-1])) {
requires_sort = 1;
}
sffs_len++;
}
// resize
sffs_mem = sffs_len;
sffs = ion_realloc(sffs, sizeof(sff_sort_t) * sffs_mem, __func__, "sffs");
if(1 == requires_sort) {
// sort
ion_sort_introsort(sff_sort, sffs_len, sffs);
}
// write
for(i=0;i<sffs_len;i++) {
if(0 == sff_write(fp_out, sffs[i].sff)) {
ion_error(__func__, "sff_write", Exit, WriteFileError);
}
}
// destroy
for(i=0;i<sffs_len;i++) {
sff_destroy(sffs[i].sff);
}
free(sffs);
}
int main(int argc, char *argv[])
{
FILE *inSFF = NULL;
FILE *outSFF = NULL;
FILE *listFP = NULL;
int n = 0; //number elements read
char *inFileName = NULL;
char *outFileName = NULL;
char *listFileName = NULL;
char *errFileName = {"./SFFFilter_err.txt"};
int numReads;
int matchCnt = 0;
int got = 0;
bool debugflag = false;
bool qualflag = false;
int qual_offset = 33;
bool listMatch = false;
char name[256];
uint16_t *flowgram_values; // [NUMBER_OF_FLOWS_PER_READ];
uint8_t *flow_index_per_base; // * number_of_bases;
char *bases; // * number_of_bases;
uint8_t *quality_scores; // * number_of_bases;
char *flow_chars;
char *key_sequence;
// Parse command line arguments
int argcc = 1;
while (argcc < argc) {
if (argv[argcc][0] == '-') {
switch (argv[argcc][1]) {
case 'd': // print debug info
debugflag = true;
break;
case 'q': // print debug info
qualflag = true;
break;
case 'f': // list of locations to filter
argcc++;
listFileName = strdup (argv[argcc]);
break;
case 's': // Offset to apply to quality scores
argcc++;
qual_offset = atoi(argv[argcc]);
if(qual_offset==0) {
fprintf (stderr, "-s option should specify a nonzero quality offset\n");
exit (1);
}
break;
case 'o': // output file name
argcc++;
outFileName = strdup(argv[argcc]);
break;
default:
fprintf (stderr, "Unknown option %s\n", argv[argcc]);
exit (1);
break;
}
}
else {
inFileName = argv[argcc];
}
argcc++;
}
if (!inFileName) {
fprintf (stdout, "No input sff file specified\n");
fprintf (stdout, "Usage: %s [-f filename] [-d] sff-filename\n", argv[0]);
fprintf (stdout, "\t-f Specify input file list.\n");
fprintf (stdout, "\t-o Specify output sff file name.\n");
fprintf (stdout, "\t-d Prints debug information.\n");
fprintf (stdout, "\t-q Take qualities from 4th field of file specified by -f.\n");
fprintf (stdout, "\t-s To use in conjunction with -q option, specifies an offset to be applied to quality scores.\n");
exit (1);
}
if (!listFileName) {
fprintf (stdout, "No input list file specified\n");
fprintf (stdout, "Usage: %s [-f filename] [-d] sff-filename\n", argv[0]);
fprintf (stdout, "\t-f Specify input file list.\n");
fprintf (stdout, "\t-o Specify output sff file name.\n");
fprintf (stdout, "\t-d Prints debug information.\n");
fprintf (stdout, "\t-q Take qualities from 4th field of file specified by -f.\n");
fprintf (stdout, "\t-s To use in conjunction with -q option, specifies an offset to be applied to quality scores.\n");
exit (1);
}
//Create output filename from input filename if it wasn't specified
if(outFileName==NULL) {
outFileName = (char *) malloc (sizeof(char) * (strlen(dirname(inFileName)) + strlen(inFileName) + 50));
sprintf (outFileName, "%s/filtered_%s", dirname(inFileName), inFileName);
}
//Open the SFF file
inSFF = fopen(inFileName, "rb");
if (!inSFF) {
perror (inFileName);
exit (1);
}
//Open the outputSFF file
outSFF = fopen(outFileName, "wb");
if (!outSFF) {
perror (outFileName);
exit (1);
}
//Open the list file
listFP = fopen(listFileName, "rb");
if (!listFP) {
perror (listFileName);
exit (1);
}
//Read the list of locations into buffer
got = GetNumLines(listFileName);
if (got <= 0) {
fprintf (stderr, "Did not read any pixel coordinates; does the file exist? Is it formatted correctly?\n");
exit (1);
}
else {
fprintf (stdout, "Reading up to %d lines\n", got);
}
//Dynamic array allocation
int *rows = (int *) malloc (sizeof(int) * got);
int *cols = (int *) malloc (sizeof(int) * got);
int *lengths = (int *) malloc (sizeof(int) * got);
char **quals = (char **) malloc (sizeof(char*) * got);
bool *fnds = (bool *) malloc (sizeof(bool) * got); //tracks reads that were found in SFF file
for (int i=0;i<got;i++)
{
fnds[i] = false;
quals[i] = (char *) malloc (sizeof(char) * MAX_BASES);
}
int lineCnt = 0;
while (!feof(listFP)) {
if(qualflag) {
if(4 != fscanf (listFP, "%d %d %d %s\n", &rows[lineCnt], &cols[lineCnt], &lengths[lineCnt], quals[lineCnt])) {
fprintf(stderr,"%s: bad format in line %d of %s - expected 3 ints and a char string.\n",argv[0],1+lineCnt,inFileName);
exit(EXIT_FAILURE);
} else if(strlen(quals[lineCnt]) < (unsigned int) lengths[lineCnt]) {
fprintf(stderr,"%s: warning: line %d of %s - quality string is shorter than requested length.\n",argv[0],1+lineCnt,inFileName);
}
lineCnt++;
} else {
if(3 != fscanf (listFP, "%d %d %d\n", &rows[lineCnt], &cols[lineCnt], &lengths[lineCnt])) {
fprintf(stderr,"%s: bad format in line %d of %s - expected 3 ints.\n",argv[0],1+lineCnt,inFileName);
exit(EXIT_FAILURE);
} else {
lineCnt++;
}
}
}
fclose (listFP);
// Read the input file header
CommonHeader h;
n = fread(&h, 31, 1, inSFF);
assert(n==1);
//Copy the header to write the output file
CommonHeader ch_out;
ch_out.magic_number = h.magic_number;
ch_out.version[0] = 0;
ch_out.version[1] = 0;
ch_out.version[2] = 0;
ch_out.version[3] = 1;
ch_out.index_offset = h.index_offset;
ch_out.index_length = h.index_length;
ch_out.number_of_reads = h.number_of_reads;
ch_out.header_length = h.header_length;
ch_out.key_length = h.key_length;
ch_out.number_of_flows_per_read = h.number_of_flows_per_read;
ch_out.flowgram_format_code = h.flowgram_format_code;
ByteSwap8(h.index_offset);
ByteSwap4(h.index_length);
ByteSwap4(h.number_of_reads);
ByteSwap2(h.header_length);
ByteSwap2(h.key_length);
ByteSwap2(h.number_of_flows_per_read);
flow_chars = (char *)malloc(h.number_of_flows_per_read);
key_sequence = (char *)malloc(h.key_length);
n = fread(flow_chars, h.number_of_flows_per_read, 1, inSFF);
assert(n==1);
n = fread(key_sequence, h.key_length, 1, inSFF);
assert(n==1);
int padBytes = (8-((31 + h.number_of_flows_per_read + h.key_length) & 7));
char padData[8];
if (padBytes > 0) {
n = fread(padData, padBytes, 1, inSFF);
assert(n==1);
}
if (debugflag) {
//DEBUG
printf("Magic: %u %s\n", h.magic_number, (h.magic_number == MAGIC ? "Yes" : "No"));
printf("Header length: %hu\n", h.header_length);
printf("Version: %d%d%d%d\n", h.version[0], h.version[1], h.version[2], h.version[3]);
printf("Index offset: %lu length: %u\n", h.index_offset, h.index_length);
printf("Number of reads: %u\n", h.number_of_reads);
printf("Key length: %u\n", h.key_length);
printf("Flows per read: %hu\n", h.number_of_flows_per_read);
printf("Flowgram format: %hhu\n", h.flowgram_format_code);
printf ("End of Header\n\n");
}
// Write the header of the output SFF
char pad[8];
memset(pad, 0, sizeof(pad));
int bytes = 31;
fwrite (&ch_out, bytes, 1, outSFF);
for(int i=0;i<h.number_of_flows_per_read;i++) {
fwrite(&flow_chars[i%4], 1, 1, outSFF);
bytes++;
}
fwrite(key_sequence, 1, 4, outSFF);
bytes += 4;
padBytes = (8 - (bytes & 0x7)) & 0x7;
if (padBytes > 0)
fwrite(pad, padBytes, 1, outSFF);
// Prepare to process all the reads
numReads = h.number_of_reads;
flowgram_values = (uint16_t *)malloc(sizeof(uint16_t) * h.number_of_flows_per_read);
int maxBases = h.number_of_flows_per_read * 100; // problems if ever a 10-mer hits every flow!
flow_index_per_base = (uint8_t *)malloc(sizeof(uint8_t) * maxBases);
bases = (char *)malloc(maxBases);
quality_scores = (uint8_t *)malloc(sizeof(uint8_t) * maxBases);
//Loop thru the reads
for (int i=0;i<numReads;i++) {
// Read read header
ReadHeader r;
n = fread(&r, 16, 1, inSFF);
assert(n==1);
ByteSwap2(r.read_header_length);
ByteSwap4(r.number_of_bases);
ByteSwap2(r.name_length);
ByteSwap2(r.clip_qual_left);
ByteSwap2(r.clip_qual_right);
ByteSwap2(r.clip_adapter_left);
ByteSwap2(r.clip_adapter_right);
if (r.name_length > 0) {
n = fread(name, r.name_length, 1, inSFF);
assert(n==1);
name[r.name_length] = '\0';
}
int readPadLength = ((8 - ((16 + r.name_length) & 7)))%8;
if (readPadLength > 0) {
n = fread(padData, readPadLength, 1, inSFF);
assert(n==1);
}
n = fread(flowgram_values, h.number_of_flows_per_read, sizeof(uint16_t), inSFF);
assert(n==sizeof(uint16_t));
n = fread(flow_index_per_base, r.number_of_bases, sizeof(uint8_t), inSFF);
assert(n==sizeof(uint8_t));
n = fread(bases, r.number_of_bases, 1, inSFF);
assert(n==1);
bases[r.number_of_bases] = '\0';
n = fread(quality_scores, r.number_of_bases, sizeof(uint8_t), inSFF);
assert(n==sizeof(uint8_t));
int bytesRead = h.number_of_flows_per_read * sizeof(uint16_t) + 3 * r.number_of_bases;
readPadLength = (8 - (bytesRead & 7))%8;
if (readPadLength > 0) {
n = fread(padData, readPadLength, 1, inSFF);
assert(n==1);
}
int f;
if (debugflag) {
// DEBUG
printf("Read: %s has %d bases\n",
(r.name_length > 0 ? name : "NONAME"),
r.number_of_bases);
//printf("Read header length: %d\n", r.read_header_length);
printf("Clip left: %d qual: %d right: %d qual: %d\n",
r.clip_adapter_left, r.clip_qual_left,
r.clip_adapter_right, r.clip_qual_right);
printf("Flowgram bases:\n");
for(f=0;f<h.number_of_flows_per_read;f++)
printf("%d ", (int) floor (ByteSwap2(flowgram_values[f])/100.0 + 0.5));
printf("\n");
//printf("\nFlow index per base:\n");
unsigned int b;
//for(b=0;b<r.number_of_bases;b++)
// printf("%d ", flow_index_per_base[b]);
printf("Bases called:\n");
for(b=0;b<r.number_of_bases;b++)
printf("%c", bases[b]);
//printf("\nQuality scores:\n");
//for(b=0;b<r.number_of_bases;b++)
// printf("%d ", quality_scores[b]);
} else {
for(f=0;f<h.number_of_flows_per_read;f++)
ByteSwap2(flowgram_values[f]);
}
//Get the row column for this read
int row;
int col;
if(1 != ion_readname_to_rowcol(name, &row, &col)) {
fprintf (stderr, "Error parsing read name: '%s'\n", name);
continue;
}
//Look for matching row column in the list
listMatch = false;
//fprintf (stdout, "Looking for %d %d\n", row, col);
int readMatch=0;
for (;readMatch<got;readMatch++) {
if (row == rows[readMatch] && col == cols[readMatch]) {
//fprintf (stdout, "\there it is %d %d\n", rows[readMatch],cols[readMatch]);
listMatch = true;
fnds[readMatch] = true;
matchCnt++;
break;
}
}
if (listMatch) {
//
// Update the output file
//
int nameLen = r.name_length;
int numBasesCalled = r.number_of_bases;
if(r.clip_qual_right == 0 || r.clip_qual_right > lengths[readMatch])
r.clip_qual_right = lengths[readMatch];
// write the header
ByteSwap2(r.read_header_length);
ByteSwap4(r.number_of_bases);
ByteSwap2(r.name_length);
ByteSwap2(r.clip_qual_left);
ByteSwap2(r.clip_qual_right);
ByteSwap2(r.clip_adapter_left);
ByteSwap2(r.clip_adapter_right);
fwrite (&r, 16, 1, outSFF);
fwrite(name, nameLen, 1, outSFF);
int writePadLength = (8 - (nameLen & 7)) & 7;
if (writePadLength)
fwrite(padData, writePadLength, 1, outSFF);
if(qualflag) {
for(int iBase=0; iBase < lengths[readMatch]; iBase++) {
quality_scores[iBase] = (uint8_t) quals[readMatch][iBase] + qual_offset;
}
}
for(int iBase=lengths[readMatch]; iBase < numBasesCalled; iBase++) {
flow_index_per_base[iBase] = 0;
bases[iBase] = 'N';
quality_scores[iBase] = 0;
}
for (f=0;f<h.number_of_flows_per_read;f++)
ByteSwap2(flowgram_values[f]);
fwrite(flowgram_values, h.number_of_flows_per_read, sizeof(uint16_t), outSFF);
fwrite(flow_index_per_base, numBasesCalled, sizeof(uint8_t), outSFF);
fwrite(bases, numBasesCalled, 1, outSFF);
fwrite(quality_scores, numBasesCalled, sizeof(uint8_t), outSFF);
int bytesWritten = h.number_of_flows_per_read * sizeof(uint16_t) + 3 * numBasesCalled;
writePadLength = (8 - (bytesWritten & 7)) & 7;
if (writePadLength)
fwrite(padData, writePadLength, 1, outSFF);
}
else {
//Skip this read
}
}
//Update Read Count in output SFF file
ch_out.number_of_reads = BYTE_SWAP_4(matchCnt);
fseek (outSFF, 0, SEEK_SET);
bytes=31;
fwrite(&ch_out, bytes, 1, outSFF);
//User message
fprintf (stdout, "Created file: %s\n", outFileName);
//
//Write out report on unfound reads
//
bool printErrorLog = false;
for (int i=0;i<got;i++)
{
if (fnds[i] == false) {
printErrorLog = true;
break;
}
}
if (printErrorLog)
{
fprintf (stdout, "There are reads that were not found. See %s\n", errFileName);
FILE *fpErr = fopen (errFileName, "wb");
if (fpErr) {
fprintf (fpErr, "# SFF file: %s\n", inFileName);
fprintf (fpErr, "# Read positions source: %s\n", listFileName);
fprintf (fpErr, "# Reads not found in SFF:\n");
fprintf (fpErr, "# Row Column\n");
for (int i=0;i<got;i++)
{
if (fnds[i] == false) {
fprintf (fpErr, "%d %d\n", rows[i], cols[i]);
}
}
fclose (fpErr);
}
}
//Cleanup
fclose (inSFF);
fclose (outSFF);
free (rows);
free (cols);
free (fnds);
free (flow_chars);
free (key_sequence);
free (flowgram_values);
free (flow_index_per_base);
free (bases);
free (quality_scores);
free (listFileName);
free (outFileName);
return 0;
}
int main(int argc, char *argv[])
{
char *fastqFileName = NULL;
char *sffFileName = NULL;
bool forceClip = false;
bool keyPass = false;
bool allReads = false;
int minReadLen = 8; // min read length after key-pass that we will write out to fastq file
int readCol = -1;
int readRow = -1;
bool findRead = false;
int row, col;
// process command-line args
int argcc = 1;
while (argcc < argc) {
if (argv[argcc][0] == '-') {
switch (argv[argcc][1]) {
case 'a': // output all reads
allReads = true;
break;
case 'R': // report read at row & column
argcc++;
readRow = atoi(argv[argcc]);
break;
case 'C': // report read at row & column
argcc++;
readCol = atoi(argv[argcc]);
break;
case 'q': // convert to fastq
argcc++;
fastqFileName = argv[argcc];
break;
case 'c': // force qual clip left to 5
forceClip = true;
break;
case 'k': // force keypass
keyPass = true;
argcc++;
hackkey = argv[argcc];
hackkeylen = strlen(hackkey);
break;
case 'l': // set min readlength for fastq file output filter
argcc++;
minReadLen = atoi(argv[argcc]);
break;
default:
//sffFileName = argv[argcc];
break;
}
}
else {
sffFileName = argv[argcc];
}
argcc++;
}
if (!sffFileName) {
printf("Usage: SFFRead [args] sffFile.sff\n");
exit(0);
}
if (readCol > -1 && readRow > -1) {
findRead = true;
allReads = true;// makes it search all reads
}
FILE *fp;
fp = fopen(sffFileName, "r+");
if (fp) {
if (!findRead && !fastqFileName)
printf("Reading file: %s\n", sffFileName);
CommonHeader h;
// Fix the flow_format_code problem: make sure it is set to 1
fpos_t p, start;
fgetpos (fp, &p);
fgetpos (fp, &start);
start = p;
int elements_read = fread(&h, 31, 1, fp);
assert(elements_read == 1);
h.flowgram_format_code = 1;
fsetpos (fp, &p);
fwrite (&h, 31, 1, fp);
fsetpos (fp, &p);
elements_read = fread(&h, 31, 1, fp);
assert(elements_read == 1);
ByteSwap8(h.index_offset);
ByteSwap4(h.index_length);
ByteSwap4(h.number_of_reads);
ByteSwap2(h.header_length);
ByteSwap2(h.key_length);
ByteSwap2(h.number_of_flows_per_read);
if (!findRead && !fastqFileName) {
printf("Magic: %u %s\n", h.magic_number, (h.magic_number == MAGIC ? "Yes" : "No"));
printf("Version: %d%d%d%d\n", h.version[0], h.version[1], h.version[2], h.version[3]);
printf("Index offset: %lu length: %u\n", h.index_offset, h.index_length);
printf("Number of reads: %u\n", h.number_of_reads);
printf("Header length: %hu\n", h.header_length);
printf("Key length: %u\n", h.key_length);
printf("Flows per read: %hu\n", h.number_of_flows_per_read);
printf("Flowgram format: %hhu\n", h.flowgram_format_code);
}
flow_chars = (char *)malloc(h.number_of_flows_per_read);
key_sequence = (char *)malloc(h.key_length);
elements_read = fread(flow_chars, h.number_of_flows_per_read, 1, fp);
assert(elements_read == 1);
elements_read = fread(key_sequence, h.key_length, 1, fp);
assert(elements_read == 1);
int i;
if (!findRead && !fastqFileName) {
printf("Key sequence: ");
for(i=0;i<h.key_length;i++)
printf("%c", key_sequence[i]);
printf("\nFlow chars:\n");
for(i=0;i<h.number_of_flows_per_read;i++)
printf("%c", flow_chars[i]);
printf("\n");
}
int padBytes = (8-((31 + h.number_of_flows_per_read + h.key_length) & 7));
char padData[8];
// fprintf (stdout, "Pad Bytes = %d\n", padBytes);
elements_read = fread(padData, padBytes, 1, fp);
assert(elements_read == 1);
fgetpos(fp, &p);
// fprintf (stdout, "We are at %ld\n", (p.__pos - start.__pos));
// -- read the reads
int numReads = h.number_of_reads;
// pre-allocate space so we be fast
flowgram_values = (uint16_t *)malloc(sizeof(uint16_t) * h.number_of_flows_per_read);
int maxBases = h.number_of_flows_per_read * 10; // problems if ever a 10-mer hits every flow!
flow_index_per_base = (uint8_t *)malloc(sizeof(uint8_t) * maxBases);
bases = (char *)malloc(maxBases);
quality_scores = (uint8_t *)malloc(sizeof(uint8_t) * maxBases);
for(i=0;i<numReads;i++) {
ReadHeader r;
#define FIXIT
#ifdef FIXIT
fpos_t pos;
// Get position ready-to-read header
fgetpos (fp, &pos);
// Read header
elements_read = fread(&r, 16, 1, fp);
assert(elements_read == 1);
// byte swap
ByteSwap2(r.read_header_length);
ByteSwap2(r.name_length);
// fprintf (stdout, "Old read header length = %d\n", r.read_header_length);
// Fix the read_header_length
// read_header_length is "16 + name_length" rounded up to nearest divisible by 8
r.read_header_length = 16 + r.name_length;
r.read_header_length += (8 - (r.read_header_length & 0x7)) & 0x7;
// fprintf (stdout, "New read header length = %d\n", r.read_header_length);
// Byte swap
ByteSwap2(r.read_header_length);
ByteSwap2(r.name_length);
// Rewind file pointer
fsetpos (fp, &pos);
// Write header out again
fwrite (&r, 16, 1, fp);
// Rewind again
fsetpos (fp, &pos);
#endif
// Read it in and continue
fpos_t readStart;
fgetpos(fp, &readStart);
elements_read = fread(&r, 16, 1, fp);
assert(elements_read == 1);
//ByteSwap2(r.read_header_length);
//ByteSwap2(r.name_length);
ByteSwap2(r.clip_qual_left);
ByteSwap2(r.clip_qual_right);
ByteSwap2(r.clip_adapter_left);
ByteSwap2(r.clip_adapter_right);
// Fix clipping values
r.clip_qual_left = 5;
r.clip_adapter_left = 0;
r.clip_qual_right = 0;
r.clip_adapter_right = 0;
ByteSwap2(r.clip_qual_left);
ByteSwap2(r.clip_qual_right);
ByteSwap2(r.clip_adapter_left);
ByteSwap2(r.clip_adapter_right);
// Rewind file pointer to beginning of read header
fsetpos (fp, &pos);
// Write read header
fwrite (&r, 16, 1, fp);
// Rewind file pointer to beginning of read header
fsetpos (fp, &pos);
// Read corrected header
elements_read = fread(&r, 16, 1, fp);
assert(elements_read == 1);
ByteSwap2(r.read_header_length);
ByteSwap4(r.number_of_bases);
ByteSwap2(r.name_length);
ByteSwap2(r.clip_qual_left);
ByteSwap2(r.clip_qual_right);
ByteSwap2(r.clip_adapter_left);
ByteSwap2(r.clip_adapter_right);
//printf("Read header length: %d\n", r.read_header_length);
//printf("Read name length: %d\n", r.name_length);
/*
flow_index_per_base = (uint8_t *)malloc(sizeof(uint8_t) * r.number_of_bases);
bases = (char *)malloc(r.number_of_bases);
quality_scores = (uint8_t *)malloc(sizeof(uint8_t) * r.number_of_bases);
*/
if (r.name_length > 0) {
elements_read = fread(name, r.name_length, 1, fp);
assert(elements_read == 1);
name[r.name_length] = 0; // so we can easily print it
}
if(1 != ion_readname_to_rowcol(name, &row, &col)) {
fprintf (stderr, "Error parsing read name: '%s'\n", name);
continue;
}
int readPadLength = ((8 - ((16 + r.name_length) & 7)))%8;
elements_read = fread(padData, readPadLength, 1, fp);
assert(elements_read == 1);
/*
printf("Read: %s (r%d|c%d) has %d bases\n",
(r.name_length > 0 ? name : "NONAME"),
row, col,
r.number_of_bases);
printf("Clip left: %d qual: %d right: %d qual: %d\n",
r.clip_adapter_left, r.clip_qual_left,
r.clip_adapter_right, r.clip_qual_right);
printf("Flowgram values:\n");
*/
elements_read = fread(flowgram_values, h.number_of_flows_per_read, sizeof(uint16_t), fp);
assert(elements_read == sizeof(uint16_t));
elements_read = fread(flow_index_per_base, r.number_of_bases, sizeof(uint8_t), fp);
assert(elements_read == sizeof(uint8_t));
elements_read = fread(bases, r.number_of_bases, 1, fp);
assert(elements_read == 1);
elements_read = fread(quality_scores, r.number_of_bases, sizeof(uint8_t), fp);
int bytesRead = h.number_of_flows_per_read * sizeof(uint16_t) + 3 * r.number_of_bases;
readPadLength = (8 - (bytesRead & 7))%8;
elements_read = fread(padData, readPadLength, 1, fp);
assert(elements_read == 1);
fpos_t readEnd;
fgetpos(fp, &readEnd);
// fprintf (stdout, "At end of read. Size: %ld\n", readEnd.__pos-readStart.__pos);
//if ((readEnd.__pos-readStart.__pos) != r.read_header_length) {
// fprintf (stdout, "mismatch in read_header_length\n");
// exit (1);
//}
// parse the name to get the row & col, if matched, print out read
if(1 != ion_readname_to_rowcol(name, &row, &col)) {
fprintf (stderr, "Error parsing read name: '%s'\n", name);
continue;
}
if (row == readRow && col == readCol) {
//printf("Ionogram: ");
int i;
for(i=0;i<h.number_of_flows_per_read;i++) {
printf("%.2lf ", (double)(ByteSwap2(flowgram_values[i]))/100.0);
}
printf("\n");
}
/*
int f;
for(f=0;f<h.number_of_flows_per_read;f++)
printf("%d ", ByteSwap2(flowgram_values[f]));
printf("\nFlow index per base:\n");
unsigned int b;
for(b=0;b<r.number_of_bases;b++)
printf("%d ", flow_index_per_base[b]);
printf("\nBases called:\n");
for(b=0;b<r.number_of_bases;b++)
printf("%c", bases[b]);
printf("\nQuality scores:\n");
for(b=0;b<r.number_of_bases;b++)
printf("%d ", quality_scores[b]);
printf("\nDone with this read\n\n");
*/
/*
if (name) free(name);
free(flowgram_values);
free(flow_index_per_base);
free(bases);
free(quality_scores);
*/
}
free(flowgram_values);
free(flow_index_per_base);
free(bases);
free(quality_scores);
fclose(fp);
}
return 0;
}
int main(int argc, char *argv[])
{
char *fastqFileName = NULL;
char *sffFileName = NULL;
bool forceClip = false;
bool keyPass = false;
bool allReads = false;
int readCol = -1;
int readRow = -1;
bool findRead = false;
int row, col;
int numKeypassedReads = 0;
int qual_offset = DEFAULT_QUAL_OFFSET;
bool legacyFASTQName = false; // enable if you want r10|c100 format name in fastq file
bool debug = false;
bool legacyReadName = false;
bool adapterTrim = true;
bool ignoreLeftQualTrim = false;
// process command-line args
int argcc = 1;
while (argcc < argc) {
if (argv[argcc][0] == '-') {
switch (argv[argcc][1]) {
case 'a': // output all reads
allReads = true;
break;
case 'R': // report read at row & column
argcc++;
readRow = atoi(argv[argcc]);
break;
case 'C': // report read at row & column
argcc++;
readCol = atoi(argv[argcc]);
break;
case 'q': // convert to fastq
argcc++;
fastqFileName = argv[argcc];
break;
case 'c': // force qual clip left to 5
forceClip = true;
break;
case 's': // Offset to apply to quality scores
argcc++;
qual_offset = atoi(argv[argcc]);
if(qual_offset==0) {
fprintf (stderr, "-s option should specify a nonzero quality offset\n");
exit (1);
}
break;
case 'k': // force keypass
keyPass = true;
argcc++;
hackkey = argv[argcc];
hackkeylen = strlen(hackkey);
break;
case 'L': // don't record name of read in comment
legacyFASTQName = true;
break;
case 'd': // enable debug print outs
debug = true;
break;
case 'h': // help info
printHelp ();
exit (0);
break;
case 'u': // prevent read clipping
adapterTrim = false;
break;
case 'b': // ignore barcodes (ok really its ignoring the left qual trim)
ignoreLeftQualTrim = true;
break;
case 'v': // version info
fprintf (stdout, "%s", IonVersion::GetFullVersion("SFFRead").c_str());
exit (0);
break;
default:
//sffFileName = argv[argcc];
break;
}
}
else {
sffFileName = argv[argcc];
}
argcc++;
}
if (!sffFileName) {
printHelp();
exit(0);
}
if (readCol > -1 && readRow > -1) {
findRead = true;
allReads = true;// makes it search all reads
}
sff_file_t* sff_file_in = NULL;
sff_file_in = sff_fopen(sffFileName, "rb", NULL, NULL);
if (sff_file_in) {
if (!findRead && !fastqFileName) {
printf("Reading file: %s\n", sffFileName);
sff_header_print(stdout, sff_file_in->header);
}
// -- read the reads
int numReads;
if (allReads) {
numReads = sff_file_in->header->n_reads;
}
else {
numReads = (sff_file_in->header->n_reads < 10 ? sff_file_in->header->n_reads:10);
}
FILE *fpq = NULL;
if (fastqFileName) {
numReads = sff_file_in->header->n_reads;
fpq = fopen(fastqFileName, "w");
if (!fpq){
perror (fastqFileName);
exit (1);
}
}
for(int i=0;i<numReads;i++) {
sff_read_header_t* rh = sff_read_header_read(sff_file_in->fp);
sff_read_t* rr = sff_read_read(sff_file_in->fp, sff_file_in->header, rh);
// optional - ignore the left & right adapter clipping by simply setting these values to 0
if (!adapterTrim) {
rh->clip_adapter_left = 0;
rh->clip_adapter_right = 0;
}
if (!fpq && !findRead) {
printf("Read header length: %d\n", rh->rheader_length);
printf("Read name length: %d\n", rh->name_length);
}
// Extract the row and column popsition info for this read
if (1 != ion_readname_to_rowcol(rh->name->s, &row, &col)) {
fprintf (stderr, "Error parsing read name: '%s'\n", rh->name->s);
continue;
}
if(1 == ion_readname_legacy(rh->name->s)) {
legacyReadName = true;
}
else {
legacyReadName = false;
}
if (!fpq && !findRead) {
printf("Read: %s (r%05d|c%05d) has %d bases\n",
(rh->name_length > 0 ? rh->name->s : "NONAME"),
row, col,
rh->n_bases);
printf("Clip left: %d qual: %d right: %d qual: %d\n",
rh->clip_adapter_left, rh->clip_qual_left,
rh->clip_adapter_right, rh->clip_qual_right);
printf("Flowgram values:\n");
}
if (findRead) {
if (row == readRow && col == readCol) {
//printf("Ionogram: ");
int i;
for(i=0;i<sff_file_in->header->flow_length;i++) {
printf("%.2lf ", (double)(rr->flowgram[i])/100.0);
}
printf("\n");
//// now print the bases - all the bases, not clipped!
//// these bases correspond to the raw flowgram data. in essence
//for (int b=0;b<r.number_of_bases;b++)
// fprintf(stdout, "%c", bases[b]);
//fprintf(stdout, "\n");
}
}
else if (fpq) {
bool ok = true;
if (keyPass) {
// if (r.number_of_bases > h.key_length) {
if ((int)rh->n_bases > hackkeylen) {
int b;
// for(b=0;b<h.key_length;b++) {
for(b=0;b<hackkeylen;b++) {
// if (key_sequence[b] != bases[b]) {
if (hackkey[b] != rr->bases->s[b]) {
ok = false;
break;
}
}
} else
ok = false; // not long enough
}
int clip_left_index = 0;
int clip_right_index = 0;
if (ok) {
//numKeypassedReads++;
// If force-clip option is set, we want to ensure the key gets trimmed
if (forceClip && rh->clip_adapter_left < 4)
rh->clip_adapter_left = hackkeylen+1;
if (ignoreLeftQualTrim)
clip_left_index = max (1, rh->clip_adapter_left);
else
clip_left_index = max (1, max (rh->clip_qual_left, rh->clip_adapter_left));
clip_right_index = min ((rh->clip_qual_right == 0 ? rh->n_bases:rh->clip_qual_right),
(rh->clip_adapter_right == 0 ? rh->n_bases:rh->clip_adapter_right));
if (debug)
fprintf (stdout, "debug clip: left = %d right = %d\n", clip_left_index, clip_right_index);
numKeypassedReads++;
if (clip_left_index > clip_right_index)
// Suppress output of zero-mer reads (left > right)
ok = false;
}
if (ok) {
//print id string
if (legacyFASTQName) {
fprintf (fpq, "@r%d|c%d\n", row, col);
}
else {
if (legacyReadName){
//Override legacy name
char runId[6] = {'\0'};
strncpy (runId, &rh->name->s[7], 5);
fprintf (fpq, "@%s:%d:%d\n", runId, row, col);
}
else {
//Copy name verbatim
fprintf (fpq, "@%s\n", rh->name->s);
}
}
//print bases
for (int b=clip_left_index-1;b<clip_right_index;b++)
fprintf(fpq, "%c", rr->bases->s[b]);
fprintf(fpq, "\n");
//print '+'
fprintf(fpq, "+\n");
//print quality scores
for (int b=clip_left_index-1;b<clip_right_index;b++)
fprintf(fpq, "%c", QualToFastQ((int)(rr->quality->s[b]),qual_offset));
fprintf(fpq, "\n");
}
}
else {
int f;
for(f=0;f<sff_file_in->header->flow_length;f++)
printf("%d ", rr->flowgram[f]);
printf("\nFlow index per base:\n");
unsigned int b;
for(b=0;b<rh->n_bases;b++)
printf("%d ", rr->flow_index[b]);
printf("\nBases called:\n");
for(b=0;b<rh->n_bases;b++)
printf("%c", rr->bases->s[b]);
printf("\nQuality scores:\n");
for(b=0;b<rh->n_bases;b++)
printf("%d ", rr->quality->s[b]);
printf("\nDone with this read\n\n");
}
sff_read_header_destroy(rh);
sff_read_destroy(rr);
}
// debug print - keypass reads written to the fastq file
if (fpq) {
static char *printkey = "All";
if (keyPass) printkey = hackkey;
fprintf (stdout, "Keypass Reads(%s) = %d\n", printkey, numKeypassedReads);
fprintf (stdout, "Total Reads = %d\n", numReads);
fprintf (stdout, "Percentage = %.2f%%\n", ((float) numKeypassedReads/ (float) numReads) * 100.0);
}
sff_fclose(sff_file_in);
if (fpq)
fclose(fpq);
}
else {
perror (sffFileName);
exit (1);
}
return 0;
}
// TODO: should we change the header:
// - must trake index_length
// - assumes row-major order
sff_index_t*
sff_index_create(sff_file_t *fp_in, sff_header_t *fp_out_header, int32_t num_rows, int32_t num_cols, int32_t type)
{
int64_t len = 0;
int32_t i, prev_row, prev_col, row, col;
sff_index_t *idx;
sff_t *sff;
uint64_t fp_in_start, prev_pos;
idx = sff_index_init();
idx->num_rows = num_rows;
idx->num_cols = num_cols;
idx->type = type;
// alloc
switch(type) {
case SFF_INDEX_ROW_ONLY:
len = 1 + idx->num_rows;
idx->offset = ion_malloc(len * sizeof(uint64_t), __func__, "idx->offset");
break;
case SFF_INDEX_ALL:
len = 1 + (idx->num_rows * idx->num_cols);
idx->offset = ion_malloc(len * sizeof(uint64_t), __func__, "idx->offset");
break;
default:
ion_error(__func__, "this index type is currently not supported", Exit, OutOfRange);
}
// save where the sff entries started
prev_pos = fp_in_start = ftell(fp_in->fp);
if(-1L == fp_in_start) {
ion_error(__func__, "ftell", Exit, ReadFileError);
}
// go through the input file
i = 0;
prev_row = prev_col = 0;
while(NULL != (sff = sff_read(fp_in))) {
// out of range
if(len-1 <= i) {
ion_error(__func__, "bug encountered", Exit, OutOfRange);
}
// get the row/col co-ordinates
if(0 == ion_readname_to_rowcol(sff->rheader->name->s, &row, &col)) {
ion_error(__func__, "could not understand the read name", Exit, OutOfRange);
}
// assumes row-major order, skips over reads that are not present
if(row < prev_row || (row == prev_row && col < prev_col)) {
ion_error(__func__, "SFF file was not sorted in row-major order", Exit, OutOfRange);
}
while(row != prev_row || col != prev_col) {
// add in empty entry
switch(type) {
case SFF_INDEX_ROW_ONLY:
if(0 == prev_col) { // first column
idx->offset[i] = UINT64_MAX;
// do not increment i, since we only do this when moving to a new row
}
break;
case SFF_INDEX_ALL:
// all rows and columns
idx->offset[i] = UINT64_MAX;
i++;
break;
default:
ion_error(__func__, "this index type is currently not supported", Exit, OutOfRange);
}
if(len-1 <= i) {
ion_error(__func__, "x/y was out of range", Exit, OutOfRange);
}
prev_col++;
if(prev_col == idx->num_cols) {
// new row
prev_col = 0;
prev_row++;
if(SFF_INDEX_ROW_ONLY == type) {
i++;
}
}
}
// add to the index
switch(type) {
case SFF_INDEX_ROW_ONLY:
if(0 == col) { // first column
idx->offset[i] = prev_pos;
}
else if(0 < col && UINT64_MAX == idx->offset[i]) {
idx->offset[i] = prev_pos;
// do not move onto the next
}
break;
case SFF_INDEX_ALL:
// all rows and columns
idx->offset[i] = prev_pos;
i++;
break;
default:
ion_error(__func__, "this index type is currently not supported", Exit, OutOfRange);
}
prev_row = row;
prev_col = col;
// destroy
sff_destroy(sff);
// next
prev_col++;
if(prev_col == idx->num_cols) {
// new row
prev_col = 0;
prev_row++;
if(SFF_INDEX_ROW_ONLY == type) {
i++;
}
}
prev_pos = ftell(fp_in->fp);
if(-1L == prev_pos) {
ion_error(__func__, "ftell", Exit, ReadFileError);
}
}
// get the last offset
idx->offset[len-1] = prev_pos;
// update the index offset in the header
fp_out_header->index_offset = fp_in_start; // insert between the header and sff entries
// update the index length in the header
fp_out_header->index_length = sff_index_length(idx);
// update the offsets based on the index length
for(i=0;i<len;i++) {
if(UINT64_MAX != idx->offset[i]) {
idx->offset[i] += fp_out_header->index_length;
}
}
return idx;
}
void UnpackOnLoad(Alignment *rai, const InputStructures &global_context)
{
// No need to waste time if the read is filtered
if (rai->filtered)
return;
rai->is_reverse_strand = rai->alignment.IsReverseStrand();
// Parse read name, run id & flow order index
rai->runid.clear();
if (not rai->alignment.Name.empty()) {
rai->well_rowcol.resize(2);
ion_readname_to_rowcol(rai->alignment.Name.c_str(), &rai->well_rowcol[0], &rai->well_rowcol[1]);
// extract runid while we are at it
rai->runid = rai->alignment.Name.substr(0,rai->alignment.Name.find(":"));
}
if (rai->runid.empty()){
cerr << "WARNING: Unable to determine run id of read " << rai->alignment.Name << endl;
rai->filtered = true;
return;
}
std::map<string,int>::const_iterator fo_it = global_context.flow_order_index_by_run_id.find(rai->runid);
if (fo_it == global_context.flow_order_index_by_run_id.end()){
cerr << "WARNING: No matching flow oder found for read " << rai->alignment.Name << endl;
rai->filtered = true;
return;
}
rai->flow_order_index = fo_it->second;
const ion::FlowOrder & flow_order = global_context.flow_order_vector.at(rai->flow_order_index);
// Retrieve measurements from ZM tag
vector<int16_t> quantized_measurements;
if (not rai->alignment.GetTag("ZM", quantized_measurements)) {
cerr << "ERROR: Normalized measurements ZM:tag is not present in read " << rai->alignment.Name << endl;
exit(1);
}
if ((int)quantized_measurements.size() > global_context.num_flows_by_run_id.at(rai->runid)) {
cerr << "ERROR: Normalized measurements ZM:tag length " << quantized_measurements.size()
<< " exceeds flow order length " << global_context.num_flows_by_run_id.at(rai->runid)
<<" in read " << rai->alignment.Name << endl;
exit(1);
}
rai->measurements.assign(global_context.num_flows_by_run_id.at(rai->runid), 0.0);
for (size_t counter = 0; counter < quantized_measurements.size(); ++counter)
rai->measurements[counter] = (float)quantized_measurements[counter]/256;
rai->measurements_length = quantized_measurements.size();
// Retrieve phasing parameters from ZP tag
if (not rai->alignment.GetTag("ZP", rai->phase_params)) {
cerr << "ERROR: Phasing Parameters ZP:tag is not present in read " << rai->alignment.Name << endl;
exit(1);
}
if (rai->phase_params.size() != 3) {
cerr << "ERROR: Phasing Parameters ZP:tag does not have 3 phase parameters in read " << rai->alignment.Name << endl;
exit(1);
}
if (rai->phase_params[0] < 0 or rai->phase_params[0] > 1 or rai->phase_params[1] < 0 or rai->phase_params[1] > 1
or rai->phase_params[2] < 0 or rai->phase_params[2] > 1) {
cerr << "ERROR: Phasing Parameters ZP:tag outside of [0,1] range in read " << rai->alignment.Name << endl;
exit(1);
}
rai->phase_params[2] = 0.0f; // ad-hoc corrector: zero droop
// Populate read_bases (bases without rev-comp on reverse-mapped reads) and flow_index
rai->read_bases = rai->alignment.QueryBases;
if (rai->is_reverse_strand)
RevComplementInPlace(rai->read_bases);
if (rai->read_bases.empty()){
cerr << "WARNING: Ignoring length zero read " << rai->alignment.Name << endl;
rai->filtered = true;
return;
}
// Unpack alignment
rai->pretty_aln.reserve(global_context.num_flows_by_run_id.at(rai->runid));
UnpackAlignmentInfo(rai);
if (rai->is_reverse_strand)
rai->start_sc = rai->right_sc;
else
rai->start_sc = rai->left_sc;
// Generate flow index
rai->start_flow = 0;
if (not rai->alignment.GetTag("ZF", rai->start_flow)) {
uint8_t start_flow_byte = 0;
if (not rai->alignment.GetTag("ZF", start_flow_byte)) {
cerr << "ERROR: Start Flow ZF:tag not found in read " << rai->alignment.Name << endl;
exit(1);
}
rai->start_flow = (int)start_flow_byte;
}
if (rai->start_flow == 0) {
cerr << "WARNING: Start Flow ZF:tag has zero value in read " << rai->alignment.Name << endl;
rai->filtered = true;
return;
}
CreateFlowIndex(rai, flow_order);
if (global_context.resolve_clipped_bases) {
// Increment start flow to first aligned base
rai->start_flow = rai->flow_index[rai->start_sc];
}
// Check validity of input arguments
if (rai->start_flow < 0 or rai->start_flow >= global_context.num_flows_by_run_id.at(rai->runid)) {
cerr << "ERROR: Start flow outside of [0,num_flows) range in read " << rai->alignment.Name << endl;
cerr << "Start flow: " << rai->start_flow << " Number of flows: " << global_context.flow_order_vector.at(rai->flow_order_index).num_flows();
exit(1);
}
// Retrieve read group name & generate prefix flow
if (not rai->alignment.GetTag("RG",rai->read_group)) {
cerr << "WARNING: No read group found in read " << rai->alignment.Name << endl;
// No big problem, we'll just have to solve the prefix like it's 2013!
rai->read_group.clear();
}
// Get read prefix - hard clipped start of the read: [KS][ZT][ZE]
rai->prefix_flow = -1;
std::map<string,string>::const_iterator key_it = global_context.key_by_read_group.find(rai->read_group);
if (key_it != global_context.key_by_read_group.end()) {
rai->prefix_bases = key_it->second;
string temp_zt, temp_ze;
if (rai->alignment.GetTag("ZT", temp_zt))
rai->prefix_bases += temp_zt;
if (rai->alignment.GetTag("ZE", temp_ze))
rai->prefix_bases += temp_ze;
if (not rai->prefix_bases.empty())
GetPrefixFlow(rai, rai->prefix_bases, flow_order);
}
// Check consistency of prefix_flow and start_flow - maybe we don't have all info about hard clipped bases
if (rai->prefix_flow >= 0) {
int check_start_flow = rai->prefix_flow;
while (check_start_flow < flow_order.num_flows() and flow_order.nuc_at(check_start_flow) != rai->read_bases.at(0))
check_start_flow++;
if (check_start_flow != rai->start_flow) {
rai->prefix_flow = -1;
rai->prefix_bases.clear();
}
}
}