Ejemplo n.º 1
0
static
rc_t SFFLoaderFmtWriteDataFile(SFFLoaderFmt* self, const SRALoaderFile* file)
{
    rc_t rc = 0;

    while( rc == 0 ) {
        if( self->curr_read_number == 0 ) {
            if( (rc = SFFLoaderFmtReadCommonHeader(self, file)) == 0 ) {
                DEBUG_MSG (5, ("%s: Common header ok: %u reads\n", self->file_name, self->header.number_of_reads));
                DEBUG_MSG (8, ("%s: flow_chars: [%hu] %s\n", self->file_name, self->header.num_flows_per_read, self->flow_chars.data));
                DEBUG_MSG (8, ("%s: key_seq: [%hu] %s\n", self->file_name, self->header.key_length, self->key_seq.data));
            } else if( GetRCObject(rc) == (enum RCObject)rcData && GetRCState(rc) == rcIgnored ) {
                rc = 0;
                break;
            }
        }
        if( rc == 0 && self->header.number_of_reads != 0 && 
            (rc = SFFLoaderFmtSkipIndex(self, file)) == 0 &&
            (rc = SFFLoaderFmtReadDataHeader(self, file)) == 0 &&
            (rc = SFFLoaderFmtReadData(self, file)) == 0 ) {
            if( self->w454 ) {
                rc = SRAWriter454_WriteRead(self->w454, file, &self->name, &self->read, &self->quality,
                                         self->skip_signal ? NULL : &self->signal,
                                         self->skip_signal ? NULL : &self->position,
                                         self->read_header.clip_quality_left, self->read_header.clip_quality_right,
                                         self->read_header.clip_adapter_left, self->read_header.clip_adapter_right);
            } else {
                rc = SRAWriterIonTorrent_WriteRead(self->wIonTorrent, file, &self->name, &self->read, &self->quality,
                         self->skip_signal ? NULL : &self->signal,
                         self->skip_signal ? NULL : &self->position,
                         self->read_header.clip_quality_left, self->read_header.clip_quality_right,
                         self->read_header.clip_adapter_left, self->read_header.clip_adapter_right);
            }
            if( rc == 0 ) {
                ++self->curr_read_number;
            }
        }
        if( rc != 0 && (GetRCObject(rc) != rcTransfer && GetRCState(rc) != rcDone) ) {
            SRALoaderFile_LOG(file, klogErr, rc, "on or about read #$(i)", PLOG_U32(i), self->curr_read_number + 1);
        } else if( self->curr_read_number == self->header.number_of_reads ) {
            DEBUG_MSG(5, ("%s: done loading declared %u reads\n", self->file_name, self->curr_read_number));
            self->curr_read_number = 0;
            /* will skip indexes if they are at eof */
            if( (rc = SFFLoaderFmtSkipIndex(self, file)) == 0 ) {
                /* This should be the end of file and/or beginning of next */
                if( (rc = SFFLoaderFmt_ReadBlock(self, file, 0, "EOF", false)) == 0 ) {
                    if( self->file_buf == NULL ) {
                        DEBUG_MSG(5, ("%s: EOF detected\n", self->file_name));
                        self->index_correction = 0;
                        break;
                    }
                }
            }
        }
    }
    return rc;
}
Ejemplo n.º 2
0
static
rc_t FastqLoaderFmt_WriteData(FastqLoaderFmt* self, uint32_t argc, const SRALoaderFile* const argv[], int64_t* spots_bad_count)
{
    rc_t rc = 0;
    uint32_t i, g = 0;
    FastqFileInfo* files = NULL;
    bool done;
    static IlluminaSpot spot;
 
    if( (files = calloc(argc, sizeof(*files))) == NULL ) {
        rc = RC(rcSRA, rcFormatter, rcReading, rcMemory, rcInsufficient);
    }

    for(i = 0; rc == 0 && i < argc; i++) {
        ExperimentQualityType qType;
        FastqFileInfo* file = &files[i];

        file->file = argv[i];
        FileReadData_init(file->spot, false);
        FileReadData_init(&file->spot[1], false);
        if( (rc = SRALoaderFile_QualityScoringSystem(file->file, &qType)) == 0 &&
            (rc = SRALoaderFile_QualityEncoding(file->file, &file->qualEnc)) == 0 &&
            (rc = SRALoaderFile_AsciiOffset(file->file, &file->qualOffset)) == 0 ) {

            file->qualType = ILLUMINAWRITER_COLMASK_NOTSET;

            if( qType == eExperimentQualityType_Undefined ) {
                qType = self->processing->quality_type;
                file->qualOffset = self->processing->quality_offset;
            }
            switch(qType) {
                case eExperimentQualityType_LogOdds:
                case eExperimentQualityType_Other:
                    if( self->w454 != NULL || self->wIonTorrent != NULL ) {
                        rc = RC(rcSRA, rcFormatter, rcConstructing, rcParam, rcInvalid);
                        LOGERR(klogInt, rc, "quality type other than Phred is not supported for this PLATFORM");
                    }
                    file->qualMin = -40;
                    file->qualMax = 41;
                    file->qualType = ILLUMINAWRITER_COLMASK_QUALITY_LOGODDS1;
                    break;
                default:
                    SRALoaderFile_LOG(file->file, klogWarn, rc, 
                        "quality_scoring_system attribute not set for this file, using Phred as default", NULL);
                case eExperimentQualityType_Phred:
                    file->qualType = ILLUMINAWRITER_COLMASK_QUALITY_PHRED;
                    file->qualMin = 0;
                    file->qualMax = (self->wIllumina) ? 61: 127;
                    break;
            }
        }
    }
    do {
        done = true;
        for(i = 0; rc == 0 && i < argc; i++) {
            FastqFileInfo* file = &files[i];
            if( (rc = read_next_spot(self, file)) != 0 || !file->spot->ready ) {
                continue;
            }
            done = false;
#if _DEBUGGING
            {{
                FileReadData* ss = file->spot;
                do {
                    DEBUG_MSG(3, ("file-%u: name:'%s', bc:%s, rd:%i, flt:%hu, seq '%.*s', qual %u bytes\n",
                                  i + 1, ss->name.data, ss->barcode.data, ss->read.read_id, ss->read.filter,
                                  ss->read.seq.len, ss->read.seq.data, ss->read.qual.len));
                    if( ss == &file->spot[1]){ break; }
                    ss = file->spot[1].ready ? &file->spot[1] : NULL;
                } while( ss != NULL );
            }}
#endif
        }
        if( rc != 0 || done ) {
            break;
        }
        /* collect spot reads, matching by spot name
         * spot data may be split across multiple files
         */
        IlluminaSpot_Init(&spot);
        for(i = 0; rc == 0 && i < argc; i++) {
            FileReadData* fspot = files[i].spot[0].ready ? &files[i].spot[0] : NULL;
            while(rc == 0 && fspot != NULL ) {
                rc = IlluminaSpot_Add(&spot, &fspot->name, &fspot->barcode, &fspot->read);
                if( rc == 0 ) {
                    g = i;
                    fspot->ready = false;
                } else if( GetRCState(rc) == rcIgnored ) {
                    rc = 0;
                } else {
                    SRALoaderFile_LOG(files[i].file, klogErr, rc, "$(msg)", "msg=adding data to spot");
                }
                if( fspot == &files[i].spot[1]) { break; }
                fspot = files[i].spot[1].ready ? &files[i].spot[1] : NULL;
            }
        }
        if( rc == 0 ) {
            if( self->wIllumina != NULL ) {
                if( (rc = SRAWriterIllumina_Write(self->wIllumina, argv[0], &spot)) != 0 &&
                    GetRCTarget(rc) == rcFormatter && GetRCContext(rc) == rcValidating ) {
                    SRALoaderFile_LOG(files[g].file, klogWarn, rc, "$(msg) $(spot_name)", "msg=bad spot,spot_name=%.*s",
                                                spot.name->len, spot.name->data);
                    self->spots_bad_count++;
                    if( self->spots_bad_allowed < 0 ||
                        self->spots_bad_count <= self->spots_bad_allowed ) {
                        rc = 0;
                    }
                }
            } else if( spot.nreads != 1 ) {
                rc = RC(rcSRA, rcFormatter, rcReading, rcData, rcUnsupported);
                SRALoaderFile_LOG(files[g].file, klogErr, rc, "$(msg)", "msg=multiple reads for this platform");
            } else if( self->wIonTorrent != NULL ) {
                rc = SRAWriterIonTorrent_WriteRead(self->wIonTorrent, argv[0], spot.name,
                                                   spot.reads[0].seq, spot.reads[0].qual, NULL, NULL, 0, 0, 0, 0);
            } else {
                rc = SRAWriter454_WriteRead(self->w454, argv[0], spot.name,
                                            spot.reads[0].seq, spot.reads[0].qual, NULL, NULL, 0, 0, 0, 0);
            }
        }
    } while( rc == 0 );
    free(files);
    *spots_bad_count = self->spots_bad_count;
    return rc;
}