//==================================================================
void Parser::doParse( DUT::MemFile &file )
{
char lineBuff[1024];
Section curSection = SEC_UNDEF;
int lineCnt = 0;
while ( file.ReadTextLine( lineBuff, sizeof(lineBuff) ) )
{
char lineWork[1024];
strcpy_s( lineWork, lineBuff );
stripComments( lineWork );
DUT::StrStripBeginEndWhite( lineWork );
if NOT( lineWork[0] )
{
++lineCnt;
continue;
}
try
{
if ( handleShaderTypeDef( lineWork, curSection ) )
{
++lineCnt;
continue;
}
if ( 0 == strcasecmp( lineWork, ".data" ) )
curSection = SEC_DATA;
else
if ( 0 == strcasecmp( lineWork, ".code" ) )
curSection = SEC_CODE;
else
if ( curSection == SEC_DATA )
{
parseDataLine( lineWork, lineCnt );
}
else
if ( curSection == SEC_CODE )
{
parseCodeLine( lineWork, lineCnt );
}
}
catch ( ... )
{
printf( "For shader '%s' at line %i\n%i) %s\n", mpName, lineCnt+1, lineCnt+1, lineBuff );
throw;
}
++lineCnt;
}
if ( mpShader->mType == SVM::Shader::TYPE_UNKNOWN )
onError( "Shader type undefined !" );
}
int ReplayRobot::getSensors(sensor* s, int sensornumber){
assert(sensornumber == (sensorEnd-sensorStart + 1));
if(!parseDataLine(sensors,f)){
cout << "ReplayRobot: no datafile in file" << endl;
}else{
sensors=sensors.rows(sensorStart, sensorEnd);
}
sensors.convertToBuffer(s,sensorEnd-sensorStart + 1);
return sensorEnd-sensorStart + 1;
};
void Genome::read ( std::ifstream& input ) {
log("Genome: reading file...");
unsigned int L(1); // line counter
LockFreeQueue<std::vector<std::string>> q;
auto readerTask = [&q,&input]() {
debug("Starting reader thread...");
for (std::string line; getline(input, line);) {
q.push(strsplit(line, "\t"));
}
q.done();
debug("All data read and tokenized, closing reader rhread");
};
auto parserTask = [&q,&L,this]() {
debug("Starting parser thread...");
std::vector<std::string> splits;
while(q.pop(splits)) {
assume(parseDataLine(splits), "Could not parse data in line " + std::to_string(L), false);
L++;
}
debug("All tokens processed, closing parser thread");
};
// read and parse header lines sequentially
for (std::string line; getline(input, line);) {
// std::cout << "Line #" << L << std::endl;
if (line[0] == '@') {
assume(parseHeaderLine(line), "Could not parse header in line " + std::to_string(L), false);
}
else {
break; // header lines parsed, break and begin threaded processing
}
L++;
}
// init and start threads
std::thread readerThread(readerTask);
std::thread parserThread(parserTask);
// wait for threads to finish
readerThread.join();
parserThread.join();
if (multistrand != nullptr) {
log("Found " + std::to_string(multistrand->size()) + " strand switching events");
}
if (circular != nullptr) {
log("Found " + std::to_string(circular->size()) + " circular transcripts");
}
}
/*------------------------------------------------------------------*/
void AzSvDataS::readData_Large(const char *data_fn,
int expected_f_num,
/*--- output ---*/
AzSmat *m_feat,
int max_data_num)
{
const char *eyec = "AzSvDataS::readData_Large";
/*--- find the number of lines and the maximum line length ---*/
AzIntArr ia_line_len;
AzFile::scan(data_fn, 1024*1024, &ia_line_len, max_data_num+1); /* +1 for sparse indicator */
int data_num = ia_line_len.size();
int max_line_len = ia_line_len.max();
if (data_num <= 0) {
throw new AzException(AzInputNotValid, eyec, "Empty data");
}
AzBytArr ba_buff;
AzByte *buff = ba_buff.reset(max_line_len+256, 0); /* +256 just in case */
/*--- 1st line indicates sparse/dense ---*/
AzFile file(data_fn);
file.open("rb");
int line0_len = file.gets(buff, max_line_len);
AzBytArr s_line0(buff, line0_len);
int line_no = 0;
bool isSparse = false;
int f_num = if_sparse(s_line0, expected_f_num);
if (f_num > 0) {
isSparse = true;
--data_num; /* b/c 1st line is information */
if (data_num <= 0) {
throw new AzException(AzInputNotValid, eyec, "Empty sparse data file");
}
line_no = 1; /* first data line */
}
else {
f_num = expected_f_num;
if (f_num <= 0) {
const AzByte *line0 = s_line0.point();
f_num = countFeatures(line0, line0+line0_len);
}
if (f_num <= 0) {
throw new AzException(AzInputNotValid, eyec, "No feature in the first line");
}
file.seek(0); /* rewind to the beginning */
}
/*--- read features ---*/
if (max_data_num > 0) {
data_num = MIN(data_num, max_data_num);
}
m_feat->reform(f_num, data_num);
int dx;
for (dx = 0; dx < data_num; ++dx, ++line_no) {
int len = ia_line_len.get(line_no);
file.readBytes(buff, len);
buff[len] = '\0'; /* to make it a C string */
if (isSparse) {
parseDataLine_Sparse(buff, len, f_num, data_fn, line_no+1,
m_feat, dx);
}
else {
parseDataLine(buff, len, f_num, data_fn, line_no+1,
m_feat, dx);
}
}
file.close();
}
/*------------------------------------------------------------------*/
void AzSvDataS::readData_Small(const char *data_fn,
int expected_f_num,
/*--- output ---*/
AzSmat *m_feat)
{
const char *eyec = "AzSvDataS::readData_Small";
AzFile file(data_fn);
file.open("rb");
int file_size = file.size();
AzBytArr bq_data;
AzByte *data = bq_data.reset(file_size+1, 0);
file.seekReadBytes(0, file_size, data);
file.close();
const char *data_end = (char *)data + file_size;
const char *wp = (char *)data;
AzIIarr iia_begin_end;
for ( ; ; ) {
if (wp >= data_end) break;
const char *next_wp = strchr(wp, '\n');
if (next_wp == NULL) next_wp = data_end;
iia_begin_end.put(Az64::ptr_diff(wp-(char *)data),
Az64::ptr_diff(next_wp-(char *)data));
wp = next_wp+1;
}
int data_num = iia_begin_end.size();
if (data_num <= 0) {
throw new AzException(AzInputNotValid, eyec, "Empty data");
}
bool isSparse = false;
int offs0, offs1;
iia_begin_end.get(0, &offs0, &offs1);
AzBytArr s_first_line(data+offs0, offs1-offs0);
int f_num = if_sparse(s_first_line, expected_f_num);
if (f_num > 0) {
isSparse = true;
--data_num; /* b/c 1st line is information */
if (data_num <= 0) {
throw new AzException(AzInputNotValid, eyec, "Empty sparse data file");
}
}
else {
f_num = expected_f_num;
if (f_num <= 0) {
f_num = countFeatures(data+offs0, data+offs1);
}
if (f_num <= 0) {
throw new AzException(AzInputNotValid, eyec, "No feature in the first line");
}
}
m_feat->reform(f_num, data_num);
/*--- read features ---*/
int dx;
for (dx = 0; dx < data_num; ++dx) {
int line_no = dx + 1;
if (isSparse) {
int offs0, offs1;
iia_begin_end.get(dx+1, &offs0, &offs1); /* +1 for 1st line */
parseDataLine_Sparse(data+offs0, offs1-offs0, f_num, data_fn,
line_no+1, /* "+1" for the header */
m_feat, dx);
}
else {
int offs0, offs1;
iia_begin_end.get(dx, &offs0, &offs1);
parseDataLine(data+offs0, offs1-offs0, f_num, data_fn, line_no,
m_feat, dx);
}
}
}