void Genome::read ( std::ifstream& input ) {
log("Genome: reading file...");
unsigned int L(1); // line counter
LockFreeQueue<std::vector<std::string>> q;
auto readerTask = [&q,&input]() {
debug("Starting reader thread...");
for (std::string line; getline(input, line);) {
q.push(strsplit(line, "\t"));
}
q.done();
debug("All data read and tokenized, closing reader rhread");
};
auto parserTask = [&q,&L,this]() {
debug("Starting parser thread...");
std::vector<std::string> splits;
while(q.pop(splits)) {
assume(parseDataLine(splits), "Could not parse data in line " + std::to_string(L), false);
L++;
}
debug("All tokens processed, closing parser thread");
};
// read and parse header lines sequentially
for (std::string line; getline(input, line);) {
// std::cout << "Line #" << L << std::endl;
if (line[0] == '@') {
assume(parseHeaderLine(line), "Could not parse header in line " + std::to_string(L), false);
}
else {
break; // header lines parsed, break and begin threaded processing
}
L++;
}
// init and start threads
std::thread readerThread(readerTask);
std::thread parserThread(parserTask);
// wait for threads to finish
readerThread.join();
parserThread.join();
if (multistrand != nullptr) {
log("Found " + std::to_string(multistrand->size()) + " strand switching events");
}
if (circular != nullptr) {
log("Found " + std::to_string(circular->size()) + " circular transcripts");
}
}
// -----------------------------------------------------------------------------
//
// -----------------------------------------------------------------------------
int CSVGrainDataReader::readHeaderOnly()
{
int err = 1;
char buf[kBufferSize];
std::ifstream in(m_FileName.c_str());
m_headerComplete = false;
if (!in.is_open())
{
std::cout << "CSV file could not be opened: " << m_FileName << std::endl;
return -100;
}
while (!in.eof() && !m_headerComplete)
{
::memset(buf, 0, kBufferSize);
in.getline(buf, kBufferSize);
parseHeaderLine(buf, kBufferSize);
}
return err;
}
static int appendByte(char c, HttpMessage *msg)
{
int ret = 1;
ParserData *pd = (ParserData *)msg->pri_data;
switch (pd->state) {
case HTTP_STARTLINE:
_append(c, msg);
if (c == '\r') {
pd->last_state = HTTP_STARTLINE;
pd->state = HTTP_CR;
}
break;
case HTTP_HEADERS:
_append(c, msg);
if (c == '\r') {
pd->last_state = HTTP_HEADERS;
pd->state = HTTP_CR;
}
break;
case HTTP_CR:
_append(c, msg);
if (c == '\n') {
pd->state = HTTP_CRLF;
// blank line ?
if (!strncmp(pd->linedata, "\r\n", 2)) {
if (_getmsglen(msg) > 0) {
pd->state = HTTP_BODY;
}
else {
pd->state = HTTP_COMPLETE;
}
}
}
else {
pd->state = pd->last_state;
}
break;
case HTTP_CRLF:
if (isblank(c)) {
char *p;
// LSW
pd->state = HTTP_LSW;
p = strstr(pd->linedata, "\r\n");
if (p) {
pd->line_datasize = p-pd->linedata;
}
}
else {
if (pd->last_state == HTTP_STARTLINE) {
parseStartLine(msg);
}
else {
parseHeaderLine(msg);
}
pd->line_datasize = 0;
_append(c, msg);
if (c == '\r')
pd->state = HTTP_CR;
else
pd->state = HTTP_HEADERS;
}
break;
case HTTP_LSW:
if (!isblank(c)) {
pd->state = pd->last_state;
_append(c, msg);
}
break;
case HTTP_BODY:
msg->body = (char *)realloc(msg->body, msg->bodylen+1);
msg->body[msg->bodylen] = c;
msg->bodylen++;
if (_getmsglen(msg) == msg->bodylen)
pd->state = HTTP_COMPLETE;
break;
case HTTP_COMPLETE:
break;
}
return ret;
}
// -----------------------------------------------------------------------------
//
// -----------------------------------------------------------------------------
int CSVGrainDataReader::readFile()
{
int err = 1;
char buf[kBufferSize];
m_headerComplete = false;
std::ifstream in(m_FileName.c_str());
if (!in.is_open())
{
std::cout << "Ang file could not be opened: " << m_FileName << std::endl;
return -100;
}
::memset(buf, 0, kBufferSize);
in.getline(buf, kBufferSize);
if (sscanf(buf, "%lu", &m_NumberOfElements) != 1)
{
std::cout << "First Line of file not parsed." << std::endl;
}
else
{
std::cout << "m_NumberOfElements = " << m_NumberOfElements << std::endl;
}
::memset(buf, 0, kBufferSize);
in.getline(buf, kBufferSize);
parseHeaderLine(buf, kBufferSize);
// Delete any currently existing pointers
deletePointers();
initPointers(m_NumberOfElements);
if (NULL == m_GrainId ||
NULL == m_Phi1 ||
NULL == m_Phi ||
NULL == m_Phi2 ||
NULL == m_EquivDiam ||
NULL == m_NumNeighbors ||
NULL == m_B_Over_A ||
NULL == m_C_Over_A ||
m_Omega3 == NULL ||
m_OutsideBoundingBox == NULL)
{
return -1;
}
size_t counter = 0;
for(size_t row = 0; row < m_NumberOfElements && in.eof() == false; ++row)
{
in.getline(buf, kBufferSize);
this->readData(buf, static_cast<int>(row), counter);
// Read the next line of data
++counter;
}
if (counter != m_NumberOfElements && in.eof() == true)
{
std::cout << "Premature End Of File reached.\n" << m_FileName
<< "\nNumRows=" << m_NumberOfElements
<< "\ncounter=" << counter << " m_NumberOfElements=" << m_NumberOfElements
<< "\nTotal Data Points Read=" << counter << std::endl;
}
return err;
}
HttpParserImpl::ParseState HttpParserImpl::parseHttp(const char*& cur_pos, const char* end)
{
const char* line = nullptr;
const char* line_end = nullptr;
bool b_line = false;
if(HTTP_READ_LINE == read_state_)
{// try to get status line
while ((b_line = getLine(cur_pos, end, line, line_end)) && line == line_end && str_buf_.empty())
;
if(b_line && (line != line_end || !str_buf_.empty())) {
if(!parseStartLine(line, line_end)) {
read_state_ = HTTP_READ_ERROR;
return PARSE_STATE_ERROR;
}
read_state_ = HTTP_READ_HEAD;
} else {
// need more data
if(saveData(cur_pos, end) != KUMA_ERROR_NOERR) {
return PARSE_STATE_ERROR;
}
cur_pos = end; // all data was consumed
return PARSE_STATE_CONTINUE;
}
}
if(HTTP_READ_HEAD == read_state_)
{
while ((b_line = getLine(cur_pos, end, line, line_end)))
{
if(line == line_end && bufferEmpty())
{// blank line, header completed
onHeaderComplete();
if(paused_) {
return PARSE_STATE_CONTINUE;
}
if(hasBody() && !upgrade_) {
read_state_ = HTTP_READ_BODY;
} else {
read_state_ = HTTP_READ_DONE;
onComplete();
return PARSE_STATE_DONE;
}
break;
}
parseHeaderLine(line, line_end);
}
if(HTTP_READ_HEAD == read_state_)
{// need more data
if(saveData(cur_pos, end) != KUMA_ERROR_NOERR) {
return PARSE_STATE_ERROR;
}
cur_pos = end; // all data was consumed
return PARSE_STATE_CONTINUE;
}
}
if(HTTP_READ_BODY == read_state_ && cur_pos < end)
{// try to get body
if(is_chunked_) {
return parseChunk(cur_pos, end);
} else {
uint32_t cur_len = uint32_t(end - cur_pos);
if(has_content_length_ && (content_length_ - total_bytes_read_) <= cur_len)
{// data enough
const char* notify_data = cur_pos;
uint32_t notify_len = content_length_ - total_bytes_read_;
cur_pos += notify_len;
total_bytes_read_ = content_length_;
read_state_ = HTTP_READ_DONE;
KUMA_ASSERT(!destroy_flag_ptr_);
bool destroyed = false;
destroy_flag_ptr_ = &destroyed;
if(cb_data_) cb_data_(notify_data, notify_len);
if(destroyed) {
return PARSE_STATE_DESTROY;
}
destroy_flag_ptr_ = nullptr;
onComplete();
return PARSE_STATE_DONE;
}
else
{// need more data, or read untill EOF
const char* notify_data = cur_pos;
total_bytes_read_ += cur_len;
cur_pos = end;
if(cb_data_) cb_data_(notify_data, cur_len);
return PARSE_STATE_CONTINUE;
}
}
}
return HTTP_READ_DONE == read_state_?PARSE_STATE_DONE:PARSE_STATE_CONTINUE;
}
// initializes a SAM file
// will consume the SAM header and prepare for fetching reads from the file
// returns false on failure
bool ReadDataFile_SAM::init(void)
{
bool ret;
if (m_file_state != FILE_OK)
{
// file failed to open
return false;
}
// read the header section
do {
ret = readLine();
if (!ret)
{
return false;
}
if (linebuf[0] != '@')
{
break;
}
char *delim;
delim = strchr(linebuf, '\t');
if (delim)
{
if (strncmp(linebuf, "@HD\t", strlen("@HD\t")) == 0)
{
ret = parseHeaderLine(delim + 1);
if (!ret)
{
return false;
}
} else if (strncmp(linebuf, "@SQ\t", strlen("@SQ\t")) == 0)
{
ret = parseReferenceSequenceLine(delim + 1);
if (!ret)
{
return false;
}
} else if (strncmp(linebuf, "@RG\t", strlen("@RG\t")) == 0) {
// ignored
continue;
} else if (strncmp(linebuf, "@PG\t", strlen("@PG\t")) == 0) {
// ignored
continue;
} else if (strncmp(linebuf, "@CO\t", strlen("@CO\t")) == 0) {
// ignored
continue;
} else {
log_warning(stderr, "SAM file warning: unknown header at line %d\n", numLines);
}
} else {
log_warning(stderr, "SAM file warning: malformed line %d\n", numLines);
}
} while(linebuf[0] == '@');
// rewind the last line
rewindLine();
return true;
}
