int64 CFileCollection::GetTimestamp(const char *pFilename) { if(m_aFileDesc[0] == '\0') { int FilenameLength = str_length(pFilename); return ExtractTimestamp(pFilename+FilenameLength-m_FileExtLength-TIMESTAMP_LENGTH); } else { return ExtractTimestamp(pFilename+m_FileDescLength+1); } }
link *GetData(std::string symbol) { SOCKET s; char command[1000]; WriteLog("Getting data for %s\n", symbol.c_str()); GetSocket(&s); //get the historical data getCommand(command, symbol.c_str(), o); int rc = send(s, command, (int) strlen(command), 0); if (rc == SOCKET_ERROR) { WriteLog("error sending data to data feed, error Code %d\n", WSAGetLastError()); throw DataException(__FILE__, __LINE__); } char *buffer = (char *) calloc(BUFFER_SIZE, sizeof(char)); if (buffer == NULL) { WriteLog("calloc error for buffer\n"); throw DataException(__FILE__, __LINE__); } char *bufferPtr = buffer; char *begin = buffer; char *end = NULL; link *head = NULL; unsigned long startTime = timeGetTime(); unsigned long currentTime = 0; unsigned long totalTime = 0; link *currentLink = NULL; int numBlocks = 0; int totalRows = 0; bool exit = false; while (exit == false) { char oneLine[5120]; char beginTimestamp[128]; char endTimestamp[128]; rc = recv(s, bufferPtr, (int) (BUFFER_SIZE - (bufferPtr - buffer)-1), 0); if (rc == 0 || rc == SOCKET_ERROR) { WriteLog("error receiving data from data feed, error code = %d\n", WSAGetLastError()); throw DataException(__FILE__, __LINE__); } if (rc > BUFFER_SIZE - (bufferPtr - buffer)-1 || rc > BUFFER_SIZE) { WriteLog("Something f'ed with the data received, it's bigger than it should be\n"); throw DataException(__FILE__, __LINE__); } begin = buffer; int numRows = 0; //validate each row, looking for the end of data message while ((end = strchr(begin, '\n')) != NULL) { //copy the line into oneLine so that we can check //for any errors. I guess I could have just checked //it from the line itself but whatever, easier to debug memset(oneLine, 0, sizeof(oneLine)); strncpy(oneLine, begin, end-begin + 1); //printf("%s\n", oneLine); //check for an error first if (isError(oneLine, symbol.c_str()) == true) { //end of data exit = true; break; } //always extract the timestamp to endTimestamp since //we don't know when it will end, but if it's the first //row of the block, then copy it to beginTimestamp ExtractTimestamp(oneLine, endTimestamp); if (numRows == 0) strcpy(beginTimestamp, endTimestamp); begin = end + 1; numRows++; } //if the very first line of the buffer is an !ENDMSG!, //then just break out of this loop if (buffer == begin) { break; } //create a new link link *newLink = (link *) calloc(1, sizeof(link)); if (newLink == NULL) { WriteLog("Calloc error for newLink\n"); throw DataException(__FILE__, __LINE__); } //create a new buffer for the new block that contains only the //valid data for that block int blockSize = begin - buffer + 1; newLink->buffer = (char *) calloc(blockSize, sizeof(char)); if (newLink->buffer == NULL) { WriteLog("Calloc error for newLink->buffer\n"); throw DataException(__FILE__, __LINE__); } //copy everything up to the beginning of begin, because //that is likely an uncompleted line of data memcpy(newLink->buffer, buffer, begin - buffer); strcpy(newLink->beginTimestamp, beginTimestamp); strcpy(newLink->endTimestamp, endTimestamp); if (head == NULL) { head = newLink; currentLink = head; } else { currentLink->next = newLink; } currentLink->numRows = numRows; currentLink = newLink; //move any other stragglers to the front of the buffer memmove(buffer, begin, strlen(begin)); bufferPtr = buffer + strlen(begin); //clear everything else after that to prevent errors memset(bufferPtr, 0, BUFFER_SIZE - (bufferPtr - buffer)); //stats gathering shit totalRows += numRows; currentTime = timeGetTime(); totalTime = currentTime - startTime; numBlocks++; if (numBlocks % 100 == 0) { WriteLog("%s: reading rows=%d, total time = %d, rows/s = %.2f\n", symbol.c_str(), totalRows, totalTime/1000, 1000.0* totalRows / totalTime); } } WriteLog("%s: FINAL: reading rows=%d, total time = %d, rows/s = %.2f, numBlocks = %d\n", symbol.c_str(), totalRows, totalTime/1000, 1000.0* totalRows / totalTime, numBlocks); free(buffer); closesocket(s); return head; }