Beispiel #1
0
int64 CFileCollection::GetTimestamp(const char *pFilename)
{
	if(m_aFileDesc[0] == '\0')
	{
		int FilenameLength = str_length(pFilename);
		return ExtractTimestamp(pFilename+FilenameLength-m_FileExtLength-TIMESTAMP_LENGTH);
	}
	else
	{
		return ExtractTimestamp(pFilename+m_FileDescLength+1);
	}
}
Beispiel #2
0
link *GetData(std::string symbol)
{
    SOCKET s;
    char command[1000];

    WriteLog("Getting data for %s\n", symbol.c_str());

    GetSocket(&s);

    //get the historical data
    getCommand(command, symbol.c_str(), o);

    int rc = send(s, command, (int) strlen(command), 0);
    if (rc == SOCKET_ERROR)
    {
        WriteLog("error sending data to data feed, error Code %d\n", WSAGetLastError());
        throw DataException(__FILE__, __LINE__);
    }

    char *buffer = (char *) calloc(BUFFER_SIZE, sizeof(char));
    if (buffer == NULL)
    {
        WriteLog("calloc error for buffer\n");
        throw DataException(__FILE__, __LINE__);
    }

    char *bufferPtr = buffer;
    char *begin = buffer;
    char *end = NULL;

    link *head = NULL;

    unsigned long startTime = timeGetTime();
    unsigned long currentTime = 0;
    unsigned long totalTime = 0;
    link *currentLink = NULL;

    int numBlocks = 0;
    int totalRows = 0;

    bool exit = false;

    while (exit == false)
    {
        char oneLine[5120];
        char beginTimestamp[128];
        char endTimestamp[128];

        rc = recv(s, bufferPtr, (int) (BUFFER_SIZE - (bufferPtr - buffer)-1), 0);
        if (rc == 0 || rc == SOCKET_ERROR) {
            WriteLog("error receiving data from data feed, error code = %d\n", WSAGetLastError());
            throw DataException(__FILE__, __LINE__);
        }

        if (rc > BUFFER_SIZE - (bufferPtr - buffer)-1 || rc > BUFFER_SIZE) {
            WriteLog("Something f'ed with the data received, it's bigger than it should be\n");
            throw DataException(__FILE__, __LINE__);
        }

        begin = buffer;

        int numRows = 0;

        //validate each row, looking for the end of data message
        while ((end = strchr(begin, '\n')) != NULL)
        {
            //copy the line into oneLine so that we can check
            //for any errors.  I guess I could have just checked
            //it from the line itself but whatever, easier to debug
            memset(oneLine, 0, sizeof(oneLine));
            strncpy(oneLine, begin, end-begin + 1);
            //printf("%s\n", oneLine);

            //check for an error first
            if (isError(oneLine, symbol.c_str()) == true)
            {
                //end of data
                exit = true;
                break;
            }

            //always extract the timestamp to endTimestamp since
            //we don't know when it will end, but if it's the first
            //row of the block, then copy it to beginTimestamp
            ExtractTimestamp(oneLine, endTimestamp);
            if (numRows == 0)
                strcpy(beginTimestamp, endTimestamp);

            begin = end + 1;

            numRows++;
        }

        //if the very first line of the buffer is an !ENDMSG!,
        //then just break out of this loop
        if (buffer == begin)
        {
            break;
        }

        //create a new link
        link *newLink = (link *) calloc(1, sizeof(link));
        if (newLink == NULL)
        {
            WriteLog("Calloc error for newLink\n");
            throw DataException(__FILE__, __LINE__);
        }

        //create a new buffer for the new block that contains only the
        //valid data for that block
        int blockSize = begin - buffer + 1;

        newLink->buffer = (char *) calloc(blockSize, sizeof(char));
        if (newLink->buffer == NULL)
        {
            WriteLog("Calloc error for newLink->buffer\n");
            throw DataException(__FILE__, __LINE__);
        }

        //copy everything up to the beginning of begin, because
        //that is likely an uncompleted line of data
        memcpy(newLink->buffer, buffer, begin - buffer);

        strcpy(newLink->beginTimestamp, beginTimestamp);
        strcpy(newLink->endTimestamp, endTimestamp);

        if (head == NULL)
        {
            head = newLink;
            currentLink = head;
        }
        else
        {
            currentLink->next = newLink;
        }
        currentLink->numRows = numRows;
        currentLink = newLink;

        //move any other stragglers to the front of the buffer
        memmove(buffer, begin, strlen(begin));
        bufferPtr = buffer + strlen(begin);
        //clear everything else after that to prevent errors
        memset(bufferPtr, 0, BUFFER_SIZE - (bufferPtr - buffer));

        //stats gathering shit
        totalRows += numRows;
        currentTime = timeGetTime();
        totalTime = currentTime - startTime;

        numBlocks++;

        if (numBlocks % 100 == 0)
        {
            WriteLog("%s: reading rows=%d, total time = %d, rows/s = %.2f\n",
                     symbol.c_str(), totalRows, totalTime/1000, 1000.0* totalRows / totalTime);
        }
    }

    WriteLog("%s: FINAL: reading rows=%d, total time = %d, rows/s = %.2f, numBlocks = %d\n",
             symbol.c_str(), totalRows, totalTime/1000, 1000.0* totalRows / totalTime, numBlocks);

    free(buffer);
    closesocket(s);

    return head;
}