static int parseDate( const char *str, TA_Timestamp *timestamp ) { TA_Timestamp temp; int day, month, year; TA_RetCode retCode; int found; static const char *monthStr[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; if( !timestamp || !str ) return 0; /* Parse day */ if( sscanf( str, "%d", &day ) != 1 ) return 0; /* Parse Month */ found = 0; month = 0; while( !found && (month < 12) ) { if( strstr(str, monthStr[month] ) ) found = 1; else month++; } if( !found ) return 0; /* Parse Year */ str = strchr( str, '-' ); if( !str || str[1] == '\0' ) return 0; str = strchr( str+1, '-' ); if( !str || str[1] == '\0' ) return 0; if( sscanf( str+1, "%d", &year ) != 1 ) return 0; /* Build timestamp */ TA_SetDefault( &temp ); retCode = TA_SetDate( year, month+1, day, &temp ); if( retCode != TA_SUCCESS ) return 0; retCode = TA_TimestampCopy( timestamp, &temp ); if( retCode != TA_SUCCESS ) return 0; return 1; }
/* Check if t0 is within the provided [t1..t2] range (inclusive check) */ static unsigned int TA_DateWithinRange( unsigned int year, unsigned int month, unsigned int day, const TA_Timestamp *t1, const TA_Timestamp *t2 ) { TA_Timestamp stamp; const TA_Timestamp *lowBorder; const TA_Timestamp *highBorder; /* Inverse t1 and t2 if not chronilogical order. */ if( TA_TimestampLess( t2, t1 ) ) { lowBorder = t2; highBorder = t1; } else { lowBorder = t1; highBorder = t2; } /* Build a timestamp for the date to be check */ TA_SetDate( year, month, day, &stamp ); /* Check if exactly on boundary */ if( TA_TimestampEqual( &stamp, lowBorder ) && TA_TimestampEqual( &stamp, highBorder ) ) { return 1; } /* Check if within range. */ if( TA_TimestampGreater( &stamp, lowBorder ) && TA_TimestampLess( &stamp, highBorder ) ) { return 1; } return 0; /* Out-of-range */ }
TA_RetCode TA_ReadOp_Do( TA_FileHandle *fileHandle, const TA_ReadOpInfo *readOpInfo, TA_Period period, const TA_Timestamp *start, const TA_Timestamp *end, unsigned int minimumNbBar, TA_Field fieldToAlloc, TA_ParamForAddData *paramForAddData, unsigned int *nbBarAdded ) { TA_PROLOG TA_RetCode retCode; TA_EstimateInfo estimationInfo; unsigned int nbElementToAllocate; unsigned int memoryNeeded; /* Boolean */ unsigned int timeNeeded; /* Boolean */ TA_Real *arrayReal[TA_REAL_ARRAY_SIZE]; TA_Integer *arrayInteger[TA_INTEGER_ARRAY_SIZE]; TA_Timestamp *timestamp; TA_Real *openBeg, *highBeg, *lowBeg, *closeBeg; TA_Integer *volumeBeg, *openInterestBeg; TA_Timestamp *timestampBeg; TA_Timestamp tmpTimestamp; TA_ReadOp op; TA_Field fieldToProcess; unsigned int year, month, day, hour, min, sec; TA_Integer curOp; unsigned int nbTotalByteDone, nbTotalBarDone; unsigned int nbBarAddedInTheBlock; char monthChar[4]; char cnvtArray[CNVT_ARRAY_SIZE]; unsigned int cnvtArrayIdx; unsigned int nbByteToAllocReal; unsigned int nbByteToAllocInteger; unsigned int fileSize; unsigned int skipField; unsigned int lineToSkip; unsigned int nbByteRead; unsigned int nbLetter; TA_Real lastValidClose; const char *car; register TA_Real tmpReal; register TA_Integer tmpInt; register unsigned int tmpIdx; register unsigned int nbCharToRead; unsigned int lastOpFieldIncremented; TA_TRACE_BEGIN( TA_PriceBarRead ); /* Initialization of local variables. */ openBeg = highBeg = lowBeg = closeBeg = NULL; timestampBeg = NULL; volumeBeg = openInterestBeg = NULL; timestamp = NULL; retCode = TA_SUCCESS; lastValidClose = 0.0; fieldToProcess = readOpInfo->fieldProvided & fieldToAlloc; if( (fieldToProcess & fieldToAlloc) != fieldToAlloc ) { /* Nothing to read because not all the requested * fields are provided by this data source! */ return TA_SUCCESS; } /* Estimate the initial amount of memory to allocate. */ fileSize = TA_FileSize( fileHandle ); retCode = TA_EstimateAllocInit( start, end, period, minimumNbBar, 2048, &estimationInfo, &nbElementToAllocate ); if( retCode != TA_SUCCESS ) { TA_TRACE_RETURN( retCode ); } if( nbElementToAllocate == 0 ) { TA_TRACE_RETURN( TA_SUCCESS ); /* Nothing to read!? Just return... */ } memset( arrayInteger, 0, sizeof( arrayInteger ) ); memset( arrayReal, 0, sizeof( arrayReal ) ); /* Set the date/time pointers to where the information will be stored. */ arrayInteger[TA_HOUR_IDX] = (TA_Integer *)&hour; arrayInteger[TA_MIN_IDX] = (TA_Integer *)&min; arrayInteger[TA_SEC_IDX] = (TA_Integer *)&sec; arrayInteger[TA_MONTH_IDX] = (TA_Integer *)&month; arrayInteger[TA_YEAR_IDX] = (TA_Integer *)&year; arrayInteger[TA_DAY_IDX] = (TA_Integer *)&day; /* Set default time/date. */ year = 1900; month = day = 1; hour = 23; min = sec = 59; /* Check if the processing of the time will be needed. */ timeNeeded = isTimeNeeded( readOpInfo->arrayReadOp); /* 'car' always point to the character being currently handled. */ nbByteRead = 0; car = TA_FileSeqRead( fileHandle, &nbByteRead ); if( (car == NULL) || (nbByteRead == 0) ) return TA_SUCCESS; /* End of file! */ --nbByteRead; nbTotalByteDone = 0; nbTotalBarDone = 0; nbBarAddedInTheBlock = 0; memoryNeeded = 1; curOp = 0; skipField = 0; monthChar[3] = '\0'; /* When requested, skip header lines. */ lineToSkip = readOpInfo->nbHeaderLineToSkip; while( lineToSkip-- ) { while( *car != '\n' ) { GET_CHAR; if( car == NULL ) goto exit_loops; } } line_loop: /* Always jump here when end-of-line is found (EOL). */ /* If curOp != 0, the last operations are canceled. */ REVERT_OPERATIONS; curOp = 0; lastOpFieldIncremented = 0; /* Start over a new line. */ if( memoryNeeded ) { /* Allocate the memory. */ nbByteToAllocReal = nbElementToAllocate * sizeof( TA_Real ); nbByteToAllocInteger = nbElementToAllocate * sizeof( TA_Integer ); timestamp = (TA_Timestamp *)TA_Malloc( nbElementToAllocate * sizeof( TA_Timestamp ) ); timestampBeg = timestamp; if( !timestampBeg ) { retCode = TA_ALLOC_ERR; goto exit_loops; } #define TA_ALLOC_MEM(upperc,lowerc,typepar) \ { \ if( fieldToProcess & TA_##upperc ) \ { \ lowerc##Beg = (TA_##typepar *)TA_Malloc( nbByteToAlloc##typepar ); \ array##typepar[TA_##upperc##_IDX] = lowerc##Beg; \ if( !lowerc##Beg ) \ { \ retCode = TA_ALLOC_ERR; \ goto exit_loops; \ } \ } \ } TA_ALLOC_MEM( OPEN, open, Real ); TA_ALLOC_MEM( HIGH, high, Real ); TA_ALLOC_MEM( LOW, low, Real ); TA_ALLOC_MEM( CLOSE, close, Real ); TA_ALLOC_MEM( VOLUME, volume, Integer ); TA_ALLOC_MEM( OPENINTEREST, openInterest, Integer ); #undef TA_ALLOC_MEM memoryNeeded = 0; } op_loop: /* Jump here when ready to proceed with the next command. */ op = readOpInfo->arrayReadOp[curOp]; if( !(op&TA_CMD_READ_MONTH_CHAR) ) { /* Skip leading non-numeric character. */ SKIP_UNTIL_NUMERIC; } /* Shall we skip this field? */ if( TA_IS_SKIP_SET(op) ) { if( (op&(TA_CMD_READ_REAL|TA_CMD_READ_INTEGER)) == 0 ) { tmpInt = TA_GET_NB_NUMERIC(op); curOp++; while( tmpInt-- ) { GET_CHAR; CHECK_EOL_EOF; } } else { if( skipField == 0 ) { skipField = TA_GET_NB_NUMERIC(op); TA_ASSERT( skipField > 0 ); } if( --skipField == 0 ) curOp++; SKIP_NUMERIC; if( (op&TA_CMD_READ_REAL) && (*car == '.') ) { GET_CHAR; CHECK_EOL_EOF; SKIP_NUMERIC; } } } else { cnvtArrayIdx = 0; if( TA_IS_REAL_CMD(op) ) { /* Extract a numeric into cnvtArray. */ READ_IN_CNVT_ARRAY; /* This is a TA_Real. */ if( car && (*car == '.') ) { /* Read rest of the float after the '.' */ READ_IN_CNVT_ARRAY; } cnvtArray[cnvtArrayIdx] = '\0'; tmpReal = atof( &cnvtArray[0] ); /* Write the TA_Real in memory. */ tmpIdx = TA_GET_IDX(op); TA_ASSERT( tmpIdx < TA_REAL_ARRAY_SIZE ); TA_ASSERT( arrayReal[tmpIdx] != NULL ); if( tmpReal != 0.0 ) { *(arrayReal[tmpIdx]) = tmpReal; if( tmpIdx == TA_CLOSE_IDX ) lastValidClose = tmpReal; } else if( TA_IS_REPLACE_ZERO(op) ) { /* Replace this zero value with the last known close. * If there is no previous close, this line is ignored. */ if( lastValidClose != 0.0 ) *(arrayReal[tmpIdx]) = lastValidClose; else { SKIP_LINE; } } else { /* Zero are not expected, consider this as a failure * and ignore all further data from this file. */ retCode = TA_PRICE_BAR_CONTAINS_ZERO; goto exit_loops; } arrayReal[tmpIdx]++; curOp++; } else { /* This is a TA_Integer. */ if( !(op&TA_CMD_READ_MONTH_CHAR) ) { nbCharToRead = TA_GET_NB_NUMERIC(op); if( nbCharToRead ) { READ_N_CHAR_IN_CNVT_ARRAY(nbCharToRead); } else { READ_IN_CNVT_ARRAY; } cnvtArray[cnvtArrayIdx] = '\0'; tmpInt = atoi( &cnvtArray[0] ); } else { /* Try to find a 3 letters month string. * Translate it to a [1..12] integer. */ nbLetter = 1; do { CHECK_EOL_EOF; monthChar[nbLetter] = (char)toupper(*car); GET_CHAR; nbLetter++; } while( nbLetter != 3 ); do { CHECK_EOL_EOF; monthChar[0] = monthChar[1]; monthChar[1] = monthChar[2]; monthChar[2] = (char)toupper(*car); if( strncmp("JAN",monthChar,3) == 0 ) tmpInt = 1; else if( strncmp("FEB",monthChar,3) == 0 ) tmpInt = 2; else if( strncmp("MAR",monthChar,3) == 0 ) tmpInt = 3; else if( strncmp("APR",monthChar,3) == 0 ) tmpInt = 4; else if( strncmp("MAY",monthChar,3) == 0 ) tmpInt = 5; else if( strncmp("JUN",monthChar,3) == 0 ) tmpInt = 6; else if( strncmp("JUL",monthChar,3) == 0 ) tmpInt = 7; else if( strncmp("AUG",monthChar,3) == 0 ) tmpInt = 8; else if( strncmp("SEP",monthChar,3) == 0 ) tmpInt = 9; else if( strncmp("OCT",monthChar,3) == 0 ) tmpInt = 10; else if( strncmp("NOV",monthChar,3) == 0 ) tmpInt = 11; else if( strncmp("DEC",monthChar,3) == 0 ) tmpInt = 12; else tmpInt = 0; GET_CHAR; } while( tmpInt == 0 ); } /* Write the TA_Integer in memory. */ tmpIdx = TA_GET_IDX(op); TA_ASSERT( tmpIdx < TA_INTEGER_ARRAY_SIZE ); TA_ASSERT( arrayInteger[tmpIdx] != NULL ); *(arrayInteger[tmpIdx]) = tmpInt; if( tmpIdx > TA_YEAR_IDX ) arrayInteger[tmpIdx]++; curOp++; if( TA_IS_TIMESTAMP_COMPLETE(op) ) { /* Build the timestamp. */ retCode = TA_SetDate( year, month, day, &tmpTimestamp ); if( retCode != TA_SUCCESS ) goto exit_loops; /* Invalid date */ if( !timeNeeded ) { /* Ignore time in comparison and use default to build the price bar. */ tmpTimestamp.time = 23595900; /* Default EOD time */ if( start && (tmpTimestamp.date < start->date) ) { /* This price bar is not needed, jump to the next line. */ retCode = TA_SUCCESS; SKIP_LINE; } if( end && (tmpTimestamp.date > end->date) ) { /* This price bar is beyond the upper limit, just exit. */ goto exit_loops; } } else { retCode = TA_SetTime( hour, min, sec, &tmpTimestamp ); if( retCode != TA_SUCCESS ) goto exit_loops; /* Invalid time */ if( start && TA_TimestampLess(&tmpTimestamp,start) ) { /* This price bar is not needed, jump to the next line. */ retCode = TA_SUCCESS; SKIP_LINE; } if( end && TA_TimestampGreater(&tmpTimestamp, end) ) { /* This price bar is beyond the upper limit, just exit. */ goto exit_loops; } } /* Write the timestamp in memory. */ *timestamp = tmpTimestamp; } } if( TA_IS_READ_STOP_FLAG_SET(op) ) { #ifdef DEBUG_PRINTF printf( "(%d%d%d,%e,%e,%e,%e,%d)\n", timestampBeg?TA_GetYear(×tampBeg[nbBarAddedInTheBlock]):0, timestampBeg?TA_GetMonth(×tampBeg[nbBarAddedInTheBlock]):0, timestampBeg?TA_GetDay(×tampBeg[nbBarAddedInTheBlock]):0, openBeg?openBeg[nbBarAddedInTheBlock]:0.0, highBeg?highBeg[nbBarAddedInTheBlock]:0.0, lowBeg?lowBeg[nbBarAddedInTheBlock]:0.0, closeBeg?closeBeg[nbBarAddedInTheBlock]:0.0, volumeBeg?volumeBeg[nbBarAddedInTheBlock]:0 ); #endif /* At this point, the price bar is completely written in memory. */ timestamp++; curOp = 0; nbBarAddedInTheBlock++; if( nbBarAddedInTheBlock < nbElementToAllocate ) { /* Go to next line. */ SKIP_LINE; } else { /* There is not enough memory for another bar, so re-allocate * some more. */ retCode = TA_HistoryAddData( paramForAddData, nbBarAddedInTheBlock, period, timestampBeg, openBeg, highBeg, lowBeg, closeBeg, volumeBeg, openInterestBeg ); /* TA_HistoryAddData is ALWAYS the owner of these memory block * when called. So we set these to NULL to make sure there will * be no attempt to free these from this function. */ openBeg = highBeg = lowBeg = closeBeg = NULL; timestampBeg = NULL; volumeBeg = openInterestBeg = NULL; nbTotalBarDone += nbBarAddedInTheBlock; nbBarAddedInTheBlock = 0; if( retCode != TA_SUCCESS ) goto exit_loops; retCode = TA_EstimateAllocNext( &estimationInfo, &nbElementToAllocate ); if( retCode != TA_SUCCESS ) goto exit_loops; memoryNeeded = 1; SKIP_LINE; } } else { /* Make sure we did not hit an EOL or EOF prematurly, if yes, * this line will be silently ignored. */ CHECK_EOL_EOF; } } goto op_loop; exit_loops: /* Jump here when the end-of-file is hit (or an error occured) */ /* On succesful exit, process possibly remaining data. */ if( (retCode == TA_SUCCESS) && (nbBarAddedInTheBlock != 0) ) { retCode = TA_HistoryAddData( paramForAddData, nbBarAddedInTheBlock, period, timestampBeg, openBeg, highBeg, lowBeg, closeBeg, volumeBeg, openInterestBeg ); openBeg = highBeg = lowBeg = closeBeg = NULL; timestampBeg = NULL; volumeBeg = openInterestBeg = NULL; nbTotalBarDone += nbBarAddedInTheBlock; } /* ALWAYS verify if locally allocated memory needs to be freed. ALWAYS. */ FREE_IF_NOT_NULL( openBeg ); FREE_IF_NOT_NULL( highBeg ); FREE_IF_NOT_NULL( lowBeg ); FREE_IF_NOT_NULL( closeBeg ); FREE_IF_NOT_NULL( volumeBeg ); FREE_IF_NOT_NULL( openInterestBeg ); FREE_IF_NOT_NULL( timestampBeg ); /* An indication that no more data needs to be provided is not a failure. */ if( retCode == TA_ENOUGH_DATA ) retCode = TA_SUCCESS; /* Return the number of added price bar to the caller. */ if( nbBarAdded ) *nbBarAdded = nbTotalBarDone; TA_TRACE_RETURN( retCode ); }
/**** Global functions definitions. ****/ TA_RetCode TA_GetHistoryDataFromWeb( TA_Libc *libHandle, TA_DataSourceHandle *handle, TA_CategoryHandle *categoryHandle, TA_SymbolHandle *symbolHandle, TA_Period period, const TA_Timestamp *start, const TA_Timestamp *end, TA_Field fieldToAlloc, TA_ParamForAddData *paramForAddData ) { TA_PROLOG; TA_RetCode retCode; TA_StringCache *stringCache; TA_String *yahooName; TA_WebPage *webPage; TA_PrivateYahooHandle *yahooHandle; TA_DecodingParam localDecodingParam; const TA_DecodingParam *decodingParam; TA_FileHandle *fileHandle; TA_ReadOpInfo *readOpInfo; UIRSuffixParsing suffixParsing; TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate; TA_InfoFromAddedData infoFromAddedData; TA_DayOfWeek dayOfWeek; int nbEstimateBar; int nbField; unsigned int nbBarAdded, nbTotalBarAdded; int again, firstTime, nbBatch; int zeroBarAddedAttempt; TA_TRACE_BEGIN( libHandle, TA_GetHistoryDataFromWeb ); /* Initialize some local variables. */ stringCache = TA_GetGlobalStringCache( libHandle ); yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData; readOpInfo = NULL; nbEstimateBar = 0; TA_ASSERT( libHandle, categoryHandle != NULL ); TA_ASSERT( libHandle, symbolHandle != NULL ); TA_ASSERT( libHandle, categoryHandle->string != NULL ); TA_ASSERT( libHandle, symbolHandle->string != NULL ); /* Set the initial first/last timestamp */ if( start ) TA_TimestampCopy( &firstBarTimestamp, start ); else { TA_SetDate( 1950, 1, 1, &firstBarTimestamp ); TA_SetTime( 0, 0, 0, &firstBarTimestamp ); } if( end ) TA_TimestampCopy( &lastBarTimestamp, end ); else { TA_SetDateNow( &lastBarTimestamp ); TA_SetTime( 0, 0, 0, &lastBarTimestamp ); } /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); /* Map the TA-Lib name into the Yahoo! name. */ retCode = TA_AllocStringFromLibName( libHandle, categoryHandle->string, symbolHandle->string, &yahooName ); if( retCode != TA_SUCCESS ) { TA_TRACE_RETURN( retCode ); } TA_ASSERT( libHandle, yahooName != NULL ); TA_ASSERT( libHandle, yahooHandle != NULL ); /* Get the decoding parameter for the CSV web page. */ decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CVS_PAGE ); if( !decodingParam ) { TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) ); } /* Use a local copy of the decoding param. * This is because the uirSuffix is replaced with * an allocated buffer (so the date field can be * manipulated). */ localDecodingParam = *decodingParam; /* Parse the uirSuffix so the start/end date can be changed. */ if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) ) { /* This should never happen unless the * Yahoo! index protocol has been broken. */ /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) ); } /* Replace the uirSuffix with a large local buffer. */ localDecodingParam.uirSuffix = TA_Malloc( libHandle, suffixParsing.maxTotalLength ); if( !localDecodingParam.uirSuffix ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_ALLOC_ERR ); } /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* nbBatch is a safety net to make sure that * TA-Lib won't stay forever in the while loop * in case Yahoo! changes their protocol. */ nbBatch = 0; /* Sometime Yahoo! return an empty csv file. Make * multiple attempts in that case. */ zeroBarAddedAttempt = 0; again = 1; firstTime = 1; nbTotalBarAdded = 0; while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) ) { if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) ) { /* Get out of this loop if all the requested data * has been retreived already. */ again = 0; break; } retCode = TA_WebPageAllocFromYahooName( libHandle, &localDecodingParam, TA_StringToChar(yahooName), &webPage ); if( retCode != TA_SUCCESS ) { TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Disguise the webPage stream into a "file". That way the speed * optimized ASCII decoder can be re-used (TA_ReadOp stuff). */ retCode = TA_FileSeqOpenFromStream( libHandle, webPage->content, &fileHandle ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } if( firstTime ) { /* Make assumption of the data provided * base on the number of fields in the CSV file. */ nbField = nbCommaField( webPage->content ); switch( nbField ) { case 2: readOpInfo = yahooHandle->readOp2Fields; break; case 5: readOpInfo = yahooHandle->readOp5Fields; break; default: readOpInfo = yahooHandle->readOp6Fields; } /* User asking for all the fields? */ if( fieldToAlloc == TA_ALL ) { switch( nbField ) { case 2: fieldToAlloc = TA_CLOSE|TA_TIMESTAMP; break; case 5: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP; break; default: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP; } } /* Optimize the read op for the requested data. */ retCode = TA_ReadOp_Optimize( libHandle, readOpInfo, period, fieldToAlloc ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Make an estimation of the number of price bar. */ nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1; if( nbEstimateBar < 100 ) nbEstimateBar = 100; } /* Interpret the CSV data. */ retCode = TA_ReadOp_Do( libHandle, fileHandle, readOpInfo, period, &firstBarTimestamp, &lastBarTimestamp, nbEstimateBar, fieldToAlloc, paramForAddData, &nbBarAdded ); TA_FileSeqClose( libHandle, fileHandle ); TA_WebPageFree( webPage ); nbTotalBarAdded += nbBarAdded; if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Yahoo! does not always return all the data it could, up to * the requested end date. It is important to detect these occurence * and cancel the usage of all data accumulated up to now. */ TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData ); if( infoFromAddedData.barAddedSinceLastCall ) { /* Do some more checking by considering holidays, week-end etc... */ if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( TA_DATA_GAP ); } TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp ); } #if DEBUG_PRINTF printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded ); #endif /* Verify if more data should be processed. * Yahoo! sometimes slice their data, in * batch of 200 price bars. */ if( firstTime && (nbBarAdded > 200) ) { again = 0; /* Assume all the data extracted... exit the loop. */ } else if( nbBarAdded == 0 ) { /* Make multiple attempts when retreiving data succeed, * but somehow there is zero bar returned. * * Sometimes this might be correct when there is truly no * more data available, so choosing an algorithm before * giving up is a comprimise between reliability and * usability. The data source is free... and you get * what you pay for after all ;) */ if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) ) { /* I did choose to add a delay when insufficient total data is returned. When * there is already ~5 years of data, most likely there is "Zero" returned * because there is NO more data available, so just do the retry without delay. */ TA_Sleep(zeroBarAddedAttempt*2); } #if DEBUG_PRINTF printf( "Retry %d", zeroBarAddedAttempt ); #endif zeroBarAddedAttempt++; } else { zeroBarAddedAttempt = 0; if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) ) { /* prevEndDate is a "safety net" to * exit the loop early in case Yahoo! starts * to return always the same batch of data. * Just ignore the repetitive data and exit. */ TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_SUCCESS ); } TA_TimestampCopy( &prevEndDate, &lastBarTimestamp ); /* Request the data up to the day BEFORE * the last batch of data received. */ TA_PrevDay( &lastBarTimestamp ); /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* From that point, data is expected to be most likely * sent in batch of 200. */ nbEstimateBar = 200; } firstTime = 0; } /* Clean-up and exit */ TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( retCode ); }
/* Period transformation is highly dependable on * the function evaluating the 'delta' between * two timestamp, so this is verified here. */ static ErrorNumber testTimestampDelta( void ) { TA_RetCode retCode; unsigned int i, delta; /* !!! A lot more of testing could be added !!! */ /* Test weekday delta. */ TA_SetDate( 2002, 12, 29, &sundayTS ); TA_SetDate( 2002, 12, 30, &mondayTS ); TA_SetDate( 2002, 12, 31, &tuesdayTS ); TA_SetDate( 2003, 1, 1, &wednesdayTS ); TA_SetDate( 2003, 1, 2, &thursdayTS ); TA_SetDate( 2003, 1, 3, &fridayTS ); TA_SetDate( 2003, 1, 4, &saturdayTS ); TA_SetDate( 2003, 1, 5, &sunday2TS ); TA_SetDate( 2003, 1, 6, &monday2TS ); TA_SetDate( 2003, 1, 7, &tuesday2TS ); TA_SetDate( 2003, 1, 8, &wednesday2TS ); TA_SetDate( 2003, 1, 9, &thursday2TS ); TA_SetDate( 2003, 1, 10, &friday2TS ); TA_SetDate( 2003, 1, 11, &saturday2TS ); for( i=0; i < NB_WEEKDAY_CHECK_TO_DO; i++ ) { retCode = TA_TimestampDeltaWeekday( toCheck[i].start, toCheck[i].end, &delta ); if( retCode != TA_SUCCESS ) { printf( "Failed: Weekday delta test #%d\n", i ); return TA_PERIOD_DELTA_WEEKDAY_FAILED; } if( delta != toCheck[i].expectedDelta ) { printf( "Failed: Expected delta != delta (%d!=%d) for test #%d\n", toCheck[i].expectedDelta, delta, i ); return TA_PERIOD_DELTA_WEEKDAY_FAILED_1; } } return TA_TEST_PASS; /* Success. */ }