/**** Global functions definitions. ****/ TA_RetCode TA_WebPageAlloc( const char *webSiteAddr, const char *webSitePage, const char *proxyName, const char *proxyPort, TA_WebPage **webPageAllocated, unsigned int nbAttempt ) { TA_RetCode retCode; unsigned int i,j; /* Make sure there is at least one attempt. */ if( nbAttempt == 0 ) nbAttempt = 1; /* Do not attempt more than 50 times (realistic upper limit). */ if( nbAttempt > 50 ) nbAttempt = 50; retCode = TA_SUCCESS; for( i=0; i < nbAttempt; i++ ) { if( i > 0 ) { /* Some delay before a new attempt. * We do not want to irritate the server. * Wait up to 20 seconds. */ j = i*2; if( j > 20 ) j = 20; TA_Sleep( j ); } /* On data retreival problems, keep retrying * many times. TA_INTERNET_READ_DATA_FAILED means * that the server can be reached, but somehow the * transmission of the data was interupted. So it is * worth to give multiple re-try immediatly. * * All other type of failure will retry * only "nbAttempt" times. */ retCode = TA_INTERNET_READ_DATA_FAILED; for( j=0; (retCode == TA_INTERNET_READ_DATA_FAILED) && (j < 100); j++ ) { retCode = internalWebPageAlloc( webSiteAddr, webSitePage, proxyName, proxyPort, webPageAllocated ); if( retCode == TA_INTERNET_READ_DATA_FAILED ) TA_Sleep( 1 ); /* 1 second */ } if( retCode == TA_SUCCESS ) return TA_SUCCESS; } return retCode; }
TA_RetCode TA_YAHOO_GetHistoryData( TA_DataSourceHandle *handle, TA_CategoryHandle *categoryHandle, TA_SymbolHandle *symbolHandle, TA_Period period, const TA_Timestamp *start, const TA_Timestamp *end, TA_Field fieldToAlloc, TA_ParamForAddData *paramForAddData ) { TA_PROLOG TA_RetCode tempRetCode, retCode; TA_PrivateYahooHandle *yahooHandle; int again, j; TA_TRACE_BEGIN( TA_YAHOO_GetHistoryData ); TA_ASSERT( handle != NULL ); TA_ASSERT( paramForAddData != NULL ); TA_ASSERT( categoryHandle != NULL ); TA_ASSERT( symbolHandle != NULL ); yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData; TA_ASSERT( yahooHandle != NULL ); /* If the requested period is too precise for the * period that can be provided by this data source, * simply return without error. * Since no data has been added, the TA-LIB will ignore * this data source. */ if( period < TA_DAILY ) { TA_TRACE_RETURN( TA_SUCCESS ); } /* Get the data from the WEB. * * Yahoo! sometimes have "gaps" in its data (like one * week missing), when this is being detected, we throw * away all the data up to now and start over (up to * 5 times before giving up). */ again = 5; do { retCode = TA_GetHistoryDataFromWeb( handle, categoryHandle, symbolHandle, TA_DAILY, start, end, fieldToAlloc, paramForAddData ); if( retCode == TA_DATA_GAP ) { retCode = TA_HistoryAddDataReset( paramForAddData ); if( retCode != TA_SUCCESS ) again = 0; /* Give up */ else { --again; /* Try again */ /* Sometimes giving Yahoo! a break helps. */ tempRetCode = TA_DriverShouldContinue(paramForAddData); j = 0; while( (j++ < 5) && (tempRetCode != TA_DATA_RETREIVE_TIMEOUT) ) { TA_Sleep(1); tempRetCode = TA_DriverShouldContinue(paramForAddData); } if( tempRetCode == TA_DATA_RETREIVE_TIMEOUT ) { retCode = tempRetCode; again = 0; } } } else { again = 0; /* Exit the loop */ } } while( again > 0 ); TA_TRACE_RETURN( retCode ); }
/**** Global functions definitions. ****/ TA_RetCode TA_GetHistoryDataFromWeb( TA_Libc *libHandle, TA_DataSourceHandle *handle, TA_CategoryHandle *categoryHandle, TA_SymbolHandle *symbolHandle, TA_Period period, const TA_Timestamp *start, const TA_Timestamp *end, TA_Field fieldToAlloc, TA_ParamForAddData *paramForAddData ) { TA_PROLOG; TA_RetCode retCode; TA_StringCache *stringCache; TA_String *yahooName; TA_WebPage *webPage; TA_PrivateYahooHandle *yahooHandle; TA_DecodingParam localDecodingParam; const TA_DecodingParam *decodingParam; TA_FileHandle *fileHandle; TA_ReadOpInfo *readOpInfo; UIRSuffixParsing suffixParsing; TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate; TA_InfoFromAddedData infoFromAddedData; TA_DayOfWeek dayOfWeek; int nbEstimateBar; int nbField; unsigned int nbBarAdded, nbTotalBarAdded; int again, firstTime, nbBatch; int zeroBarAddedAttempt; TA_TRACE_BEGIN( libHandle, TA_GetHistoryDataFromWeb ); /* Initialize some local variables. */ stringCache = TA_GetGlobalStringCache( libHandle ); yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData; readOpInfo = NULL; nbEstimateBar = 0; TA_ASSERT( libHandle, categoryHandle != NULL ); TA_ASSERT( libHandle, symbolHandle != NULL ); TA_ASSERT( libHandle, categoryHandle->string != NULL ); TA_ASSERT( libHandle, symbolHandle->string != NULL ); /* Set the initial first/last timestamp */ if( start ) TA_TimestampCopy( &firstBarTimestamp, start ); else { TA_SetDate( 1950, 1, 1, &firstBarTimestamp ); TA_SetTime( 0, 0, 0, &firstBarTimestamp ); } if( end ) TA_TimestampCopy( &lastBarTimestamp, end ); else { TA_SetDateNow( &lastBarTimestamp ); TA_SetTime( 0, 0, 0, &lastBarTimestamp ); } /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); /* Map the TA-Lib name into the Yahoo! name. */ retCode = TA_AllocStringFromLibName( libHandle, categoryHandle->string, symbolHandle->string, &yahooName ); if( retCode != TA_SUCCESS ) { TA_TRACE_RETURN( retCode ); } TA_ASSERT( libHandle, yahooName != NULL ); TA_ASSERT( libHandle, yahooHandle != NULL ); /* Get the decoding parameter for the CSV web page. */ decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CVS_PAGE ); if( !decodingParam ) { TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) ); } /* Use a local copy of the decoding param. * This is because the uirSuffix is replaced with * an allocated buffer (so the date field can be * manipulated). */ localDecodingParam = *decodingParam; /* Parse the uirSuffix so the start/end date can be changed. */ if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) ) { /* This should never happen unless the * Yahoo! index protocol has been broken. */ /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) ); } /* Replace the uirSuffix with a large local buffer. */ localDecodingParam.uirSuffix = TA_Malloc( libHandle, suffixParsing.maxTotalLength ); if( !localDecodingParam.uirSuffix ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_ALLOC_ERR ); } /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* nbBatch is a safety net to make sure that * TA-Lib won't stay forever in the while loop * in case Yahoo! changes their protocol. */ nbBatch = 0; /* Sometime Yahoo! return an empty csv file. Make * multiple attempts in that case. */ zeroBarAddedAttempt = 0; again = 1; firstTime = 1; nbTotalBarAdded = 0; while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) ) { if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) ) { /* Get out of this loop if all the requested data * has been retreived already. */ again = 0; break; } retCode = TA_WebPageAllocFromYahooName( libHandle, &localDecodingParam, TA_StringToChar(yahooName), &webPage ); if( retCode != TA_SUCCESS ) { TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Disguise the webPage stream into a "file". That way the speed * optimized ASCII decoder can be re-used (TA_ReadOp stuff). */ retCode = TA_FileSeqOpenFromStream( libHandle, webPage->content, &fileHandle ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } if( firstTime ) { /* Make assumption of the data provided * base on the number of fields in the CSV file. */ nbField = nbCommaField( webPage->content ); switch( nbField ) { case 2: readOpInfo = yahooHandle->readOp2Fields; break; case 5: readOpInfo = yahooHandle->readOp5Fields; break; default: readOpInfo = yahooHandle->readOp6Fields; } /* User asking for all the fields? */ if( fieldToAlloc == TA_ALL ) { switch( nbField ) { case 2: fieldToAlloc = TA_CLOSE|TA_TIMESTAMP; break; case 5: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP; break; default: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP; } } /* Optimize the read op for the requested data. */ retCode = TA_ReadOp_Optimize( libHandle, readOpInfo, period, fieldToAlloc ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Make an estimation of the number of price bar. */ nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1; if( nbEstimateBar < 100 ) nbEstimateBar = 100; } /* Interpret the CSV data. */ retCode = TA_ReadOp_Do( libHandle, fileHandle, readOpInfo, period, &firstBarTimestamp, &lastBarTimestamp, nbEstimateBar, fieldToAlloc, paramForAddData, &nbBarAdded ); TA_FileSeqClose( libHandle, fileHandle ); TA_WebPageFree( webPage ); nbTotalBarAdded += nbBarAdded; if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Yahoo! does not always return all the data it could, up to * the requested end date. It is important to detect these occurence * and cancel the usage of all data accumulated up to now. */ TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData ); if( infoFromAddedData.barAddedSinceLastCall ) { /* Do some more checking by considering holidays, week-end etc... */ if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( TA_DATA_GAP ); } TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp ); } #if DEBUG_PRINTF printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded ); #endif /* Verify if more data should be processed. * Yahoo! sometimes slice their data, in * batch of 200 price bars. */ if( firstTime && (nbBarAdded > 200) ) { again = 0; /* Assume all the data extracted... exit the loop. */ } else if( nbBarAdded == 0 ) { /* Make multiple attempts when retreiving data succeed, * but somehow there is zero bar returned. * * Sometimes this might be correct when there is truly no * more data available, so choosing an algorithm before * giving up is a comprimise between reliability and * usability. The data source is free... and you get * what you pay for after all ;) */ if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) ) { /* I did choose to add a delay when insufficient total data is returned. When * there is already ~5 years of data, most likely there is "Zero" returned * because there is NO more data available, so just do the retry without delay. */ TA_Sleep(zeroBarAddedAttempt*2); } #if DEBUG_PRINTF printf( "Retry %d", zeroBarAddedAttempt ); #endif zeroBarAddedAttempt++; } else { zeroBarAddedAttempt = 0; if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) ) { /* prevEndDate is a "safety net" to * exit the loop early in case Yahoo! starts * to return always the same batch of data. * Just ignore the repetitive data and exit. */ TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_SUCCESS ); } TA_TimestampCopy( &prevEndDate, &lastBarTimestamp ); /* Request the data up to the day BEFORE * the last batch of data received. */ TA_PrevDay( &lastBarTimestamp ); /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* From that point, data is expected to be most likely * sent in batch of 200. */ nbEstimateBar = 200; } firstTime = 0; } /* Clean-up and exit */ TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( retCode ); }
static ErrorNumber checkRangeSame( TA_UDBase *udb, const TA_History *historyRef, const TA_Timestamp *start, const TA_Timestamp *end, unsigned int startIdx, unsigned int nbPriceBar ) { TA_History *history; unsigned int i; TA_RetCode retCode; #if 1 /* Do not retry... no forgiveness! */ retCode = TA_HistoryAlloc( udb, "CA.CDNX.STOCK", "MRY", TA_DAILY, start, end, TA_ALL, &history ); #else int retry, again; /* Try up to 10 times to get the requested number of * price bar (sometimes Yahoo! or the Web do fail to * return the data). */ again = 1; for( retry=0; (retry < 10) && again; retry++ ) { retCode = TA_HistoryAlloc( udb, "CA.CDNX.STOCK", "MRY", TA_DAILY, start, end, TA_ALL, &history ); if( (retCode == TA_SUCCESS) && (history->nbBars == nbPriceBar) ) again = 0; else { printf( "Warning: Yahoo! history alloc retry #%d of 10\n", retry+1 ); if( retCode == TA_SUCCESS ) { TA_HistoryFree( history ); } TA_Sleep( 10 /* seconds */ ); } } #endif if( retCode != TA_SUCCESS ) { reportError( "TA_HistoryAlloc", retCode ); return TA_YAHOO_CRS_HISTORYALLOC_FAILED; } /* Check that the expected number of price bar is returned. */ if( history->nbBars != nbPriceBar ) { printf( "Failed: nbBars (received != requested)=(%d != %d)\n", history->nbBars, nbPriceBar ); TA_HistoryFree( history ); return TA_YAHOO_CRS_NBBARSBAD; } /* Check that the data is the same for the range. */ for( i=0; i < nbPriceBar; i++ ) { if( !TA_TimestampEqual( &history->timestamp[i], &historyRef->timestamp[startIdx+i] ) || (history->open[i] != historyRef->open[startIdx+i]) || (history->high[i] != historyRef->high[startIdx+i]) || (history->low[i] != historyRef->low[startIdx+i]) || (history->close[i] != historyRef->close[startIdx+i]) || (history->volume[i] != historyRef->volume[startIdx+i]) ) { printf( "Failed: Price Bar value different\n" ); printf( "Failed: Data = %f,%f,%f,%f,%d\n", history->open[i], history->high[i], history->low[i], history->close[i], history->volume[i] ); printf( "Failed: Ref = %f,%f,%f,%f,%d\n", historyRef->open[startIdx+i], historyRef->high[startIdx+i], historyRef->low[startIdx+i], historyRef->close[startIdx+i], historyRef->volume[startIdx+i] ); TA_HistoryFree( history ); return TA_YAHOO_CRS_PRICEBARBAD; } } retCode = TA_HistoryFree( history ); if( retCode != TA_SUCCESS ) { reportError( "TA_HistoryFree", retCode ); return TA_YAHOO_HISTORYFREE_FAILED; } return TA_TEST_PASS; }
TA_RetCode TA_WebPageAllocFromYahooName( TA_Libc *libHandle, const TA_DecodingParam *decodingParam, const char *yahooName, TA_WebPage **allocatedWebPage ) { TA_PROLOG; TA_RetCode retCode; char webSitePage[300]; unsigned int prefixLength, suffixLength, symbolLength, i; const char *webSiteAddr; const char *uirPrefix, *uirSuffix; TA_WebPage *webPage; TA_TRACE_BEGIN( libHandle, TA_WebPageAllocFromYahooName ); retCode = TA_INTERNAL_ERROR(117); if( !decodingParam || !yahooName || !allocatedWebPage ) { TA_TRACE_RETURN( TA_BAD_PARAM ); } webSiteAddr = decodingParam->webSiteServer; TA_ASSERT( libHandle, webSiteAddr != NULL ); uirPrefix = decodingParam->uirPrefix; TA_ASSERT( libHandle, uirPrefix != NULL ); prefixLength = strlen( uirPrefix ); uirSuffix = decodingParam->uirSuffix; TA_ASSERT( libHandle, uirSuffix != NULL ); suffixLength = strlen( uirSuffix ); symbolLength = strlen( yahooName ); if( (symbolLength + suffixLength + prefixLength) >= 299 ) { TA_TRACE_RETURN( TA_INVALID_SECURITY_EXCHANGE ); } sprintf( webSitePage, "%s%s%s", uirPrefix, yahooName, uirSuffix ); /* Get the Web Page */ for( i=0; i < 10; i++ ) { retCode = TA_WebPageAlloc( libHandle, webSiteAddr, webSitePage, NULL, NULL, &webPage, 10 ); if( retCode == TA_SUCCESS ) break; else { /* Yahoo! is may be slow, let's sleep 1 minute */ TA_Sleep( 60 ); } } if( retCode != TA_SUCCESS ) { TA_TRACE_RETURN( retCode ); } *allocatedWebPage = webPage; TA_TRACE_RETURN( TA_SUCCESS ); }
static ErrorNumber checkRangeSame( TA_UDBase *udb, const TA_History *historyRef, const TA_Timestamp *start, const TA_Timestamp *end, TA_Period period, unsigned int startIdx, unsigned int nbPriceBar ) { TA_History *history; unsigned int i; TA_RetCode retCode; TA_HistoryAllocParam histParam; int retry, again; /* Try up to 10 times to get the requested number of * price bar (sometimes Yahoo! or the Web do fail to * return the data). */ again = 1; for( retry=0; (retry < 10) && again; retry++ ) { memset( &histParam, 0, sizeof( TA_HistoryAllocParam ) ); histParam.category = "CA.TSE.STOCK"; histParam.symbol = "NT"; histParam.field = TA_ALL; histParam.start = *start; histParam.end = *end; histParam.period = period; retCode = TA_HistoryAlloc( udb, &histParam, &history ); if( (retCode == TA_SUCCESS) && (history->nbBars == nbPriceBar) ) again = 0; else { printf( "Warning: Yahoo! history alloc retry #%d of 10\n", retry+1 ); if( retCode == TA_SUCCESS ) { TA_HistoryFree( history ); } TA_Sleep( 10 /* seconds */ ); } } if( retCode != TA_SUCCESS ) { reportError( "TA_HistoryAlloc", retCode ); return TA_YAHOO_CRS_HISTORYALLOC_FAILED; } /* Check that the expected number of price bar is returned. */ if( history->nbBars != nbPriceBar ) { printf( "Failed: nbBars (received != requested)=(%d != %d)\n", history->nbBars, nbPriceBar ); TA_HistoryFree( history ); return TA_YAHOO_CRS_NBBARSBAD; } /* printf( "startIdx=%d\n", startIdx ); */ /* Check that the data is the same for the range. */ for( i=0; i < nbPriceBar; i++ ) { if( !TA_TimestampEqual( &history->timestamp[i], &historyRef->timestamp[startIdx+i] ) || (history->open[i] != historyRef->open[startIdx+i]) || (history->high[i] != historyRef->high[startIdx+i]) || (history->low[i] != historyRef->low[startIdx+i]) || (history->close[i] != historyRef->close[startIdx+i]) || (history->volume[i] != historyRef->volume[startIdx+i]) ) { printf( "Failed: Price Bar value different\n" ); printf( "Failed: Data = %d/%d/%d : %f,%f,%f,%f,%d\n", TA_GetMonth(&history->timestamp[i]), TA_GetDay(&history->timestamp[i]), TA_GetYear(&history->timestamp[i]), history->open[i], history->high[i], history->low[i], history->close[i], history->volume[i] ); printf( "Failed: Ref = %d/%d/%d : %f,%f,%f,%f,%d\n", TA_GetMonth(&historyRef->timestamp[i]), TA_GetDay(&historyRef->timestamp[i]), TA_GetYear(&historyRef->timestamp[i]), historyRef->open[startIdx+i], historyRef->high[startIdx+i], historyRef->low[startIdx+i], historyRef->close[startIdx+i], historyRef->volume[startIdx+i] ); TA_HistoryFree( history ); return TA_YAHOO_CRS_PRICEBARBAD; } } retCode = TA_HistoryFree( history ); if( retCode != TA_SUCCESS ) { reportError( "TA_HistoryFree", retCode ); return TA_YAHOO_HISTORYFREE_FAILED; } return TA_TEST_PASS; }