/**** Global functions definitions. ****/ TA_RetCode TA_GetHistoryDataFromWeb( TA_DataSourceHandle *handle, TA_CategoryHandle *categoryHandle, TA_SymbolHandle *symbolHandle, TA_Period period, const TA_Timestamp *start, const TA_Timestamp *end, TA_Field fieldToAlloc, TA_ParamForAddData *paramForAddData ) { TA_PROLOG TA_RetCode retCode; TA_StringCache *stringCache; TA_String *yahooName; TA_WebPage *webPage; TA_PrivateYahooHandle *yahooHandle; TA_DecodingParam localDecodingParam; TA_DecodingParam directYahooDecodingParam; const TA_DecodingParam *decodingParam; TA_FileHandle *fileHandle; TA_ReadOpInfo *readOpInfo; TA_UIRSuffixParsing suffixParsing; TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate; TA_InfoFromAddedData infoFromAddedData; TA_DayOfWeek dayOfWeek; TA_Timestamp curAdjustYear, lastAdjustYear; const char *overideServerAddr; int nbEstimateBar; int nbField; unsigned int nbBarAdded, nbTotalBarAdded; int again, firstTime, nbBatch; int zeroBarAddedAttempt; int doAdjustment; TA_TRACE_BEGIN( TA_GetHistoryDataFromWeb ); /* Initialize some local variables. */ stringCache = TA_GetGlobalStringCache(); yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData; readOpInfo = NULL; nbEstimateBar = 0; TA_ASSERT( categoryHandle != NULL ); TA_ASSERT( symbolHandle != NULL ); TA_ASSERT( categoryHandle->string != NULL ); TA_ASSERT( symbolHandle->string != NULL ); retCode = TA_BAD_PARAM; /* Set the initial first/last timestamp */ if( start ) TA_TimestampCopy( &firstBarTimestamp, start ); else TA_SetDate( 1950, 1, 1, &firstBarTimestamp ); if( end ) TA_TimestampCopy( &lastBarTimestamp, end ); else TA_SetDateNow( &lastBarTimestamp ); /* Time component is not important for Yahoo! but all end of day * price bar use 00:00:00, so do the same here. */ TA_SetTime( 0, 0, 0, &firstBarTimestamp ); TA_SetTime( 0, 0, 0, &lastBarTimestamp ); /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); TA_ASSERT( yahooHandle != NULL ); if( yahooHandle->param->id == TA_YAHOO_ONE_SYMBOL ) { /* User specified Yahoo! name. */ yahooName = TA_StringDup(stringCache,yahooHandle->webSiteSymbol); TA_ASSERT( yahooName != NULL ); } else { /* Map the TA-Lib name into the Yahoo! name. */ retCode = TA_AllocStringFromLibName( categoryHandle->string, symbolHandle->string, &yahooName ); if( retCode != TA_SUCCESS ) { TA_TRACE_RETURN( retCode ); } TA_ASSERT( yahooName != NULL ); } /* Check if the user did overide the server address in the location parameter. * (as needed, convert from TA_String to char *) */ if( yahooHandle->userSpecifiedServer ) overideServerAddr = TA_StringToChar(yahooHandle->userSpecifiedServer); else overideServerAddr = NULL; /* Get the decoding parameter for the CSV web page. */ if( yahooHandle->param->id == TA_YAHOO_ONE_SYMBOL ) { retCode = TA_YahooIdxDataDecoding( yahooHandle->webSiteCountry, TA_YAHOOIDX_CSV_PAGE, &directYahooDecodingParam ); if( retCode != TA_SUCCESS ) { TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( retCode ); } decodingParam = &directYahooDecodingParam; } else { decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CSV_PAGE ); if( !decodingParam ) { TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) ); } } localDecodingParam = *decodingParam; /* Check if split/value adjustment are necessary. */ if( (yahooHandle->param->flags & TA_DO_NOT_SPLIT_ADJUST) && (yahooHandle->param->flags & TA_DO_NOT_VALUE_ADJUST) ) { doAdjustment = 0; } else doAdjustment = 1; /* Parse the uirSuffix so the start/end date can be changed. */ if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) ) { /* This should never happen unless the * Yahoo! index protocol has been broken. */ /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) ); } /* Use a local copy of the decoding param. * This is because the uirSuffix is replaced with * an allocated buffer (so the date field can be * manipulated). */ /* Replace the uirSuffix with a large local buffer. */ localDecodingParam.uirSuffix = TA_Malloc( suffixParsing.maxTotalLength ); if( !localDecodingParam.uirSuffix ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_ALLOC_ERR ); } /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* nbBatch is a safety net to make sure that * TA-Lib won't stay forever in the while loop * in case Yahoo! changes their protocol. */ nbBatch = 0; /* Sometime Yahoo! return an empty csv file. Make * multiple attempts in that case. */ zeroBarAddedAttempt = 0; again = 1; firstTime = 1; nbTotalBarAdded = 0; while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) ) { retCode = TA_DriverShouldContinue(paramForAddData); if( retCode != TA_SUCCESS ) { TA_StringFree( stringCache, yahooName ); TA_Free( (char *)localDecodingParam.uirSuffix ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif TA_TRACE_RETURN( retCode ); } if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) ) { /* Get out of this loop if all the requested data * has been retreived already. */ again = 0; break; } retCode = TA_WebPageAllocFromYahooName( &localDecodingParam, TA_StringToChar(yahooName), overideServerAddr, &webPage, paramForAddData ); if( retCode != TA_SUCCESS ) { TA_StringFree( stringCache, yahooName ); TA_Free( (char *)localDecodingParam.uirSuffix ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif TA_TRACE_RETURN( retCode ); } /* Disguise the webPage stream into a "file". That way the speed * optimized ASCII decoder can be re-used (TA_ReadOp stuff). */ retCode = TA_FileSeqOpenFromStream( webPage->content, &fileHandle ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( (char *)localDecodingParam.uirSuffix ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif TA_TRACE_RETURN( retCode ); } if( firstTime ) { /* Make assumption of the data provided * base on the number of fields in the CSV file. */ nbField = nbCommaField( webPage->content ); switch( nbField ) { case 2: readOpInfo = yahooHandle->readOp2Fields; break; case 5: readOpInfo = yahooHandle->readOp5Fields; break; default: readOpInfo = yahooHandle->readOp6Fields; } #if !defined( TA_SINGLE_THREAD ) /* Must use a local copy if multi-threaded */ retCode = TA_ReadOpInfoClone(&readOpInfo, readOpInfo); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } #endif /* User asking for all the fields? */ if( fieldToAlloc == TA_ALL ) { switch( nbField ) { case 2: fieldToAlloc = TA_CLOSE|TA_TIMESTAMP; break; case 5: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP; break; default: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP; } } /* Optimize the read op for the requested data. */ retCode = TA_ReadOp_Optimize( readOpInfo, period, fieldToAlloc ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( (char *)localDecodingParam.uirSuffix ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif TA_TRACE_RETURN( retCode ); } /* Make an estimation of the number of price bar. */ nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1; if( nbEstimateBar < 100 ) nbEstimateBar = 100; } /* Interpret the CSV data. */ retCode = TA_ReadOp_Do( fileHandle, readOpInfo, period, &firstBarTimestamp, &lastBarTimestamp, nbEstimateBar, fieldToAlloc, paramForAddData, &nbBarAdded ); TA_FileSeqClose( fileHandle ); TA_WebPageFree( webPage ); nbTotalBarAdded += nbBarAdded; if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( (char *)localDecodingParam.uirSuffix ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif TA_TRACE_RETURN( retCode ); } /* Yahoo! does not always return all the data it could, up to * the requested end date. It is important to detect these occurence * and cancel the usage of all data accumulated up to now. */ TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData ); if( infoFromAddedData.barAddedSinceLastCall ) { /* Do some more checking by considering holidays, week-end etc... */ if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( (char *)localDecodingParam.uirSuffix ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif TA_TRACE_RETURN( TA_DATA_GAP ); } TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp ); } #if DEBUG_PRINTF printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded ); #endif /* Verify if more data should be processed. * Yahoo! sometimes slice their data, in * batch of 200 price bars. */ if( firstTime && (nbBarAdded > 200) ) { again = 0; /* Assume all the data extracted... exit the loop. */ } else if( nbBarAdded == 0 ) { /* Make multiple attempts when retreiving data succeed, * but somehow there is zero bar returned. * * Sometimes this might be correct when there is truly no * more data available, so choosing an algorithm before * giving up is a comprimise between reliability and * usability. The data source is free... and you get * what you pay for after all ;) */ if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) ) { /* I did choose to add a delay when insufficient total data is returned. When * there is already ~5 years of data, most likely there is "Zero" returned * because there is NO more data available, so just do the retry without delay. */ TA_Sleep(zeroBarAddedAttempt*2); } #if DEBUG_PRINTF printf( "Retry %d", zeroBarAddedAttempt ); #endif zeroBarAddedAttempt++; } else { zeroBarAddedAttempt = 0; if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) ) { /* prevEndDate is a "safety net" to * exit the loop early in case Yahoo! starts * to return always the same batch of data. * Just ignore the repetitive data and exit. */ TA_Free( (char *)localDecodingParam.uirSuffix ); TA_StringFree( stringCache, yahooName ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif TA_TRACE_RETURN( TA_SUCCESS ); } TA_TimestampCopy( &prevEndDate, &lastBarTimestamp ); /* Request the data up to the day BEFORE * the last batch of data received. */ TA_PrevDay( &lastBarTimestamp ); /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* From that point, data is expected to be most likely * sent in batch of 200. */ nbEstimateBar = 200; } firstTime = 0; } /* Get rid of some memory not used anymore. */ TA_Free( (char *)localDecodingParam.uirSuffix ); #if !defined( TA_SINGLE_THREAD ) if( readOpInfo ) TA_ReadOpInfoFree( readOpInfo ); #endif /* If adjusted data is requested, use splits and dividend info from Yahoo!. */ if( doAdjustment && (nbTotalBarAdded >= 1) ) { /* Get the decoding parameter for the adjustment page. */ if( yahooHandle->param->id == TA_YAHOO_ONE_SYMBOL ) { retCode = TA_YahooIdxDataDecoding( yahooHandle->webSiteCountry, TA_YAHOOIDX_ADJUSTMENT, &directYahooDecodingParam ); if( retCode != TA_SUCCESS ) { TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( retCode ); } decodingParam = &directYahooDecodingParam; } else { decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_ADJUSTMENT ); if( !decodingParam ) { TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(140) ); } } localDecodingParam = *decodingParam; if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) ) { /* This should never happen unless the * Yahoo! index protocol has been broken. */ /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(141) ); } /* Use a local copy of the decoding param. * This is because the uirSuffix is replaced with * an allocated buffer (so the date field can be * manipulated). */ /* Replace the uirSuffix with a large local buffer. */ localDecodingParam.uirSuffix = TA_Malloc( suffixParsing.maxTotalLength ); if( !localDecodingParam.uirSuffix ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_ALLOC_ERR ); } /* curAdjustYear indicates for which year the download is * taking place. */ TA_SetDefault( &curAdjustYear ); TA_SetDateNow( &curAdjustYear ); /* Identify the oldest year for which data was downloaded */ TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData ); TA_TimestampCopy( &lastAdjustYear, &infoFromAddedData.lowestTimestamp ); TA_PrevYear( &lastAdjustYear ); /* Get one more year to be on the safe side. */ while( TA_TimestampLess( &lastAdjustYear, &curAdjustYear ) ) { /* Set prevEndDate to two years earlier. */ TA_TimestampCopy( &prevEndDate, &curAdjustYear ); TA_PrevYear( &prevEndDate ); TA_PrevYear( &prevEndDate ); /* Change the dates in the uirSuffix. */ TA_SetDate( TA_GetYear(&curAdjustYear), 12, 31, &curAdjustYear ); TA_SetDate( TA_GetYear(&prevEndDate), 1, 1, &prevEndDate ); buildUIRSuffix( &suffixParsing, &prevEndDate, &curAdjustYear, (char *)localDecodingParam.uirSuffix ); retCode = doAdjustments( &localDecodingParam, yahooName, yahooHandle, NULL, paramForAddData ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Move 3 years earlier. */ TA_PrevYear( &prevEndDate ); TA_TimestampCopy( &curAdjustYear, &prevEndDate ); } /* Clean-up what was allocated for the adjustment logic. */ TA_Free( (char *)localDecodingParam.uirSuffix ); } /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( retCode ); }
TA_RetCode TA_YAHOO_GetHistoryData( TA_DataSourceHandle *handle, TA_CategoryHandle *categoryHandle, TA_SymbolHandle *symbolHandle, TA_Period period, const TA_Timestamp *start, const TA_Timestamp *end, TA_Field fieldToAlloc, TA_ParamForAddData *paramForAddData ) { TA_PROLOG TA_RetCode tempRetCode, retCode; TA_PrivateYahooHandle *yahooHandle; int again, j; TA_TRACE_BEGIN( TA_YAHOO_GetHistoryData ); TA_ASSERT( handle != NULL ); TA_ASSERT( paramForAddData != NULL ); TA_ASSERT( categoryHandle != NULL ); TA_ASSERT( symbolHandle != NULL ); yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData; TA_ASSERT( yahooHandle != NULL ); /* If the requested period is too precise for the * period that can be provided by this data source, * simply return without error. * Since no data has been added, the TA-LIB will ignore * this data source. */ if( period < TA_DAILY ) { TA_TRACE_RETURN( TA_SUCCESS ); } /* Get the data from the WEB. * * Yahoo! sometimes have "gaps" in its data (like one * week missing), when this is being detected, we throw * away all the data up to now and start over (up to * 5 times before giving up). */ again = 5; do { retCode = TA_GetHistoryDataFromWeb( handle, categoryHandle, symbolHandle, TA_DAILY, start, end, fieldToAlloc, paramForAddData ); if( retCode == TA_DATA_GAP ) { retCode = TA_HistoryAddDataReset( paramForAddData ); if( retCode != TA_SUCCESS ) again = 0; /* Give up */ else { --again; /* Try again */ /* Sometimes giving Yahoo! a break helps. */ tempRetCode = TA_DriverShouldContinue(paramForAddData); j = 0; while( (j++ < 5) && (tempRetCode != TA_DATA_RETREIVE_TIMEOUT) ) { TA_Sleep(1); tempRetCode = TA_DriverShouldContinue(paramForAddData); } if( tempRetCode == TA_DATA_RETREIVE_TIMEOUT ) { retCode = tempRetCode; again = 0; } } } else { again = 0; /* Exit the loop */ } } while( again > 0 ); TA_TRACE_RETURN( retCode ); }
/**** Global functions definitions. ****/ TA_RetCode TA_WebPageAlloc( const char *webSiteAddr, const char *webSitePage, const char *proxyName, const char *proxyPort, TA_WebPage **webPageAllocated, unsigned int nbAttempt, void *paramForAddData ) { TA_RetCode retCode; unsigned int i,j, k; TA_ParamForAddData *paramForAddDataPtr; paramForAddDataPtr = (TA_ParamForAddData *)paramForAddData; /* Make sure there is at least one attempt. */ if( nbAttempt == 0 ) nbAttempt = 1; /* Do not attempt more than 50 times (realistic upper limit). */ if( nbAttempt > 50 ) nbAttempt = 50; retCode = TA_SUCCESS; for( i=0; i < nbAttempt; i++ ) { if( i > 0 ) { /* Some delay before a new attempt. * We do not want to irritate the server. * Wait up to 20 seconds. */ j = i*2; if( j > 20 ) j = 20; retCode = TA_DriverShouldContinue(paramForAddDataPtr); k = 0; while( (retCode == TA_SUCCESS) && (k++<j) ) { TA_Sleep( 1 ); retCode = TA_DriverShouldContinue(paramForAddDataPtr); } if( retCode != TA_SUCCESS ) return retCode; } /* On data retreival problems, keep retrying * many times. TA_INTERNET_READ_DATA_FAILED means * that the server can be reached, but somehow the * transmission of the data was interupted. So it is * worth to give multiple re-try immediatly. * * All other type of failure will retry * only "nbAttempt" times. */ retCode = TA_INTERNET_READ_DATA_FAILED; for( j=0; (retCode == TA_INTERNET_READ_DATA_FAILED) && (j < 100); j++ ) { retCode = internalWebPageAlloc( webSiteAddr, webSitePage, proxyName, proxyPort, webPageAllocated ); if( retCode == TA_INTERNET_READ_DATA_FAILED ) TA_Sleep( 1 ); /* 1 second */ } if( retCode == TA_SUCCESS ) return TA_SUCCESS; } return retCode; }