TA_RetCode TA_YAHOO_OpenSource( const TA_AddDataSourceParamPriv *param, TA_DataSourceHandle **handle ) { TA_PROLOG TA_DataSourceHandle *tmpHandle; TA_PrivateYahooHandle *privData; TA_RetCode retCode; TA_StringCache *stringCache; TA_CountryId countryId; TA_CountryId countryIdTemp; TA_Timestamp now; const char *locationPtr; char locationBuffer[3]; int timeout_set; /* boolean */ int i, again; unsigned int strLength, strServerLength; const char *strTemp; *handle = NULL; TA_TRACE_BEGIN( TA_YAHOO_OpenSource ); stringCache = TA_GetGlobalStringCache(); /* Verify that the requested functionality is supported or not. */ if( param->flags & TA_REPLACE_ZERO_PRICE_BAR ) { TA_TRACE_RETURN( TA_NOT_SUPPORTED ); } /* Allocate and initialize the handle. This function will also allocate the * private handle (opaque data). */ tmpHandle = TA_YAHOO_DataSourceHandleAlloc(); if( tmpHandle == NULL ) { TA_TRACE_RETURN( TA_ALLOC_ERR ); } privData = (TA_PrivateYahooHandle *)(tmpHandle->opaqueData); /* Copy some parameters in the private handle. */ privData->param = param; /* Indentify the country and replace with the default as needed. * At the same time, identify optional "server=" modifier. */ countryId = TA_Country_ID_INVALID; if( privData->param->location ) { locationPtr = TA_StringToChar(privData->param->location); /* Split into token (seperator is ';'). Check for a 2 character country * string and the optional "server=" modifier. */ i = -1; strTemp = &locationPtr[0]; strLength = 0; strServerLength = strlen(TA_SERVER_STR); again = 1; while( (++i < 1024) && again ) { if( strLength == 0 ) strTemp = &locationPtr[i]; if( locationPtr[i] == '\0' ) again = 0; if( locationPtr[i] == ';' || !again ) { if( strLength == 2 ) { locationBuffer[0] = locationPtr[i-2]; locationBuffer[1] = locationPtr[i-1]; locationBuffer[2] = '\0'; countryIdTemp = TA_CountryAbbrevToId(locationBuffer); if( countryIdTemp == TA_Country_ID_INVALID ) TA_TRACE_RETURN( TA_UNSUPPORTED_COUNTRY ); if( countryId != TA_Country_ID_INVALID ) TA_TRACE_RETURN( TA_LIMIT_OF_ONE_COUNTRY_ID_EXCEEDED ); countryId = countryIdTemp; } else if( strLength>strServerLength && (strncmp(strTemp,TA_SERVER_STR,strServerLength)==0) ) { if( privData->userSpecifiedServer ) TA_TRACE_RETURN( TA_LIMIT_OF_ONE_SERVER_EXCEEDED ); privData->userSpecifiedServer = TA_StringAllocN( stringCache, &strTemp[strServerLength], strLength-strServerLength ); TA_ASSERT( privData->userSpecifiedServer != NULL ); } else TA_TRACE_RETURN( TA_LOCATION_PARAM_INVALID ); strLength = 0; } else strLength++; } } if( countryId == TA_Country_ID_INVALID ) { /* Default is United States. */ countryId = TA_Country_ID_US; } if( privData->param->id == TA_YAHOO_ONE_SYMBOL ) { privData->index = NULL; privData->webSiteCountry = countryId; privData->webSiteSymbol = TA_StringDup(stringCache,privData->param->info); tmpHandle->nbCategory = 1; } else { /* Build the index using .dat files */ switch( countryId ) { case TA_Country_ID_US: /* United States */ case TA_Country_ID_CA: /* Canada */ case TA_Country_ID_UK: /* United Kingdom */ case TA_Country_ID_DE: /* Germany */ case TA_Country_ID_DK: /* Denmark */ case TA_Country_ID_ES: /* Spain */ case TA_Country_ID_FR: /* France */ case TA_Country_ID_IT: /* Italy */ case TA_Country_ID_SE: /* Sweden */ case TA_Country_ID_NO: /* Norway */ /* These country are currently supported. */ break; default: TA_YAHOO_DataSourceHandleFree( tmpHandle ); TA_TRACE_RETURN( TA_UNSUPPORTED_COUNTRY ); } /* Establish the timeout for local cache of the index. * Let's make it 4 business days. */ timeout_set = 0; TA_SetDefault( &now ); retCode = TA_SetDateNow( &now ); for( i=0; (i < 4) && (retCode == TA_SUCCESS); i++ ) retCode = TA_PrevWeekday( &now ); if( (i == 4) && (retCode == TA_SUCCESS) ) timeout_set = 1; /* At this point, we got all the information we * need in the handle. * Now build the TA_YahooIdx. */ retCode = TA_YahooIdxAlloc( countryId, &privData->index, TA_USE_LOCAL_CACHE|TA_USE_REMOTE_CACHE, NULL, timeout_set?&now:NULL, NULL ); if( retCode != TA_SUCCESS ) { TA_YAHOO_DataSourceHandleFree( tmpHandle ); TA_TRACE_RETURN( retCode ); } /* Set the total number of distinct category. */ tmpHandle->nbCategory = privData->index->nbCategory; } *handle = tmpHandle; TA_TRACE_RETURN( TA_SUCCESS ); }
/**** Global functions definitions. ****/ TA_RetCode TA_GetHistoryDataFromWeb( TA_Libc *libHandle, TA_DataSourceHandle *handle, TA_CategoryHandle *categoryHandle, TA_SymbolHandle *symbolHandle, TA_Period period, const TA_Timestamp *start, const TA_Timestamp *end, TA_Field fieldToAlloc, TA_ParamForAddData *paramForAddData ) { TA_PROLOG; TA_RetCode retCode; TA_StringCache *stringCache; TA_String *yahooName; TA_WebPage *webPage; TA_PrivateYahooHandle *yahooHandle; TA_DecodingParam localDecodingParam; const TA_DecodingParam *decodingParam; TA_FileHandle *fileHandle; TA_ReadOpInfo *readOpInfo; UIRSuffixParsing suffixParsing; TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate; TA_InfoFromAddedData infoFromAddedData; TA_DayOfWeek dayOfWeek; int nbEstimateBar; int nbField; unsigned int nbBarAdded, nbTotalBarAdded; int again, firstTime, nbBatch; int zeroBarAddedAttempt; TA_TRACE_BEGIN( libHandle, TA_GetHistoryDataFromWeb ); /* Initialize some local variables. */ stringCache = TA_GetGlobalStringCache( libHandle ); yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData; readOpInfo = NULL; nbEstimateBar = 0; TA_ASSERT( libHandle, categoryHandle != NULL ); TA_ASSERT( libHandle, symbolHandle != NULL ); TA_ASSERT( libHandle, categoryHandle->string != NULL ); TA_ASSERT( libHandle, symbolHandle->string != NULL ); /* Set the initial first/last timestamp */ if( start ) TA_TimestampCopy( &firstBarTimestamp, start ); else { TA_SetDate( 1950, 1, 1, &firstBarTimestamp ); TA_SetTime( 0, 0, 0, &firstBarTimestamp ); } if( end ) TA_TimestampCopy( &lastBarTimestamp, end ); else { TA_SetDateNow( &lastBarTimestamp ); TA_SetTime( 0, 0, 0, &lastBarTimestamp ); } /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); /* Map the TA-Lib name into the Yahoo! name. */ retCode = TA_AllocStringFromLibName( libHandle, categoryHandle->string, symbolHandle->string, &yahooName ); if( retCode != TA_SUCCESS ) { TA_TRACE_RETURN( retCode ); } TA_ASSERT( libHandle, yahooName != NULL ); TA_ASSERT( libHandle, yahooHandle != NULL ); /* Get the decoding parameter for the CSV web page. */ decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CVS_PAGE ); if( !decodingParam ) { TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) ); } /* Use a local copy of the decoding param. * This is because the uirSuffix is replaced with * an allocated buffer (so the date field can be * manipulated). */ localDecodingParam = *decodingParam; /* Parse the uirSuffix so the start/end date can be changed. */ if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) ) { /* This should never happen unless the * Yahoo! index protocol has been broken. */ /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) ); } /* Replace the uirSuffix with a large local buffer. */ localDecodingParam.uirSuffix = TA_Malloc( libHandle, suffixParsing.maxTotalLength ); if( !localDecodingParam.uirSuffix ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_ALLOC_ERR ); } /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* nbBatch is a safety net to make sure that * TA-Lib won't stay forever in the while loop * in case Yahoo! changes their protocol. */ nbBatch = 0; /* Sometime Yahoo! return an empty csv file. Make * multiple attempts in that case. */ zeroBarAddedAttempt = 0; again = 1; firstTime = 1; nbTotalBarAdded = 0; while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) ) { if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) ) { /* Get out of this loop if all the requested data * has been retreived already. */ again = 0; break; } retCode = TA_WebPageAllocFromYahooName( libHandle, &localDecodingParam, TA_StringToChar(yahooName), &webPage ); if( retCode != TA_SUCCESS ) { TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Disguise the webPage stream into a "file". That way the speed * optimized ASCII decoder can be re-used (TA_ReadOp stuff). */ retCode = TA_FileSeqOpenFromStream( libHandle, webPage->content, &fileHandle ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } if( firstTime ) { /* Make assumption of the data provided * base on the number of fields in the CSV file. */ nbField = nbCommaField( webPage->content ); switch( nbField ) { case 2: readOpInfo = yahooHandle->readOp2Fields; break; case 5: readOpInfo = yahooHandle->readOp5Fields; break; default: readOpInfo = yahooHandle->readOp6Fields; } /* User asking for all the fields? */ if( fieldToAlloc == TA_ALL ) { switch( nbField ) { case 2: fieldToAlloc = TA_CLOSE|TA_TIMESTAMP; break; case 5: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP; break; default: fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP; } } /* Optimize the read op for the requested data. */ retCode = TA_ReadOp_Optimize( libHandle, readOpInfo, period, fieldToAlloc ); if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_WebPageFree( webPage ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Make an estimation of the number of price bar. */ nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1; if( nbEstimateBar < 100 ) nbEstimateBar = 100; } /* Interpret the CSV data. */ retCode = TA_ReadOp_Do( libHandle, fileHandle, readOpInfo, period, &firstBarTimestamp, &lastBarTimestamp, nbEstimateBar, fieldToAlloc, paramForAddData, &nbBarAdded ); TA_FileSeqClose( libHandle, fileHandle ); TA_WebPageFree( webPage ); nbTotalBarAdded += nbBarAdded; if( retCode != TA_SUCCESS ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( retCode ); } /* Yahoo! does not always return all the data it could, up to * the requested end date. It is important to detect these occurence * and cancel the usage of all data accumulated up to now. */ TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData ); if( infoFromAddedData.barAddedSinceLastCall ) { /* Do some more checking by considering holidays, week-end etc... */ if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) ) { /* Clean-up and exit */ TA_StringFree( stringCache, yahooName ); TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_TRACE_RETURN( TA_DATA_GAP ); } TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp ); } #if DEBUG_PRINTF printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded ); #endif /* Verify if more data should be processed. * Yahoo! sometimes slice their data, in * batch of 200 price bars. */ if( firstTime && (nbBarAdded > 200) ) { again = 0; /* Assume all the data extracted... exit the loop. */ } else if( nbBarAdded == 0 ) { /* Make multiple attempts when retreiving data succeed, * but somehow there is zero bar returned. * * Sometimes this might be correct when there is truly no * more data available, so choosing an algorithm before * giving up is a comprimise between reliability and * usability. The data source is free... and you get * what you pay for after all ;) */ if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) ) { /* I did choose to add a delay when insufficient total data is returned. When * there is already ~5 years of data, most likely there is "Zero" returned * because there is NO more data available, so just do the retry without delay. */ TA_Sleep(zeroBarAddedAttempt*2); } #if DEBUG_PRINTF printf( "Retry %d", zeroBarAddedAttempt ); #endif zeroBarAddedAttempt++; } else { zeroBarAddedAttempt = 0; if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) ) { /* prevEndDate is a "safety net" to * exit the loop early in case Yahoo! starts * to return always the same batch of data. * Just ignore the repetitive data and exit. */ TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( TA_SUCCESS ); } TA_TimestampCopy( &prevEndDate, &lastBarTimestamp ); /* Request the data up to the day BEFORE * the last batch of data received. */ TA_PrevDay( &lastBarTimestamp ); /* Make sure that lastBarTimestamp is a week-day. */ dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp ); if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) ) TA_PrevWeekday( &lastBarTimestamp ); /* Change the dates in the uirSuffix. */ buildUIRSuffix( &suffixParsing, &firstBarTimestamp, &lastBarTimestamp, (char *)localDecodingParam.uirSuffix ); /* From that point, data is expected to be most likely * sent in batch of 200. */ nbEstimateBar = 200; } firstTime = 0; } /* Clean-up and exit */ TA_Free( libHandle, (char *)localDecodingParam.uirSuffix ); TA_StringFree( stringCache, yahooName ); TA_TRACE_RETURN( retCode ); }
TA_RetCode TA_YAHOO_OpenSource( const TA_AddDataSourceParamPriv *param, TA_DataSourceHandle **handle ) { TA_PROLOG TA_DataSourceHandle *tmpHandle; TA_PrivateYahooHandle *privData; TA_RetCode retCode; TA_StringCache *stringCache; TA_CountryId countryId; TA_Timestamp now; const char *locationPtr; int timeout_set; /* boolean */ int i; *handle = NULL; TA_TRACE_BEGIN( TA_YAHOO_OpenSource ); stringCache = TA_GetGlobalStringCache(); /* Verify that the requested functionality is supported or not. */ if( param->flags & TA_REPLACE_ZERO_PRICE_BAR ) { TA_TRACE_RETURN( TA_NOT_SUPPORTED ); } /* Allocate and initialize the handle. This function will also allocate the * private handle (opaque data). */ tmpHandle = TA_YAHOO_DataSourceHandleAlloc(); if( tmpHandle == NULL ) { TA_TRACE_RETURN( TA_ALLOC_ERR ); } privData = (TA_PrivateYahooHandle *)(tmpHandle->opaqueData); /* Copy some parameters in the private handle. */ privData->param = param; /* Validate if this is a supported country. */ if( !privData->param->location ) { countryId = TA_Country_ID_US; } else { locationPtr = TA_StringToChar(privData->param->location); countryId = TA_CountryAbbrevToId(locationPtr); switch( countryId ) { case TA_Country_ID_US: /* United States */ case TA_Country_ID_CA: /* Canada */ case TA_Country_ID_UK: /* United Kingdom */ case TA_Country_ID_DE: /* Germany */ case TA_Country_ID_DK: /* Denmark */ case TA_Country_ID_ES: /* Spain */ case TA_Country_ID_FR: /* France */ case TA_Country_ID_IT: /* Italy */ case TA_Country_ID_SE: /* Sweden */ case TA_Country_ID_NO: /* Norway */ /* These country are currently supported. */ break; default: TA_YAHOO_DataSourceHandleFree( tmpHandle ); TA_TRACE_RETURN( TA_UNSUPPORTED_COUNTRY ); } } /* Establish the timeout for local cache of the index. * Let's make it 3 business days. */ timeout_set = 0; TA_SetDefault( &now ); retCode = TA_SetDateNow( &now ); for( i=0; (i < 3) && (retCode == TA_SUCCESS); i++ ) retCode = TA_PrevWeekday( &now ); if( (i == 3) && (retCode == TA_SUCCESS) ) timeout_set = 1; /* At this point, we got all the information we * need in the handle. * Now build the TA_YahooIdx. */ retCode = TA_YahooIdxAlloc( countryId, &privData->index, TA_USE_LOCAL_CACHE|TA_USE_REMOTE_CACHE, NULL, timeout_set?&now:NULL, NULL ); if( retCode != TA_SUCCESS ) { TA_YAHOO_DataSourceHandleFree( tmpHandle ); TA_TRACE_RETURN( retCode ); } /* Set the total number of distinct category. */ tmpHandle->nbCategory = privData->index->nbCategory; *handle = tmpHandle; TA_TRACE_RETURN( TA_SUCCESS ); }
/**** Global functions definitions. ****/ ErrorNumber test_pm( void ) { ErrorNumber errorNumber; TA_UDBase *udb; unsigned int i, j; printf( "Testing Performance Measurement\n" ); /* Side Note: * Why all these allocLib/freeLib in this function? * Each time freeLib is being called, it is verified * that all ressource has been freed. So that's a good * way to verify for any potential memory leak. */ /* Initialize some globals used throughout these tests. */ TA_SetTimeNow( ×tampNow ); TA_SetDateNow( ×tampNow ); TA_NextWeekday( ×tampNow ); TA_TimestampCopy( ×tampNowPlusOneYear, ×tampNow ); TA_NextYear( ×tampNowPlusOneYear ); TA_PrevDay( ×tampNowPlusOneYear ); /* Using a user defined kkey */ TA_InstrumentInitWithUserKey( &id1_1, 12 ); TA_InstrumentInitWithUserKey( &id1_2, 9 ); /* Using a category / symbol strings. */ TA_InstrumentInit( &id2_1, "AB", "CD" ); TA_InstrumentInit( &id2_1, "AB", "CE" ); /* Using a category only. */ TA_InstrumentInit( &id3_1, "ABCD", NULL ); TA_InstrumentInit( &id3_2, "EFGH", NULL ); /* Using only a symbol string */ TA_InstrumentInit( &id4_1, NULL, "A" ); TA_InstrumentInit( &id4_2, NULL, "B" ); /* Test limit cases with empty TA_TradeLog */ errorNumber = allocLib( &udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; errorNumber = test_emptytradelog(); if( errorNumber != TA_TEST_PASS ) { printf( "Failed: Empty trade log cases\n" ); return errorNumber; } errorNumber = freeLib( udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; /* Test with only one TA_Transaction. * Repeat all tests for each possible * TA_Instrument key type. */ for( i=0; i < NB_TA_KEY_TYPE; i++ ) { errorNumber = allocLib( &udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; errorNumber = test_onetransaction_only( (TA_KEY_TYPE)i ); if( errorNumber != TA_TEST_PASS ) { printf( "Failed: one transaction cases (key=%d,errorNumber=%d)\n", (TA_KEY_TYPE)i, errorNumber ); return errorNumber; } errorNumber = freeLib( udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; } /* Tests with two TA_Transaction for the * same given TA_Instrument. * * Repeat the test for each combination * of: * - TA_Instrument key type * - long and short trade. * - winning and losing trade */ for( i=0; i <= 1; i++ ) { /* 0 = test a loosing trade * 1 = test a winning trade */ for( j=0; j < NB_TA_KEY_TYPE; j++ ) { /* Test Long */ errorNumber = allocLib( &udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; errorNumber = test_onetrade_only( (TA_KEY_TYPE)j, TA_LONG_ENTRY, i ); if( errorNumber != TA_TEST_PASS ) { printf( "Failed: one trade only (key=%d,type=%d,winning=%d)\n", (TA_KEY_TYPE)j, TA_LONG_ENTRY, i ); return errorNumber; } errorNumber = freeLib( udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; /* Test Short */ errorNumber = allocLib( &udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; errorNumber = test_onetrade_only( (TA_KEY_TYPE)j, TA_SHORT_ENTRY, i ); if( errorNumber != TA_TEST_PASS ) { printf( "Failed: one trade only (key=%d,type=%d,winning=%d)\n", (TA_KEY_TYPE)j, TA_SHORT_ENTRY, i ); return errorNumber; } errorNumber = freeLib( udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; } } /* Test TA_PMValueId using a list of tests * defined in static variables. */ for( i=0; i < NB_PMVALUEID_TEST; i++ ) { errorNumber = allocLib( &udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; errorNumber = test_valueId( &pmValueIdTests[i] ); if( errorNumber != TA_TEST_PASS ) { printf( "Failed: test_valueId #%d\n", i); return errorNumber; } errorNumber = freeLib( udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; } /* Test TA_PMArrayId using a list of tests * defined in static variables. */ for( i=0; i < NB_PMARRAYID_TEST; i++ ) { errorNumber = allocLib( &udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; errorNumber = test_arrayId( &pmArrayIdTests[i] ); if( errorNumber != TA_TEST_PASS ) { printf( "Failed: test_arrayId #%d\n", i); return errorNumber; } errorNumber = freeLib( udb ); if( errorNumber != TA_TEST_PASS ) return errorNumber; } return TA_TEST_PASS; }