/**** Global functions definitions. ****/
TA_RetCode TA_GetHistoryDataFromWeb( TA_Libc *libHandle,
TA_DataSourceHandle *handle,
TA_CategoryHandle *categoryHandle,
TA_SymbolHandle *symbolHandle,
TA_Period period,
const TA_Timestamp *start,
const TA_Timestamp *end,
TA_Field fieldToAlloc,
TA_ParamForAddData *paramForAddData )
{
TA_PROLOG;
TA_RetCode retCode;
TA_StringCache *stringCache;
TA_String *yahooName;
TA_WebPage *webPage;
TA_PrivateYahooHandle *yahooHandle;
TA_DecodingParam localDecodingParam;
const TA_DecodingParam *decodingParam;
TA_FileHandle *fileHandle;
TA_ReadOpInfo *readOpInfo;
UIRSuffixParsing suffixParsing;
TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate;
TA_InfoFromAddedData infoFromAddedData;
TA_DayOfWeek dayOfWeek;
int nbEstimateBar;
int nbField;
unsigned int nbBarAdded, nbTotalBarAdded;
int again, firstTime, nbBatch;
int zeroBarAddedAttempt;
TA_TRACE_BEGIN( libHandle, TA_GetHistoryDataFromWeb );
/* Initialize some local variables. */
stringCache = TA_GetGlobalStringCache( libHandle );
yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData;
readOpInfo = NULL;
nbEstimateBar = 0;
TA_ASSERT( libHandle, categoryHandle != NULL );
TA_ASSERT( libHandle, symbolHandle != NULL );
TA_ASSERT( libHandle, categoryHandle->string != NULL );
TA_ASSERT( libHandle, symbolHandle->string != NULL );
/* Set the initial first/last timestamp */
if( start )
TA_TimestampCopy( &firstBarTimestamp, start );
else
{
TA_SetDate( 1950, 1, 1, &firstBarTimestamp );
TA_SetTime( 0, 0, 0, &firstBarTimestamp );
}
if( end )
TA_TimestampCopy( &lastBarTimestamp, end );
else
{
TA_SetDateNow( &lastBarTimestamp );
TA_SetTime( 0, 0, 0, &lastBarTimestamp );
}
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
/* Map the TA-Lib name into the Yahoo! name. */
retCode = TA_AllocStringFromLibName( libHandle,
categoryHandle->string,
symbolHandle->string,
&yahooName );
if( retCode != TA_SUCCESS )
{
TA_TRACE_RETURN( retCode );
}
TA_ASSERT( libHandle, yahooName != NULL );
TA_ASSERT( libHandle, yahooHandle != NULL );
/* Get the decoding parameter for the CSV web page. */
decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CVS_PAGE );
if( !decodingParam )
{
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) );
}
/* Use a local copy of the decoding param.
* This is because the uirSuffix is replaced with
* an allocated buffer (so the date field can be
* manipulated).
*/
localDecodingParam = *decodingParam;
/* Parse the uirSuffix so the start/end date can be changed. */
if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) )
{
/* This should never happen unless the
* Yahoo! index protocol has been broken.
*/
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) );
}
/* Replace the uirSuffix with a large local buffer. */
localDecodingParam.uirSuffix = TA_Malloc( libHandle, suffixParsing.maxTotalLength );
if( !localDecodingParam.uirSuffix )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* nbBatch is a safety net to make sure that
* TA-Lib won't stay forever in the while loop
* in case Yahoo! changes their protocol.
*/
nbBatch = 0;
/* Sometime Yahoo! return an empty csv file. Make
* multiple attempts in that case.
*/
zeroBarAddedAttempt = 0;
again = 1;
firstTime = 1;
nbTotalBarAdded = 0;
while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) )
{
if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) )
{
/* Get out of this loop if all the requested data
* has been retreived already.
*/
again = 0;
break;
}
retCode = TA_WebPageAllocFromYahooName( libHandle,
&localDecodingParam,
TA_StringToChar(yahooName),
&webPage );
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Disguise the webPage stream into a "file". That way the speed
* optimized ASCII decoder can be re-used (TA_ReadOp stuff).
*/
retCode = TA_FileSeqOpenFromStream( libHandle, webPage->content, &fileHandle );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
if( firstTime )
{
/* Make assumption of the data provided
* base on the number of fields in the CSV file.
*/
nbField = nbCommaField( webPage->content );
switch( nbField )
{
case 2:
readOpInfo = yahooHandle->readOp2Fields;
break;
case 5:
readOpInfo = yahooHandle->readOp5Fields;
break;
default:
readOpInfo = yahooHandle->readOp6Fields;
}
/* User asking for all the fields? */
if( fieldToAlloc == TA_ALL )
{
switch( nbField )
{
case 2:
fieldToAlloc = TA_CLOSE|TA_TIMESTAMP;
break;
case 5:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP;
break;
default:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP;
}
}
/* Optimize the read op for the requested data. */
retCode = TA_ReadOp_Optimize( libHandle,
readOpInfo,
period,
fieldToAlloc );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Make an estimation of the number of price bar. */
nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1;
if( nbEstimateBar < 100 )
nbEstimateBar = 100;
}
/* Interpret the CSV data. */
retCode = TA_ReadOp_Do( libHandle, fileHandle,
readOpInfo,
period, &firstBarTimestamp, &lastBarTimestamp,
nbEstimateBar, fieldToAlloc,
paramForAddData,
&nbBarAdded );
TA_FileSeqClose( libHandle, fileHandle );
TA_WebPageFree( webPage );
nbTotalBarAdded += nbBarAdded;
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Yahoo! does not always return all the data it could, up to
* the requested end date. It is important to detect these occurence
* and cancel the usage of all data accumulated up to now.
*/
TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData );
if( infoFromAddedData.barAddedSinceLastCall )
{
/* Do some more checking by considering holidays, week-end etc... */
if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( TA_DATA_GAP );
}
TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp );
}
#if DEBUG_PRINTF
printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded );
#endif
/* Verify if more data should be processed.
* Yahoo! sometimes slice their data, in
* batch of 200 price bars.
*/
if( firstTime && (nbBarAdded > 200) )
{
again = 0; /* Assume all the data extracted... exit the loop. */
}
else if( nbBarAdded == 0 )
{
/* Make multiple attempts when retreiving data succeed,
* but somehow there is zero bar returned.
*
* Sometimes this might be correct when there is truly no
* more data available, so choosing an algorithm before
* giving up is a comprimise between reliability and
* usability. The data source is free... and you get
* what you pay for after all ;)
*/
if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) )
{
/* I did choose to add a delay when insufficient total data is returned. When
* there is already ~5 years of data, most likely there is "Zero" returned
* because there is NO more data available, so just do the retry without delay.
*/
TA_Sleep(zeroBarAddedAttempt*2);
}
#if DEBUG_PRINTF
printf( "Retry %d", zeroBarAddedAttempt );
#endif
zeroBarAddedAttempt++;
}
else
{
zeroBarAddedAttempt = 0;
if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) )
{
/* prevEndDate is a "safety net" to
* exit the loop early in case Yahoo! starts
* to return always the same batch of data.
* Just ignore the repetitive data and exit.
*/
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_SUCCESS );
}
TA_TimestampCopy( &prevEndDate, &lastBarTimestamp );
/* Request the data up to the day BEFORE
* the last batch of data received.
*/
TA_PrevDay( &lastBarTimestamp );
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* From that point, data is expected to be most likely
* sent in batch of 200.
*/
nbEstimateBar = 200;
}
firstTime = 0;
}
/* Clean-up and exit */
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( retCode );
}
/**** Global functions definitions. ****/
ErrorNumber test_pm( void )
{
ErrorNumber errorNumber;
TA_UDBase *udb;
unsigned int i, j;
printf( "Testing Performance Measurement\n" );
/* Side Note:
* Why all these allocLib/freeLib in this function?
* Each time freeLib is being called, it is verified
* that all ressource has been freed. So that's a good
* way to verify for any potential memory leak.
*/
/* Initialize some globals used throughout these tests. */
TA_SetTimeNow( ×tampNow );
TA_SetDateNow( ×tampNow );
TA_NextWeekday( ×tampNow );
TA_TimestampCopy( ×tampNowPlusOneYear, ×tampNow );
TA_NextYear( ×tampNowPlusOneYear );
TA_PrevDay( ×tampNowPlusOneYear );
/* Using a user defined kkey */
TA_InstrumentInitWithUserKey( &id1_1, 12 );
TA_InstrumentInitWithUserKey( &id1_2, 9 );
/* Using a category / symbol strings. */
TA_InstrumentInit( &id2_1, "AB", "CD" );
TA_InstrumentInit( &id2_1, "AB", "CE" );
/* Using a category only. */
TA_InstrumentInit( &id3_1, "ABCD", NULL );
TA_InstrumentInit( &id3_2, "EFGH", NULL );
/* Using only a symbol string */
TA_InstrumentInit( &id4_1, NULL, "A" );
TA_InstrumentInit( &id4_2, NULL, "B" );
/* Test limit cases with empty TA_TradeLog */
errorNumber = allocLib( &udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
errorNumber = test_emptytradelog();
if( errorNumber != TA_TEST_PASS )
{
printf( "Failed: Empty trade log cases\n" );
return errorNumber;
}
errorNumber = freeLib( udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
/* Test with only one TA_Transaction.
* Repeat all tests for each possible
* TA_Instrument key type.
*/
for( i=0; i < NB_TA_KEY_TYPE; i++ )
{
errorNumber = allocLib( &udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
errorNumber = test_onetransaction_only( (TA_KEY_TYPE)i );
if( errorNumber != TA_TEST_PASS )
{
printf( "Failed: one transaction cases (key=%d,errorNumber=%d)\n", (TA_KEY_TYPE)i, errorNumber );
return errorNumber;
}
errorNumber = freeLib( udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
}
/* Tests with two TA_Transaction for the
* same given TA_Instrument.
*
* Repeat the test for each combination
* of:
* - TA_Instrument key type
* - long and short trade.
* - winning and losing trade
*/
for( i=0; i <= 1; i++ )
{
/* 0 = test a loosing trade
* 1 = test a winning trade
*/
for( j=0; j < NB_TA_KEY_TYPE; j++ )
{
/* Test Long */
errorNumber = allocLib( &udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
errorNumber = test_onetrade_only( (TA_KEY_TYPE)j, TA_LONG_ENTRY, i );
if( errorNumber != TA_TEST_PASS )
{
printf( "Failed: one trade only (key=%d,type=%d,winning=%d)\n", (TA_KEY_TYPE)j, TA_LONG_ENTRY, i );
return errorNumber;
}
errorNumber = freeLib( udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
/* Test Short */
errorNumber = allocLib( &udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
errorNumber = test_onetrade_only( (TA_KEY_TYPE)j, TA_SHORT_ENTRY, i );
if( errorNumber != TA_TEST_PASS )
{
printf( "Failed: one trade only (key=%d,type=%d,winning=%d)\n", (TA_KEY_TYPE)j, TA_SHORT_ENTRY, i );
return errorNumber;
}
errorNumber = freeLib( udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
}
}
/* Test TA_PMValueId using a list of tests
* defined in static variables.
*/
for( i=0; i < NB_PMVALUEID_TEST; i++ )
{
errorNumber = allocLib( &udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
errorNumber = test_valueId( &pmValueIdTests[i] );
if( errorNumber != TA_TEST_PASS )
{
printf( "Failed: test_valueId #%d\n", i);
return errorNumber;
}
errorNumber = freeLib( udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
}
/* Test TA_PMArrayId using a list of tests
* defined in static variables.
*/
for( i=0; i < NB_PMARRAYID_TEST; i++ )
{
errorNumber = allocLib( &udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
errorNumber = test_arrayId( &pmArrayIdTests[i] );
if( errorNumber != TA_TEST_PASS )
{
printf( "Failed: test_arrayId #%d\n", i);
return errorNumber;
}
errorNumber = freeLib( udb );
if( errorNumber != TA_TEST_PASS )
return errorNumber;
}
return TA_TEST_PASS;
}
