TA_RetCode TA_DictAddPair_S2( TA_Dict *dict,
TA_String *key1,
TA_String *key2,
void *value )
{
TA_PrivDictInfo *theDict;
dnode_t *node;
TA_String *dupKey;
TA_Dict *subDict;
TA_Libc *libHandle;
dict_t *kazlibDict;
theDict = (TA_PrivDictInfo *)dict;
if( (theDict == NULL) ||
(key1 == NULL) || (key2 == NULL) || (value == NULL) )
return TA_BAD_PARAM;
kazlibDict = &theDict->d;
libHandle = theDict->libHandle;
/* Verify if a a dictionary already exist for key1. */
node = dict_lookup( libHandle, kazlibDict, TA_StringToChar(key1) );
if( node )
{
/* A dictionary already exist with the same key1... re-use it. */
subDict = (TA_Dict *)dnode_get( node );
}
else
{
/* Alloc a new directory corresponding to key1. */
subDict = TA_DictAlloc( libHandle, TA_DICT_KEY_ONE_STRING, theDict->freeValueFunc );
if( !subDict )
return TA_ALLOC_ERR;
dupKey = TA_StringDup( TA_GetGlobalStringCache( libHandle ), key1 );
if( !dupKey )
{
TA_DictFree( subDict );
return TA_ALLOC_ERR;
}
if( !dict_alloc_insert( libHandle, kazlibDict, TA_StringToChar(dupKey), subDict ) )
{
TA_DictFree( subDict );
TA_StringFree( TA_GetGlobalStringCache( libHandle ), dupKey );
return TA_ALLOC_ERR;
}
}
/* Insert the string in the subDict using key2 */
return TA_DictAddPair_S( subDict, key2, value );
}
TA_RetCode TA_FileIndexAddTokenInfo( TA_FileIndexPriv *data,
TA_TokenId id,
TA_String *value,
TA_TokenInfo *optBefore )
{
TA_PROLOG;
TA_RetCode retCode;
TA_TokenInfo *info;
TA_Libc *libHandle;
TA_StringCache *stringCache;
libHandle = data->libHandle;
TA_TRACE_BEGIN( libHandle, TA_FileIndexAddTokenInfo );
stringCache = TA_GetGlobalStringCache( libHandle );
info = TA_Malloc( libHandle, sizeof( TA_TokenInfo ) );
if( !info )
{
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
info->id = id;
if( value == NULL )
info->value = NULL;
else
{
info->value = TA_StringDup( stringCache, value );
if( !info->value )
{
TA_Free( libHandle, info );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
}
if( optBefore )
retCode = TA_ListAddBefore( data->listLocationToken, optBefore, info );
else
retCode = TA_ListAddTail( data->listLocationToken, info );
if( retCode != TA_SUCCESS )
{
if( info->value )
TA_StringFree( stringCache, info->value );
TA_Free( libHandle, info );
TA_TRACE_RETURN( retCode );
}
TA_TRACE_RETURN( TA_SUCCESS );
}
TA_RetCode TA_DictAddPair_S( TA_Dict *dict,
TA_String *key,
void *value )
{
TA_PrivDictInfo *theDict;
dnode_t *node;
TA_String *dupKey;
TA_Libc *libHandle;
dict_t *kazlibDict;
theDict = (TA_PrivDictInfo *)dict;
if( (theDict == NULL) || (key == NULL) || (value == NULL) )
return TA_BAD_PARAM;
kazlibDict = &theDict->d;
libHandle = theDict->libHandle;
/* Verify if an entry exist already with the same key. */
node = dict_lookup( libHandle, kazlibDict, TA_StringToChar(key) );
if( node )
{
/* An entry already exist with the same key...
* Re-use the existing node. Just replace the
* 'value' part.
* De-allocate the older 'value'.
*/
if( theDict->freeValueFunc )
(*theDict->freeValueFunc)( libHandle, dnode_get( node ) );
dnode_put( node, value );
}
else
{
/* Alloc/insert a new key-value pair in the dictionary. */
dupKey = TA_StringDup( TA_GetGlobalStringCache( libHandle ), key );
if( !dupKey )
return TA_ALLOC_ERR;
if( !dict_alloc_insert( libHandle, kazlibDict, TA_StringToChar(dupKey), value ) )
{
TA_StringFree( TA_GetGlobalStringCache( libHandle ), dupKey );
return TA_ALLOC_ERR;
}
}
return TA_SUCCESS;
}
/**** Local functions definitions. ****/
static TA_ValueTreeNode *allocTreeNode( TA_Libc *libHandle,
TA_ValueTreeNode *parent,
TA_String *string )
{
TA_ValueTreeNode *node;
TA_RetCode retCode;
TA_StringCache *stringCache;
stringCache = TA_GetGlobalStringCache( libHandle );
node = (TA_ValueTreeNode *)TA_Malloc( libHandle, sizeof( TA_ValueTreeNode ) );
if( !node )
return (TA_ValueTreeNode *)NULL;
node->string = NULL;
node->parent = NULL;
node->child = TA_ListAlloc( libHandle );
if( !node->child )
{
freeTreeNode( libHandle, node );
return NULL;
}
if( string )
{
node->string = TA_StringDup( stringCache, string );
if( !node->string )
{
freeTreeNode( libHandle, node );
return NULL;
}
}
if( parent )
{
retCode = TA_ListAddTail( parent->child, node );
if( retCode != TA_SUCCESS )
{
freeTreeNode( libHandle, node );
return NULL;
}
node->parent = parent;
}
return node;
}
/* Allows to change the value associated to a TA_ValueTreeNode. */
TA_RetCode TA_FileIndexChangeValueTreeNodeValue( TA_Libc *libHandle,
TA_ValueTreeNode *nodeToChange,
TA_String *newValue )
{
TA_PROLOG;
TA_String *dup;
TA_StringCache *stringCache;
TA_TRACE_BEGIN( libHandle, TA_FileIndexChangeValueTreeNodeValue );
stringCache = TA_GetGlobalStringCache( libHandle );
if( !nodeToChange )
{
TA_TRACE_RETURN( TA_BAD_PARAM );
}
if( nodeToChange->string )
TA_StringFree( stringCache, nodeToChange->string );
if( !newValue )
nodeToChange->string = NULL;
else
{
dup = TA_StringDup( stringCache, newValue );
if( !dup )
{
nodeToChange->string = NULL;
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
else
nodeToChange->string = dup;
}
TA_TRACE_RETURN( TA_SUCCESS );
}
TA_RetCode TA_YAHOO_OpenSource( const TA_AddDataSourceParamPriv *param,
TA_DataSourceHandle **handle )
{
TA_PROLOG
TA_DataSourceHandle *tmpHandle;
TA_PrivateYahooHandle *privData;
TA_RetCode retCode;
TA_StringCache *stringCache;
TA_CountryId countryId;
TA_CountryId countryIdTemp;
TA_Timestamp now;
const char *locationPtr;
char locationBuffer[3];
int timeout_set; /* boolean */
int i, again;
unsigned int strLength, strServerLength;
const char *strTemp;
*handle = NULL;
TA_TRACE_BEGIN( TA_YAHOO_OpenSource );
stringCache = TA_GetGlobalStringCache();
/* Verify that the requested functionality is supported or not. */
if( param->flags & TA_REPLACE_ZERO_PRICE_BAR )
{
TA_TRACE_RETURN( TA_NOT_SUPPORTED );
}
/* Allocate and initialize the handle. This function will also allocate the
* private handle (opaque data).
*/
tmpHandle = TA_YAHOO_DataSourceHandleAlloc();
if( tmpHandle == NULL )
{
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
privData = (TA_PrivateYahooHandle *)(tmpHandle->opaqueData);
/* Copy some parameters in the private handle. */
privData->param = param;
/* Indentify the country and replace with the default as needed.
* At the same time, identify optional "server=" modifier.
*/
countryId = TA_Country_ID_INVALID;
if( privData->param->location )
{
locationPtr = TA_StringToChar(privData->param->location);
/* Split into token (seperator is ';'). Check for a 2 character country
* string and the optional "server=" modifier.
*/
i = -1;
strTemp = &locationPtr[0];
strLength = 0;
strServerLength = strlen(TA_SERVER_STR);
again = 1;
while( (++i < 1024) && again )
{
if( strLength == 0 )
strTemp = &locationPtr[i];
if( locationPtr[i] == '\0' )
again = 0;
if( locationPtr[i] == ';' || !again )
{
if( strLength == 2 )
{
locationBuffer[0] = locationPtr[i-2];
locationBuffer[1] = locationPtr[i-1];
locationBuffer[2] = '\0';
countryIdTemp = TA_CountryAbbrevToId(locationBuffer);
if( countryIdTemp == TA_Country_ID_INVALID )
TA_TRACE_RETURN( TA_UNSUPPORTED_COUNTRY );
if( countryId != TA_Country_ID_INVALID )
TA_TRACE_RETURN( TA_LIMIT_OF_ONE_COUNTRY_ID_EXCEEDED );
countryId = countryIdTemp;
}
else if( strLength>strServerLength && (strncmp(strTemp,TA_SERVER_STR,strServerLength)==0) )
{
if( privData->userSpecifiedServer )
TA_TRACE_RETURN( TA_LIMIT_OF_ONE_SERVER_EXCEEDED );
privData->userSpecifiedServer = TA_StringAllocN( stringCache,
&strTemp[strServerLength],
strLength-strServerLength );
TA_ASSERT( privData->userSpecifiedServer != NULL );
}
else
TA_TRACE_RETURN( TA_LOCATION_PARAM_INVALID );
strLength = 0;
}
else
strLength++;
}
}
if( countryId == TA_Country_ID_INVALID )
{
/* Default is United States. */
countryId = TA_Country_ID_US;
}
if( privData->param->id == TA_YAHOO_ONE_SYMBOL )
{
privData->index = NULL;
privData->webSiteCountry = countryId;
privData->webSiteSymbol = TA_StringDup(stringCache,privData->param->info);
tmpHandle->nbCategory = 1;
}
else
{
/* Build the index using .dat files */
switch( countryId )
{
case TA_Country_ID_US: /* United States */
case TA_Country_ID_CA: /* Canada */
case TA_Country_ID_UK: /* United Kingdom */
case TA_Country_ID_DE: /* Germany */
case TA_Country_ID_DK: /* Denmark */
case TA_Country_ID_ES: /* Spain */
case TA_Country_ID_FR: /* France */
case TA_Country_ID_IT: /* Italy */
case TA_Country_ID_SE: /* Sweden */
case TA_Country_ID_NO: /* Norway */
/* These country are currently supported. */
break;
default:
TA_YAHOO_DataSourceHandleFree( tmpHandle );
TA_TRACE_RETURN( TA_UNSUPPORTED_COUNTRY );
}
/* Establish the timeout for local cache of the index.
* Let's make it 4 business days.
*/
timeout_set = 0;
TA_SetDefault( &now );
retCode = TA_SetDateNow( &now );
for( i=0; (i < 4) && (retCode == TA_SUCCESS); i++ )
retCode = TA_PrevWeekday( &now );
if( (i == 4) && (retCode == TA_SUCCESS) )
timeout_set = 1;
/* At this point, we got all the information we
* need in the handle.
* Now build the TA_YahooIdx.
*/
retCode = TA_YahooIdxAlloc( countryId,
&privData->index,
TA_USE_LOCAL_CACHE|TA_USE_REMOTE_CACHE,
NULL, timeout_set?&now:NULL, NULL );
if( retCode != TA_SUCCESS )
{
TA_YAHOO_DataSourceHandleFree( tmpHandle );
TA_TRACE_RETURN( retCode );
}
/* Set the total number of distinct category. */
tmpHandle->nbCategory = privData->index->nbCategory;
}
*handle = tmpHandle;
TA_TRACE_RETURN( TA_SUCCESS );
}
/**** Global functions definitions. ****/
TA_RetCode TA_GetHistoryDataFromWeb( TA_DataSourceHandle *handle,
TA_CategoryHandle *categoryHandle,
TA_SymbolHandle *symbolHandle,
TA_Period period,
const TA_Timestamp *start,
const TA_Timestamp *end,
TA_Field fieldToAlloc,
TA_ParamForAddData *paramForAddData )
{
TA_PROLOG
TA_RetCode retCode;
TA_StringCache *stringCache;
TA_String *yahooName;
TA_WebPage *webPage;
TA_PrivateYahooHandle *yahooHandle;
TA_DecodingParam localDecodingParam;
TA_DecodingParam directYahooDecodingParam;
const TA_DecodingParam *decodingParam;
TA_FileHandle *fileHandle;
TA_ReadOpInfo *readOpInfo;
TA_UIRSuffixParsing suffixParsing;
TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate;
TA_InfoFromAddedData infoFromAddedData;
TA_DayOfWeek dayOfWeek;
TA_Timestamp curAdjustYear, lastAdjustYear;
const char *overideServerAddr;
int nbEstimateBar;
int nbField;
unsigned int nbBarAdded, nbTotalBarAdded;
int again, firstTime, nbBatch;
int zeroBarAddedAttempt;
int doAdjustment;
TA_TRACE_BEGIN( TA_GetHistoryDataFromWeb );
/* Initialize some local variables. */
stringCache = TA_GetGlobalStringCache();
yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData;
readOpInfo = NULL;
nbEstimateBar = 0;
TA_ASSERT( categoryHandle != NULL );
TA_ASSERT( symbolHandle != NULL );
TA_ASSERT( categoryHandle->string != NULL );
TA_ASSERT( symbolHandle->string != NULL );
retCode = TA_BAD_PARAM;
/* Set the initial first/last timestamp */
if( start )
TA_TimestampCopy( &firstBarTimestamp, start );
else
TA_SetDate( 1950, 1, 1, &firstBarTimestamp );
if( end )
TA_TimestampCopy( &lastBarTimestamp, end );
else
TA_SetDateNow( &lastBarTimestamp );
/* Time component is not important for Yahoo! but all end of day
* price bar use 00:00:00, so do the same here.
*/
TA_SetTime( 0, 0, 0, &firstBarTimestamp );
TA_SetTime( 0, 0, 0, &lastBarTimestamp );
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
TA_ASSERT( yahooHandle != NULL );
if( yahooHandle->param->id == TA_YAHOO_ONE_SYMBOL )
{
/* User specified Yahoo! name. */
yahooName = TA_StringDup(stringCache,yahooHandle->webSiteSymbol);
TA_ASSERT( yahooName != NULL );
}
else
{
/* Map the TA-Lib name into the Yahoo! name. */
retCode = TA_AllocStringFromLibName( categoryHandle->string,
symbolHandle->string,
&yahooName );
if( retCode != TA_SUCCESS )
{
TA_TRACE_RETURN( retCode );
}
TA_ASSERT( yahooName != NULL );
}
/* Check if the user did overide the server address in the location parameter.
* (as needed, convert from TA_String to char *)
*/
if( yahooHandle->userSpecifiedServer )
overideServerAddr = TA_StringToChar(yahooHandle->userSpecifiedServer);
else
overideServerAddr = NULL;
/* Get the decoding parameter for the CSV web page. */
if( yahooHandle->param->id == TA_YAHOO_ONE_SYMBOL )
{
retCode = TA_YahooIdxDataDecoding( yahooHandle->webSiteCountry,
TA_YAHOOIDX_CSV_PAGE,
&directYahooDecodingParam );
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( retCode );
}
decodingParam = &directYahooDecodingParam;
}
else
{
decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CSV_PAGE );
if( !decodingParam )
{
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) );
}
}
localDecodingParam = *decodingParam;
/* Check if split/value adjustment are necessary. */
if( (yahooHandle->param->flags & TA_DO_NOT_SPLIT_ADJUST) &&
(yahooHandle->param->flags & TA_DO_NOT_VALUE_ADJUST) )
{
doAdjustment = 0;
}
else
doAdjustment = 1;
/* Parse the uirSuffix so the start/end date can be changed. */
if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) )
{
/* This should never happen unless the
* Yahoo! index protocol has been broken.
*/
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) );
}
/* Use a local copy of the decoding param.
* This is because the uirSuffix is replaced with
* an allocated buffer (so the date field can be
* manipulated).
*/
/* Replace the uirSuffix with a large local buffer. */
localDecodingParam.uirSuffix = TA_Malloc( suffixParsing.maxTotalLength );
if( !localDecodingParam.uirSuffix )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* nbBatch is a safety net to make sure that
* TA-Lib won't stay forever in the while loop
* in case Yahoo! changes their protocol.
*/
nbBatch = 0;
/* Sometime Yahoo! return an empty csv file. Make
* multiple attempts in that case.
*/
zeroBarAddedAttempt = 0;
again = 1;
firstTime = 1;
nbTotalBarAdded = 0;
while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) )
{
retCode = TA_DriverShouldContinue(paramForAddData);
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, yahooName );
TA_Free( (char *)localDecodingParam.uirSuffix );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
TA_TRACE_RETURN( retCode );
}
if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) )
{
/* Get out of this loop if all the requested data
* has been retreived already.
*/
again = 0;
break;
}
retCode = TA_WebPageAllocFromYahooName( &localDecodingParam,
TA_StringToChar(yahooName),
overideServerAddr,
&webPage, paramForAddData );
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, yahooName );
TA_Free( (char *)localDecodingParam.uirSuffix );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
TA_TRACE_RETURN( retCode );
}
/* Disguise the webPage stream into a "file". That way the speed
* optimized ASCII decoder can be re-used (TA_ReadOp stuff).
*/
retCode = TA_FileSeqOpenFromStream( webPage->content, &fileHandle );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( (char *)localDecodingParam.uirSuffix );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
TA_TRACE_RETURN( retCode );
}
if( firstTime )
{
/* Make assumption of the data provided
* base on the number of fields in the CSV file.
*/
nbField = nbCommaField( webPage->content );
switch( nbField )
{
case 2:
readOpInfo = yahooHandle->readOp2Fields;
break;
case 5:
readOpInfo = yahooHandle->readOp5Fields;
break;
default:
readOpInfo = yahooHandle->readOp6Fields;
}
#if !defined( TA_SINGLE_THREAD )
/* Must use a local copy if multi-threaded */
retCode = TA_ReadOpInfoClone(&readOpInfo, readOpInfo);
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
#endif
/* User asking for all the fields? */
if( fieldToAlloc == TA_ALL )
{
switch( nbField )
{
case 2:
fieldToAlloc = TA_CLOSE|TA_TIMESTAMP;
break;
case 5:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP;
break;
default:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP;
}
}
/* Optimize the read op for the requested data. */
retCode = TA_ReadOp_Optimize( readOpInfo,
period,
fieldToAlloc );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( (char *)localDecodingParam.uirSuffix );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
TA_TRACE_RETURN( retCode );
}
/* Make an estimation of the number of price bar. */
nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1;
if( nbEstimateBar < 100 )
nbEstimateBar = 100;
}
/* Interpret the CSV data. */
retCode = TA_ReadOp_Do( fileHandle,
readOpInfo,
period, &firstBarTimestamp, &lastBarTimestamp,
nbEstimateBar, fieldToAlloc,
paramForAddData,
&nbBarAdded );
TA_FileSeqClose( fileHandle );
TA_WebPageFree( webPage );
nbTotalBarAdded += nbBarAdded;
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( (char *)localDecodingParam.uirSuffix );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
TA_TRACE_RETURN( retCode );
}
/* Yahoo! does not always return all the data it could, up to
* the requested end date. It is important to detect these occurence
* and cancel the usage of all data accumulated up to now.
*/
TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData );
if( infoFromAddedData.barAddedSinceLastCall )
{
/* Do some more checking by considering holidays, week-end etc... */
if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( (char *)localDecodingParam.uirSuffix );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
TA_TRACE_RETURN( TA_DATA_GAP );
}
TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp );
}
#if DEBUG_PRINTF
printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded );
#endif
/* Verify if more data should be processed.
* Yahoo! sometimes slice their data, in
* batch of 200 price bars.
*/
if( firstTime && (nbBarAdded > 200) )
{
again = 0; /* Assume all the data extracted... exit the loop. */
}
else if( nbBarAdded == 0 )
{
/* Make multiple attempts when retreiving data succeed,
* but somehow there is zero bar returned.
*
* Sometimes this might be correct when there is truly no
* more data available, so choosing an algorithm before
* giving up is a comprimise between reliability and
* usability. The data source is free... and you get
* what you pay for after all ;)
*/
if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) )
{
/* I did choose to add a delay when insufficient total data is returned. When
* there is already ~5 years of data, most likely there is "Zero" returned
* because there is NO more data available, so just do the retry without delay.
*/
TA_Sleep(zeroBarAddedAttempt*2);
}
#if DEBUG_PRINTF
printf( "Retry %d", zeroBarAddedAttempt );
#endif
zeroBarAddedAttempt++;
}
else
{
zeroBarAddedAttempt = 0;
if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) )
{
/* prevEndDate is a "safety net" to
* exit the loop early in case Yahoo! starts
* to return always the same batch of data.
* Just ignore the repetitive data and exit.
*/
TA_Free( (char *)localDecodingParam.uirSuffix );
TA_StringFree( stringCache, yahooName );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
TA_TRACE_RETURN( TA_SUCCESS );
}
TA_TimestampCopy( &prevEndDate, &lastBarTimestamp );
/* Request the data up to the day BEFORE
* the last batch of data received.
*/
TA_PrevDay( &lastBarTimestamp );
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* From that point, data is expected to be most likely
* sent in batch of 200.
*/
nbEstimateBar = 200;
}
firstTime = 0;
}
/* Get rid of some memory not used anymore. */
TA_Free( (char *)localDecodingParam.uirSuffix );
#if !defined( TA_SINGLE_THREAD )
if( readOpInfo )
TA_ReadOpInfoFree( readOpInfo );
#endif
/* If adjusted data is requested, use splits and dividend info from Yahoo!. */
if( doAdjustment && (nbTotalBarAdded >= 1) )
{
/* Get the decoding parameter for the adjustment page. */
if( yahooHandle->param->id == TA_YAHOO_ONE_SYMBOL )
{
retCode = TA_YahooIdxDataDecoding( yahooHandle->webSiteCountry,
TA_YAHOOIDX_ADJUSTMENT,
&directYahooDecodingParam );
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( retCode );
}
decodingParam = &directYahooDecodingParam;
}
else
{
decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_ADJUSTMENT );
if( !decodingParam )
{
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(140) );
}
}
localDecodingParam = *decodingParam;
if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) )
{
/* This should never happen unless the
* Yahoo! index protocol has been broken.
*/
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(141) );
}
/* Use a local copy of the decoding param.
* This is because the uirSuffix is replaced with
* an allocated buffer (so the date field can be
* manipulated).
*/
/* Replace the uirSuffix with a large local buffer. */
localDecodingParam.uirSuffix = TA_Malloc( suffixParsing.maxTotalLength );
if( !localDecodingParam.uirSuffix )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* curAdjustYear indicates for which year the download is
* taking place.
*/
TA_SetDefault( &curAdjustYear );
TA_SetDateNow( &curAdjustYear );
/* Identify the oldest year for which data was downloaded */
TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData );
TA_TimestampCopy( &lastAdjustYear, &infoFromAddedData.lowestTimestamp );
TA_PrevYear( &lastAdjustYear ); /* Get one more year to be on the safe side. */
while( TA_TimestampLess( &lastAdjustYear, &curAdjustYear ) )
{
/* Set prevEndDate to two years earlier. */
TA_TimestampCopy( &prevEndDate, &curAdjustYear );
TA_PrevYear( &prevEndDate );
TA_PrevYear( &prevEndDate );
/* Change the dates in the uirSuffix. */
TA_SetDate( TA_GetYear(&curAdjustYear), 12, 31, &curAdjustYear );
TA_SetDate( TA_GetYear(&prevEndDate), 1, 1, &prevEndDate );
buildUIRSuffix( &suffixParsing,
&prevEndDate, &curAdjustYear,
(char *)localDecodingParam.uirSuffix );
retCode = doAdjustments( &localDecodingParam, yahooName,
yahooHandle, NULL, paramForAddData );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Move 3 years earlier. */
TA_PrevYear( &prevEndDate );
TA_TimestampCopy( &curAdjustYear, &prevEndDate );
}
/* Clean-up what was allocated for the adjustment logic. */
TA_Free( (char *)localDecodingParam.uirSuffix );
}
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( retCode );
}
/**** Global functions definitions. ****/
TA_FileIndexPriv *TA_FileIndexPrivAlloc( TA_Libc *libHandle,
TA_String *initialCategory,
TA_String *initialCategoryCountry,
TA_String *initialCategoryExchange,
TA_String *initialCategoryType )
{
TA_FileIndexPriv *fileIndexPrivData;
TA_StringCache *stringCache;
stringCache = TA_GetGlobalStringCache( libHandle );
/* Initialize the TA_FileIndexPriv element. */
fileIndexPrivData = (TA_FileIndexPriv *)TA_Malloc( libHandle, sizeof( TA_FileIndexPriv ) );
if( !fileIndexPrivData )
return NULL;
/* initialize all fields to NULL. */
memset( fileIndexPrivData, 0, sizeof( TA_FileIndexPriv ) );
/* Now attempt to allocate all sub-elements. */
fileIndexPrivData->libHandle = libHandle;
stringCache = TA_GetGlobalStringCache( libHandle );
fileIndexPrivData->initialCategoryString = TA_StringDup( stringCache, initialCategory );
fileIndexPrivData->initialCategoryCountryString = TA_StringDup( stringCache, initialCategoryCountry);
fileIndexPrivData->initialCategoryExchangeString = TA_StringDup( stringCache, initialCategoryExchange );
fileIndexPrivData->initialCategoryTypeString = TA_StringDup( stringCache, initialCategoryType );
if( (fileIndexPrivData->initialCategoryString == NULL) ||
(fileIndexPrivData->initialCategoryCountryString == NULL) ||
(fileIndexPrivData->initialCategoryExchangeString == NULL) ||
(fileIndexPrivData->initialCategoryTypeString == NULL) )
{
freeFileIndexPriv( (void *)fileIndexPrivData );
return NULL;
}
fileIndexPrivData->scratchPad = (char *)TA_Malloc( libHandle, TA_SOURCELOCATION_MAX_LENGTH+2 );
if( !fileIndexPrivData->scratchPad )
{
freeFileIndexPriv( (void *)fileIndexPrivData );
return NULL;
}
fileIndexPrivData->listLocationToken = TA_ListAlloc( libHandle );
if( !fileIndexPrivData->listLocationToken )
{
freeFileIndexPriv( (void *)fileIndexPrivData );
return NULL;
}
fileIndexPrivData->listCategory = TA_ListAlloc( libHandle );
if( !fileIndexPrivData->listCategory )
{
freeFileIndexPriv( (void *)fileIndexPrivData );
return NULL;
}
fileIndexPrivData->root = allocTreeNode( libHandle, NULL, NULL );
if( !fileIndexPrivData->root )
{
freeFileIndexPriv( (void *)fileIndexPrivData );
return NULL;
}
fileIndexPrivData->currentNode = fileIndexPrivData->root;
fileIndexPrivData->wildOneChar = TA_StringAlloc( stringCache, "?" );
fileIndexPrivData->wildZeroOrMoreChar = TA_StringAlloc( stringCache, "*" );
fileIndexPrivData->wildOneOrMoreChar = TA_StringAlloc( stringCache, "?*" );
if( (!fileIndexPrivData->wildOneChar) ||
(!fileIndexPrivData->wildZeroOrMoreChar) ||
(!fileIndexPrivData->wildOneOrMoreChar) )
{
freeFileIndexPriv( (void *)fileIndexPrivData );
return NULL;
}
return fileIndexPrivData;
}
TA_RetCode TA_FileIndexAddSymbolData( TA_Libc *libHandle,
TA_FileIndexCategoryData *categoryData,
TA_String *stringSymbol,
TA_ValueTreeNode *treeNodeValue,
TA_FileIndexSymbolData **added )
{
TA_PROLOG;
TA_RetCode retCode;
TA_FileIndexSymbolData *symbolData;
unsigned int tmpInt;
unsigned int symbolFound; /* Boolean */
TA_StringCache *stringCache;
TA_TRACE_BEGIN( libHandle, TA_FileIndexAddSymbolData );
stringCache = TA_GetGlobalStringCache( libHandle );
TA_ASSERT( libHandle, categoryData != NULL );
TA_ASSERT( libHandle, categoryData->listSymbol != NULL );
TA_ASSERT( libHandle, stringSymbol != NULL );
TA_ASSERT( libHandle, treeNodeValue != NULL );
if( added )
*added = NULL;
/* Trap the case where this symbol is already there for that
* category. In that case, the information is ignored.
* Under the same category, for the same datasource, only one file
* is supported for a category-symbol pair.
*/
symbolData = (TA_FileIndexSymbolData *)TA_ListAccessTail( categoryData->listSymbol );
symbolFound = 0;
while( symbolData && !symbolFound )
{
TA_ASSERT( libHandle, symbolData->string != NULL );
tmpInt = strcmp( TA_StringToChar( stringSymbol ),
TA_StringToChar( symbolData->string ) );
if( tmpInt == 0 )
symbolFound = 1;
else
symbolData = (TA_FileIndexSymbolData *)TA_ListAccessPrev( categoryData->listSymbol );
}
if( !symbolFound )
{
/* This is a new symbol, so allocate the TA_FileIndexSymbolData */
symbolData = (TA_FileIndexSymbolData *)TA_Malloc( libHandle, sizeof(TA_FileIndexSymbolData) );
if( !symbolData )
{
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* Initialize the TA_FileIndexSymbolData */
symbolData->parent = categoryData;
symbolData->node = treeNodeValue;
symbolData->string = TA_StringDup( stringCache, stringSymbol );
if( !symbolData->string )
{
TA_Free( libHandle, symbolData );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* Add it to the category list. */
retCode = TA_ListAddTail( categoryData->listSymbol, symbolData );
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, symbolData->string );
TA_Free( libHandle, symbolData );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
}
/* Return the address of the object representing that symbol. */
if( added )
*added = symbolData;
TA_TRACE_RETURN( TA_SUCCESS );
}
TA_RetCode TA_FileIndexAddCategoryData( TA_FileIndexPriv *data,
TA_String *stringCategory,
TA_FileIndexCategoryData **added )
{
TA_PROLOG;
TA_RetCode retCode;
TA_FileIndexCategoryData *categoryData;
unsigned int tmpInt;
unsigned int categoryFound; /* Boolean */
TA_Libc *libHandle;
TA_StringCache *stringCache;
libHandle = data->libHandle;
TA_TRACE_BEGIN( libHandle, TA_FileIndexAddCategoryData );
stringCache = TA_GetGlobalStringCache( libHandle );
TA_ASSERT( libHandle, data != NULL );
TA_ASSERT( libHandle, stringCategory != NULL );
/* Trap the case where the category is already added. */
categoryData = (TA_FileIndexCategoryData *)TA_ListAccessTail( data->listCategory );
categoryFound = 0;
while( categoryData && !categoryFound )
{
TA_ASSERT( libHandle, categoryData->string != NULL );
tmpInt = strcmp( TA_StringToChar( stringCategory ),
TA_StringToChar( categoryData->string ) );
if( tmpInt == 0 )
categoryFound = 1;
else
categoryData = (TA_FileIndexCategoryData *)TA_ListAccessPrev( data->listCategory );
}
if( !categoryFound )
{
/* This is a new category, so allocate the TA_FileIndexCategoryData */
categoryData = (TA_FileIndexCategoryData *)TA_Malloc( libHandle, sizeof(TA_FileIndexCategoryData) );
if( !categoryData )
TA_TRACE_RETURN( TA_ALLOC_ERR );
/* Initialize the TA_FileIndexCategoryData */
categoryData->parent = data;
if( stringCategory )
{
categoryData->string = TA_StringDup( stringCache, stringCategory); /* String for this category. Can be NULL. */
if( !categoryData->string )
{
TA_Free( libHandle, categoryData );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
}
else
categoryData->string = NULL;
categoryData->listSymbol = TA_ListAlloc( libHandle );
if( !categoryData->listSymbol )
{
if( categoryData->string )
TA_StringFree( stringCache, categoryData->string );
TA_Free( libHandle, categoryData );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* Add it to the TA_FileIndexPriv */
retCode = TA_ListAddTail( data->listCategory, categoryData );
if( retCode != TA_SUCCESS )
{
TA_ListFree( categoryData->listSymbol );
if( categoryData->string )
TA_StringFree( stringCache, categoryData->string );
TA_Free( libHandle, categoryData );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
#if 0
printf( "**ADDING CATEGORY[%s]\n", TA_StringToChar( categoryData->string ) );
#endif
}
/* Return the address of the object representing that category. */
if( added )
*added = categoryData;
TA_TRACE_RETURN( TA_SUCCESS );
}
