TA_RetCode TA_ReadOp_Do( TA_FileHandle *fileHandle,
const TA_ReadOpInfo *readOpInfo,
TA_Period period,
const TA_Timestamp *start,
const TA_Timestamp *end,
unsigned int minimumNbBar,
TA_Field fieldToAlloc,
TA_ParamForAddData *paramForAddData,
unsigned int *nbBarAdded )
{
TA_PROLOG
TA_RetCode retCode;
TA_EstimateInfo estimationInfo;
unsigned int nbElementToAllocate;
unsigned int memoryNeeded; /* Boolean */
unsigned int timeNeeded; /* Boolean */
TA_Real *arrayReal[TA_REAL_ARRAY_SIZE];
TA_Integer *arrayInteger[TA_INTEGER_ARRAY_SIZE];
TA_Timestamp *timestamp;
TA_Real *openBeg, *highBeg, *lowBeg, *closeBeg;
TA_Integer *volumeBeg, *openInterestBeg;
TA_Timestamp *timestampBeg;
TA_Timestamp tmpTimestamp;
TA_ReadOp op;
TA_Field fieldToProcess;
unsigned int year, month, day, hour, min, sec;
TA_Integer curOp;
unsigned int nbTotalByteDone, nbTotalBarDone;
unsigned int nbBarAddedInTheBlock;
char monthChar[4];
char cnvtArray[CNVT_ARRAY_SIZE];
unsigned int cnvtArrayIdx;
unsigned int nbByteToAllocReal;
unsigned int nbByteToAllocInteger;
unsigned int fileSize;
unsigned int skipField;
unsigned int lineToSkip;
unsigned int nbByteRead;
unsigned int nbLetter;
TA_Real lastValidClose;
const char *car;
register TA_Real tmpReal;
register TA_Integer tmpInt;
register unsigned int tmpIdx;
register unsigned int nbCharToRead;
unsigned int lastOpFieldIncremented;
TA_TRACE_BEGIN( TA_PriceBarRead );
/* Initialization of local variables. */
openBeg = highBeg = lowBeg = closeBeg = NULL;
timestampBeg = NULL;
volumeBeg = openInterestBeg = NULL;
timestamp = NULL;
retCode = TA_SUCCESS;
lastValidClose = 0.0;
fieldToProcess = readOpInfo->fieldProvided & fieldToAlloc;
if( (fieldToProcess & fieldToAlloc) != fieldToAlloc )
{
/* Nothing to read because not all the requested
* fields are provided by this data source!
*/
return TA_SUCCESS;
}
/* Estimate the initial amount of memory to allocate. */
fileSize = TA_FileSize( fileHandle );
retCode = TA_EstimateAllocInit( start, end, period,
minimumNbBar, 2048,
&estimationInfo,
&nbElementToAllocate );
if( retCode != TA_SUCCESS )
{
TA_TRACE_RETURN( retCode );
}
if( nbElementToAllocate == 0 )
{
TA_TRACE_RETURN( TA_SUCCESS ); /* Nothing to read!? Just return... */
}
memset( arrayInteger, 0, sizeof( arrayInteger ) );
memset( arrayReal, 0, sizeof( arrayReal ) );
/* Set the date/time pointers to where the information will be stored. */
arrayInteger[TA_HOUR_IDX] = (TA_Integer *)&hour;
arrayInteger[TA_MIN_IDX] = (TA_Integer *)&min;
arrayInteger[TA_SEC_IDX] = (TA_Integer *)&sec;
arrayInteger[TA_MONTH_IDX] = (TA_Integer *)&month;
arrayInteger[TA_YEAR_IDX] = (TA_Integer *)&year;
arrayInteger[TA_DAY_IDX] = (TA_Integer *)&day;
/* Set default time/date. */
year = 1900;
month = day = 1;
hour = 23;
min = sec = 59;
/* Check if the processing of the time will be needed. */
timeNeeded = isTimeNeeded( readOpInfo->arrayReadOp);
/* 'car' always point to the character being currently handled. */
nbByteRead = 0;
car = TA_FileSeqRead( fileHandle, &nbByteRead );
if( (car == NULL) || (nbByteRead == 0) )
return TA_SUCCESS; /* End of file! */
--nbByteRead;
nbTotalByteDone = 0;
nbTotalBarDone = 0;
nbBarAddedInTheBlock = 0;
memoryNeeded = 1;
curOp = 0;
skipField = 0;
monthChar[3] = '\0';
/* When requested, skip header lines. */
lineToSkip = readOpInfo->nbHeaderLineToSkip;
while( lineToSkip-- )
{
while( *car != '\n' )
{
GET_CHAR;
if( car == NULL )
goto exit_loops;
}
}
line_loop: /* Always jump here when end-of-line is found (EOL). */
/* If curOp != 0, the last operations are canceled. */
REVERT_OPERATIONS;
curOp = 0;
lastOpFieldIncremented = 0;
/* Start over a new line. */
if( memoryNeeded )
{
/* Allocate the memory. */
nbByteToAllocReal = nbElementToAllocate * sizeof( TA_Real );
nbByteToAllocInteger = nbElementToAllocate * sizeof( TA_Integer );
timestamp = (TA_Timestamp *)TA_Malloc( nbElementToAllocate * sizeof( TA_Timestamp ) );
timestampBeg = timestamp;
if( !timestampBeg )
{
retCode = TA_ALLOC_ERR;
goto exit_loops;
}
#define TA_ALLOC_MEM(upperc,lowerc,typepar) \
{ \
if( fieldToProcess & TA_##upperc ) \
{ \
lowerc##Beg = (TA_##typepar *)TA_Malloc( nbByteToAlloc##typepar ); \
array##typepar[TA_##upperc##_IDX] = lowerc##Beg; \
if( !lowerc##Beg ) \
{ \
retCode = TA_ALLOC_ERR; \
goto exit_loops; \
} \
} \
}
TA_ALLOC_MEM( OPEN, open, Real );
TA_ALLOC_MEM( HIGH, high, Real );
TA_ALLOC_MEM( LOW, low, Real );
TA_ALLOC_MEM( CLOSE, close, Real );
TA_ALLOC_MEM( VOLUME, volume, Integer );
TA_ALLOC_MEM( OPENINTEREST, openInterest, Integer );
#undef TA_ALLOC_MEM
memoryNeeded = 0;
}
op_loop: /* Jump here when ready to proceed with the next command. */
op = readOpInfo->arrayReadOp[curOp];
if( !(op&TA_CMD_READ_MONTH_CHAR) )
{
/* Skip leading non-numeric character. */
SKIP_UNTIL_NUMERIC;
}
/* Shall we skip this field? */
if( TA_IS_SKIP_SET(op) )
{
if( (op&(TA_CMD_READ_REAL|TA_CMD_READ_INTEGER)) == 0 )
{
tmpInt = TA_GET_NB_NUMERIC(op);
curOp++;
while( tmpInt-- )
{
GET_CHAR;
CHECK_EOL_EOF;
}
}
else
{
if( skipField == 0 )
{
skipField = TA_GET_NB_NUMERIC(op);
TA_ASSERT( skipField > 0 );
}
if( --skipField == 0 )
curOp++;
SKIP_NUMERIC;
if( (op&TA_CMD_READ_REAL) && (*car == '.') )
{
GET_CHAR;
CHECK_EOL_EOF;
SKIP_NUMERIC;
}
}
}
else
{
cnvtArrayIdx = 0;
if( TA_IS_REAL_CMD(op) )
{
/* Extract a numeric into cnvtArray. */
READ_IN_CNVT_ARRAY;
/* This is a TA_Real. */
if( car && (*car == '.') )
{
/* Read rest of the float after the '.' */
READ_IN_CNVT_ARRAY;
}
cnvtArray[cnvtArrayIdx] = '\0';
tmpReal = atof( &cnvtArray[0] );
/* Write the TA_Real in memory. */
tmpIdx = TA_GET_IDX(op);
TA_ASSERT( tmpIdx < TA_REAL_ARRAY_SIZE );
TA_ASSERT( arrayReal[tmpIdx] != NULL );
if( tmpReal != 0.0 )
{
*(arrayReal[tmpIdx]) = tmpReal;
if( tmpIdx == TA_CLOSE_IDX )
lastValidClose = tmpReal;
}
else if( TA_IS_REPLACE_ZERO(op) )
{
/* Replace this zero value with the last known close.
* If there is no previous close, this line is ignored.
*/
if( lastValidClose != 0.0 )
*(arrayReal[tmpIdx]) = lastValidClose;
else
{
SKIP_LINE;
}
}
else
{
/* Zero are not expected, consider this as a failure
* and ignore all further data from this file.
*/
retCode = TA_PRICE_BAR_CONTAINS_ZERO;
goto exit_loops;
}
arrayReal[tmpIdx]++;
curOp++;
}
else
{
/* This is a TA_Integer. */
if( !(op&TA_CMD_READ_MONTH_CHAR) )
{
nbCharToRead = TA_GET_NB_NUMERIC(op);
if( nbCharToRead )
{
READ_N_CHAR_IN_CNVT_ARRAY(nbCharToRead);
}
else
{
READ_IN_CNVT_ARRAY;
}
cnvtArray[cnvtArrayIdx] = '\0';
tmpInt = atoi( &cnvtArray[0] );
}
else
{
/* Try to find a 3 letters month string.
* Translate it to a [1..12] integer.
*/
nbLetter = 1;
do
{
CHECK_EOL_EOF;
monthChar[nbLetter] = (char)toupper(*car);
GET_CHAR;
nbLetter++;
} while( nbLetter != 3 );
do
{
CHECK_EOL_EOF;
monthChar[0] = monthChar[1];
monthChar[1] = monthChar[2];
monthChar[2] = (char)toupper(*car);
if( strncmp("JAN",monthChar,3) == 0 )
tmpInt = 1;
else if( strncmp("FEB",monthChar,3) == 0 )
tmpInt = 2;
else if( strncmp("MAR",monthChar,3) == 0 )
tmpInt = 3;
else if( strncmp("APR",monthChar,3) == 0 )
tmpInt = 4;
else if( strncmp("MAY",monthChar,3) == 0 )
tmpInt = 5;
else if( strncmp("JUN",monthChar,3) == 0 )
tmpInt = 6;
else if( strncmp("JUL",monthChar,3) == 0 )
tmpInt = 7;
else if( strncmp("AUG",monthChar,3) == 0 )
tmpInt = 8;
else if( strncmp("SEP",monthChar,3) == 0 )
tmpInt = 9;
else if( strncmp("OCT",monthChar,3) == 0 )
tmpInt = 10;
else if( strncmp("NOV",monthChar,3) == 0 )
tmpInt = 11;
else if( strncmp("DEC",monthChar,3) == 0 )
tmpInt = 12;
else
tmpInt = 0;
GET_CHAR;
} while( tmpInt == 0 );
}
/* Write the TA_Integer in memory. */
tmpIdx = TA_GET_IDX(op);
TA_ASSERT( tmpIdx < TA_INTEGER_ARRAY_SIZE );
TA_ASSERT( arrayInteger[tmpIdx] != NULL );
*(arrayInteger[tmpIdx]) = tmpInt;
if( tmpIdx > TA_YEAR_IDX )
arrayInteger[tmpIdx]++;
curOp++;
if( TA_IS_TIMESTAMP_COMPLETE(op) )
{
/* Build the timestamp. */
retCode = TA_SetDate( year, month, day, &tmpTimestamp );
if( retCode != TA_SUCCESS )
goto exit_loops; /* Invalid date */
if( !timeNeeded )
{
/* Ignore time in comparison and use default to build the price bar. */
tmpTimestamp.time = 23595900; /* Default EOD time */
if( start && (tmpTimestamp.date < start->date) )
{
/* This price bar is not needed, jump to the next line. */
retCode = TA_SUCCESS;
SKIP_LINE;
}
if( end && (tmpTimestamp.date > end->date) )
{
/* This price bar is beyond the upper limit, just exit. */
goto exit_loops;
}
}
else
{
retCode = TA_SetTime( hour, min, sec, &tmpTimestamp );
if( retCode != TA_SUCCESS )
goto exit_loops; /* Invalid time */
if( start && TA_TimestampLess(&tmpTimestamp,start) )
{
/* This price bar is not needed, jump to the next line. */
retCode = TA_SUCCESS;
SKIP_LINE;
}
if( end && TA_TimestampGreater(&tmpTimestamp, end) )
{
/* This price bar is beyond the upper limit, just exit. */
goto exit_loops;
}
}
/* Write the timestamp in memory. */
*timestamp = tmpTimestamp;
}
}
if( TA_IS_READ_STOP_FLAG_SET(op) )
{
#ifdef DEBUG_PRINTF
printf( "(%d%d%d,%e,%e,%e,%e,%d)\n",
timestampBeg?TA_GetYear(×tampBeg[nbBarAddedInTheBlock]):0,
timestampBeg?TA_GetMonth(×tampBeg[nbBarAddedInTheBlock]):0,
timestampBeg?TA_GetDay(×tampBeg[nbBarAddedInTheBlock]):0,
openBeg?openBeg[nbBarAddedInTheBlock]:0.0,
highBeg?highBeg[nbBarAddedInTheBlock]:0.0,
lowBeg?lowBeg[nbBarAddedInTheBlock]:0.0,
closeBeg?closeBeg[nbBarAddedInTheBlock]:0.0,
volumeBeg?volumeBeg[nbBarAddedInTheBlock]:0 );
#endif
/* At this point, the price bar is completely written in memory. */
timestamp++;
curOp = 0;
nbBarAddedInTheBlock++;
if( nbBarAddedInTheBlock < nbElementToAllocate )
{
/* Go to next line. */
SKIP_LINE;
}
else
{
/* There is not enough memory for another bar, so re-allocate
* some more.
*/
retCode = TA_HistoryAddData( paramForAddData, nbBarAddedInTheBlock,
period, timestampBeg,
openBeg, highBeg, lowBeg, closeBeg,
volumeBeg, openInterestBeg );
/* TA_HistoryAddData is ALWAYS the owner of these memory block
* when called. So we set these to NULL to make sure there will
* be no attempt to free these from this function.
*/
openBeg = highBeg = lowBeg = closeBeg = NULL;
timestampBeg = NULL;
volumeBeg = openInterestBeg = NULL;
nbTotalBarDone += nbBarAddedInTheBlock;
nbBarAddedInTheBlock = 0;
if( retCode != TA_SUCCESS )
goto exit_loops;
retCode = TA_EstimateAllocNext( &estimationInfo, &nbElementToAllocate );
if( retCode != TA_SUCCESS )
goto exit_loops;
memoryNeeded = 1;
SKIP_LINE;
}
}
else
{
/* Make sure we did not hit an EOL or EOF prematurly, if yes,
* this line will be silently ignored.
*/
CHECK_EOL_EOF;
}
}
goto op_loop;
exit_loops: /* Jump here when the end-of-file is hit (or an error occured) */
/* On succesful exit, process possibly remaining data. */
if( (retCode == TA_SUCCESS) && (nbBarAddedInTheBlock != 0) )
{
retCode = TA_HistoryAddData( paramForAddData, nbBarAddedInTheBlock,
period, timestampBeg,
openBeg, highBeg, lowBeg, closeBeg,
volumeBeg, openInterestBeg );
openBeg = highBeg = lowBeg = closeBeg = NULL;
timestampBeg = NULL;
volumeBeg = openInterestBeg = NULL;
nbTotalBarDone += nbBarAddedInTheBlock;
}
/* ALWAYS verify if locally allocated memory needs to be freed. ALWAYS. */
FREE_IF_NOT_NULL( openBeg );
FREE_IF_NOT_NULL( highBeg );
FREE_IF_NOT_NULL( lowBeg );
FREE_IF_NOT_NULL( closeBeg );
FREE_IF_NOT_NULL( volumeBeg );
FREE_IF_NOT_NULL( openInterestBeg );
FREE_IF_NOT_NULL( timestampBeg );
/* An indication that no more data needs to be provided is not a failure. */
if( retCode == TA_ENOUGH_DATA )
retCode = TA_SUCCESS;
/* Return the number of added price bar to the caller. */
if( nbBarAdded )
*nbBarAdded = nbTotalBarDone;
TA_TRACE_RETURN( retCode );
}
/**** Global functions definitions. ****/
TA_RetCode TA_GetHistoryDataFromWeb( TA_Libc *libHandle,
TA_DataSourceHandle *handle,
TA_CategoryHandle *categoryHandle,
TA_SymbolHandle *symbolHandle,
TA_Period period,
const TA_Timestamp *start,
const TA_Timestamp *end,
TA_Field fieldToAlloc,
TA_ParamForAddData *paramForAddData )
{
TA_PROLOG;
TA_RetCode retCode;
TA_StringCache *stringCache;
TA_String *yahooName;
TA_WebPage *webPage;
TA_PrivateYahooHandle *yahooHandle;
TA_DecodingParam localDecodingParam;
const TA_DecodingParam *decodingParam;
TA_FileHandle *fileHandle;
TA_ReadOpInfo *readOpInfo;
UIRSuffixParsing suffixParsing;
TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate;
TA_InfoFromAddedData infoFromAddedData;
TA_DayOfWeek dayOfWeek;
int nbEstimateBar;
int nbField;
unsigned int nbBarAdded, nbTotalBarAdded;
int again, firstTime, nbBatch;
int zeroBarAddedAttempt;
TA_TRACE_BEGIN( libHandle, TA_GetHistoryDataFromWeb );
/* Initialize some local variables. */
stringCache = TA_GetGlobalStringCache( libHandle );
yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData;
readOpInfo = NULL;
nbEstimateBar = 0;
TA_ASSERT( libHandle, categoryHandle != NULL );
TA_ASSERT( libHandle, symbolHandle != NULL );
TA_ASSERT( libHandle, categoryHandle->string != NULL );
TA_ASSERT( libHandle, symbolHandle->string != NULL );
/* Set the initial first/last timestamp */
if( start )
TA_TimestampCopy( &firstBarTimestamp, start );
else
{
TA_SetDate( 1950, 1, 1, &firstBarTimestamp );
TA_SetTime( 0, 0, 0, &firstBarTimestamp );
}
if( end )
TA_TimestampCopy( &lastBarTimestamp, end );
else
{
TA_SetDateNow( &lastBarTimestamp );
TA_SetTime( 0, 0, 0, &lastBarTimestamp );
}
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
/* Map the TA-Lib name into the Yahoo! name. */
retCode = TA_AllocStringFromLibName( libHandle,
categoryHandle->string,
symbolHandle->string,
&yahooName );
if( retCode != TA_SUCCESS )
{
TA_TRACE_RETURN( retCode );
}
TA_ASSERT( libHandle, yahooName != NULL );
TA_ASSERT( libHandle, yahooHandle != NULL );
/* Get the decoding parameter for the CSV web page. */
decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CVS_PAGE );
if( !decodingParam )
{
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) );
}
/* Use a local copy of the decoding param.
* This is because the uirSuffix is replaced with
* an allocated buffer (so the date field can be
* manipulated).
*/
localDecodingParam = *decodingParam;
/* Parse the uirSuffix so the start/end date can be changed. */
if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) )
{
/* This should never happen unless the
* Yahoo! index protocol has been broken.
*/
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) );
}
/* Replace the uirSuffix with a large local buffer. */
localDecodingParam.uirSuffix = TA_Malloc( libHandle, suffixParsing.maxTotalLength );
if( !localDecodingParam.uirSuffix )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* nbBatch is a safety net to make sure that
* TA-Lib won't stay forever in the while loop
* in case Yahoo! changes their protocol.
*/
nbBatch = 0;
/* Sometime Yahoo! return an empty csv file. Make
* multiple attempts in that case.
*/
zeroBarAddedAttempt = 0;
again = 1;
firstTime = 1;
nbTotalBarAdded = 0;
while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) )
{
if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) )
{
/* Get out of this loop if all the requested data
* has been retreived already.
*/
again = 0;
break;
}
retCode = TA_WebPageAllocFromYahooName( libHandle,
&localDecodingParam,
TA_StringToChar(yahooName),
&webPage );
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Disguise the webPage stream into a "file". That way the speed
* optimized ASCII decoder can be re-used (TA_ReadOp stuff).
*/
retCode = TA_FileSeqOpenFromStream( libHandle, webPage->content, &fileHandle );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
if( firstTime )
{
/* Make assumption of the data provided
* base on the number of fields in the CSV file.
*/
nbField = nbCommaField( webPage->content );
switch( nbField )
{
case 2:
readOpInfo = yahooHandle->readOp2Fields;
break;
case 5:
readOpInfo = yahooHandle->readOp5Fields;
break;
default:
readOpInfo = yahooHandle->readOp6Fields;
}
/* User asking for all the fields? */
if( fieldToAlloc == TA_ALL )
{
switch( nbField )
{
case 2:
fieldToAlloc = TA_CLOSE|TA_TIMESTAMP;
break;
case 5:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP;
break;
default:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP;
}
}
/* Optimize the read op for the requested data. */
retCode = TA_ReadOp_Optimize( libHandle,
readOpInfo,
period,
fieldToAlloc );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Make an estimation of the number of price bar. */
nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1;
if( nbEstimateBar < 100 )
nbEstimateBar = 100;
}
/* Interpret the CSV data. */
retCode = TA_ReadOp_Do( libHandle, fileHandle,
readOpInfo,
period, &firstBarTimestamp, &lastBarTimestamp,
nbEstimateBar, fieldToAlloc,
paramForAddData,
&nbBarAdded );
TA_FileSeqClose( libHandle, fileHandle );
TA_WebPageFree( webPage );
nbTotalBarAdded += nbBarAdded;
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Yahoo! does not always return all the data it could, up to
* the requested end date. It is important to detect these occurence
* and cancel the usage of all data accumulated up to now.
*/
TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData );
if( infoFromAddedData.barAddedSinceLastCall )
{
/* Do some more checking by considering holidays, week-end etc... */
if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( TA_DATA_GAP );
}
TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp );
}
#if DEBUG_PRINTF
printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded );
#endif
/* Verify if more data should be processed.
* Yahoo! sometimes slice their data, in
* batch of 200 price bars.
*/
if( firstTime && (nbBarAdded > 200) )
{
again = 0; /* Assume all the data extracted... exit the loop. */
}
else if( nbBarAdded == 0 )
{
/* Make multiple attempts when retreiving data succeed,
* but somehow there is zero bar returned.
*
* Sometimes this might be correct when there is truly no
* more data available, so choosing an algorithm before
* giving up is a comprimise between reliability and
* usability. The data source is free... and you get
* what you pay for after all ;)
*/
if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) )
{
/* I did choose to add a delay when insufficient total data is returned. When
* there is already ~5 years of data, most likely there is "Zero" returned
* because there is NO more data available, so just do the retry without delay.
*/
TA_Sleep(zeroBarAddedAttempt*2);
}
#if DEBUG_PRINTF
printf( "Retry %d", zeroBarAddedAttempt );
#endif
zeroBarAddedAttempt++;
}
else
{
zeroBarAddedAttempt = 0;
if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) )
{
/* prevEndDate is a "safety net" to
* exit the loop early in case Yahoo! starts
* to return always the same batch of data.
* Just ignore the repetitive data and exit.
*/
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_SUCCESS );
}
TA_TimestampCopy( &prevEndDate, &lastBarTimestamp );
/* Request the data up to the day BEFORE
* the last batch of data received.
*/
TA_PrevDay( &lastBarTimestamp );
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* From that point, data is expected to be most likely
* sent in batch of 200.
*/
nbEstimateBar = 200;
}
firstTime = 0;
}
/* Clean-up and exit */
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( retCode );
}
