/* Check if t0 is within the provided [t1..t2] range (inclusive check) */
static unsigned int TA_DateWithinRange( unsigned int year,
unsigned int month,
unsigned int day,
const TA_Timestamp *t1,
const TA_Timestamp *t2 )
{
TA_Timestamp stamp;
const TA_Timestamp *lowBorder;
const TA_Timestamp *highBorder;
/* Inverse t1 and t2 if not chronilogical order. */
if( TA_TimestampLess( t2, t1 ) )
{
lowBorder = t2;
highBorder = t1;
}
else
{
lowBorder = t1;
highBorder = t2;
}
/* Build a timestamp for the date to be check */
TA_SetDate( year, month, day, &stamp );
/* Check if exactly on boundary */
if( TA_TimestampEqual( &stamp, lowBorder ) && TA_TimestampEqual( &stamp, highBorder ) )
{
return 1;
}
/* Check if within range. */
if( TA_TimestampGreater( &stamp, lowBorder ) && TA_TimestampLess( &stamp, highBorder ) )
{
return 1;
}
return 0; /* Out-of-range */
}
static int findTimestampIndex( const TA_PMPriv *pmPriv,
const TA_Timestamp *exitTimestamp,
int *idx )
{
const TA_Timestamp *startDate;
const TA_Timestamp *endDate;
TA_DayOfWeek dayOfTheWeek;
/* Return 0 when no index can be resolved. */
startDate = &pmPriv->startDate;
endDate = &pmPriv->endDate;
/* Make sure the exitTimestamp is within the start/end date. */
if( (TA_TimestampGreater(exitTimestamp,startDate)&&TA_TimestampLess(exitTimestamp,endDate)) ||
TA_TimestampEqual(exitTimestamp, startDate) ||
TA_TimestampEqual(exitTimestamp, endDate) )
{
/* Make sure the exitTimestamp is NOT on week-end. Week-end
* trades are currently ignored.
*/
dayOfTheWeek = TA_GetDayOfTheWeek( exitTimestamp );
if( (dayOfTheWeek != TA_SUNDAY) && (dayOfTheWeek != TA_SATURDAY) )
{
TA_TimestampDeltaWeekday( startDate, exitTimestamp, (unsigned int *)idx );
*idx -= 1;
#ifdef TA_DEBUG
TA_ASSERT_RET( *idx >= 0, 0 );
TA_ASSERT_RET( (unsigned int)*idx < pmPriv->nbDailyBars, 0 );
TA_ASSERT_RET( TA_TimestampEqual(&pmPriv->arrayTimestamp[*idx], exitTimestamp ), 0 );
#endif
return 1;
}
}
/* No index can be found, initialize to zero, just to
* be safe.
*/
*idx = 0;
return 0;
}
TA_RetCode TA_ReadOp_Do( TA_FileHandle *fileHandle,
const TA_ReadOpInfo *readOpInfo,
TA_Period period,
const TA_Timestamp *start,
const TA_Timestamp *end,
unsigned int minimumNbBar,
TA_Field fieldToAlloc,
TA_ParamForAddData *paramForAddData,
unsigned int *nbBarAdded )
{
TA_PROLOG
TA_RetCode retCode;
TA_EstimateInfo estimationInfo;
unsigned int nbElementToAllocate;
unsigned int memoryNeeded; /* Boolean */
unsigned int timeNeeded; /* Boolean */
TA_Real *arrayReal[TA_REAL_ARRAY_SIZE];
TA_Integer *arrayInteger[TA_INTEGER_ARRAY_SIZE];
TA_Timestamp *timestamp;
TA_Real *openBeg, *highBeg, *lowBeg, *closeBeg;
TA_Integer *volumeBeg, *openInterestBeg;
TA_Timestamp *timestampBeg;
TA_Timestamp tmpTimestamp;
TA_ReadOp op;
TA_Field fieldToProcess;
unsigned int year, month, day, hour, min, sec;
TA_Integer curOp;
unsigned int nbTotalByteDone, nbTotalBarDone;
unsigned int nbBarAddedInTheBlock;
char monthChar[4];
char cnvtArray[CNVT_ARRAY_SIZE];
unsigned int cnvtArrayIdx;
unsigned int nbByteToAllocReal;
unsigned int nbByteToAllocInteger;
unsigned int fileSize;
unsigned int skipField;
unsigned int lineToSkip;
unsigned int nbByteRead;
unsigned int nbLetter;
TA_Real lastValidClose;
const char *car;
register TA_Real tmpReal;
register TA_Integer tmpInt;
register unsigned int tmpIdx;
register unsigned int nbCharToRead;
unsigned int lastOpFieldIncremented;
TA_TRACE_BEGIN( TA_PriceBarRead );
/* Initialization of local variables. */
openBeg = highBeg = lowBeg = closeBeg = NULL;
timestampBeg = NULL;
volumeBeg = openInterestBeg = NULL;
timestamp = NULL;
retCode = TA_SUCCESS;
lastValidClose = 0.0;
fieldToProcess = readOpInfo->fieldProvided & fieldToAlloc;
if( (fieldToProcess & fieldToAlloc) != fieldToAlloc )
{
/* Nothing to read because not all the requested
* fields are provided by this data source!
*/
return TA_SUCCESS;
}
/* Estimate the initial amount of memory to allocate. */
fileSize = TA_FileSize( fileHandle );
retCode = TA_EstimateAllocInit( start, end, period,
minimumNbBar, 2048,
&estimationInfo,
&nbElementToAllocate );
if( retCode != TA_SUCCESS )
{
TA_TRACE_RETURN( retCode );
}
if( nbElementToAllocate == 0 )
{
TA_TRACE_RETURN( TA_SUCCESS ); /* Nothing to read!? Just return... */
}
memset( arrayInteger, 0, sizeof( arrayInteger ) );
memset( arrayReal, 0, sizeof( arrayReal ) );
/* Set the date/time pointers to where the information will be stored. */
arrayInteger[TA_HOUR_IDX] = (TA_Integer *)&hour;
arrayInteger[TA_MIN_IDX] = (TA_Integer *)&min;
arrayInteger[TA_SEC_IDX] = (TA_Integer *)&sec;
arrayInteger[TA_MONTH_IDX] = (TA_Integer *)&month;
arrayInteger[TA_YEAR_IDX] = (TA_Integer *)&year;
arrayInteger[TA_DAY_IDX] = (TA_Integer *)&day;
/* Set default time/date. */
year = 1900;
month = day = 1;
hour = 23;
min = sec = 59;
/* Check if the processing of the time will be needed. */
timeNeeded = isTimeNeeded( readOpInfo->arrayReadOp);
/* 'car' always point to the character being currently handled. */
nbByteRead = 0;
car = TA_FileSeqRead( fileHandle, &nbByteRead );
if( (car == NULL) || (nbByteRead == 0) )
return TA_SUCCESS; /* End of file! */
--nbByteRead;
nbTotalByteDone = 0;
nbTotalBarDone = 0;
nbBarAddedInTheBlock = 0;
memoryNeeded = 1;
curOp = 0;
skipField = 0;
monthChar[3] = '\0';
/* When requested, skip header lines. */
lineToSkip = readOpInfo->nbHeaderLineToSkip;
while( lineToSkip-- )
{
while( *car != '\n' )
{
GET_CHAR;
if( car == NULL )
goto exit_loops;
}
}
line_loop: /* Always jump here when end-of-line is found (EOL). */
/* If curOp != 0, the last operations are canceled. */
REVERT_OPERATIONS;
curOp = 0;
lastOpFieldIncremented = 0;
/* Start over a new line. */
if( memoryNeeded )
{
/* Allocate the memory. */
nbByteToAllocReal = nbElementToAllocate * sizeof( TA_Real );
nbByteToAllocInteger = nbElementToAllocate * sizeof( TA_Integer );
timestamp = (TA_Timestamp *)TA_Malloc( nbElementToAllocate * sizeof( TA_Timestamp ) );
timestampBeg = timestamp;
if( !timestampBeg )
{
retCode = TA_ALLOC_ERR;
goto exit_loops;
}
#define TA_ALLOC_MEM(upperc,lowerc,typepar) \
{ \
if( fieldToProcess & TA_##upperc ) \
{ \
lowerc##Beg = (TA_##typepar *)TA_Malloc( nbByteToAlloc##typepar ); \
array##typepar[TA_##upperc##_IDX] = lowerc##Beg; \
if( !lowerc##Beg ) \
{ \
retCode = TA_ALLOC_ERR; \
goto exit_loops; \
} \
} \
}
TA_ALLOC_MEM( OPEN, open, Real );
TA_ALLOC_MEM( HIGH, high, Real );
TA_ALLOC_MEM( LOW, low, Real );
TA_ALLOC_MEM( CLOSE, close, Real );
TA_ALLOC_MEM( VOLUME, volume, Integer );
TA_ALLOC_MEM( OPENINTEREST, openInterest, Integer );
#undef TA_ALLOC_MEM
memoryNeeded = 0;
}
op_loop: /* Jump here when ready to proceed with the next command. */
op = readOpInfo->arrayReadOp[curOp];
if( !(op&TA_CMD_READ_MONTH_CHAR) )
{
/* Skip leading non-numeric character. */
SKIP_UNTIL_NUMERIC;
}
/* Shall we skip this field? */
if( TA_IS_SKIP_SET(op) )
{
if( (op&(TA_CMD_READ_REAL|TA_CMD_READ_INTEGER)) == 0 )
{
tmpInt = TA_GET_NB_NUMERIC(op);
curOp++;
while( tmpInt-- )
{
GET_CHAR;
CHECK_EOL_EOF;
}
}
else
{
if( skipField == 0 )
{
skipField = TA_GET_NB_NUMERIC(op);
TA_ASSERT( skipField > 0 );
}
if( --skipField == 0 )
curOp++;
SKIP_NUMERIC;
if( (op&TA_CMD_READ_REAL) && (*car == '.') )
{
GET_CHAR;
CHECK_EOL_EOF;
SKIP_NUMERIC;
}
}
}
else
{
cnvtArrayIdx = 0;
if( TA_IS_REAL_CMD(op) )
{
/* Extract a numeric into cnvtArray. */
READ_IN_CNVT_ARRAY;
/* This is a TA_Real. */
if( car && (*car == '.') )
{
/* Read rest of the float after the '.' */
READ_IN_CNVT_ARRAY;
}
cnvtArray[cnvtArrayIdx] = '\0';
tmpReal = atof( &cnvtArray[0] );
/* Write the TA_Real in memory. */
tmpIdx = TA_GET_IDX(op);
TA_ASSERT( tmpIdx < TA_REAL_ARRAY_SIZE );
TA_ASSERT( arrayReal[tmpIdx] != NULL );
if( tmpReal != 0.0 )
{
*(arrayReal[tmpIdx]) = tmpReal;
if( tmpIdx == TA_CLOSE_IDX )
lastValidClose = tmpReal;
}
else if( TA_IS_REPLACE_ZERO(op) )
{
/* Replace this zero value with the last known close.
* If there is no previous close, this line is ignored.
*/
if( lastValidClose != 0.0 )
*(arrayReal[tmpIdx]) = lastValidClose;
else
{
SKIP_LINE;
}
}
else
{
/* Zero are not expected, consider this as a failure
* and ignore all further data from this file.
*/
retCode = TA_PRICE_BAR_CONTAINS_ZERO;
goto exit_loops;
}
arrayReal[tmpIdx]++;
curOp++;
}
else
{
/* This is a TA_Integer. */
if( !(op&TA_CMD_READ_MONTH_CHAR) )
{
nbCharToRead = TA_GET_NB_NUMERIC(op);
if( nbCharToRead )
{
READ_N_CHAR_IN_CNVT_ARRAY(nbCharToRead);
}
else
{
READ_IN_CNVT_ARRAY;
}
cnvtArray[cnvtArrayIdx] = '\0';
tmpInt = atoi( &cnvtArray[0] );
}
else
{
/* Try to find a 3 letters month string.
* Translate it to a [1..12] integer.
*/
nbLetter = 1;
do
{
CHECK_EOL_EOF;
monthChar[nbLetter] = (char)toupper(*car);
GET_CHAR;
nbLetter++;
} while( nbLetter != 3 );
do
{
CHECK_EOL_EOF;
monthChar[0] = monthChar[1];
monthChar[1] = monthChar[2];
monthChar[2] = (char)toupper(*car);
if( strncmp("JAN",monthChar,3) == 0 )
tmpInt = 1;
else if( strncmp("FEB",monthChar,3) == 0 )
tmpInt = 2;
else if( strncmp("MAR",monthChar,3) == 0 )
tmpInt = 3;
else if( strncmp("APR",monthChar,3) == 0 )
tmpInt = 4;
else if( strncmp("MAY",monthChar,3) == 0 )
tmpInt = 5;
else if( strncmp("JUN",monthChar,3) == 0 )
tmpInt = 6;
else if( strncmp("JUL",monthChar,3) == 0 )
tmpInt = 7;
else if( strncmp("AUG",monthChar,3) == 0 )
tmpInt = 8;
else if( strncmp("SEP",monthChar,3) == 0 )
tmpInt = 9;
else if( strncmp("OCT",monthChar,3) == 0 )
tmpInt = 10;
else if( strncmp("NOV",monthChar,3) == 0 )
tmpInt = 11;
else if( strncmp("DEC",monthChar,3) == 0 )
tmpInt = 12;
else
tmpInt = 0;
GET_CHAR;
} while( tmpInt == 0 );
}
/* Write the TA_Integer in memory. */
tmpIdx = TA_GET_IDX(op);
TA_ASSERT( tmpIdx < TA_INTEGER_ARRAY_SIZE );
TA_ASSERT( arrayInteger[tmpIdx] != NULL );
*(arrayInteger[tmpIdx]) = tmpInt;
if( tmpIdx > TA_YEAR_IDX )
arrayInteger[tmpIdx]++;
curOp++;
if( TA_IS_TIMESTAMP_COMPLETE(op) )
{
/* Build the timestamp. */
retCode = TA_SetDate( year, month, day, &tmpTimestamp );
if( retCode != TA_SUCCESS )
goto exit_loops; /* Invalid date */
if( !timeNeeded )
{
/* Ignore time in comparison and use default to build the price bar. */
tmpTimestamp.time = 23595900; /* Default EOD time */
if( start && (tmpTimestamp.date < start->date) )
{
/* This price bar is not needed, jump to the next line. */
retCode = TA_SUCCESS;
SKIP_LINE;
}
if( end && (tmpTimestamp.date > end->date) )
{
/* This price bar is beyond the upper limit, just exit. */
goto exit_loops;
}
}
else
{
retCode = TA_SetTime( hour, min, sec, &tmpTimestamp );
if( retCode != TA_SUCCESS )
goto exit_loops; /* Invalid time */
if( start && TA_TimestampLess(&tmpTimestamp,start) )
{
/* This price bar is not needed, jump to the next line. */
retCode = TA_SUCCESS;
SKIP_LINE;
}
if( end && TA_TimestampGreater(&tmpTimestamp, end) )
{
/* This price bar is beyond the upper limit, just exit. */
goto exit_loops;
}
}
/* Write the timestamp in memory. */
*timestamp = tmpTimestamp;
}
}
if( TA_IS_READ_STOP_FLAG_SET(op) )
{
#ifdef DEBUG_PRINTF
printf( "(%d%d%d,%e,%e,%e,%e,%d)\n",
timestampBeg?TA_GetYear(×tampBeg[nbBarAddedInTheBlock]):0,
timestampBeg?TA_GetMonth(×tampBeg[nbBarAddedInTheBlock]):0,
timestampBeg?TA_GetDay(×tampBeg[nbBarAddedInTheBlock]):0,
openBeg?openBeg[nbBarAddedInTheBlock]:0.0,
highBeg?highBeg[nbBarAddedInTheBlock]:0.0,
lowBeg?lowBeg[nbBarAddedInTheBlock]:0.0,
closeBeg?closeBeg[nbBarAddedInTheBlock]:0.0,
volumeBeg?volumeBeg[nbBarAddedInTheBlock]:0 );
#endif
/* At this point, the price bar is completely written in memory. */
timestamp++;
curOp = 0;
nbBarAddedInTheBlock++;
if( nbBarAddedInTheBlock < nbElementToAllocate )
{
/* Go to next line. */
SKIP_LINE;
}
else
{
/* There is not enough memory for another bar, so re-allocate
* some more.
*/
retCode = TA_HistoryAddData( paramForAddData, nbBarAddedInTheBlock,
period, timestampBeg,
openBeg, highBeg, lowBeg, closeBeg,
volumeBeg, openInterestBeg );
/* TA_HistoryAddData is ALWAYS the owner of these memory block
* when called. So we set these to NULL to make sure there will
* be no attempt to free these from this function.
*/
openBeg = highBeg = lowBeg = closeBeg = NULL;
timestampBeg = NULL;
volumeBeg = openInterestBeg = NULL;
nbTotalBarDone += nbBarAddedInTheBlock;
nbBarAddedInTheBlock = 0;
if( retCode != TA_SUCCESS )
goto exit_loops;
retCode = TA_EstimateAllocNext( &estimationInfo, &nbElementToAllocate );
if( retCode != TA_SUCCESS )
goto exit_loops;
memoryNeeded = 1;
SKIP_LINE;
}
}
else
{
/* Make sure we did not hit an EOL or EOF prematurly, if yes,
* this line will be silently ignored.
*/
CHECK_EOL_EOF;
}
}
goto op_loop;
exit_loops: /* Jump here when the end-of-file is hit (or an error occured) */
/* On succesful exit, process possibly remaining data. */
if( (retCode == TA_SUCCESS) && (nbBarAddedInTheBlock != 0) )
{
retCode = TA_HistoryAddData( paramForAddData, nbBarAddedInTheBlock,
period, timestampBeg,
openBeg, highBeg, lowBeg, closeBeg,
volumeBeg, openInterestBeg );
openBeg = highBeg = lowBeg = closeBeg = NULL;
timestampBeg = NULL;
volumeBeg = openInterestBeg = NULL;
nbTotalBarDone += nbBarAddedInTheBlock;
}
/* ALWAYS verify if locally allocated memory needs to be freed. ALWAYS. */
FREE_IF_NOT_NULL( openBeg );
FREE_IF_NOT_NULL( highBeg );
FREE_IF_NOT_NULL( lowBeg );
FREE_IF_NOT_NULL( closeBeg );
FREE_IF_NOT_NULL( volumeBeg );
FREE_IF_NOT_NULL( openInterestBeg );
FREE_IF_NOT_NULL( timestampBeg );
/* An indication that no more data needs to be provided is not a failure. */
if( retCode == TA_ENOUGH_DATA )
retCode = TA_SUCCESS;
/* Return the number of added price bar to the caller. */
if( nbBarAdded )
*nbBarAdded = nbTotalBarDone;
TA_TRACE_RETURN( retCode );
}
/**** Global functions definitions. ****/
TA_RetCode TA_GetHistoryDataFromWeb( TA_Libc *libHandle,
TA_DataSourceHandle *handle,
TA_CategoryHandle *categoryHandle,
TA_SymbolHandle *symbolHandle,
TA_Period period,
const TA_Timestamp *start,
const TA_Timestamp *end,
TA_Field fieldToAlloc,
TA_ParamForAddData *paramForAddData )
{
TA_PROLOG;
TA_RetCode retCode;
TA_StringCache *stringCache;
TA_String *yahooName;
TA_WebPage *webPage;
TA_PrivateYahooHandle *yahooHandle;
TA_DecodingParam localDecodingParam;
const TA_DecodingParam *decodingParam;
TA_FileHandle *fileHandle;
TA_ReadOpInfo *readOpInfo;
UIRSuffixParsing suffixParsing;
TA_Timestamp firstBarTimestamp, lastBarTimestamp, prevEndDate;
TA_InfoFromAddedData infoFromAddedData;
TA_DayOfWeek dayOfWeek;
int nbEstimateBar;
int nbField;
unsigned int nbBarAdded, nbTotalBarAdded;
int again, firstTime, nbBatch;
int zeroBarAddedAttempt;
TA_TRACE_BEGIN( libHandle, TA_GetHistoryDataFromWeb );
/* Initialize some local variables. */
stringCache = TA_GetGlobalStringCache( libHandle );
yahooHandle = (TA_PrivateYahooHandle *)handle->opaqueData;
readOpInfo = NULL;
nbEstimateBar = 0;
TA_ASSERT( libHandle, categoryHandle != NULL );
TA_ASSERT( libHandle, symbolHandle != NULL );
TA_ASSERT( libHandle, categoryHandle->string != NULL );
TA_ASSERT( libHandle, symbolHandle->string != NULL );
/* Set the initial first/last timestamp */
if( start )
TA_TimestampCopy( &firstBarTimestamp, start );
else
{
TA_SetDate( 1950, 1, 1, &firstBarTimestamp );
TA_SetTime( 0, 0, 0, &firstBarTimestamp );
}
if( end )
TA_TimestampCopy( &lastBarTimestamp, end );
else
{
TA_SetDateNow( &lastBarTimestamp );
TA_SetTime( 0, 0, 0, &lastBarTimestamp );
}
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
/* Map the TA-Lib name into the Yahoo! name. */
retCode = TA_AllocStringFromLibName( libHandle,
categoryHandle->string,
symbolHandle->string,
&yahooName );
if( retCode != TA_SUCCESS )
{
TA_TRACE_RETURN( retCode );
}
TA_ASSERT( libHandle, yahooName != NULL );
TA_ASSERT( libHandle, yahooHandle != NULL );
/* Get the decoding parameter for the CSV web page. */
decodingParam = TA_YahooIdxDecodingParam( yahooHandle->index, TA_YAHOOIDX_CVS_PAGE );
if( !decodingParam )
{
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(103) );
}
/* Use a local copy of the decoding param.
* This is because the uirSuffix is replaced with
* an allocated buffer (so the date field can be
* manipulated).
*/
localDecodingParam = *decodingParam;
/* Parse the uirSuffix so the start/end date can be changed. */
if( !setUIRSuffixParsing( decodingParam->uirSuffix, &suffixParsing ) )
{
/* This should never happen unless the
* Yahoo! index protocol has been broken.
*/
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_INTERNAL_ERROR(104) );
}
/* Replace the uirSuffix with a large local buffer. */
localDecodingParam.uirSuffix = TA_Malloc( libHandle, suffixParsing.maxTotalLength );
if( !localDecodingParam.uirSuffix )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_ALLOC_ERR );
}
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* nbBatch is a safety net to make sure that
* TA-Lib won't stay forever in the while loop
* in case Yahoo! changes their protocol.
*/
nbBatch = 0;
/* Sometime Yahoo! return an empty csv file. Make
* multiple attempts in that case.
*/
zeroBarAddedAttempt = 0;
again = 1;
firstTime = 1;
nbTotalBarAdded = 0;
while( again && (++nbBatch < 100) && (zeroBarAddedAttempt < 10) )
{
if( TA_TimestampLess(&lastBarTimestamp,&firstBarTimestamp) )
{
/* Get out of this loop if all the requested data
* has been retreived already.
*/
again = 0;
break;
}
retCode = TA_WebPageAllocFromYahooName( libHandle,
&localDecodingParam,
TA_StringToChar(yahooName),
&webPage );
if( retCode != TA_SUCCESS )
{
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Disguise the webPage stream into a "file". That way the speed
* optimized ASCII decoder can be re-used (TA_ReadOp stuff).
*/
retCode = TA_FileSeqOpenFromStream( libHandle, webPage->content, &fileHandle );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
if( firstTime )
{
/* Make assumption of the data provided
* base on the number of fields in the CSV file.
*/
nbField = nbCommaField( webPage->content );
switch( nbField )
{
case 2:
readOpInfo = yahooHandle->readOp2Fields;
break;
case 5:
readOpInfo = yahooHandle->readOp5Fields;
break;
default:
readOpInfo = yahooHandle->readOp6Fields;
}
/* User asking for all the fields? */
if( fieldToAlloc == TA_ALL )
{
switch( nbField )
{
case 2:
fieldToAlloc = TA_CLOSE|TA_TIMESTAMP;
break;
case 5:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_TIMESTAMP;
break;
default:
fieldToAlloc = TA_OPEN|TA_HIGH|TA_LOW|TA_CLOSE|TA_VOLUME|TA_TIMESTAMP;
}
}
/* Optimize the read op for the requested data. */
retCode = TA_ReadOp_Optimize( libHandle,
readOpInfo,
period,
fieldToAlloc );
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_WebPageFree( webPage );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Make an estimation of the number of price bar. */
nbEstimateBar = TA_StreamCountChar( webPage->content, '\n' ) + 1;
if( nbEstimateBar < 100 )
nbEstimateBar = 100;
}
/* Interpret the CSV data. */
retCode = TA_ReadOp_Do( libHandle, fileHandle,
readOpInfo,
period, &firstBarTimestamp, &lastBarTimestamp,
nbEstimateBar, fieldToAlloc,
paramForAddData,
&nbBarAdded );
TA_FileSeqClose( libHandle, fileHandle );
TA_WebPageFree( webPage );
nbTotalBarAdded += nbBarAdded;
if( retCode != TA_SUCCESS )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( retCode );
}
/* Yahoo! does not always return all the data it could, up to
* the requested end date. It is important to detect these occurence
* and cancel the usage of all data accumulated up to now.
*/
TA_GetInfoFromAddedData( paramForAddData, &infoFromAddedData );
if( infoFromAddedData.barAddedSinceLastCall )
{
/* Do some more checking by considering holidays, week-end etc... */
if( !isGapAcceptable(&infoFromAddedData.highestTimestampAddedSinceLastCall, &lastBarTimestamp) )
{
/* Clean-up and exit */
TA_StringFree( stringCache, yahooName );
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_TRACE_RETURN( TA_DATA_GAP );
}
TA_TimestampCopy( &lastBarTimestamp, &infoFromAddedData.lowestTimestamp );
}
#if DEBUG_PRINTF
printf( "NB BAR ADDED=%d, TOTAL=%d\n", nbBarAdded, nbTotalBarAdded );
#endif
/* Verify if more data should be processed.
* Yahoo! sometimes slice their data, in
* batch of 200 price bars.
*/
if( firstTime && (nbBarAdded > 200) )
{
again = 0; /* Assume all the data extracted... exit the loop. */
}
else if( nbBarAdded == 0 )
{
/* Make multiple attempts when retreiving data succeed,
* but somehow there is zero bar returned.
*
* Sometimes this might be correct when there is truly no
* more data available, so choosing an algorithm before
* giving up is a comprimise between reliability and
* usability. The data source is free... and you get
* what you pay for after all ;)
*/
if( (nbTotalBarAdded < 1000) && (zeroBarAddedAttempt >= 1) && (zeroBarAddedAttempt < 7) )
{
/* I did choose to add a delay when insufficient total data is returned. When
* there is already ~5 years of data, most likely there is "Zero" returned
* because there is NO more data available, so just do the retry without delay.
*/
TA_Sleep(zeroBarAddedAttempt*2);
}
#if DEBUG_PRINTF
printf( "Retry %d", zeroBarAddedAttempt );
#endif
zeroBarAddedAttempt++;
}
else
{
zeroBarAddedAttempt = 0;
if( TA_TimestampEqual( &lastBarTimestamp, &prevEndDate ) )
{
/* prevEndDate is a "safety net" to
* exit the loop early in case Yahoo! starts
* to return always the same batch of data.
* Just ignore the repetitive data and exit.
*/
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( TA_SUCCESS );
}
TA_TimestampCopy( &prevEndDate, &lastBarTimestamp );
/* Request the data up to the day BEFORE
* the last batch of data received.
*/
TA_PrevDay( &lastBarTimestamp );
/* Make sure that lastBarTimestamp is a week-day. */
dayOfWeek = TA_GetDayOfTheWeek( &lastBarTimestamp );
if( (dayOfWeek == TA_SUNDAY) || (dayOfWeek == TA_SATURDAY) )
TA_PrevWeekday( &lastBarTimestamp );
/* Change the dates in the uirSuffix. */
buildUIRSuffix( &suffixParsing,
&firstBarTimestamp, &lastBarTimestamp,
(char *)localDecodingParam.uirSuffix );
/* From that point, data is expected to be most likely
* sent in batch of 200.
*/
nbEstimateBar = 200;
}
firstTime = 0;
}
/* Clean-up and exit */
TA_Free( libHandle, (char *)localDecodingParam.uirSuffix );
TA_StringFree( stringCache, yahooName );
TA_TRACE_RETURN( retCode );
}
/* Perform the very first level of calculation among all
* the trades allocated.
*
* Note: A trade is a TA_DataLog with the quantity > 0
*/
static TA_RetCode processTradeLog_BasicCalculation( TA_Timestamp *startDate,
TA_Timestamp *endDate,
TA_TradeLogPriv *tradeLog )
{
TA_AllocatorForDataLog *allocator;
TA_DataLogBlock *block;
TA_List *list;
TA_DataLog *invalidDataLog, *curDataLog;
int i;
TA_PMValueCache *shortValueCache, *longValueCache;
/* Temporary values for calculation. */
register TA_Real tempReal1, tempReal2, tempReal3;
register int tempInt1;
/* The following variables are all the
* accumulators.
*
* Some are suggested to be kept in
* registers, most of the others will
* be maintain within the local TA_PMValueCache.
*
* All these value are then merge within the
* tradeLog at the very end.
*/
register int long_nbLosingTrade;
register int short_nbLosingTrade;
register int long_nbWinningTrade;
register int short_nbWinningTrade;
TA_PMValueCache shortV, longV;
/* Initialize all accumulators. */
initValueCache( &shortV );
initValueCache( &longV );
long_nbLosingTrade =
short_nbLosingTrade =
long_nbWinningTrade =
short_nbWinningTrade = 0;
/* Simply iterate through all the TA_TradeLog
* and update the accumulators.
*/
allocator = &tradeLog->allocator;
if( allocator )
{
list = &allocator->listOfDataLogBlock;
block = TA_ListAccessHead( list );
while( block )
{
/* Process each blocks. */
invalidDataLog = allocator->nextAvailableTrade;
curDataLog = block->array;
for( i=0; i < TA_TRADE_BLOCK_SIZE; i++ )
{
if( curDataLog == invalidDataLog )
{
break;
}
else
{
/* Process each TA_DataLog being
* a trade (not an entry)
* An entry have a negative 'quantity'.
*/
tempInt1 = curDataLog->u.trade.quantity;
if( (tempInt1 > 0) &&
!TA_TimestampLess( &curDataLog->u.trade.entryTimestamp, startDate ) &&
!TA_TimestampGreater( &curDataLog->u.trade.exitTimestamp, endDate ) )
{
tempReal1 = curDataLog->u.trade.entryPrice; /* Positive = long, negative = short */
tempReal2 = curDataLog->u.trade.profit; /* Positive = winning, negative = losing */
if( tempReal1 > 0.0 )
{
/* This is a long trade. */
if( tempReal2 > 0.0 )
{
/* This is a winning long trade */
longV.sumInvestmentProfit += tempReal1;
longV.sumProfit += tempReal2;
TA_SET_MAX(longV.largestProfit, tempReal2 );
tempReal1 = tempReal2/tempReal1;
TA_SET_MAX(longV.largestProfitPercent, tempReal1 );
longV.sumProfitPercent += tempReal1;
long_nbWinningTrade++;
}
else
{
/* This is a losing long trade */
longV.sumInvestmentLoss += tempReal1;
longV.sumLoss += tempReal2;
TA_SET_MIN(longV.largestLoss, tempReal2 );
tempReal1 = tempReal2/tempReal1;
TA_SET_MIN(longV.largestLossPercent, tempReal1 );
longV.sumLossPercent += tempReal1;
long_nbLosingTrade++;
}
}
else
{
/* This is a short trade. */
if( tempReal2 > 0.0 )
{
/* This is a winning short trade */
tempReal1 = -tempReal1;
shortV.sumInvestmentProfit += tempReal1;
shortV.sumProfit += tempReal2;
TA_SET_MAX(shortV.largestProfit, tempReal2 );
tempReal1 = tempReal2/tempReal1;
TA_SET_MAX(shortV.largestProfitPercent, tempReal1 );
shortV.sumProfitPercent += tempReal1;
short_nbWinningTrade++;
}
else
{
/* This is a losing short trade */
tempReal1 = -tempReal1;
shortV.sumInvestmentLoss += tempReal1;
shortV.sumLoss += tempReal2;
TA_SET_MIN(shortV.largestLoss, tempReal2 );
tempReal1 = tempReal2/tempReal1;
TA_SET_MIN(shortV.largestLossPercent, tempReal1 );
shortV.sumLossPercent += tempReal1;
short_nbLosingTrade++;
}
}
}
}
curDataLog++;
}
block = TA_ListAccessNext( list );
}
}
/* Initialize the output with the accumulated results. */
shortValueCache = &tradeLog->shortValueCache;
longValueCache = &tradeLog->longValueCache;
*shortValueCache = shortV;
*longValueCache = longV;
shortValueCache->nbLosingTrade = short_nbLosingTrade;
shortValueCache->nbWinningTrade = short_nbWinningTrade;
longValueCache->nbLosingTrade = long_nbLosingTrade;
longValueCache->nbWinningTrade = long_nbWinningTrade;
/* Indicate that the value are now calculated and set
* in the "cache".
*/
tradeLog->flags |= TA_PMVALUECACHE_CALCULATED;
return TA_SUCCESS;
}
TA_RetCode TA_TradeReportAlloc( TA_PM *pm, TA_TradeReport **tradeReportAllocated )
{
TA_PMPriv *pmPriv;
TA_TradeReport *tradeReport;
TA_TradeReportPriv *tradeReportPriv;
TA_List *tradeLogList;
TA_TradeLogPriv *tradeLogPriv;
TA_AllocatorForDataLog *allocator;
TA_DataLogBlock *block;
TA_List *listOfBlock;
TA_DataLog *invalidDataLog;
TA_DataLog *curDataLog;
TA_Trade **tradePtr;
TA_Timestamp *startDate;
TA_Timestamp *endDate;
TA_RetCode retCode;
TA_Real tempReal;
int nbTrade, nbTradeAdded, i;
if( !tradeReportAllocated )
return TA_BAD_PARAM;
*tradeReportAllocated = NULL;
if( !pm )
return TA_BAD_PARAM;
/* Make sure this TA_PM is a valid object */
pmPriv = (TA_PMPriv *)pm->hiddenData;
if( !pmPriv || (pmPriv->magicNb != TA_PMPRIV_MAGIC_NB) )
return TA_BAD_OBJECT;
tradeReport = TA_Malloc( sizeof( TA_TradeReport ) + sizeof( TA_TradeReportPriv ) );
if( !tradeReport )
return TA_ALLOC_ERR;
memset( tradeReport, 0, sizeof( TA_TradeReport ) + sizeof( TA_TradeReportPriv ) );
tradeReportPriv = (TA_TradeReportPriv *)(((char *)tradeReport)+sizeof(TA_TradeReport));
tradeReportPriv->magicNb = TA_TRADEREPORT_MAGIC_NB;
tradeReport->hiddenData = tradeReportPriv;
/* TA_TradeReportFree can be safely called from this point. */
/* Get the number of closed trades */
tempReal = 0;
retCode = TA_PMValue( pm, TA_PM_TOTAL_NB_OF_TRADE, TA_PM_ALL_TRADES, &tempReal );
if( retCode != TA_SUCCESS )
{
TA_TradeReportFree( tradeReport );
return retCode;
}
nbTrade = (unsigned int)tempReal;
tradeReport->nbTrades = nbTrade;
if( nbTrade != 0 )
{
startDate = &pmPriv->startDate;
endDate = &pmPriv->endDate;
tradePtr = (TA_Trade **)TA_Malloc( nbTrade*sizeof(const TA_Trade *));
tradeReport->trades = (const TA_Trade **)tradePtr;
if( !tradePtr )
{
TA_TradeReportFree( tradeReport );
return TA_ALLOC_ERR;
}
/* Iterate through all the closed trades. */
nbTradeAdded = 0;
tradeLogList = &pmPriv->tradeLogList;
tradeLogPriv = TA_ListAccessHead( tradeLogList );
if( !tradeLogPriv )
{
TA_TradeReportFree( tradeReport );
return TA_NO_TRADE_LOG;
}
do
{
allocator = &tradeLogPriv->allocator;
listOfBlock = &allocator->listOfDataLogBlock;
block = TA_ListAccessHead( listOfBlock );
while( block )
{
/* Process each blocks. */
invalidDataLog = allocator->nextAvailableTrade;
curDataLog = block->array;
for( i=0; i < TA_TRADE_BLOCK_SIZE; i++ )
{
if( curDataLog == invalidDataLog )
{
break;
}
else
{
/* Process each TA_DataLog being a trade (not an entry)
* An entry have a negative 'quantity'.
*/
if( (curDataLog->u.trade.quantity > 0) &&
!TA_TimestampLess( &curDataLog->u.trade.entryTimestamp, startDate ) &&
!TA_TimestampGreater( &curDataLog->u.trade.exitTimestamp, endDate ) )
{
/* Make sure not to exceed array size */
if( nbTradeAdded >= nbTrade )
{
TA_TradeReportFree( tradeReport );
return TA_ALLOC_ERR;
}
tradePtr[nbTradeAdded++] = &curDataLog->u.trade;
}
}
curDataLog++;
}
block = TA_ListAccessNext( listOfBlock );
}
tradeLogPriv = TA_ListAccessNext( tradeLogList );
} while( tradeLogPriv );
/* Make sure all trades were initialized. */
if( nbTradeAdded != nbTrade )
{
TA_TradeReportFree( tradeReport );
return TA_ALLOC_ERR;
}
/* Sort all trades in chronological order of exit. */
qsort( tradePtr, (size_t)nbTrade, sizeof(TA_Trade *), compareTrade );
}
/* All succeed. Return pointer to caller. */
*tradeReportAllocated = tradeReport;
return TA_SUCCESS;
}
static TA_Timestamp *allocTimestampArray( const TA_Timestamp *start,
const TA_Timestamp *end,
int *nbDays )
{
TA_RetCode retCode;
TA_Timestamp *array;
int outIdx;
TA_Timestamp curDate;
TA_DayOfWeek dayOfTheWeek;
TA_ASSERT_RET( TA_TimestampValidate(start) == TA_SUCCESS, (TA_Timestamp *)NULL );
TA_ASSERT_RET( TA_TimestampValidate(end ) == TA_SUCCESS, (TA_Timestamp *)NULL );
TA_ASSERT_RET( nbDays != NULL, (TA_Timestamp *)NULL );
/* Calculate the exact number of week days
* between start and end inclusive.
* Excluding week-ends.
*/
retCode = TA_TimestampDeltaWeekday( start, end, (unsigned int *)nbDays );
if( retCode != TA_SUCCESS )
return (TA_Timestamp *)NULL;
/* Allocate the array. Add two element just to be on the safe side. */
array = TA_Malloc( sizeof( TA_Timestamp ) * ((*nbDays)+2) );
if( !array )
return (TA_Timestamp *)NULL;
/* Fill up the array. */
TA_TimestampCopy( &curDate, start );
/* Write the start point, if it is a weekday. */
outIdx = 0;
dayOfTheWeek = TA_GetDayOfTheWeek( &curDate );
if( (dayOfTheWeek != TA_SUNDAY) && (dayOfTheWeek != TA_SATURDAY) )
{
TA_TimestampCopy( &array[outIdx], &curDate );
outIdx++;
TA_NextWeekday( &curDate );
TA_ASSERT_RET( TA_TimestampValidate(&curDate) == TA_SUCCESS, (TA_Timestamp *)NULL );
}
/* Count the number of weekday */
while( TA_TimestampLess( &curDate, end ) )
{
TA_TimestampCopy( &array[outIdx], &curDate );
outIdx++;
TA_NextWeekday( &curDate );
TA_ASSERT_RET( TA_TimestampValidate(&curDate) == TA_SUCCESS, (TA_Timestamp *)NULL );
}
/* Check if the ending point is a weekday. */
if( TA_TimestampEqual( &curDate, end ) )
{
dayOfTheWeek = TA_GetDayOfTheWeek( &curDate );
if( (dayOfTheWeek != TA_SUNDAY) && (dayOfTheWeek != TA_SATURDAY) )
TA_TimestampCopy( &array[outIdx++], &curDate );
}
TA_ASSERT_RET( outIdx == (*nbDays), (TA_Timestamp *)NULL );
return array;
}
