void CSV::parse ()
{
Setting rule;
getRule(rule);
if (! rule.count())
{
QString s(tr("Empty rule"));
printStatusLogMessage(s);
downloadComplete();
return;
}
QString ts = "Rule";
QString ts2, s;
rule.getData(ts, ts2);
if (! ts2.contains("Date:"))
{
QString ss(tr("Rule missing Date field"));
printStatusLogMessage(ss);
downloadComplete();
return;
}
ts = "Delimiter";
rule.getData(ts, s);
if (! s.length())
{
QString ss(tr("Delimiter not found"));
printStatusLogMessage(ss);
downloadComplete();
return;
}
setDelimiter(s);
if (dateRange->isChecked())
{
if (sdate->date() >= edate->date() || edate->date() <= sdate->date())
{
QString ss = stringDone + " " + QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss");
printStatusLogMessage(ss);
downloadComplete();
return;
}
}
QString type;
ts = "Type";
rule.getData(ts, type);
if (! type.length())
{
QString ss(tr("Type not found"));
printStatusLogMessage(ss);
downloadComplete();
return;
}
ts = "Rule";
rule.getData(ts, ts2);
QStringList fieldList = QStringList::split(",", ts2, FALSE);
if (! fieldList.count())
{
QString ss(tr("No rule found"));
printStatusLogMessage(ss);
downloadComplete();
return;
}
// get the directory path offset
ts = "Directory";
QString directory;
rule.getData(ts, directory);
if (! directory.length())
{
QString ss(tr("Directory not found"));
printStatusLogMessage(ss);
downloadComplete();
return;
}
// get the symbol filter
ts = "SymbolFilter";
rule.getData(ts, ts2);
QStringList symbolFilter = QStringList::split(",", ts2, FALSE);
// check for time field and set the tickflag
bool tickFlag = FALSE;
ts = "Rule";
rule.getData(ts, ts2);
if (ts2.contains("Time"))
tickFlag = TRUE;
else
{
if (ts2.contains("HHMMSS"))
tickFlag = TRUE;
}
QStringList list;
file->getFile(list);
int loop;
for (loop = 0; loop < (int) list.count(); loop++)
{
if (cancelFlag)
break;
QFile f(list[loop]);
if (! f.open(QIODevice::ReadOnly))
continue;
Q3TextStream stream(&f);
QString symbol = symbolOveride;
if (! symbol.length() && fieldList.findIndex("Symbol") == -1)
{
QStringList l = QStringList::split("/", list[loop], FALSE);
symbol = l[l.count() - 1];
if (symbol.right(4).contains(".txt"))
symbol.truncate(symbol.length() - 4);
if (symbol.right(4).contains(".TXT"))
symbol.truncate(symbol.length() - 4);
while (symbol.contains("_"))
symbol = symbol.remove(symbol.find("_", 0, TRUE), 1);
}
QString path;
if (! type.compare("Stocks"))
{
QString tmp = "Stocks/" + directory;
createDirectory(tmp, path);
if (! path.length())
{
QString ss(tr("CSVPlugin::Unable to create directory"));
printStatusLogMessage(ss);
downloadComplete();
f.close();
return;
}
}
else
{
if (! type.compare("Futures"))
{
QString tmp = "Futures/" + directory;
createDirectory(tmp, path);
if (! path.length())
{
QString ss(tr("CSVPlugin::Unable to create directory"));
printStatusLogMessage(ss);
downloadComplete();
f.close();
return;
}
}
}
path.append("/");
if (symbol.length())
{
QString s = path;
s.append(symbol);
if (openDb(s, symbol, type, tickFlag))
continue;
// QString ss = tr("Updating") + " " + symbol;
// printStatusLogMessage(ss);
progressBar->setProgress(loop, (int) list.count());
}
int lineCount = 0;
QFileInfo fi(f);
QString fName = fi.fileName();
while(stream.atEnd() == 0)
{
QString ts = stream.readLine();
QStringList l = QStringList::split( '\n', substituteSeparator(ts, delim, '\n'), FALSE);
lineCount++;
if (l.count() != fieldList.count())
{
QString ss = QString().sprintf(
"%s - %s - %s: %s Number of fields in file (%i) != rule format (%i)",
fName.latin1(), symbol.latin1(), tr("Line").latin1(),
QString::number(lineCount).latin1(), l.count(), fieldList.count()
);
printStatusLogMessage(ss);
continue;
}
int fieldLoop;
bool flag = FALSE;
Setting r;
for (fieldLoop = 0; fieldLoop < (int) fieldList.count(); fieldLoop++)
{
QString listItem = l[fieldLoop].stripWhiteSpace();
if (fieldList[fieldLoop].contains("Date:"))
{
QDate dt;
getDate(fieldList[fieldLoop], listItem, r, dt);
if (! dt.isValid())
{
QString ss = QString().sprintf("%s - %s - %s: %s %s: %s",
fName.latin1(), symbol.latin1(), tr("Line").latin1(), QString::number(lineCount).latin1(),
tr("Bad date").latin1(), listItem.latin1()
);
printStatusLogMessage(ss);
flag = TRUE;
break;
}
if (dateRange->isChecked())
{
if (dt < sdate->date() || dt > edate->date())
{
flag = TRUE;
break;
}
}
ts = "Date";
ts2 = dt.toString("yyyyMMdd");
r.setData(ts, ts2);
continue;
}
if (! fieldList[fieldLoop].compare("Time"))
{
getTime(listItem, s);
if (! s.length())
{
QString ss = QString().sprintf("%s - %s - %s: %s %s: %s",
fName.latin1(), symbol.latin1(), tr("Line").latin1(), QString::number(lineCount).latin1(),
tr("Bad time").latin1(), listItem.latin1()
);
printStatusLogMessage(ss);
flag = TRUE;
break;
}
ts = "Time";
r.setData(ts, s);
continue;
}
if (! fieldList[fieldLoop].compare("Symbol"))
{
if (symbolFilter.count())
{
if (symbolFilter.findIndex(listItem) == -1)
{
flag = TRUE;
break;
}
}
ts = "Symbol";
r.setData(ts, listItem);
continue;
}
if (! fieldList[fieldLoop].compare("Name"))
{
ts = "Name";
r.setData(ts, listItem);
continue;
}
if (! fieldList[fieldLoop].compare("Open") ||
! fieldList[fieldLoop].compare("High") ||
! fieldList[fieldLoop].compare("Low") ||
! fieldList[fieldLoop].compare("Close"))
{
if (setTFloat(listItem, TRUE))
{
QString ss = QString().sprintf("%s - %s - %s: %i %s: %s",
fName.latin1(), symbol.latin1(), tr("Line").latin1(), lineCount,
tr("Bad value").latin1(), tr(fieldList[fieldLoop]).latin1()
);
printStatusLogMessage(ss);
flag = TRUE;
break;
}
ts = QString::number(tfloat);
r.setData(fieldList[fieldLoop], ts);
continue;
}
if (! fieldList[fieldLoop].compare("Volume") ||
! fieldList[fieldLoop].compare("OI"))
{
if (setTFloat(listItem, FALSE))
{
QString ss = QString().sprintf("%s - %s - %s: %s %s: %s",
fName.latin1(), symbol.latin1(), tr("Line").latin1(), QString::number(lineCount).latin1(),
tr("Bad value").latin1(), tr(fieldList[fieldLoop]).latin1()
);
printStatusLogMessage(ss);
flag = TRUE;
break;
}
ts = QString::number(tfloat);
r.setData(fieldList[fieldLoop], ts);
continue;
}
}
if (flag)
continue;
ts = "Date";
r.getData(ts, s);
if (! s.length())
continue;
ts = "Time";
r.getData(ts, ts2);
if (ts2.length())
s.append(ts2);
else
s.append("000000");
Bar bar;
if (bar.setDate(s))
{
ts = "Date";
r.getData(ts, ts2);
QString ss = tr("Bad date") + " " + ts2;
printStatusLogMessage(ss);
continue;
}
bar.setTickFlag(tickFlag);
ts = "Open";
bar.setOpen(r.getDouble(ts));
ts = "High";
bar.setHigh(r.getDouble(ts));
ts = "Low";
bar.setLow(r.getDouble(ts));
ts = "Close";
bar.setClose(r.getDouble(ts));
ts = "Volume";
bar.setVolume(r.getDouble(ts));
ts = "OI";
bar.setOI(r.getInt(ts));
DBIndexItem item;
if (! symbol.length())
{
ts = "Symbol";
QString t;
r.getData(ts, t);
s = path;
s.append(t);
if (openDb(s, t, type, tickFlag))
continue;
ts = "Name";
r.getData(ts, s);
if (s.length())
{
chartIndex->getIndexItem(t, item);
item.setTitle(s);
chartIndex->setIndexItem(t, item);
}
db.setBar(bar);
ts = "Symbol";
r.getData(ts, ts2);
// QString ss = tr("Updating") + " " + ts2;
// printStatusLogMessage(ss);
config.closePlugin(type);
db.close();
emit signalWakeup();
}
else
{
ts = "Name";
r.getData(ts, s);
if (s.length())
{
chartIndex->getIndexItem(symbol, item);
item.setTitle(s);
chartIndex->setIndexItem(symbol, item);
}
db.setBar(bar);
emit signalWakeup();
}
}
db.close();
f.close();
}
progressBar->setProgress((int) list.count(), (int) list.count());
downloadComplete();
if (cancelFlag)
{
cancelFlag = FALSE;
printStatusLogMessage(stringCanceled);
}
else {
QString ss = stringDone + " " + QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss");
printStatusLogMessage(ss);
}
progressBar->reset();
}
void Yahoo::parseQuote ()
{
if (! data.length())
return;
QFile f(file);
if (! f.open(QIODevice::WriteOnly))
return;
Q3TextStream stream(&f);
stream << data;
f.close();
f.setName(file);
if (! f.open(QIODevice::ReadOnly))
return;
stream.setDevice(&f);
QString s = dataPath + "/";
QString ts = "symbol";
QString ts2;
currentUrl->getData(ts, ts2);
QFileInfo fi(ts2);
if (fi.extension(FALSE).length())
s.append(fi.extension(FALSE).upper());
else
s.append("US");
s.append("/");
s.append(ts2);
if (plug.open(s, chartIndex))
{
QString ss(tr("Could not open db"));
printStatusLogMessage(ss);
f.close();
return;
}
QString fn = ts2;
// verify if this chart can be updated by this plugin
DBIndexItem item;
chartIndex->getIndexItem(fn, item);
item.getSymbol(s);
if (! s.length())
{
if (plug.createNewStock())
{
f.close();
plug.close();
return;
}
chartIndex->getIndexItem(fn, item);
item.setSymbol(ts2);
item.setTitle(ts2);
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(fn, item);
}
// item.getQuotePlugin(s);
// if (s.compare(pluginName))
// {
// s = ts2 + " - " + tr("skipping update. Source does not match destination");
// printStatusLogMessage(s);
// f.close();
// plug.close();
// return;
// }
while(stream.atEnd() == 0)
{
ts = stream.readLine();
QStringList l = QStringList::split( '\n', substituteSeparator( ts, ',', '\n' ), FALSE);
if (l.count() < 9 || l.count() > 10)
{
QString ss = tr("Parse: invalid number of parameters") + " '" + ts2 + "' " + tr("skipped");
printStatusLogMessage(ss);
errorList.append(ts2);
continue;
}
// get date
QStringList l2 = QStringList::split("/", l[3], FALSE);
if (l2.count() != 3)
continue;
QString date = l2[2];
if (l2[0].toInt() < 10)
date.append("0");
date.append(l2[0]);
if (l2[1].toInt() < 10)
date.append("0");
date.append(l2[1]);
date.append("000000");
Bar bar;
if (bar.setDate(date))
{
QString ss = ts2 + " - " + tr("Bad date") + " " + l[3];
qDebug("Yahoo::parseQuote: %s - Bad date %s", ts2.latin1(), l[3].latin1());
printStatusLogMessage(ss);
continue;
}
if (setTFloat(l[6], FALSE))
continue;
else
bar.setOpen(tfloat);
if (setTFloat(l[7], FALSE))
continue;
else
bar.setHigh(tfloat);
// make Low price equal to Open if Low is not available
if (! l[8].compare("N/A"))
l[8] = l[6];
if (setTFloat(l[8], FALSE))
continue;
else
bar.setLow(tfloat);
if (setTFloat(l[2], FALSE))
continue;
else
bar.setClose(tfloat);
if (l.count() == 10)
{
if (setTFloat(l[9], FALSE))
continue;
else
bar.setVolume(tfloat);
}
if (bar.verify())
continue;
plug.setBar(bar);
emit signalWakeup();
}
f.close();
plug.close();
}
void NYBOT::parse ()
{
if (cancelFlag)
return;
QFile f(file);
if (! f.open(IO_ReadOnly))
return;
QTextStream stream(&f);
QString ts = stream.readLine();
QString s;
stripJunk(ts, s);
QStringList keys = QStringList::split(",", s, FALSE);
while(stream.atEnd() == 0)
{
ts = stream.readLine();
stripJunk(ts, s);
QStringList l = QStringList::split(",", s, FALSE);
if (l.count() != keys.count())
continue;
Setting data;
int loop2;
for (loop2 = 0; loop2 < (int) keys.count(); loop2++)
data.setData(keys[loop2], l[loop2]);
// symbol
QString symbol;
QString ts = "commoditySymbol";
data.getData(ts, symbol);
symbol = symbol.stripWhiteSpace();
if (symbol.length() == 0)
continue;
if (! symbol.compare("CC") || ! symbol.compare("CR") || ! symbol.compare("CT") ||
! symbol.compare("DX") || ! symbol.compare("KC") || ! symbol.compare("OJ") ||
! symbol.compare("SB") || ! symbol.compare("YX"))
{
}
else
continue;
// date
QString date;
ts = "tradeDate";
data.getData(ts, date);
date.append("000000");
Bar bar;
if (bar.setDate(date))
{
QString ss = tr("Bad date") + " " + date;
printStatusLogMessage(ss);
continue;
}
ts = "dailyOpenPrice1";
data.getData(ts, s);
if (s.toFloat() == 0)
{
ts = "dailyOpenPrice2";
data.getData(ts, s);
}
if (setTFloat(s, FALSE))
continue;
else
bar.setOpen(tfloat);
ts = "dailyHigh";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setHigh(tfloat);
ts = "dailyLow";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setLow(tfloat);
ts = "settlementPrice";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setClose(tfloat);
ts = "tradeVolume";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setVolume(tfloat);
ts = "openInterest";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setOI((int) tfloat);
if (symbol.compare("CC"))
{
bar.setOpen(bar.getOpen() / 100);
bar.setHigh(bar.getHigh() / 100);
bar.setLow(bar.getLow() / 100);
bar.setClose(bar.getClose() / 100);
}
if (bar.verify())
continue;
//futures month
ts = "contractMonth";
data.getData(ts, s);
QString year = s.left(4);
QString month = s.right(2);
QString fmonth;
switch (month.toInt())
{
case 1:
fmonth = "F";
break;
case 2:
fmonth = "G";
break;
case 3:
fmonth = "H";
break;
case 4:
fmonth = "J";
break;
case 5:
fmonth = "K";
break;
case 6:
fmonth = "M";
break;
case 7:
fmonth = "N";
break;
case 8:
fmonth = "Q";
break;
case 9:
fmonth = "U";
break;
case 10:
fmonth = "V";
break;
case 11:
fmonth = "X";
break;
case 12:
fmonth = "Z";
break;
default:
break;
}
if (fd.setSymbol(symbol))
continue;
if (year.length())
{
symbol.append(year);
if (fmonth.length())
symbol.append(fmonth);
else
continue;
}
else
continue;
s = "Futures/";
QString s2;
fd.getExchange(s2);
s.append(s2 + "/");
fd.getSymbol(s2);
s.append(s2);
QString path;
createDirectory(s, path);
if (! path.length())
{
QString ss(tr("Unable to create futures directory"));
printStatusLogMessage(ss);
return;
}
s = path + "/" + symbol;
if (plug.open(s, chartIndex))
{
QString ss(tr("Could not open db"));
printStatusLogMessage(ss);
return;
}
DBIndexItem item;
chartIndex->getIndexItem(symbol, item);
item.getSymbol(s);
if (! s.length())
{
if (plug.createNewFutures())
return;
chartIndex->getIndexItem(symbol, item);
s = QString::number(Exchange::NYBOT);
item.setExchange(s);
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(symbol, item);
}
plug.setBar(bar);
plug.close();
emit signalWakeup();
}
f.close();
downloadComplete();
if (cancelFlag)
{
cancelFlag = FALSE;
QString ss(tr("Update cancelled"));
printStatusLogMessage(ss);
}
else
{
QString ss(tr("Done"));
printStatusLogMessage(ss);
}
}
void Yahoo::parseHistory ()
{
if (! data.length())
return;
if (data.contains("No data available"))
return;
if (data.contains("No Prices in this date range"))
return;
// strip off the header
QString s = "Date,Open,High,Low,Close,Volume,Adj Close\n";
int p = data.find(s, 0, TRUE);
if (p != -1)
data.remove(0, p + s.length());
QFile f(file);
if (! f.open(QIODevice::WriteOnly))
return;
Q3TextStream stream(&f);
stream << data;
f.close();
f.setName(file);
if (! f.open(QIODevice::ReadOnly))
return;
stream.setDevice(&f);
s = dataPath + "/";
QString ts = "symbol";
QString ts2;
currentUrl->getData(ts, ts2);
QFileInfo fi(ts2);
if (fi.extension(FALSE).length())
s.append(fi.extension(FALSE).upper());
else
s.append("US");
s.append("/");
s.append(ts2);
if (plug.open(s, chartIndex))
{
QString ss(tr("Could not open db"));
printStatusLogMessage(ss);
f.close();
return;
}
QString fn = ts2;
// verify if this chart can be updated by this plugin
DBIndexItem item;
chartIndex->getIndexItem(fn, item);
item.getSymbol(s);
if (! s.length())
{
if (plug.createNewStock())
{
f.close();
plug.close();
return;
}
chartIndex->getIndexItem(fn, item);
item.setSymbol(ts2);
item.setTitle(ts2);
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(fn, item);
}
// item.getQuotePlugin(s);
// if (s.compare(pluginName))
// {
// s = ts2 + " - " + tr("skipping update. Source does not match destination");
// printStatusLogMessage(s);
// f.close();
// plug.close();
// return;
// }
while(stream.atEnd() == 0)
{
ts = stream.readLine();
QStringList l = QStringList::split( '\n', substituteSeparator( ts, ',', '\n' ), FALSE);
if (l.count() < 5)
{
QString ss = tr("Parse: invalid number of parameters") + " '" + ts2 + "' " + tr("skipped");
printStatusLogMessage(ss);
errorList.append(ts2);
continue;
}
// date
QString date = parseDate(l[0]);
Bar bar;
if (bar.setDate(date))
{
QString ss = ts2 + " - " + tr("Bad date") + " " + l[0];
qDebug("Yahoo::parseHistory: %s - Bad date %s", ts2.latin1(), l[0].latin1());
printStatusLogMessage(ss);
continue;
}
if (setTFloat(l[1], FALSE))
continue;
else
bar.setOpen(tfloat);
if (setTFloat(l[2], FALSE))
continue;
else
bar.setHigh(tfloat);
if (setTFloat(l[3], FALSE))
continue;
else
bar.setLow(tfloat);
if (setTFloat(l[4], FALSE))
continue;
else
bar.setClose(tfloat);
if (l.count() >= 6)
{
if (setTFloat(l[5], FALSE))
continue;
else
bar.setVolume(tfloat);
}
if (bar.verify())
continue;
// adjusted close
if (adjustment->isChecked())
{
double adjclose = 0;
if (l.count() >= 7)
{
if (setTFloat(l[6], FALSE))
continue;
else
adjclose = tfloat;
// apply yahoo's adjustments through all the data, not just closing price
// i.e. adjust for stock splits and dividends
float factor = bar.getClose() / adjclose;
if (factor != 1)
{
bar.setHigh(bar.getHigh() / factor);
bar.setLow(bar.getLow() / factor);
bar.setOpen(bar.getOpen() / factor);
bar.setClose(bar.getClose() / factor);
bar.setVolume(bar.getVolume() * factor);
}
}
}
plug.setBar(bar);
emit signalWakeup();
}
f.close();
plug.close();
}
