bool CSV::openDb (QString &path, QString &symbol, QString &type, bool tickFlag)
{
if (db.open(path, chartIndex))
{
qDebug("CSV::openDb:can't open chart");
QString ss("CSV::OpenDb:Could not open db.");
printStatusLogMessage(ss);
db.close();
return TRUE;
}
QString s;
DBIndexItem item;
chartIndex->getIndexItem(symbol, item);
item.getSymbol(s);
if (! s.length())
{
if (! type.compare("Futures"))
{
db.setType(DbPlugin::Futures1);
if (db.createNewFutures())
{
db.close();
return TRUE;
}
}
else
{
if(db.createNewStock())
{
db.close();
return TRUE;
}
}
item.setSymbol(symbol);
item.setBarType(tickFlag);
chartIndex->setIndexItem(symbol, item);
}
// verify if this chart can be updated by this plugin
item.getQuotePlugin(s);
if (! s.length())
{
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(symbol, item);
}
else
{
if (s.compare(pluginName))
{
s = symbol + " - skipping update. Source does not match destination.";
printStatusLogMessage(s);
db.close();
return TRUE;
}
}
return FALSE;
}
void UpgradeMessage::saveHeaderData (DbPlugin &db, QString &k, QString &d, QString &sym, DBIndexItem &item)
{
// is this a co key?
bool ok = FALSE;
double t = k.toDouble(&ok);
if (ok)
{
if (t < 10000)
{
// its a chart object
Setting t;
t.parse(d);
QString s = "Plugin";
QString s2;
t.getData(s,s2);
if (s2.length())
{
t.remove(s);
s = "Type";
t.setData(s, s2);
}
s = "Plot";
t.getData(s, s2);
if (! s2.compare("Main Plot"))
{
s2 = "Bars";
t.setData(s, s2);
}
else
return;
index.setChartObject(sym, k, t);
return;
}
}
if (! k.compare("Type"))
{
item.setType(d);
return;
}
if (! k.compare("FuturesType"))
{
item.setFuturesType(d);
return;
}
if (! k.compare("FuturesMonth"))
{
item.setFuturesMonth(d);
return;
}
if (! k.compare("BarType"))
{
item.setBarType(d);
return;
}
if (! k.compare("Fundamentals"))
{
index.setFundamentals(sym, d);
return;
}
if (! k.compare("LocalIndicators"))
{
index.addIndicator(sym, d);
return;
}
if (! k.compare("QuotePlugin"))
{
item.setQuotePlugin(d);
return;
}
if (! k.compare("Symbol"))
{
item.setSymbol(d);
return;
}
if (! k.compare("Title"))
{
item.setTitle(d);
return;
}
if (! k.compare("Path"))
{
item.setPath(d);
return;
}
if (! k.compare("SpreadFirstSymbol"))
{
int t = d.find("/data0/", 0, TRUE);
d.replace(t + 5, 1, "1");
QString ts = "FirstSymbol";
db.setData(ts, d);
return;
}
if (! k.compare("SpreadSecondSymbol"))
{
int t = d.find("/data0/", 0, TRUE);
d.replace(t + 5, 1, "1");
QString ts = "SecondSymbol";
db.setData(ts, d);
return;
}
if (! k.compare("IndexList"))
{
while (1)
{
int t = d.find("/data0/", 0, TRUE);
if (t == -1)
break;
else
d.replace(t + 5, 1, "1");
}
QString ts = "List";
db.setData(ts, d);
return;
}
if (! k.compare("CCAdjustment"))
{
QString ts = "Adjustment";
db.setData(ts, d);
return;
}
}
void NYBOT::parse ()
{
if (cancelFlag)
return;
QFile f(file);
if (! f.open(IO_ReadOnly))
return;
QTextStream stream(&f);
QString ts = stream.readLine();
QString s;
stripJunk(ts, s);
QStringList keys = QStringList::split(",", s, FALSE);
while(stream.atEnd() == 0)
{
ts = stream.readLine();
stripJunk(ts, s);
QStringList l = QStringList::split(",", s, FALSE);
if (l.count() != keys.count())
continue;
Setting data;
int loop2;
for (loop2 = 0; loop2 < (int) keys.count(); loop2++)
data.setData(keys[loop2], l[loop2]);
// symbol
QString symbol;
QString ts = "commoditySymbol";
data.getData(ts, symbol);
symbol = symbol.stripWhiteSpace();
if (symbol.length() == 0)
continue;
if (! symbol.compare("CC") || ! symbol.compare("CR") || ! symbol.compare("CT") ||
! symbol.compare("DX") || ! symbol.compare("KC") || ! symbol.compare("OJ") ||
! symbol.compare("SB") || ! symbol.compare("YX"))
{
}
else
continue;
// date
QString date;
ts = "tradeDate";
data.getData(ts, date);
date.append("000000");
Bar bar;
if (bar.setDate(date))
{
QString ss = tr("Bad date") + " " + date;
printStatusLogMessage(ss);
continue;
}
ts = "dailyOpenPrice1";
data.getData(ts, s);
if (s.toFloat() == 0)
{
ts = "dailyOpenPrice2";
data.getData(ts, s);
}
if (setTFloat(s, FALSE))
continue;
else
bar.setOpen(tfloat);
ts = "dailyHigh";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setHigh(tfloat);
ts = "dailyLow";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setLow(tfloat);
ts = "settlementPrice";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setClose(tfloat);
ts = "tradeVolume";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setVolume(tfloat);
ts = "openInterest";
data.getData(ts, s);
if (setTFloat(s, FALSE))
continue;
else
bar.setOI((int) tfloat);
if (symbol.compare("CC"))
{
bar.setOpen(bar.getOpen() / 100);
bar.setHigh(bar.getHigh() / 100);
bar.setLow(bar.getLow() / 100);
bar.setClose(bar.getClose() / 100);
}
if (bar.verify())
continue;
//futures month
ts = "contractMonth";
data.getData(ts, s);
QString year = s.left(4);
QString month = s.right(2);
QString fmonth;
switch (month.toInt())
{
case 1:
fmonth = "F";
break;
case 2:
fmonth = "G";
break;
case 3:
fmonth = "H";
break;
case 4:
fmonth = "J";
break;
case 5:
fmonth = "K";
break;
case 6:
fmonth = "M";
break;
case 7:
fmonth = "N";
break;
case 8:
fmonth = "Q";
break;
case 9:
fmonth = "U";
break;
case 10:
fmonth = "V";
break;
case 11:
fmonth = "X";
break;
case 12:
fmonth = "Z";
break;
default:
break;
}
if (fd.setSymbol(symbol))
continue;
if (year.length())
{
symbol.append(year);
if (fmonth.length())
symbol.append(fmonth);
else
continue;
}
else
continue;
s = "Futures/";
QString s2;
fd.getExchange(s2);
s.append(s2 + "/");
fd.getSymbol(s2);
s.append(s2);
QString path;
createDirectory(s, path);
if (! path.length())
{
QString ss(tr("Unable to create futures directory"));
printStatusLogMessage(ss);
return;
}
s = path + "/" + symbol;
if (plug.open(s, chartIndex))
{
QString ss(tr("Could not open db"));
printStatusLogMessage(ss);
return;
}
DBIndexItem item;
chartIndex->getIndexItem(symbol, item);
item.getSymbol(s);
if (! s.length())
{
if (plug.createNewFutures())
return;
chartIndex->getIndexItem(symbol, item);
s = QString::number(Exchange::NYBOT);
item.setExchange(s);
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(symbol, item);
}
plug.setBar(bar);
plug.close();
emit signalWakeup();
}
f.close();
downloadComplete();
if (cancelFlag)
{
cancelFlag = FALSE;
QString ss(tr("Update cancelled"));
printStatusLogMessage(ss);
}
else
{
QString ss(tr("Done"));
printStatusLogMessage(ss);
}
}
void Yahoo::createAutoHistoryUrls (QString &path, QString &d)
{
if (plug.open(path, chartIndex))
{
qDebug("Yahoo::createAutoHistoryUrls:could not open db");
return;
}
QFileInfo fi(path);
QString fn = fi.fileName();
// verify if this chart can be updated by this plugin
QString s;
DBIndexItem item;
chartIndex->getIndexItem(fn, item);
item.getQuotePlugin(s);
if (! s.length())
{
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(fn, item);
}
// else
// {
// if (s.compare(pluginName))
// {
// qDebug("Yahoo::createAutoHistoryUrls:source not same as destination");
// plug.close();
// return;
// }
// }
QDate edt = QDate::currentDate();
if (edt.dayOfWeek() == 6)
edt = edt.addDays(-1);
else
{
if (edt.dayOfWeek() == 7)
edt = edt.addDays(-2);
}
Bar bar;
plug.getLastBar(bar);
if (bar.getEmptyFlag())
{
QDate dt = edt;
dt = dt.addDays(-365);
s = dt.toString("yyyyMMdd000000");
bar.setDate(s);
}
QDateTime dt;
bar.getDate(dt);
if (dt.date() == edt)
{
// qDebug("Yahoo::createAutoHistoryUrls:barDate == endDate");
plug.close();
return;
}
s = "http://table.finance.yahoo.com/table.csv?s=";
s.append(d);
s.append("&a=");
s.append(QString::number(dt.date().month() - 1));
s.append("&b=");
s.append(dt.toString("dd"));
s.append("&c=");
s.append(dt.toString("yyyy"));
s.append("&d=");
s.append(QString::number(edt.month() - 1));
s.append("&e=");
s.append(edt.toString("dd"));
s.append("&f=");
s.append(edt.toString("yyyy"));
s.append("&g=d&ignore=.csv");
plug.close();
Setting *set = new Setting;
QString ts = "url";
set->setData(ts, s);
ts = "symbol";
set->setData(ts, d);
url.append(set);
}
void Yahoo::parseFundamental ()
{
if (! data.length())
return;
if (data.contains("no data available"))
return;
QStringList l = QStringList::split("yfnc_tablehead1", data, FALSE);
int loop;
Setting fund;
for (loop = 1; loop < (int) l.count(); loop++)
{
QString k = l[loop];
int p = k.find(">", 0, TRUE);
if (p == -1)
continue;
p++;
k.remove(0, p);
p = k.find("<", 0, TRUE);
if (p == -1)
continue;
k.truncate(p);
if (k.contains("&sup"))
k.truncate(k.length() - 6);
if (k.contains("&"))
k.remove(k.find("&", 0, TRUE), 5);
k = k.stripWhiteSpace();
if (! k.length())
continue;
QString d = l[loop];
p = d.find("yfnc_tabledata1", 0, TRUE);
if (p == -1)
continue;
p = d.find(">", p, TRUE);
if (p == -1)
continue;
p++;
d.remove(0, p);
p = d.find("<", 0, TRUE);
if (p == -1)
continue;
d.truncate(p);
d = d.stripWhiteSpace();
if (! d.length())
continue;
fund.setData(k, d);
}
QString s = dataPath + "/";
QString ts = "symbol";
QString ts2;
currentUrl->getData(ts, ts2);
QFileInfo fi(ts2);
if (fi.extension(FALSE).length())
s.append(fi.extension(FALSE).upper());
else
s.append("US");
s.append("/");
s.append(ts2);
if (plug.open(s, chartIndex))
{
QString ss(tr("Could not open db"));
printStatusLogMessage(ss);
return;
}
QString fn = ts2;
// verify if this chart can be updated by this plugin
DBIndexItem item;
chartIndex->getIndexItem(fn, item);
item.getSymbol(s);
if (! s.length())
{
if(plug.createNewStock())
{
plug.close();
return;
}
chartIndex->getIndexItem(fn, item);
item.setQuotePlugin(pluginName);
item.setSymbol(ts2);
QString title = ts2;
int p = data.find("yfnc_leftnav1", 0, TRUE);
if (p != -1)
{
p = data.find("b>", p, TRUE);
if (p != -1)
{
p = p + 2;
int p2 = data.find("<", p, TRUE);
if (p2 != -1)
{
s = data.mid(p, p2 - p);
if (s.length())
title = s;
}
}
}
item.setTitle(title);
chartIndex->setIndexItem(fn, item);
}
else
{
QString s2;
item.getTitle(s2);
if (! s.compare(s2))
{
int p = data.find("yfnc_leftnav1", 0, TRUE);
if (p != -1)
{
p = data.find("b>", p, TRUE);
if (p != -1)
{
p = p + 2;
int p2 = data.find("<", p, TRUE);
if (p2 != -1)
{
s = data.mid(p, p2 - p);
if (s.length())
{
item.setTitle(s);
chartIndex->setIndexItem(fn, item);
}
}
}
}
}
}
// item.getQuotePlugin(s);
// if (s.compare(pluginName))
// {
// s = ts2 + " - " + tr("skipping update. Source does not match destination");
// printStatusLogMessage(s);
// plug.close();
// return;
// }
// include date of this update
QDate dt = QDate::currentDate();
ts = "updateDate";
ts2 = dt.toString("yyyy-MM-dd");
fund.setData(ts, ts2);
fund.getString(ts2);
ts = "Fundamentals";
chartIndex->setFundamentals(fn, ts2);
plug.close();
emit signalWakeup();
}
void Yahoo::parseQuote ()
{
if (! data.length())
return;
QFile f(file);
if (! f.open(QIODevice::WriteOnly))
return;
Q3TextStream stream(&f);
stream << data;
f.close();
f.setName(file);
if (! f.open(QIODevice::ReadOnly))
return;
stream.setDevice(&f);
QString s = dataPath + "/";
QString ts = "symbol";
QString ts2;
currentUrl->getData(ts, ts2);
QFileInfo fi(ts2);
if (fi.extension(FALSE).length())
s.append(fi.extension(FALSE).upper());
else
s.append("US");
s.append("/");
s.append(ts2);
if (plug.open(s, chartIndex))
{
QString ss(tr("Could not open db"));
printStatusLogMessage(ss);
f.close();
return;
}
QString fn = ts2;
// verify if this chart can be updated by this plugin
DBIndexItem item;
chartIndex->getIndexItem(fn, item);
item.getSymbol(s);
if (! s.length())
{
if (plug.createNewStock())
{
f.close();
plug.close();
return;
}
chartIndex->getIndexItem(fn, item);
item.setSymbol(ts2);
item.setTitle(ts2);
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(fn, item);
}
// item.getQuotePlugin(s);
// if (s.compare(pluginName))
// {
// s = ts2 + " - " + tr("skipping update. Source does not match destination");
// printStatusLogMessage(s);
// f.close();
// plug.close();
// return;
// }
while(stream.atEnd() == 0)
{
ts = stream.readLine();
QStringList l = QStringList::split( '\n', substituteSeparator( ts, ',', '\n' ), FALSE);
if (l.count() < 9 || l.count() > 10)
{
QString ss = tr("Parse: invalid number of parameters") + " '" + ts2 + "' " + tr("skipped");
printStatusLogMessage(ss);
errorList.append(ts2);
continue;
}
// get date
QStringList l2 = QStringList::split("/", l[3], FALSE);
if (l2.count() != 3)
continue;
QString date = l2[2];
if (l2[0].toInt() < 10)
date.append("0");
date.append(l2[0]);
if (l2[1].toInt() < 10)
date.append("0");
date.append(l2[1]);
date.append("000000");
Bar bar;
if (bar.setDate(date))
{
QString ss = ts2 + " - " + tr("Bad date") + " " + l[3];
qDebug("Yahoo::parseQuote: %s - Bad date %s", ts2.latin1(), l[3].latin1());
printStatusLogMessage(ss);
continue;
}
if (setTFloat(l[6], FALSE))
continue;
else
bar.setOpen(tfloat);
if (setTFloat(l[7], FALSE))
continue;
else
bar.setHigh(tfloat);
// make Low price equal to Open if Low is not available
if (! l[8].compare("N/A"))
l[8] = l[6];
if (setTFloat(l[8], FALSE))
continue;
else
bar.setLow(tfloat);
if (setTFloat(l[2], FALSE))
continue;
else
bar.setClose(tfloat);
if (l.count() == 10)
{
if (setTFloat(l[9], FALSE))
continue;
else
bar.setVolume(tfloat);
}
if (bar.verify())
continue;
plug.setBar(bar);
emit signalWakeup();
}
f.close();
plug.close();
}
void Yahoo::parseHistory ()
{
if (! data.length())
return;
if (data.contains("No data available"))
return;
if (data.contains("No Prices in this date range"))
return;
// strip off the header
QString s = "Date,Open,High,Low,Close,Volume,Adj Close\n";
int p = data.find(s, 0, TRUE);
if (p != -1)
data.remove(0, p + s.length());
QFile f(file);
if (! f.open(QIODevice::WriteOnly))
return;
Q3TextStream stream(&f);
stream << data;
f.close();
f.setName(file);
if (! f.open(QIODevice::ReadOnly))
return;
stream.setDevice(&f);
s = dataPath + "/";
QString ts = "symbol";
QString ts2;
currentUrl->getData(ts, ts2);
QFileInfo fi(ts2);
if (fi.extension(FALSE).length())
s.append(fi.extension(FALSE).upper());
else
s.append("US");
s.append("/");
s.append(ts2);
if (plug.open(s, chartIndex))
{
QString ss(tr("Could not open db"));
printStatusLogMessage(ss);
f.close();
return;
}
QString fn = ts2;
// verify if this chart can be updated by this plugin
DBIndexItem item;
chartIndex->getIndexItem(fn, item);
item.getSymbol(s);
if (! s.length())
{
if (plug.createNewStock())
{
f.close();
plug.close();
return;
}
chartIndex->getIndexItem(fn, item);
item.setSymbol(ts2);
item.setTitle(ts2);
item.setQuotePlugin(pluginName);
chartIndex->setIndexItem(fn, item);
}
// item.getQuotePlugin(s);
// if (s.compare(pluginName))
// {
// s = ts2 + " - " + tr("skipping update. Source does not match destination");
// printStatusLogMessage(s);
// f.close();
// plug.close();
// return;
// }
while(stream.atEnd() == 0)
{
ts = stream.readLine();
QStringList l = QStringList::split( '\n', substituteSeparator( ts, ',', '\n' ), FALSE);
if (l.count() < 5)
{
QString ss = tr("Parse: invalid number of parameters") + " '" + ts2 + "' " + tr("skipped");
printStatusLogMessage(ss);
errorList.append(ts2);
continue;
}
// date
QString date = parseDate(l[0]);
Bar bar;
if (bar.setDate(date))
{
QString ss = ts2 + " - " + tr("Bad date") + " " + l[0];
qDebug("Yahoo::parseHistory: %s - Bad date %s", ts2.latin1(), l[0].latin1());
printStatusLogMessage(ss);
continue;
}
if (setTFloat(l[1], FALSE))
continue;
else
bar.setOpen(tfloat);
if (setTFloat(l[2], FALSE))
continue;
else
bar.setHigh(tfloat);
if (setTFloat(l[3], FALSE))
continue;
else
bar.setLow(tfloat);
if (setTFloat(l[4], FALSE))
continue;
else
bar.setClose(tfloat);
if (l.count() >= 6)
{
if (setTFloat(l[5], FALSE))
continue;
else
bar.setVolume(tfloat);
}
if (bar.verify())
continue;
// adjusted close
if (adjustment->isChecked())
{
double adjclose = 0;
if (l.count() >= 7)
{
if (setTFloat(l[6], FALSE))
continue;
else
adjclose = tfloat;
// apply yahoo's adjustments through all the data, not just closing price
// i.e. adjust for stock splits and dividends
float factor = bar.getClose() / adjclose;
if (factor != 1)
{
bar.setHigh(bar.getHigh() / factor);
bar.setLow(bar.getLow() / factor);
bar.setOpen(bar.getOpen() / factor);
bar.setClose(bar.getClose() / factor);
bar.setVolume(bar.getVolume() * factor);
}
}
}
plug.setBar(bar);
emit signalWakeup();
}
f.close();
plug.close();
}
