void Writer::writeBegin(boost::uint64_t /*targetNumPointsToWrite*/)
{
std::string block_table = getOptions().getValueOrThrow<std::string>("block_table");
std::string cloud_table = getOptions().getValueOrThrow<std::string>("cloud_table");
std::string cloud_column = getOptions().getValueOrDefault<std::string>("cloud_column", "id");
m_block_insert_query << "INSERT INTO " << boost::to_lower_copy(block_table)
<< " ("<< boost::to_lower_copy(cloud_column) <<", block_id, num_points, points, extent, bbox) VALUES ("
<< " :obj_id, :block_id, :num_points, decode(:hex, 'hex'), ST_Force_2D(ST_GeometryFromText(:extent,:srid)), :bbox)";
m_session->begin();
bool bHaveBlockTable = CheckTableExists(block_table);
bool bHaveCloudTable = CheckTableExists(cloud_table);
if (getOptions().getValueOrDefault<bool>("overwrite", true))
{
if (bHaveBlockTable)
{
DeleteBlockTable(cloud_table, cloud_column, block_table);
bHaveBlockTable = false;
}
if (bHaveCloudTable)
{
DeleteCloudTable(cloud_table, cloud_column);
bHaveCloudTable = false;
}
}
std::string pre_sql = getOptions().getValueOrDefault<std::string>("pre_sql", "");
if (pre_sql.size())
{
std::string sql = FileUtils::readFileAsString(pre_sql);
if (!sql.size())
{
// if there was no file to read because the data in pre_sql was
// actually the sql code the user wanted to run instead of the
// filename to open, we'll use that instead.
sql = pre_sql;
}
m_session->once << sql;
}
if (!bHaveCloudTable)
{
CreateCloudTable(cloud_table, getOptions().getValueOrDefault<boost::uint32_t>("srid", 4326));
}
if (!bHaveBlockTable)
{
m_doCreateIndex = true;
CreateBlockTable(block_table, getOptions().getValueOrDefault<boost::uint32_t>("srid", 4326));
}
return;
}
int main(int argc, char* argv[])
{
#ifdef CHECK_MEMLEAK
_CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF );
#endif
//Initialize the ICU calendar
UErrorCode status = U_ZERO_ERROR;
dateGC = new GregorianCalendar(status);
//get all the command line options
GetOptionFlags(argc, argv, o);
s.username = o.username;
s.password = o.password;
s.serverName = o.serverName;
//set up the log file
std::string dateString = GetDateString(dateGC->getNow(), dateGC);
std::string fileName = dateString + o.gSymbol;
fileName = GetSymbolTableName(fileName, o);
fileName += ".log";
gLogFile = fopen(fileName.c_str(), "w+");
if (gLogFile == NULL)
{
printf("Could not open file %s, exiting. Sucks that I can't log this output\n", fileName.c_str());
exit(-1);
}
//Initialize IQ Feed
InitDataFeed();
//set the SQL pointers based on the data being downloaded
SetSQLPointers(o);
/////////////////////////////
// Get the symbols
/////////////////////////////
if (o.gSymbol.compare("dow30") == 0)
o.gSymbol = dow30;
boost::char_separator<char> sep(",");
boost::tokenizer< boost::char_separator<char> > tokens(o.gSymbol, sep);
// Create a list from the tokens
std::vector<std::string> symbolList(tokens.begin(), tokens.end());
std::vector<std::string> contractList;
std::map<std::string, std::string> contractMap;
//get all the symbols
if (o.isOption == true || o.isFutures == true)
{
contractList = convertSymbolsToContracts(symbolList, contractMap);
symbolList = contractList;
}
int numSymbols = symbolList.size();
WriteLog("There are %d symbols\n", numSymbols);
//determine if we should use the hashTable, currently only
//for ticks and non-contract-related data
if (o.useTicks == true)// && (o.isOption == false && o.isFutures == false))
o.useHashTable = true;
DBHandles s;
s.username = o.username;
s.serverName = o.serverName;
s.password = o.password;
ConnectDB(&s);
for (int i = 0; i < numSymbols; i++)
{
std::string currentSymbol = symbolList[i];
// Get the data from the data feed and stuff into link list
link *head = NULL;
try
{
head = GetData(currentSymbol);
}
catch (const DataException &d)
{
std::string errorString;
ODBCError(&s, errorString);
WriteLog("GetData error: %s\n", errorString.c_str());
WriteLog("problem on file %s, line %d\n", d._function.c_str(), d._lineNum);
exit(-1);
}
//if there's no data then move to the next symbol
if (head == NULL)
continue;
//get the symbolTableName and create table if necessary
std::string symbolTableName;
if (o.useHardcodedTable == true)
symbolTableName = o.hardcodedTable;
else if (o.isOption == true || o.isFutures == true)
symbolTableName = GetSymbolTableName(contractMap[currentSymbol], o);
else
symbolTableName = GetSymbolTableName(currentSymbol, o);
if (CheckTableExists(&s, checkTableExists, symbolTableName.c_str()) == false)
{
CreateTable(&s, o.gSqlCommand, o.gIndexSQL, (char *) symbolTableName.c_str());
}
//get the last chunk for its timestamp
//lame could do this better but it's not too expensive for now
link *lastChunk = head;
while (lastChunk->next != NULL)
lastChunk = lastChunk->next;
//write the data
try
{
boost::unordered_set<dataPoint> *hash = NULL;
if (o.useHashTable == true)
hash = GetDBHash(&s, symbolTableName, head->beginTimestamp, lastChunk->endTimestamp, currentSymbol);
WriteData(&s, head, currentSymbol, symbolTableName, hash);
delete hash;
}
catch (const DataException &d)
{
std::string errorString;
ODBCError(&s, errorString);
WriteLog("WriteData error: %s\n", errorString.c_str());
WriteLog("problem on file %s, line %d\n", d._function.c_str(), d._lineNum);
exit(-1);
}
}
DisconnectDB(&s);
RemoveClientApp(NULL);
}
