void Checkpoint::executePlanOperation() {
#ifdef PERSISTENCY_BUFFEREDLOGGER
auto checkpoint_id = io::Logger::getInstance().startCheckpoint();
if (checkpoint_id > 0) {
std::string path = Settings::getInstance()->getCheckpointDir() + std::to_string(checkpoint_id) + "/";
auto sm = io::StorageManager::getInstance();
auto tablenames = sm->getTableNames();
storage::SimpleTableDump delta_dumplor(path);
storage::SimpleTableDump main_dumplor(Settings::getInstance()->getTableDumpDir());
// go through all tables and save the size, so that we have a "most consistent snapshot" as possible.
// e.g. we do not have too many updates coming in between the first dumped table and the last.
for (auto tablename : tablenames) {
auto store = std::dynamic_pointer_cast<storage::Store>(sm->getTable(tablename));
if (store) {
store->prepareCheckpoint();
}
}
for (auto tablename : tablenames) {
std::shared_ptr<storage::AbstractTable> a__table = sm->getTable(tablename);
auto store = std::dynamic_pointer_cast<storage::Store>(a__table);
// we only work on stores
if (store) {
if (_withMain) {
main_dumplor.dump(tablename, a__table);
}
delta_dumplor.dumpDelta(tablename, a__table);
delta_dumplor.dumpCidVectors(tablename, a__table);
}
}
io::Logger::getInstance().endCheckpoint();
}
#endif
}
int checkTableCoords(char *db)
/* Check several invariants (see comments in check*() above),
* summarize errors, return nonzero if there are errors. */
{
struct sqlConnection *conn = hAllocConn(db);
struct slName *tableList = NULL, *curTable = NULL;
struct slName *allChroms = NULL;
boolean gotError = FALSE;
allChroms = hAllChromNames(db);
if (theTable == NULL)
tableList = getTableNames(conn);
else if (sqlTableExists(conn, theTable))
tableList = newSlName(theTable);
else
errAbort("Error: specified table \"%s\" does not exist in database %s.",
theTable, db);
for (curTable = tableList; curTable != NULL; curTable = curTable->next)
{
struct hTableInfo *hti = NULL;
struct slName *chromList = NULL, *chromPtr = NULL;
char *table = curTable->name;
char tableChrom[32], trackName[128], tableChromPrefix[33];
hParseTableName(db, table, trackName, tableChrom);
hti = hFindTableInfo(db, tableChrom, trackName);
if (hti != NULL && hti->isPos)
{
/* watch out for presence of both split and non-split tables;
* hti for non-split will be replaced with hti of split. */
if (splitAndNonSplitExist(conn, table, tableChrom))
continue;
safef(tableChromPrefix, sizeof(tableChromPrefix), "%s_", tableChrom);
if (hti->isSplit)
chromList = newSlName(tableChrom);
else
chromList = allChroms;
/* invariant: chrom must be described in chromInfo. */
/* items with bad chrom will be invisible to hGetBedRange(), so
* catch them here by SQL query. */
/* The SQL query is too huge for scaffold-based db's, check count: */
if (hChromCount(db) <= MAX_SEQS_SUPPORTED)
{
if (isNotEmpty(hti->chromField))
{
struct dyString *bigQuery = newDyString(1024);
dyStringClear(bigQuery);
sqlDyStringPrintf(bigQuery, "select count(*) from %s where ",
table);
for (chromPtr=chromList; chromPtr != NULL;
chromPtr=chromPtr->next)
{
sqlDyStringPrintf(bigQuery, "%s != '%s' ",
hti->chromField, chromPtr->name);
if (chromPtr->next != NULL)
dyStringAppend(bigQuery, "AND ");
}
gotError |= reportErrors(BAD_CHROM, table,
sqlQuickNum(conn, bigQuery->string));
dyStringFree(&bigQuery);
}
for (chromPtr=chromList; chromPtr != NULL; chromPtr=chromPtr->next)
{
char *chrom = chromPtr->name;
struct bed *bedList = hGetBedRange(db, table, chrom, 0, 0, NULL);
if (hti->isSplit && isNotEmpty(hti->chromField))
gotError |= checkSplitTableOnlyChrom(bedList, table, hti,
tableChrom);
gotError |= checkStartEnd(bedList, table, hti,
testChromSize(chrom));
if (hti->hasCDS)
gotError |= checkCDSStartEnd(bedList, table, hti);
if (hti->hasBlocks && !ignoreBlocks)
gotError |= checkBlocks(bedList, table, hti);
bedFreeList(&bedList);
}
}
}
}
return gotError;
}
void DBSystem::populateDBInfo()
{
//struct Page* head=createDoublyLinkedList(this);
std::vector<string> tableNames = getTableNames();
int pageSize = getPageSize();
string folderPath= getPathForData();
bool forFolder=false;
int pageCount=0;
if(folderPath.at(folderPath.length()-1)=='/')
forFolder=true;
for(int i=0;i<tableNames.size();i++)
{
string filePath;
if(forFolder)
filePath=folderPath+tableNames[i]+".csv";
else
filePath=folderPath+"/"+tableNames[i]+".csv";
FILE *fd=fopen(filePath.c_str(),"r+");
if(fd==NULL){ cout<<"Table file not found "<<endl;continue;}
int recordCounter=-1;
char* record=new char[pageSize];
int currentSizeOfPage=0;
int StartingRecordId=-1,EndingRecordId=-1;
bool needToAddPage=false;
bool ifNewlineChar=false;
while(fgets(record,pageSize,fd)!=NULL)
{
//cout<<record;
recordCounter++;
string recordStr(record);
if(ifNewlineChar)
{
recordStr="\n"+recordStr;
ifNewlineChar=false;
}
if(pageSize == currentSizeOfPage+recordStr.size()-1)
{
//cout<<recordCounter<<endl;
recordStr=recordStr.substr(0,recordStr.length()-1);
ifNewlineChar=true;
needToAddPage=true;
currentSizeOfPage+=recordStr.size();
if(StartingRecordId==-1)
{
StartingRecordId=recordCounter;
EndingRecordId=recordCounter;
}
else
EndingRecordId=recordCounter;
}
else if(pageSize>currentSizeOfPage+recordStr.size())
{
currentSizeOfPage+=recordStr.size();
needToAddPage=true;
if(StartingRecordId==-1)
{
StartingRecordId=recordCounter;
EndingRecordId=recordCounter;
}
else
EndingRecordId=recordCounter;
}
else
{
//Whenever a new page for table is added
needToAddPage=false;
std::pair <int,int> bar = std::make_pair (StartingRecordId,EndingRecordId);
pageToRecordMap.insert(std::pair< int, std::pair<int,int> >(pageCount,bar));
tableNameToPageNumbersMap[tableNames[i]].push_back(pageCount);
pageIdToSizeHash[pageCount]=currentSizeOfPage;
pageCount++;
currentSizeOfPage=recordStr.size();
StartingRecordId=recordCounter;
EndingRecordId=recordCounter;
}
}
if(needToAddPage)
{
std::pair <int,int> bar = std::make_pair (StartingRecordId,EndingRecordId);
pageToRecordMap.insert(std::pair< int, std::pair<int,int> >(pageCount,bar));
tableNameToPageNumbersMap[tableNames[i]].push_back(pageCount);
pageIdToSizeHash[pageCount]=currentSizeOfPage;
pageCount++;
currentSizeOfPage=0;
StartingRecordId=-1;
EndingRecordId=-1;
}
currentPageCount=pageCount+1;
fclose(fd);
}
/*********************************************************************************/
//printAllPages(tableNames[0]);
/*Print the content in the maps*/
/*std::map<int,pair<int,int> >::iterator it = pageToRecordMap.begin();
for(it=pageToRecordMap.begin();it!=pageToRecordMap.end();it++)
cout<<it->first<<" "<<it->second.first<<"-"<<it->second.second<<endl;
cout<<"Table Name and page id"<<endl;
std::map<string,vector<int> >::iterator itr = tableNameToPageNumbersMap.begin();
for(itr=tableNameToPageNumbersMap.begin(); itr!=tableNameToPageNumbersMap.end();itr++)
{
cout<<itr->first<<" -->"<<endl;
for(int j=0;j<itr->second.size();j++) cout<<itr->second[j]<<endl;
}
*/
}
