void DMLPackageProcessor::getColumnsForTable(uint32_t sessionID, std::string schema,
std::string table, dmlpackage::ColumnList& colList)
{
CalpontSystemCatalog::TableName tableName;
tableName.schema = schema;
tableName.table = table;
CalpontSystemCatalog* systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog( sessionID );
CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName, true);
CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin();
while (rid_iterator != ridList.end())
{
CalpontSystemCatalog::ROPair roPair = *rid_iterator;
DMLColumn* columnPtr = new DMLColumn();
CalpontSystemCatalog::TableColName tblColName = systemCatalogPtr->colName( roPair.objnum );
columnPtr->set_Name(tblColName.column);
colList.push_back(columnPtr);
++rid_iterator;
}
}
DMLPackageProcessor::DMLResult
DeletePackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage)
{
SUMMARY_INFO("DeletePackageProcessor::processPackage");
DMLResult result;
result.result = NO_ERROR;
BRM::TxnID txnid;
// set-up the transaction
txnid.id = cpackage.get_TxnID();
txnid.valid = true;
fSessionID = cpackage.get_SessionID();
//StopWatch timer;
VERBOSE_INFO("DeletePackageProcessor is processing CalpontDMLPackage ...");
TablelockData * tablelockData = TablelockData::makeTablelockData(fSessionID);
uint64_t uniqueId = 0;
//Bug 5070. Added exception handling
try {
uniqueId = fDbrm->getUnique64();
}
catch (std::exception& ex)
{
logging::Message::Args args;
logging::Message message(9);
args.add(ex.what());
message.format(args);
result.result = DELETE_ERROR;
result.message = message;
fSessionManager.rolledback(txnid);
return result;
}
catch ( ... )
{
logging::Message::Args args;
logging::Message message(9);
args.add("Unknown error occured while getting unique number.");
message.format(args);
result.result = DELETE_ERROR;
result.message = message;
fSessionManager.rolledback(txnid);
return result;
}
uint64_t tableLockId = 0;
// get the table object from the package
DMLTable* tablePtr = cpackage.get_Table();
std::string schemaName = tablePtr->get_SchemaName();
std::string tableName = tablePtr->get_TableName();
boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog( fSessionID );
CalpontSystemCatalog::TableName aTableName;
aTableName.table = tableName;
aTableName.schema = schemaName;
fWEClient->addQueue(uniqueId);
execplan::CalpontSystemCatalog::ROPair roPair;
try
{
string stmt = cpackage.get_SQLStatement() + "|" + schemaName + "|";
SQLLogger sqlLogger(stmt, DMLLoggingId, fSessionID, txnid.id);
if ( 0 != tablePtr )
{
roPair = csc->tableRID(aTableName);
tableLockId = tablelockData->getTablelockId(roPair.objnum); //check whether this table is locked already for this session
if (tableLockId == 0)
{
//cout << "tablelock is not found in cache " << endl;
uint32_t processID = ::getpid();
int32_t txnId = txnid.id;
std::string processName("DMLProc");
int32_t sessionId = fSessionID;
int i = 0;
OamCache * oamcache = OamCache::makeOamCache();
std::vector<int> pmList = oamcache->getModuleIds();
std::vector<uint32_t> pms;
for (unsigned i=0; i < pmList.size(); i++)
{
pms.push_back((uint32_t)pmList[i]);
}
try {
tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING );
}
catch (std::exception&)
{
throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE));
}
if ( tableLockId == 0 )
{
int waitPeriod = 10;
int sleepTime = 100; // sleep 100 milliseconds between checks
int numTries = 10; // try 10 times per second
waitPeriod = Config::getWaitPeriod();
numTries = waitPeriod * 10;
struct timespec rm_ts;
rm_ts.tv_sec = sleepTime/1000;
rm_ts.tv_nsec = sleepTime%1000 *1000000;
for (; i < numTries; i++)
{
#ifdef _MSC_VER
Sleep(rm_ts.tv_sec * 1000);
#else
struct timespec abs_ts;
do
{
abs_ts.tv_sec = rm_ts.tv_sec;
abs_ts.tv_nsec = rm_ts.tv_nsec;
}
while(nanosleep(&abs_ts,&rm_ts) < 0);
#endif
try {
processID = ::getpid();
txnId = txnid.id;
sessionId = fSessionID;
processName = "DMLProc";
tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING );
}
catch (std::exception&)
{
throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE));
}
if (tableLockId > 0)
break;
}
if (i >= numTries) //error out
{
result.result = DELETE_ERROR;
logging::Message::Args args;
args.add(processName);
args.add((uint64_t)processID);
args.add(sessionId);
throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED,args));
}
}
}
//cout << " tablelock is obtained with id " << tableLockId << endl;
tablelockData->setTablelock(roPair.objnum, tableLockId);
//@Bug 4491 start AI sequence for autoincrement column
const CalpontSystemCatalog::RIDList ridList = csc->columnRIDs(aTableName);
CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin();
CalpontSystemCatalog::ColType colType;
while (rid_iterator != ridList.end())
{
// If user hit ctrl+c in the mysql console, fRollbackPending will be true.
if (fRollbackPending)
{
result.result = JOB_CANCELED;
break;
}
CalpontSystemCatalog::ROPair roPair = *rid_iterator;
colType = csc->colType(roPair.objnum);
if (colType.autoincrement)
{
try
{
uint64_t nextVal = csc->nextAutoIncrValue(aTableName);
fDbrm->startAISequence(roPair.objnum, nextVal, colType.colWidth, colType.colDataType);
break; //Only one autoincrement column per table
}
catch (std::exception& ex)
{
result.result = DELETE_ERROR;
throw std::runtime_error(ex.what());
}
}
++rid_iterator;
}
uint64_t rowsProcessed = 0;
rowsProcessed = fixUpRows(cpackage, result, uniqueId, roPair.objnum);
//@Bug 4994 Cancelled job is not error
if (result.result == JOB_CANCELED)
throw std::runtime_error("Query execution was interrupted");
if ((result.result != 0) && (result.result != DMLPackageProcessor::IDBRANGE_WARNING))
throw std::runtime_error(result.message.msg());
result.rowCount = rowsProcessed;
// Log the DML statement.
logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_SQLStatement(), cpackage.get_SchemaName());
}
}
catch (exception& ex)
{
cerr << "DeletePackageProcessor::processPackage: " << ex.what() << endl;
//@Bug 4994 Cancelled job is not error
if (result.result == 0)
{
result.result = DELETE_ERROR;
}
result.message = Message(ex.what());
}
catch (...)
{
cerr << "DeletePackageProcessor::processPackage: caught unknown exception!" << endl;
logging::Message::Args args;
logging::Message message(7);
args.add( "Delete Failed: ");
args.add( "encountered unknown exception" );
args.add(result.message.msg());
args.add("");
message.format( args );
result.result = DELETE_ERROR;
result.message = message;
}
//timer.finish();
//@Bug 1886,2870 Flush VM cache only once per statement.
std::map<uint32_t,uint32_t> oids;
int rc = 0;
if (result.result == NO_ERROR)
{
rc = flushDataFiles( result.result, oids, uniqueId, txnid, roPair.objnum);
if (rc != NO_ERROR)
{
cerr << "UpdatePackageProcessor::processPackage: write data to disk failed" << endl;
if (!fRollbackPending)
{
logging::Message::Args args;
logging::Message message(7);
args.add("Delete Failed: ");
args.add("error when writing data to disk");
args.add("");
args.add("");
message.format(args);
result.result = UPDATE_ERROR;
result.message = message;
}
result.rowCount = 0;
rc = endTransaction(uniqueId, txnid, false);
if ( (rc != NO_ERROR) && (!fRollbackPending))
{
logging::Message::Args args;
logging::Message message(7);
args.add("Delete Failed: ");
args.add("error when cleaning up data files");
args.add("");
args.add("");
message.format(args);
result.result = UPDATE_ERROR;
result.message = message;
result.rowCount = 0;
}
}
else
{
if (fRollbackPending)
rc = endTransaction(uniqueId, txnid, false);
else
rc = endTransaction(uniqueId, txnid, true);
}
}
else
{
rc = flushDataFiles( result.result, oids, uniqueId, txnid, roPair.objnum);
result.rowCount = 0;
rc = endTransaction(uniqueId, txnid, false);
}
if (fRollbackPending)
{
result.result = JOB_CANCELED;
logging::Message::Args args1;
args1.add("Query execution was interrupted");
result.message.format(args1);
}
fWEClient->removeQueue(uniqueId);
VERBOSE_INFO("Finished Processing Delete DML Package");
return result;
}
int RedistributeWorkerThread::buildEntryList()
{
int ret = 0;
try
{
// get all column oids
boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0);
const CalpontSystemCatalog::TableName table = csc->tableName(fPlanEntry.table);
CalpontSystemCatalog::RIDList cols = csc->columnRIDs(table, true);
for (CalpontSystemCatalog::RIDList::iterator i = cols.begin(); i != cols.end(); i++)
fOids.push_back(i->objnum);
CalpontSystemCatalog::DictOIDList dicts = csc->dictOIDs(table);
for (CalpontSystemCatalog::DictOIDList::iterator i = dicts.begin(); i != dicts.end(); i++)
fOids.push_back(i->dictOID);
bool firstOid = true; // for adding segments, all columns have the same lay out.
uint16_t source = fPlanEntry.source;
uint16_t target = fPlanEntry.destination;
uint16_t partition = fPlanEntry.partition;
uint32_t minWidth = 8; // column width greater than 8 will be dictionary.
for (vector<int64_t>::iterator i = fOids.begin(); i != fOids.end(); i++)
{
vector<EMEntry> entries;
int rc = fDbrm->getExtents(*i, entries, false, false, true);
if (rc != 0 || entries.size() == 0)
{
ostringstream oss;
oss << "Error in DBRM getExtents; oid:" << *i << "; returnCode: " << rc;
throw runtime_error(oss.str());
}
// same oid has the same column width
uint32_t colWid = entries.front().colWid;
vector<EMEntry>::iterator targetHwmEntry = entries.end(); // for HWM_0 workaround
if (colWid > 0 && colWid < minWidth)
minWidth = colWid;
for (vector<EMEntry>::iterator j = entries.begin(); j != entries.end(); j++)
{
if (j->dbRoot == source && j->partitionNum == partition)
{
fUpdateRtEntries.push_back(BulkUpdateDBRootArg(j->range.start, target));
if (firstOid)
fSegments.insert(j->segmentNum);
}
else if (j->dbRoot == target && j->partitionNum == partition)
{
// the partition already exists on the target dbroot
fErrorCode = RED_EC_PART_EXIST_ON_TARGET;
ostringstream oss;
oss << "oid:" << *i << ", partition:" << partition << " exists, source:"
<< source << ", destination:" << target;
fErrorMsg = oss.str();
logMessage(fErrorMsg, __LINE__);
return fErrorCode;
}
// workaround for HWM_0 of highest extents of the oid on target dbroot.
if (j->dbRoot == target)
{
if (targetHwmEntry == entries.end())
{
targetHwmEntry = j;
}
else
{
if (j->partitionNum > targetHwmEntry->partitionNum)
{
targetHwmEntry = j;
}
else if (j->partitionNum == targetHwmEntry->partitionNum &&
j->blockOffset > targetHwmEntry->blockOffset)
{
targetHwmEntry = j;
}
else if (j->partitionNum == targetHwmEntry->partitionNum &&
j->blockOffset == targetHwmEntry->blockOffset &&
j->segmentNum > targetHwmEntry->segmentNum)
{
targetHwmEntry = j;
}
}
}
} // for em entries
// HWM_0 workaround
// HWM 0 has two possibilities:
// 1. segment file has one extent, and the first block is not full yet.
// 2. segment file has more than one extents, the HWM of the extents other than
// the last extent is set to 0, that is only last extent has none-zero HWM.
// In tuple-bps::makeJob, there is a check to handle last extent has 0 hwm:
// (scannedExtents[i].HWM == 0 && (int) i < lastExtent[scannedExtents[i].dbRoot-1])
// lbidsToScan = scannedExtents[i].range.size * 1024;
// Based on this check, the number of block to scan is caculated.
// After redistributing the partitions, the original case 1 extent on destination
// may not be the highest extent in the dbroot, and result in a full extent scan.
// This scan will fail because there is no enough blocks if this is an abbreviated
// extent or not enough chunks if the column is compressed.
// The workaround is to bump up the HWM to 1 if moved in partitions are greater.
if (targetHwmEntry != entries.end() && // exclude no extent case
targetHwmEntry->colWid > 0 && // exclude dictionary
targetHwmEntry->HWM == 0 &&
targetHwmEntry->partitionNum < partition)
{
BulkSetHWMArg arg;
arg.oid = *i;
arg.partNum = targetHwmEntry->partitionNum;
arg.segNum = targetHwmEntry->segmentNum;
arg.hwm = targetHwmEntry->colWid; // will correct later based on minWidth
fUpdateHwmEntries.push_back(arg);
}
} // for oids
// HWM_0 workaround
// Caculate the min(column width), the HWM(bump up to) for each column.
if (fUpdateHwmEntries.size() > 0)
{
// update the HWM based in column width, not include dictionary extents
for (vector<BRM::BulkSetHWMArg>::iterator j = fUpdateHwmEntries.begin();
j != fUpdateHwmEntries.end(); j++)
{
if (j->hwm <= 8)
j->hwm /= minWidth;
else
j->hwm = 1; // not needed, but in case
}
}
}
catch (const std::exception& ex)
{
fErrorCode = RED_EC_EXTENT_ERROR;
fErrorMsg = ex.what();
logMessage(fErrorMsg, __LINE__);
ret = fErrorCode;
}
catch (...)
{
fErrorCode = RED_EC_EXTENT_ERROR;
fErrorMsg = "get extent error.";
logMessage(fErrorMsg, __LINE__);
ret = fErrorCode;
}
return ret;
}