// PD_TRACE_DECLARE_FUNCTION ( SDB__MIGLOADJSONPS__RUN, "migMaster::run" )
INT32 migMaster::run()
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__MIGLOADJSONPS__RUN );
UINT32 startOffset = 0 ;
UINT32 size = 0 ;
UINT32 startBlock = 0 ;
UINT32 endBlock = 0 ;
pmdKRCB *krcb = pmdGetKRCB () ;
pmdEDUMgr *eduMgr = krcb->getEDUMgr () ;
SDB_DMSCB *dmsCB = krcb->getDMSCB () ;
pmdEDUCB *eduCB = eduMgr->getEDU() ;
EDUID agentEDU = PMD_INVALID_EDUID ;
BOOLEAN writable = FALSE ;
BOOLEAN noClearFlag = FALSE ;
dmsMBContext *mbContext = NULL ;
UINT32 line = 0 ;
UINT32 column = 0 ;
UINT32 success = 0 ;
UINT32 failure = 0 ;
UINT16 clFlag = 0 ;
dmsStorageUnitID suID = DMS_INVALID_CS ;
dmsStorageUnit *su = NULL ;
initWorker dataWorker ;
pmdEDUEvent event ;
dmsStorageLoadOp dmsLoadExtent ;
sendMsgToClient ( "Load start" ) ;
rc = rtnCollectionSpaceLock ( _pParameters->pCollectionSpaceName,
dmsCB,
FALSE,
&su,
suID ) ;
if ( rc )
{
if ( SDB_DMS_CS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection space not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection space, rc=%d", rc ) ;
goto error ;
}
dmsLoadExtent.init ( su ) ;
rc = su->data()->getMBContext( &mbContext, _pParameters->pCollectionName,
EXCLUSIVE ) ;
if ( rc )
{
if ( SDB_DMS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
goto error ;
}
clFlag = mbContext->mb()->_flag ;
if ( DMS_IS_MB_DROPPED( clFlag ) )
{
PD_LOG( PDERROR, "Collection is droped" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is droped" ) ;
goto error ;
}
else if ( DMS_IS_MB_LOAD ( clFlag ) )
{
PD_LOG( PDERROR, "Collection is loading" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is loading" ) ;
// we set noClearFlag to true, so that we'll convert the collection
// flag to NORMAL in done
noClearFlag = TRUE ;
goto error ;
}
dmsLoadExtent.setFlagLoad ( mbContext->mb() ) ;
dmsLoadExtent.setFlagLoadLoad ( mbContext->mb() ) ;
// unlock
mbContext->mbUnlock() ;
rc = dmsCB->writable( eduCB ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Database is not writable, rc = %d", rc ) ;
goto error;
}
writable = TRUE;
dataWorker.pMaster = this ;
dataWorker.masterEDUID = eduCB->getID() ;
dataWorker.pSu = su ;
dataWorker.clLID = mbContext->clLID() ;
dataWorker.collectionID = mbContext->mbID() ;
for ( UINT32 i = 0; i < _pParameters->workerNum; ++i )
{
eduMgr->startEDU ( EDU_TYPE_LOADWORKER, &dataWorker, &agentEDU ) ;
}
while ( TRUE )
{
rc = _checkErrAndRollback ( eduCB, &dmsLoadExtent,
mbContext, success, failure ) ;
if ( SDB_TIMEOUT != rc &&
rc )
{
PD_LOG ( PDERROR,
"Failed to call _checkErrAndRollback, rc=%d", rc ) ;
goto error ;
}
// fetch one record
rc = _parser->getNextRecord ( startOffset, size,
&line, &column, _ppBucket ) ;
if ( rc )
{
// special handle for end of file
if ( rc == SDB_EOF )
{
// when we hit end of file, let's push 0 to all worker threads,
// with num of workers ( so each worker will dispatch one 0, and
// exit )
rc = _stopAndWaitWorker ( eduCB, success, failure ) ;
PD_RC_CHECK ( rc, PDERROR,
"Failed to call _stopAndWaitWorker, rc=%d", rc ) ;
break ;
}
sendMsgToClient ( "Error: Parse Json error in line: %u,"
" column: %u", line, column ) ;
PD_LOG ( PDERROR, "Failed to parseJSONs getNextRecord,rc=%d", rc ) ;
goto error1 ;
}
// calculate the blocks to be locked, based on the length of our record
rc = getBlockFromPointer ( startOffset, size,
startBlock, endBlock ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to get block from pointer, rc=%d", rc ) ;
goto error1 ;
}
// lock them
for ( UINT32 i = startBlock; i <= endBlock; ++i )
{
_ppBucket[i]->inc() ;
}
// push the record to queue
pushToQueue ( startOffset, size, line, column ) ;
} // while ( !eduCB->isForced() )
// when all workers are finish, let's start build phase to rebuild all
// indexes
sendMsgToClient ( "build index" ) ;
rc = dmsLoadExtent.loadBuildPhase ( mbContext, eduCB, _pParameters->isAsynchronous,
this, &success, &failure ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to load data, rc=%d", rc ) ;
goto error ;
}
done:
// we only lock and clear flag if we switched to load
if ( su && mbContext && !noClearFlag )
{
rc = mbContext->mbLock( EXCLUSIVE ) ;
// we should log failure information
if ( SDB_OK == rc )
{
if ( dmsLoadExtent.isFlagLoadLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadLoad ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoadBuild ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadBuild ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoad ( mbContext->mb() ) ;
}
}
else
{
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
}
}
// send the success message to client
sendMsgToClient ( "success json: %u, failure json: %u",
success, failure ) ;
sendMsgToClient ( "Load end" ) ;
if ( su && mbContext )
{
su->data()->releaseMBContext( mbContext ) ;
}
if ( DMS_INVALID_CS != suID )
{
dmsCB->suUnlock ( suID ) ;
}
// count down
if ( writable )
{
dmsCB->writeDown( eduCB );
}
PD_TRACE_EXITRC ( SDB__MIGLOADJSONPS__RUN, rc );
return rc ;
error:
goto done ;
error1:
_stopAndWaitWorker ( eduCB, success, failure ) ;
sendMsgToClient ( "Error: rollback all data" ) ;
failure += success ;
success = 0 ;
rc = dmsLoadExtent.loadRollbackPhase ( mbContext ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to rollback, rc=%d", rc ) ;
sendMsgToClient ( "Error: Failed to rollback, rc = %d",
rc ) ;
goto error ;
}
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB_RTNREBUILDDB, "rtnRebuildDB" )
INT32 rtnRebuildDB ( pmdEDUCB *cb )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB_RTNREBUILDDB ) ;
pmdKRCB *krcb = pmdGetKRCB () ;
SDB_DMSCB *dmsCB = krcb->getDMSCB () ;
SDB_RTNCB *rtnCB = krcb->getRTNCB () ;
BSONObj dummyObj ;
std::set<monCollectionSpace> csList ;
std::set<monCollectionSpace>::iterator it ;
BOOLEAN registeredRebuild = FALSE ;
PD_LOG ( PDEVENT, "Start rebuilding database" ) ;
rc = dmsCB->registerRebuild () ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to register rebuild" ) ;
goto error ;
}
registeredRebuild = TRUE ;
dmsCB->dumpInfo ( csList, TRUE ) ;
for ( it = csList.begin(); it != csList.end(); ++it )
{
const CHAR *pCSName = (*it)._name ;
std::set<monCollection> clList ;
std::set<monCollection>::iterator itCollection ;
dmsStorageUnitID suID ;
if ( ossStrlen ( pCSName ) > DMS_COLLECTION_SPACE_NAME_SZ )
{
PD_LOG ( PDERROR, "collection space name is not valid: %s",
pCSName ) ;
continue ;
}
if ( ossStrncmp ( pCSName, SDB_DMSTEMP_NAME,
DMS_COLLECTION_SPACE_NAME_SZ ) == 0 )
{
continue ;
}
PD_LOG ( PDEVENT, "Start rebuilding collection space %s", pCSName ) ;
dmsStorageUnit *su = NULL;
rc = dmsCB->nameToSUAndLock ( pCSName, suID, &su ) ;
if ( rc != SDB_OK )
{
PD_LOG ( PDERROR, "Failed to lock collection space %s", pCSName ) ;
continue ;
}
do
{
su->dumpInfo ( clList ) ;
for ( itCollection = clList.begin();
itCollection != clList.end();
++itCollection )
{
dmsMBContext *mbContext = NULL ;
UINT16 collectionFlag ;
const CHAR *pCLNameTemp = NULL ;
const CHAR *pCLName = (*itCollection)._name ;
if ( ( ossStrlen ( pCLName ) > DMS_COLLECTION_FULL_NAME_SZ ) ||
( NULL == ( pCLNameTemp = ossStrrchr ( pCLName, '.' ))) )
{
PD_LOG ( PDERROR, "collection name is not valid: %s",
pCLName ) ;
continue ;
}
rc = su->data()->getMBContext( &mbContext, pCLNameTemp+1,
EXCLUSIVE ) ;
if ( rc )
{
PD_LOG ( PDWARNING, "Failed to lock collection %s, rc = %d",
pCLName, rc ) ;
continue ;
}
collectionFlag = mbContext->mb()->_flag ;
su->data()->releaseMBContext( mbContext ) ;
PD_LOG ( PDEVENT, "Start rebuilding collection %s", pCLName ) ;
if ( DMS_IS_MB_OFFLINE_REORG( collectionFlag ) ||
DMS_IS_MB_ONLINE_REORG ( collectionFlag ) )
{
rc = rtnReorgRecover ( pCLName, cb, dmsCB, rtnCB ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to perform reorg recover: %s, "
"rc = %d", pCLName, rc ) ;
continue ;
}
}
else
{
rc = rtnReorgOffline ( pCLName, dummyObj, cb,
dmsCB, rtnCB, TRUE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to perform offline reorg: %s, "
"rc = %d", pCLName, rc ) ;
continue ;
}
}
PD_LOG ( PDEVENT, "Complete rebuilding collection %s",
pCLName ) ;
} // for
} while ( 0 ) ;
dmsCB->suUnlock ( suID ) ;
PD_LOG ( PDEVENT, "Complete rebuilding collection space %s", pCSName ) ;
} // end for
PD_LOG ( PDEVENT, "Database rebuild is completed" ) ;
done :
if ( registeredRebuild )
{
dmsCB->rebuildDown () ;
}
PD_TRACE_EXITRC ( SDB_RTNREBUILDDB, rc ) ;
return rc ;
error :
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB_PMDPRELOADERENENTPNT, "pmdPreLoaderEntryPoint" )
INT32 pmdPreLoaderEntryPoint ( pmdEDUCB *cb, void *pData )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB_PMDPRELOADERENENTPNT );
pmdKRCB *krcb = pmdGetKRCB() ;
SDB_BPSCB *bpscb = krcb->getBPSCB() ;
SDB_DMSCB *dmscb = krcb->getDMSCB() ;
ossQueue<bpsPreLoadReq*> *prefReqQ = bpscb->getReqQueue () ;
ossQueue<bpsPreLoadReq*> *dropReqQ = bpscb->getDropQueue () ;
bpsPreLoadReq *prefReq = NULL ;
while ( !PMD_IS_DB_DOWN )
{
if ( prefReqQ->timed_wait_and_pop ( prefReq, PMD_QUEUE_WAIT_TIME ) )
{
dmsStorageUnitID csid = prefReq->_csid ;
dmsStorageUnit *su = dmscb->suLock ( csid ) ;
if ( su && su->LogicalCSID() == prefReq->_csLID )
{
ossValuePtr addr = 0 ;
UINT32 pageSizeSqureRoot = 0 ;
if ( BPS_DMS_DATA == prefReq->_type )
{
addr = su->data()->extentAddr ( prefReq->_extid ) ;
pageSizeSqureRoot = su->data()->pageSizeSquareRoot () ;
}
else if ( BPS_DMS_INDEX == prefReq->_type )
{
addr = su->index()->extentAddr ( prefReq->_extid ) ;
pageSizeSqureRoot = su->index()->pageSizeSquareRoot () ;
}
if ( addr )
{
CHAR *pAddr = (CHAR*)addr ;
UINT32 totalSize = 0 ;
if ( pAddr[0] == DMS_EXTENT_EYECATCHER0 &&
pAddr[1] == DMS_EXTENT_EYECATCHER1 )
{
totalSize = (UINT32)(((dmsExtent*)pAddr)->_blockSize <<
pageSizeSqureRoot ) ;
}
else if ( pAddr[0] == DMS_META_EXTENT_EYECATCHER0 &&
pAddr[1] == DMS_META_EXTENT_EYECATCHER1 )
{
totalSize = (UINT32)(((dmsMetaExtent*)pAddr)->_blockSize <<
pageSizeSqureRoot ) ;
}
else
{
totalSize = (UINT32)( 1 << pageSizeSqureRoot ) ;
}
totalSize = OSS_MIN ( totalSize, DMS_MAX_EXTENT_SZ ) ;
UINT32 index = 0 ;
while ( index < totalSize )
{
doPreLoad( pAddr + index ) ;
index += PMD_PRELOAD_UNIT ;
}
} // if ( addr )
dmscb->suUnlock ( csid ) ;
} // if ( su )
dropReqQ->push ( prefReq ) ;
} // if ( prefReqQ->timed_wait_and_pop
} // while ( !PMD_IS_DB_DOWN )
PD_TRACE_EXITRC ( SDB_PMDPRELOADERENENTPNT, rc );
return rc;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB__MIGLOADJSONPS__RUN, "migMaster::run" )
INT32 migMaster::run( pmdEDUCB *cb )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__MIGLOADJSONPS__RUN );
UINT32 startOffset = 0 ;
UINT32 size = 0 ;
UINT32 startBlock = 0 ;
UINT32 endBlock = 0 ;
pmdKRCB *krcb = pmdGetKRCB () ;
pmdEDUMgr *eduMgr = krcb->getEDUMgr () ;
SDB_DMSCB *dmsCB = krcb->getDMSCB () ;
EDUID agentEDU = PMD_INVALID_EDUID ;
BOOLEAN writable = FALSE ;
BOOLEAN noClearFlag = FALSE ;
dmsMBContext *mbContext = NULL ;
UINT32 line = 0 ;
UINT32 column = 0 ;
UINT32 success = 0 ;
UINT32 failure = 0 ;
UINT16 clFlag = 0 ;
dmsStorageUnitID suID = DMS_INVALID_CS ;
dmsStorageUnit *su = NULL ;
initWorker dataWorker ;
pmdEDUEvent event ;
dmsStorageLoadOp dmsLoadExtent ;
sendMsgToClient ( "Load start" ) ;
rc = dmsCB->writable( cb ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Database is not writable, rc = %d", rc ) ;
goto error;
}
writable = TRUE;
rc = rtnCollectionSpaceLock ( _pParameters->pCollectionSpaceName,
dmsCB,
FALSE,
&su,
suID ) ;
if ( rc )
{
if ( SDB_DMS_CS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection space not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection space, rc=%d", rc ) ;
goto error ;
}
dmsLoadExtent.init ( su ) ;
rc = su->data()->getMBContext( &mbContext,
_pParameters->pCollectionName,
EXCLUSIVE ) ;
if ( rc )
{
if ( SDB_DMS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
goto error ;
}
clFlag = mbContext->mb()->_flag ;
if ( DMS_IS_MB_DROPPED( clFlag ) )
{
PD_LOG( PDERROR, "Collection is droped" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is droped" ) ;
goto error ;
}
else if ( DMS_IS_MB_LOAD ( clFlag ) )
{
PD_LOG( PDERROR, "Collection is loading" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is loading" ) ;
noClearFlag = TRUE ;
goto error ;
}
dmsLoadExtent.setFlagLoad ( mbContext->mb() ) ;
dmsLoadExtent.setFlagLoadLoad ( mbContext->mb() ) ;
mbContext->mbUnlock() ;
dataWorker.pMaster = this ;
dataWorker.masterEDUID = cb->getID() ;
dataWorker.pSu = su ;
dataWorker.clLID = mbContext->clLID() ;
dataWorker.collectionID = mbContext->mbID() ;
for ( UINT32 i = 0; i < _pParameters->workerNum ; ++i )
{
eduMgr->startEDU ( EDU_TYPE_LOADWORKER, &dataWorker, &agentEDU ) ;
}
while ( TRUE )
{
rc = _checkErrAndRollback ( cb, &dmsLoadExtent,
mbContext, success, failure ) ;
if ( SDB_TIMEOUT != rc &&
rc )
{
PD_LOG ( PDERROR,
"Failed to call _checkErrAndRollback, rc=%d", rc ) ;
goto error ;
}
rc = _parser->getNextRecord ( startOffset, size,
&line, &column, _ppBucket ) ;
if ( rc )
{
if ( rc == SDB_EOF )
{
rc = _stopAndWaitWorker ( cb, success, failure ) ;
PD_RC_CHECK ( rc, PDERROR,
"Failed to call _stopAndWaitWorker, rc=%d", rc ) ;
break ;
}
sendMsgToClient ( "Error: Parse Json error in line: %u,"
" column: %u", line, column ) ;
PD_LOG ( PDERROR, "Failed to parseJSONs getNextRecord,rc=%d", rc ) ;
goto error1 ;
}
rc = getBlockFromPointer ( startOffset, size,
startBlock, endBlock ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to get block from pointer, rc=%d", rc ) ;
goto error1 ;
}
for ( UINT32 i = startBlock; i <= endBlock; ++i )
{
_ppBucket[i]->inc() ;
}
pushToQueue ( startOffset, size, line, column ) ;
} // while ( !cb->isForced() )
sendMsgToClient ( "build index" ) ;
rc = dmsLoadExtent.loadBuildPhase ( mbContext, cb,
_pParameters->isAsynchronous,
this, &success, &failure ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to load data, rc=%d", rc ) ;
goto error ;
}
done:
if ( su && mbContext && !noClearFlag )
{
rc = mbContext->mbLock( EXCLUSIVE ) ;
if ( SDB_OK == rc )
{
if ( dmsLoadExtent.isFlagLoadLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadLoad ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoadBuild ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadBuild ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoad ( mbContext->mb() ) ;
}
}
else
{
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
}
}
sendMsgToClient ( "success json: %u, failure json: %u",
success, failure ) ;
sendMsgToClient ( "Load end" ) ;
if ( su && mbContext )
{
su->data()->releaseMBContext( mbContext ) ;
}
if ( DMS_INVALID_CS != suID )
{
dmsCB->suUnlock ( suID ) ;
}
if ( writable )
{
dmsCB->writeDown( cb );
}
PD_TRACE_EXITRC ( SDB__MIGLOADJSONPS__RUN, rc );
return rc ;
error:
goto done ;
error1:
_stopAndWaitWorker ( cb, success, failure ) ;
sendMsgToClient ( "Error: rollback all data" ) ;
failure += success ;
success = 0 ;
rc = dmsLoadExtent.loadRollbackPhase ( mbContext ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to rollback, rc=%d", rc ) ;
sendMsgToClient ( "Error: Failed to rollback, rc = %d",
rc ) ;
goto error ;
}
goto done ;
}
