// PD_TRACE_DECLARE_FUNCTION ( SDB__MIGLOADJSONPS__RUN, "migMaster::run" )
INT32 migMaster::run()
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__MIGLOADJSONPS__RUN );
UINT32 startOffset = 0 ;
UINT32 size = 0 ;
UINT32 startBlock = 0 ;
UINT32 endBlock = 0 ;
pmdKRCB *krcb = pmdGetKRCB () ;
pmdEDUMgr *eduMgr = krcb->getEDUMgr () ;
SDB_DMSCB *dmsCB = krcb->getDMSCB () ;
pmdEDUCB *eduCB = eduMgr->getEDU() ;
EDUID agentEDU = PMD_INVALID_EDUID ;
BOOLEAN writable = FALSE ;
BOOLEAN noClearFlag = FALSE ;
dmsMBContext *mbContext = NULL ;
UINT32 line = 0 ;
UINT32 column = 0 ;
UINT32 success = 0 ;
UINT32 failure = 0 ;
UINT16 clFlag = 0 ;
dmsStorageUnitID suID = DMS_INVALID_CS ;
dmsStorageUnit *su = NULL ;
initWorker dataWorker ;
pmdEDUEvent event ;
dmsStorageLoadOp dmsLoadExtent ;
sendMsgToClient ( "Load start" ) ;
rc = rtnCollectionSpaceLock ( _pParameters->pCollectionSpaceName,
dmsCB,
FALSE,
&su,
suID ) ;
if ( rc )
{
if ( SDB_DMS_CS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection space not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection space, rc=%d", rc ) ;
goto error ;
}
dmsLoadExtent.init ( su ) ;
rc = su->data()->getMBContext( &mbContext, _pParameters->pCollectionName,
EXCLUSIVE ) ;
if ( rc )
{
if ( SDB_DMS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
goto error ;
}
clFlag = mbContext->mb()->_flag ;
if ( DMS_IS_MB_DROPPED( clFlag ) )
{
PD_LOG( PDERROR, "Collection is droped" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is droped" ) ;
goto error ;
}
else if ( DMS_IS_MB_LOAD ( clFlag ) )
{
PD_LOG( PDERROR, "Collection is loading" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is loading" ) ;
// we set noClearFlag to true, so that we'll convert the collection
// flag to NORMAL in done
noClearFlag = TRUE ;
goto error ;
}
dmsLoadExtent.setFlagLoad ( mbContext->mb() ) ;
dmsLoadExtent.setFlagLoadLoad ( mbContext->mb() ) ;
// unlock
mbContext->mbUnlock() ;
rc = dmsCB->writable( eduCB ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Database is not writable, rc = %d", rc ) ;
goto error;
}
writable = TRUE;
dataWorker.pMaster = this ;
dataWorker.masterEDUID = eduCB->getID() ;
dataWorker.pSu = su ;
dataWorker.clLID = mbContext->clLID() ;
dataWorker.collectionID = mbContext->mbID() ;
for ( UINT32 i = 0; i < _pParameters->workerNum; ++i )
{
eduMgr->startEDU ( EDU_TYPE_LOADWORKER, &dataWorker, &agentEDU ) ;
}
while ( TRUE )
{
rc = _checkErrAndRollback ( eduCB, &dmsLoadExtent,
mbContext, success, failure ) ;
if ( SDB_TIMEOUT != rc &&
rc )
{
PD_LOG ( PDERROR,
"Failed to call _checkErrAndRollback, rc=%d", rc ) ;
goto error ;
}
// fetch one record
rc = _parser->getNextRecord ( startOffset, size,
&line, &column, _ppBucket ) ;
if ( rc )
{
// special handle for end of file
if ( rc == SDB_EOF )
{
// when we hit end of file, let's push 0 to all worker threads,
// with num of workers ( so each worker will dispatch one 0, and
// exit )
rc = _stopAndWaitWorker ( eduCB, success, failure ) ;
PD_RC_CHECK ( rc, PDERROR,
"Failed to call _stopAndWaitWorker, rc=%d", rc ) ;
break ;
}
sendMsgToClient ( "Error: Parse Json error in line: %u,"
" column: %u", line, column ) ;
PD_LOG ( PDERROR, "Failed to parseJSONs getNextRecord,rc=%d", rc ) ;
goto error1 ;
}
// calculate the blocks to be locked, based on the length of our record
rc = getBlockFromPointer ( startOffset, size,
startBlock, endBlock ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to get block from pointer, rc=%d", rc ) ;
goto error1 ;
}
// lock them
for ( UINT32 i = startBlock; i <= endBlock; ++i )
{
_ppBucket[i]->inc() ;
}
// push the record to queue
pushToQueue ( startOffset, size, line, column ) ;
} // while ( !eduCB->isForced() )
// when all workers are finish, let's start build phase to rebuild all
// indexes
sendMsgToClient ( "build index" ) ;
rc = dmsLoadExtent.loadBuildPhase ( mbContext, eduCB, _pParameters->isAsynchronous,
this, &success, &failure ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to load data, rc=%d", rc ) ;
goto error ;
}
done:
// we only lock and clear flag if we switched to load
if ( su && mbContext && !noClearFlag )
{
rc = mbContext->mbLock( EXCLUSIVE ) ;
// we should log failure information
if ( SDB_OK == rc )
{
if ( dmsLoadExtent.isFlagLoadLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadLoad ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoadBuild ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadBuild ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoad ( mbContext->mb() ) ;
}
}
else
{
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
}
}
// send the success message to client
sendMsgToClient ( "success json: %u, failure json: %u",
success, failure ) ;
sendMsgToClient ( "Load end" ) ;
if ( su && mbContext )
{
su->data()->releaseMBContext( mbContext ) ;
}
if ( DMS_INVALID_CS != suID )
{
dmsCB->suUnlock ( suID ) ;
}
// count down
if ( writable )
{
dmsCB->writeDown( eduCB );
}
PD_TRACE_EXITRC ( SDB__MIGLOADJSONPS__RUN, rc );
return rc ;
error:
goto done ;
error1:
_stopAndWaitWorker ( eduCB, success, failure ) ;
sendMsgToClient ( "Error: rollback all data" ) ;
failure += success ;
success = 0 ;
rc = dmsLoadExtent.loadRollbackPhase ( mbContext ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to rollback, rc=%d", rc ) ;
sendMsgToClient ( "Error: Failed to rollback, rc = %d",
rc ) ;
goto error ;
}
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB__MIGLOADJSONPS__RUN, "migMaster::run" )
INT32 migMaster::run( pmdEDUCB *cb )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__MIGLOADJSONPS__RUN );
UINT32 startOffset = 0 ;
UINT32 size = 0 ;
UINT32 startBlock = 0 ;
UINT32 endBlock = 0 ;
pmdKRCB *krcb = pmdGetKRCB () ;
pmdEDUMgr *eduMgr = krcb->getEDUMgr () ;
SDB_DMSCB *dmsCB = krcb->getDMSCB () ;
EDUID agentEDU = PMD_INVALID_EDUID ;
BOOLEAN writable = FALSE ;
BOOLEAN noClearFlag = FALSE ;
dmsMBContext *mbContext = NULL ;
UINT32 line = 0 ;
UINT32 column = 0 ;
UINT32 success = 0 ;
UINT32 failure = 0 ;
UINT16 clFlag = 0 ;
dmsStorageUnitID suID = DMS_INVALID_CS ;
dmsStorageUnit *su = NULL ;
initWorker dataWorker ;
pmdEDUEvent event ;
dmsStorageLoadOp dmsLoadExtent ;
sendMsgToClient ( "Load start" ) ;
rc = dmsCB->writable( cb ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Database is not writable, rc = %d", rc ) ;
goto error;
}
writable = TRUE;
rc = rtnCollectionSpaceLock ( _pParameters->pCollectionSpaceName,
dmsCB,
FALSE,
&su,
suID ) ;
if ( rc )
{
if ( SDB_DMS_CS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection space not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection space, rc=%d", rc ) ;
goto error ;
}
dmsLoadExtent.init ( su ) ;
rc = su->data()->getMBContext( &mbContext,
_pParameters->pCollectionName,
EXCLUSIVE ) ;
if ( rc )
{
if ( SDB_DMS_NOTEXIST == rc )
{
sendMsgToClient ( "Error: collection not exist" ) ;
}
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
goto error ;
}
clFlag = mbContext->mb()->_flag ;
if ( DMS_IS_MB_DROPPED( clFlag ) )
{
PD_LOG( PDERROR, "Collection is droped" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is droped" ) ;
goto error ;
}
else if ( DMS_IS_MB_LOAD ( clFlag ) )
{
PD_LOG( PDERROR, "Collection is loading" ) ;
rc = SDB_COLLECTION_LOAD ;
sendMsgToClient ( "Collection is loading" ) ;
noClearFlag = TRUE ;
goto error ;
}
dmsLoadExtent.setFlagLoad ( mbContext->mb() ) ;
dmsLoadExtent.setFlagLoadLoad ( mbContext->mb() ) ;
mbContext->mbUnlock() ;
dataWorker.pMaster = this ;
dataWorker.masterEDUID = cb->getID() ;
dataWorker.pSu = su ;
dataWorker.clLID = mbContext->clLID() ;
dataWorker.collectionID = mbContext->mbID() ;
for ( UINT32 i = 0; i < _pParameters->workerNum ; ++i )
{
eduMgr->startEDU ( EDU_TYPE_LOADWORKER, &dataWorker, &agentEDU ) ;
}
while ( TRUE )
{
rc = _checkErrAndRollback ( cb, &dmsLoadExtent,
mbContext, success, failure ) ;
if ( SDB_TIMEOUT != rc &&
rc )
{
PD_LOG ( PDERROR,
"Failed to call _checkErrAndRollback, rc=%d", rc ) ;
goto error ;
}
rc = _parser->getNextRecord ( startOffset, size,
&line, &column, _ppBucket ) ;
if ( rc )
{
if ( rc == SDB_EOF )
{
rc = _stopAndWaitWorker ( cb, success, failure ) ;
PD_RC_CHECK ( rc, PDERROR,
"Failed to call _stopAndWaitWorker, rc=%d", rc ) ;
break ;
}
sendMsgToClient ( "Error: Parse Json error in line: %u,"
" column: %u", line, column ) ;
PD_LOG ( PDERROR, "Failed to parseJSONs getNextRecord,rc=%d", rc ) ;
goto error1 ;
}
rc = getBlockFromPointer ( startOffset, size,
startBlock, endBlock ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to get block from pointer, rc=%d", rc ) ;
goto error1 ;
}
for ( UINT32 i = startBlock; i <= endBlock; ++i )
{
_ppBucket[i]->inc() ;
}
pushToQueue ( startOffset, size, line, column ) ;
} // while ( !cb->isForced() )
sendMsgToClient ( "build index" ) ;
rc = dmsLoadExtent.loadBuildPhase ( mbContext, cb,
_pParameters->isAsynchronous,
this, &success, &failure ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to load data, rc=%d", rc ) ;
goto error ;
}
done:
if ( su && mbContext && !noClearFlag )
{
rc = mbContext->mbLock( EXCLUSIVE ) ;
if ( SDB_OK == rc )
{
if ( dmsLoadExtent.isFlagLoadLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadLoad ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoadBuild ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoadBuild ( mbContext->mb() ) ;
}
if ( dmsLoadExtent.isFlagLoad ( mbContext->mb() ) )
{
dmsLoadExtent.clearFlagLoad ( mbContext->mb() ) ;
}
}
else
{
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
}
}
sendMsgToClient ( "success json: %u, failure json: %u",
success, failure ) ;
sendMsgToClient ( "Load end" ) ;
if ( su && mbContext )
{
su->data()->releaseMBContext( mbContext ) ;
}
if ( DMS_INVALID_CS != suID )
{
dmsCB->suUnlock ( suID ) ;
}
if ( writable )
{
dmsCB->writeDown( cb );
}
PD_TRACE_EXITRC ( SDB__MIGLOADJSONPS__RUN, rc );
return rc ;
error:
goto done ;
error1:
_stopAndWaitWorker ( cb, success, failure ) ;
sendMsgToClient ( "Error: rollback all data" ) ;
failure += success ;
success = 0 ;
rc = dmsLoadExtent.loadRollbackPhase ( mbContext ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to rollback, rc=%d", rc ) ;
sendMsgToClient ( "Error: Failed to rollback, rc = %d",
rc ) ;
goto error ;
}
goto done ;
}
