// PD_TRACE_DECLARE_FUNCTION ( SDB__MIGLOADJSONPS__RUN, "migMaster::run" ) INT32 migMaster::run() { INT32 rc = SDB_OK ; PD_TRACE_ENTRY ( SDB__MIGLOADJSONPS__RUN ); UINT32 startOffset = 0 ; UINT32 size = 0 ; UINT32 startBlock = 0 ; UINT32 endBlock = 0 ; pmdKRCB *krcb = pmdGetKRCB () ; pmdEDUMgr *eduMgr = krcb->getEDUMgr () ; SDB_DMSCB *dmsCB = krcb->getDMSCB () ; pmdEDUCB *eduCB = eduMgr->getEDU() ; EDUID agentEDU = PMD_INVALID_EDUID ; BOOLEAN writable = FALSE ; BOOLEAN noClearFlag = FALSE ; dmsMBContext *mbContext = NULL ; UINT32 line = 0 ; UINT32 column = 0 ; UINT32 success = 0 ; UINT32 failure = 0 ; UINT16 clFlag = 0 ; dmsStorageUnitID suID = DMS_INVALID_CS ; dmsStorageUnit *su = NULL ; initWorker dataWorker ; pmdEDUEvent event ; dmsStorageLoadOp dmsLoadExtent ; sendMsgToClient ( "Load start" ) ; rc = rtnCollectionSpaceLock ( _pParameters->pCollectionSpaceName, dmsCB, FALSE, &su, suID ) ; if ( rc ) { if ( SDB_DMS_CS_NOTEXIST == rc ) { sendMsgToClient ( "Error: collection space not exist" ) ; } PD_LOG ( PDERROR, "Failed to lock collection space, rc=%d", rc ) ; goto error ; } dmsLoadExtent.init ( su ) ; rc = su->data()->getMBContext( &mbContext, _pParameters->pCollectionName, EXCLUSIVE ) ; if ( rc ) { if ( SDB_DMS_NOTEXIST == rc ) { sendMsgToClient ( "Error: collection not exist" ) ; } PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ; goto error ; } clFlag = mbContext->mb()->_flag ; if ( DMS_IS_MB_DROPPED( clFlag ) ) { PD_LOG( PDERROR, "Collection is droped" ) ; rc = SDB_COLLECTION_LOAD ; sendMsgToClient ( "Collection is droped" ) ; goto error ; } else if ( DMS_IS_MB_LOAD ( clFlag ) ) { PD_LOG( PDERROR, "Collection is loading" ) ; rc = SDB_COLLECTION_LOAD ; sendMsgToClient ( "Collection is loading" ) ; // we set noClearFlag to true, so that we'll convert the collection // flag to NORMAL in done noClearFlag = TRUE ; goto error ; } dmsLoadExtent.setFlagLoad ( mbContext->mb() ) ; dmsLoadExtent.setFlagLoadLoad ( mbContext->mb() ) ; // unlock mbContext->mbUnlock() ; rc = dmsCB->writable( eduCB ) ; if ( rc ) { PD_LOG ( PDERROR, "Database is not writable, rc = %d", rc ) ; goto error; } writable = TRUE; dataWorker.pMaster = this ; dataWorker.masterEDUID = eduCB->getID() ; dataWorker.pSu = su ; dataWorker.clLID = mbContext->clLID() ; dataWorker.collectionID = mbContext->mbID() ; for ( UINT32 i = 0; i < _pParameters->workerNum; ++i ) { eduMgr->startEDU ( EDU_TYPE_LOADWORKER, &dataWorker, &agentEDU ) ; } while ( TRUE ) { rc = _checkErrAndRollback ( eduCB, &dmsLoadExtent, mbContext, success, failure ) ; if ( SDB_TIMEOUT != rc && rc ) { PD_LOG ( PDERROR, "Failed to call _checkErrAndRollback, rc=%d", rc ) ; goto error ; } // fetch one record rc = _parser->getNextRecord ( startOffset, size, &line, &column, _ppBucket ) ; if ( rc ) { // special handle for end of file if ( rc == SDB_EOF ) { // when we hit end of file, let's push 0 to all worker threads, // with num of workers ( so each worker will dispatch one 0, and // exit ) rc = _stopAndWaitWorker ( eduCB, success, failure ) ; PD_RC_CHECK ( rc, PDERROR, "Failed to call _stopAndWaitWorker, rc=%d", rc ) ; break ; } sendMsgToClient ( "Error: Parse Json error in line: %u," " column: %u", line, column ) ; PD_LOG ( PDERROR, "Failed to parseJSONs getNextRecord,rc=%d", rc ) ; goto error1 ; } // calculate the blocks to be locked, based on the length of our record rc = getBlockFromPointer ( startOffset, size, startBlock, endBlock ) ; if ( rc ) { PD_LOG ( PDERROR, "Failed to get block from pointer, rc=%d", rc ) ; goto error1 ; } // lock them for ( UINT32 i = startBlock; i <= endBlock; ++i ) { _ppBucket[i]->inc() ; } // push the record to queue pushToQueue ( startOffset, size, line, column ) ; } // while ( !eduCB->isForced() ) // when all workers are finish, let's start build phase to rebuild all // indexes sendMsgToClient ( "build index" ) ; rc = dmsLoadExtent.loadBuildPhase ( mbContext, eduCB, _pParameters->isAsynchronous, this, &success, &failure ) ; if ( rc ) { PD_LOG ( PDERROR, "Failed to load data, rc=%d", rc ) ; goto error ; } done: // we only lock and clear flag if we switched to load if ( su && mbContext && !noClearFlag ) { rc = mbContext->mbLock( EXCLUSIVE ) ; // we should log failure information if ( SDB_OK == rc ) { if ( dmsLoadExtent.isFlagLoadLoad ( mbContext->mb() ) ) { dmsLoadExtent.clearFlagLoadLoad ( mbContext->mb() ) ; } if ( dmsLoadExtent.isFlagLoadBuild ( mbContext->mb() ) ) { dmsLoadExtent.clearFlagLoadBuild ( mbContext->mb() ) ; } if ( dmsLoadExtent.isFlagLoad ( mbContext->mb() ) ) { dmsLoadExtent.clearFlagLoad ( mbContext->mb() ) ; } } else { PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ; } } // send the success message to client sendMsgToClient ( "success json: %u, failure json: %u", success, failure ) ; sendMsgToClient ( "Load end" ) ; if ( su && mbContext ) { su->data()->releaseMBContext( mbContext ) ; } if ( DMS_INVALID_CS != suID ) { dmsCB->suUnlock ( suID ) ; } // count down if ( writable ) { dmsCB->writeDown( eduCB ); } PD_TRACE_EXITRC ( SDB__MIGLOADJSONPS__RUN, rc ); return rc ; error: goto done ; error1: _stopAndWaitWorker ( eduCB, success, failure ) ; sendMsgToClient ( "Error: rollback all data" ) ; failure += success ; success = 0 ; rc = dmsLoadExtent.loadRollbackPhase ( mbContext ) ; if ( rc ) { PD_LOG ( PDERROR, "Failed to rollback, rc=%d", rc ) ; sendMsgToClient ( "Error: Failed to rollback, rc = %d", rc ) ; goto error ; } goto done ; }
// PD_TRACE_DECLARE_FUNCTION ( SDB__MIGLOADJSONPS__RUN, "migMaster::run" ) INT32 migMaster::run( pmdEDUCB *cb ) { INT32 rc = SDB_OK ; PD_TRACE_ENTRY ( SDB__MIGLOADJSONPS__RUN ); UINT32 startOffset = 0 ; UINT32 size = 0 ; UINT32 startBlock = 0 ; UINT32 endBlock = 0 ; pmdKRCB *krcb = pmdGetKRCB () ; pmdEDUMgr *eduMgr = krcb->getEDUMgr () ; SDB_DMSCB *dmsCB = krcb->getDMSCB () ; EDUID agentEDU = PMD_INVALID_EDUID ; BOOLEAN writable = FALSE ; BOOLEAN noClearFlag = FALSE ; dmsMBContext *mbContext = NULL ; UINT32 line = 0 ; UINT32 column = 0 ; UINT32 success = 0 ; UINT32 failure = 0 ; UINT16 clFlag = 0 ; dmsStorageUnitID suID = DMS_INVALID_CS ; dmsStorageUnit *su = NULL ; initWorker dataWorker ; pmdEDUEvent event ; dmsStorageLoadOp dmsLoadExtent ; sendMsgToClient ( "Load start" ) ; rc = dmsCB->writable( cb ) ; if ( rc ) { PD_LOG ( PDERROR, "Database is not writable, rc = %d", rc ) ; goto error; } writable = TRUE; rc = rtnCollectionSpaceLock ( _pParameters->pCollectionSpaceName, dmsCB, FALSE, &su, suID ) ; if ( rc ) { if ( SDB_DMS_CS_NOTEXIST == rc ) { sendMsgToClient ( "Error: collection space not exist" ) ; } PD_LOG ( PDERROR, "Failed to lock collection space, rc=%d", rc ) ; goto error ; } dmsLoadExtent.init ( su ) ; rc = su->data()->getMBContext( &mbContext, _pParameters->pCollectionName, EXCLUSIVE ) ; if ( rc ) { if ( SDB_DMS_NOTEXIST == rc ) { sendMsgToClient ( "Error: collection not exist" ) ; } PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ; goto error ; } clFlag = mbContext->mb()->_flag ; if ( DMS_IS_MB_DROPPED( clFlag ) ) { PD_LOG( PDERROR, "Collection is droped" ) ; rc = SDB_COLLECTION_LOAD ; sendMsgToClient ( "Collection is droped" ) ; goto error ; } else if ( DMS_IS_MB_LOAD ( clFlag ) ) { PD_LOG( PDERROR, "Collection is loading" ) ; rc = SDB_COLLECTION_LOAD ; sendMsgToClient ( "Collection is loading" ) ; noClearFlag = TRUE ; goto error ; } dmsLoadExtent.setFlagLoad ( mbContext->mb() ) ; dmsLoadExtent.setFlagLoadLoad ( mbContext->mb() ) ; mbContext->mbUnlock() ; dataWorker.pMaster = this ; dataWorker.masterEDUID = cb->getID() ; dataWorker.pSu = su ; dataWorker.clLID = mbContext->clLID() ; dataWorker.collectionID = mbContext->mbID() ; for ( UINT32 i = 0; i < _pParameters->workerNum ; ++i ) { eduMgr->startEDU ( EDU_TYPE_LOADWORKER, &dataWorker, &agentEDU ) ; } while ( TRUE ) { rc = _checkErrAndRollback ( cb, &dmsLoadExtent, mbContext, success, failure ) ; if ( SDB_TIMEOUT != rc && rc ) { PD_LOG ( PDERROR, "Failed to call _checkErrAndRollback, rc=%d", rc ) ; goto error ; } rc = _parser->getNextRecord ( startOffset, size, &line, &column, _ppBucket ) ; if ( rc ) { if ( rc == SDB_EOF ) { rc = _stopAndWaitWorker ( cb, success, failure ) ; PD_RC_CHECK ( rc, PDERROR, "Failed to call _stopAndWaitWorker, rc=%d", rc ) ; break ; } sendMsgToClient ( "Error: Parse Json error in line: %u," " column: %u", line, column ) ; PD_LOG ( PDERROR, "Failed to parseJSONs getNextRecord,rc=%d", rc ) ; goto error1 ; } rc = getBlockFromPointer ( startOffset, size, startBlock, endBlock ) ; if ( rc ) { PD_LOG ( PDERROR, "Failed to get block from pointer, rc=%d", rc ) ; goto error1 ; } for ( UINT32 i = startBlock; i <= endBlock; ++i ) { _ppBucket[i]->inc() ; } pushToQueue ( startOffset, size, line, column ) ; } // while ( !cb->isForced() ) sendMsgToClient ( "build index" ) ; rc = dmsLoadExtent.loadBuildPhase ( mbContext, cb, _pParameters->isAsynchronous, this, &success, &failure ) ; if ( rc ) { PD_LOG ( PDERROR, "Failed to load data, rc=%d", rc ) ; goto error ; } done: if ( su && mbContext && !noClearFlag ) { rc = mbContext->mbLock( EXCLUSIVE ) ; if ( SDB_OK == rc ) { if ( dmsLoadExtent.isFlagLoadLoad ( mbContext->mb() ) ) { dmsLoadExtent.clearFlagLoadLoad ( mbContext->mb() ) ; } if ( dmsLoadExtent.isFlagLoadBuild ( mbContext->mb() ) ) { dmsLoadExtent.clearFlagLoadBuild ( mbContext->mb() ) ; } if ( dmsLoadExtent.isFlagLoad ( mbContext->mb() ) ) { dmsLoadExtent.clearFlagLoad ( mbContext->mb() ) ; } } else { PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ; } } sendMsgToClient ( "success json: %u, failure json: %u", success, failure ) ; sendMsgToClient ( "Load end" ) ; if ( su && mbContext ) { su->data()->releaseMBContext( mbContext ) ; } if ( DMS_INVALID_CS != suID ) { dmsCB->suUnlock ( suID ) ; } if ( writable ) { dmsCB->writeDown( cb ); } PD_TRACE_EXITRC ( SDB__MIGLOADJSONPS__RUN, rc ); return rc ; error: goto done ; error1: _stopAndWaitWorker ( cb, success, failure ) ; sendMsgToClient ( "Error: rollback all data" ) ; failure += success ; success = 0 ; rc = dmsLoadExtent.loadRollbackPhase ( mbContext ) ; if ( rc ) { PD_LOG ( PDERROR, "Failed to rollback, rc=%d", rc ) ; sendMsgToClient ( "Error: Failed to rollback, rc = %d", rc ) ; goto error ; } goto done ; }