// PD_TRACE_DECLARE_FUNCTION ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK, "dmsStorageLoadOp::pushToTempDataBlock" )
INT32 dmsStorageLoadOp::pushToTempDataBlock ( dmsMBContext *mbContext,
BSONObj &record,
BOOLEAN isLast,
BOOLEAN isAsynchr )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK );
UINT32 dmsrecordSize = 0 ;
ossValuePtr recordPtr = 0 ;
ossValuePtr prevPtr = 0 ;
dmsOffset offset = DMS_INVALID_OFFSET ;
dmsOffset recordOffset = DMS_INVALID_OFFSET ;
BSONElement ele ;
_IDToInsert oid ;
idToInsertEle oidEle((CHAR*)(&oid)) ;
BOOLEAN addOID = FALSE ;
INT32 oidLen = 0 ;
SDB_ASSERT( mbContext, "mb context can't be NULL" ) ;
/* (0) */
ele = record.getField ( DMS_ID_KEY_NAME ) ;
if ( ele.type() == Array )
{
PD_LOG ( PDERROR, "record id can't be array: %s",
record.toString().c_str()) ;
rc = SDB_INVALIDARG ;
goto error ;
}
if ( ele.eoo() )
{
oid._oid.init() ;
oidLen += oidEle.size() ;
addOID = TRUE ;
}
if ( ( dmsrecordSize =
(record.objsize() + DMS_RECORD_METADATA_SZ + oidLen) )
> DMS_RECORD_MAX_SZ )
{
rc = SDB_CORRUPTED_RECORD ;
goto error ;
}
dmsrecordSize *= DMS_RECORD_OVERFLOW_RATIO ;
dmsrecordSize = OSS_MIN(DMS_RECORD_MAX_SZ, ossAlignX(dmsrecordSize,4)) ;
if ( !_pCurrentExtent )
{
rc = _allocateExtent ( dmsrecordSize <<
DMS_RECORDS_PER_EXTENT_SQUARE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
_currentExtent = (dmsExtent*)_pCurrentExtent ;
}
if ( dmsrecordSize > (UINT32)_currentExtent->_freeSpace || isLast )
{
rc = mbContext->mbLock( EXCLUSIVE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
goto error ;
}
if ( !isAsynchr )
{
_currentExtent->_firstRecordOffset = DMS_INVALID_OFFSET ;
_currentExtent->_lastRecordOffset = DMS_INVALID_OFFSET ;
}
rc = _su->loadExtentA( mbContext, _pCurrentExtent,
_currentExtentSize / _pageSize,
TRUE ) ;
mbContext->mbUnlock() ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to load extent, rc = %d", rc ) ;
goto error ;
}
if ( isLast )
{
goto done ;
}
rc = _allocateExtent ( dmsrecordSize <<
DMS_RECORDS_PER_EXTENT_SQUARE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
}
recordOffset = _currentExtentSize - _currentExtent->_freeSpace ;
recordPtr = ((ossValuePtr)_currentExtent) + recordOffset ;
if ( _currentExtent->_freeSpace - (INT32)dmsrecordSize <
(INT32)DMS_MIN_RECORD_SZ &&
_currentExtent->_freeSpace <= (INT32)DMS_RECORD_MAX_SZ )
{
dmsrecordSize = _currentExtent->_freeSpace ;
}
DMS_RECORD_SETFLAG ( recordPtr, DMS_RECORD_FLAG_NORMAL ) ;
DMS_RECORD_SETMYOFFSET ( recordPtr, recordOffset ) ;
DMS_RECORD_SETSIZE ( recordPtr, dmsrecordSize ) ;
if ( !addOID )
{
DMS_RECORD_SETDATA ( recordPtr, record.objdata(), record.objsize() ) ;
}
else
{
DMS_RECORD_SETDATA_OID ( recordPtr,
record.objdata(),
record.objsize(),
oidEle ) ;
}
DMS_RECORD_SETNEXTOFFSET ( recordPtr, DMS_INVALID_OFFSET ) ;
DMS_RECORD_SETPREVOFFSET ( recordPtr, DMS_INVALID_OFFSET ) ;
if ( isAsynchr )
{
_currentExtent->_recCount++ ;
}
_currentExtent->_freeSpace -= dmsrecordSize ;
offset = _currentExtent->_lastRecordOffset ;
if ( DMS_INVALID_OFFSET != offset )
{
prevPtr = ((ossValuePtr)_currentExtent) + offset ;
DMS_RECORD_SETNEXTOFFSET ( prevPtr, recordOffset ) ;
DMS_RECORD_SETPREVOFFSET ( recordPtr, offset ) ;
}
_currentExtent->_lastRecordOffset = recordOffset ;
offset = _currentExtent->_firstRecordOffset ;
if ( DMS_INVALID_OFFSET == offset )
{
_currentExtent->_firstRecordOffset = recordOffset ;
}
done:
PD_TRACE_EXITRC ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK, rc );
return rc ;
error:
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB_RTNINSERT2, "rtnInsert" )
INT32 rtnInsert ( const CHAR *pCollectionName, BSONObj &objs, INT32 objNum,
INT32 flags, pmdEDUCB *cb, SDB_DMSCB *dmsCB,
SDB_DPSCB *dpsCB, INT16 w )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB_RTNINSERT2 ) ;
SDB_ASSERT ( pCollectionName, "collection name can't be NULL" ) ;
SDB_ASSERT ( cb, "educb can't be NULL" ) ;
SDB_ASSERT ( dmsCB, "dmsCB can't be NULL" ) ;
dmsStorageUnit *su = NULL ;
dmsStorageUnitID suID = DMS_INVALID_CS ;
const CHAR *pCollectionShortName = NULL ;
UINT32 insertCount = 0 ;
BOOLEAN writable = FALSE ;
ossValuePtr pDataPos = 0 ;
rc = dmsCB->writable( cb ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Database is not writable, rc = %d", rc ) ;
goto error;
}
writable = TRUE;
rc = rtnResolveCollectionNameAndLock ( pCollectionName, dmsCB, &su,
&pCollectionShortName, suID ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to resolve collection name %s",
pCollectionName ) ;
goto error ;
}
if ( objs.isEmpty () )
{
PD_LOG ( PDERROR, "Insert record can't be empty" ) ;
rc = SDB_INVALIDARG ;
goto error ;
}
pDataPos = (ossValuePtr)objs.objdata() ;
for ( INT32 i = 0 ; i < objNum ; ++i )
{
if ( ++insertCount > RTN_INSERT_ONCE_NUM )
{
insertCount = 0 ;
if ( cb->isInterrupted() )
{
rc = SDB_APP_INTERRUPT ;
goto error ;
}
}
try
{
BSONObj record ( (const CHAR*)pDataPos ) ;
rc = su->insertRecord ( pCollectionShortName, record, cb, dpsCB ) ;
if ( rc )
{
if ( ( SDB_IXM_DUP_KEY == rc ) &&
( FLG_INSERT_CONTONDUP & flags ) )
{
rc = SDB_OK ;
}
else
{
PD_LOG ( PDERROR, "Failed to insert record %s into "
"collection: %s, rc: %d", record.toString().c_str(),
pCollectionName, rc ) ;
goto error ;
}
}
pDataPos += ossAlignX ( (ossValuePtr)record.objsize(), 4 ) ;
}
catch ( std::exception &e )
{
PD_LOG ( PDERROR, "Failed to convert to BSON and insert to "
"collection: %s", e.what() ) ;
rc = SDB_INVALIDARG ;
goto error ;
}
}
done :
if ( DMS_INVALID_CS != suID )
{
dmsCB->suUnlock ( suID ) ;
}
if ( writable )
{
dmsCB->writeDown( cb );
}
if ( cb )
{
if ( SDB_OK == rc && dpsCB )
{
rc = dpsCB->completeOpr( cb, w ) ;
}
}
PD_TRACE_EXITRC ( SDB_RTNINSERT2, rc ) ;
return rc ;
error :
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB__DMSROUNIT_INSRCD, "_dmsReorgUnit::insertRecord" )
INT32 _dmsReorgUnit::insertRecord ( BSONObj &obj,
_pmdEDUCB *cb, UINT32 attributes )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__DMSROUNIT_INSRCD );
UINT32 dmsrecordSize = 0 ;
ossValuePtr recordPtr = 0 ;
ossValuePtr prevPtr = 0 ;
dmsOffset offset = DMS_INVALID_OFFSET ;
dmsOffset recordOffset = DMS_INVALID_OFFSET ;
dmsExtent *currentExtent = (dmsExtent*)_pCurrentExtent ;
BOOLEAN isCompressed = FALSE ;
const CHAR *compressedData = NULL ;
INT32 compressedDataSize = 0 ;
if ( obj.objsize() + DMS_RECORD_METADATA_SZ >
DMS_RECORD_MAX_SZ )
{
rc = SDB_CORRUPTED_RECORD ;
goto error ;
}
if ( OSS_BIT_TEST ( attributes, DMS_MB_ATTR_COMPRESSED ) )
{
rc = dmsCompress ( cb, obj, NULL, 0, &compressedData,
&compressedDataSize ) ;
PD_RC_CHECK ( rc, PDERROR,
"Failed to compress record, rc = %d: %s",
rc, obj.toString().c_str() ) ;
dmsrecordSize = compressedDataSize + sizeof(INT32) ;
if ( dmsrecordSize > (UINT32)(obj.objsize()) )
{
dmsrecordSize = obj.objsize() ;
}
else
{
isCompressed = TRUE ;
}
}
else
{
dmsrecordSize = obj.objsize() ;
}
dmsrecordSize += DMS_RECORD_METADATA_SZ ;
dmsrecordSize *= DMS_RECORD_OVERFLOW_RATIO ;
dmsrecordSize = OSS_MIN(DMS_RECORD_MAX_SZ, ossAlignX(dmsrecordSize,4)) ;
alloc:
if ( !_pCurrentExtent )
{
rc = _allocateExtent ( dmsrecordSize <<
DMS_RECORDS_PER_EXTENT_SQUARE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
currentExtent = (dmsExtent*)_pCurrentExtent ;
}
if ( dmsrecordSize > (UINT32)currentExtent->_freeSpace )
{
rc = _flushExtent () ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to flush extent, rc = %d", rc ) ;
goto error ;
}
goto alloc ;
}
recordOffset = _currentExtentSize - currentExtent->_freeSpace ;
recordPtr = ((ossValuePtr)currentExtent) + recordOffset ;
if ( currentExtent->_freeSpace - (INT32)dmsrecordSize <
(INT32)DMS_MIN_RECORD_SZ &&
currentExtent->_freeSpace <= (INT32)DMS_RECORD_MAX_SZ )
{
dmsrecordSize = (UINT32)currentExtent->_freeSpace ;
}
DMS_RECORD_SETSTATE ( recordPtr, DMS_RECORD_FLAG_NORMAL ) ;
DMS_RECORD_RESETATTR ( recordPtr ) ;
DMS_RECORD_SETMYOFFSET ( recordPtr, recordOffset ) ;
DMS_RECORD_SETSIZE ( recordPtr, dmsrecordSize ) ;
if ( isCompressed )
{
DMS_RECORD_SETATTR ( recordPtr, DMS_RECORD_FLAG_COMPRESSED ) ;
DMS_RECORD_SETDATA ( recordPtr, compressedData, compressedDataSize ) ;
}
else
{
DMS_RECORD_SETDATA ( recordPtr, obj.objdata(), obj.objsize() ) ;
}
DMS_RECORD_SETNEXTOFFSET ( recordPtr, DMS_INVALID_OFFSET ) ;
DMS_RECORD_SETPREVOFFSET ( recordPtr, DMS_INVALID_OFFSET ) ;
currentExtent->_recCount ++ ;
currentExtent->_freeSpace -= dmsrecordSize ;
offset = currentExtent->_lastRecordOffset ;
if ( DMS_INVALID_OFFSET != offset )
{
prevPtr = ((ossValuePtr)currentExtent) + offset ;
DMS_RECORD_SETNEXTOFFSET ( prevPtr, recordOffset ) ;
DMS_RECORD_SETPREVOFFSET ( recordPtr, offset ) ;
}
currentExtent->_lastRecordOffset = recordOffset ;
offset = currentExtent->_firstRecordOffset ;
if ( DMS_INVALID_OFFSET == offset )
{
currentExtent->_firstRecordOffset = recordOffset ;
}
done :
PD_TRACE_EXITRC ( SDB__DMSROUNIT_INSRCD, rc );
return rc ;
error :
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK, "dmsStorageLoadOp::pushToTempDataBlock" )
INT32 dmsStorageLoadOp::pushToTempDataBlock ( dmsMBContext *mbContext,
pmdEDUCB *cb,
BSONObj &record,
BOOLEAN isLast,
BOOLEAN isAsynchr )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK );
UINT32 dmsrecordSize = 0 ;
dmsRecord *pRecord = NULL ;
dmsRecord *pPreRecord = NULL ;
dmsOffset offset = DMS_INVALID_OFFSET ;
dmsOffset recordOffset = DMS_INVALID_OFFSET ;
_IDToInsert oid ;
idToInsertEle oidEle((CHAR*)(&oid)) ;
CHAR *pNewRecordData = NULL ;
dmsRecordData recordData ;
dmsCompressorEntry *compressorEntry = NULL ;
SDB_ASSERT( mbContext, "mb context can't be NULL" ) ;
compressorEntry = _su->data()->getCompressorEntry( mbContext->mbID() ) ;
/* For concurrency protection with drop CL and set compresor. */
dmsCompressorGuard compGuard( compressorEntry, SHARED ) ;
try
{
recordData.setData( record.objdata(), record.objsize(),
FALSE, TRUE ) ;
/* (0) */
BSONElement ele = record.getField ( DMS_ID_KEY_NAME ) ;
const CHAR *pCheckErr = "" ;
if ( !dmsIsRecordIDValid( ele, TRUE, &pCheckErr ) )
{
PD_LOG( PDERROR, "Record[%s] _id is error: %s",
record.toString().c_str(), pCheckErr ) ;
rc = SDB_INVALIDARG ;
goto error ;
}
if ( ele.eoo() )
{
oid._oid.init() ;
rc = cb->allocBuff( oidEle.size() + record.objsize(),
&pNewRecordData ) ;
if ( rc )
{
PD_LOG( PDERROR, "Alloc memory[size:%u] failed, rc: %d",
oidEle.size() + record.objsize(), rc ) ;
goto error ;
}
*(UINT32*)pNewRecordData = oidEle.size() + record.objsize() ;
ossMemcpy( pNewRecordData + sizeof(UINT32), oidEle.rawdata(),
oidEle.size() ) ;
ossMemcpy( pNewRecordData + sizeof(UINT32) + oidEle.size(),
record.objdata() + sizeof(UINT32),
record.objsize() - sizeof(UINT32) ) ;
recordData.setData( pNewRecordData,
oidEle.size() + record.objsize(),
FALSE, TRUE ) ;
record = BSONObj( pNewRecordData ) ;
}
dmsrecordSize = recordData.len() ;
if ( recordData.len() + DMS_RECORD_METADATA_SZ >
DMS_RECORD_USER_MAX_SZ )
{
rc = SDB_DMS_RECORD_TOO_BIG ;
goto error ;
}
if ( compressorEntry->ready() )
{
const CHAR *compressedData = NULL ;
INT32 compressedDataSize = 0 ;
UINT8 compressRatio = 0 ;
rc = dmsCompress( cb, compressorEntry,
recordData.data(), recordData.len(),
&compressedData, &compressedDataSize,
compressRatio ) ;
if ( SDB_OK == rc &&
compressedDataSize + sizeof(UINT32) < recordData.orgLen() &&
compressRatio < DMS_COMPRESS_RATIO_THRESHOLD )
{
dmsrecordSize = compressedDataSize + sizeof(UINT32) ;
recordData.setData( compressedData, compressedDataSize,
TRUE, FALSE ) ;
}
else if ( rc )
{
if ( SDB_UTIL_COMPRESS_ABORT == rc )
{
PD_LOG( PDINFO, "Record compression aborted. "
"Insert the original data. rc: %d", rc ) ;
}
else
{
PD_LOG( PDWARNING, "Record compression failed. "
"Insert the original data. rc: %d", rc ) ;
}
rc = SDB_OK ;
}
}
/*
* Release the guard to avoid deadlock with truncate/drop collection.
*/
compGuard.release() ;
dmsrecordSize *= DMS_RECORD_OVERFLOW_RATIO ;
dmsrecordSize += DMS_RECORD_METADATA_SZ ;
dmsrecordSize = OSS_MIN( DMS_RECORD_MAX_SZ,
ossAlignX ( dmsrecordSize, 4 ) ) ;
INT32 expandSize = dmsrecordSize << DMS_RECORDS_PER_EXTENT_SQUARE ;
if ( expandSize > DMS_BEST_UP_EXTENT_SZ )
{
expandSize = expandSize < DMS_BEST_UP_EXTENT_SZ ?
DMS_BEST_UP_EXTENT_SZ : expandSize ;
}
if ( !_pCurrentExtent )
{
rc = _allocateExtent ( expandSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
_currentExtent = (dmsExtent*)_pCurrentExtent ;
}
if ( dmsrecordSize > (UINT32)_currentExtent->_freeSpace || isLast )
{
rc = mbContext->mbLock( EXCLUSIVE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
goto error ;
}
if ( !isAsynchr )
{
_currentExtent->_firstRecordOffset = DMS_INVALID_OFFSET ;
_currentExtent->_lastRecordOffset = DMS_INVALID_OFFSET ;
}
rc = _su->loadExtentA( mbContext, _pCurrentExtent,
_currentExtentSize / _pageSize,
TRUE ) ;
mbContext->mbUnlock() ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to load extent, rc = %d", rc ) ;
goto error ;
}
if ( isLast )
{
goto done ;
}
rc = _allocateExtent ( expandSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
}
recordOffset = _currentExtentSize - _currentExtent->_freeSpace ;
pRecord = ( dmsRecord* )( (const CHAR*)_currentExtent + recordOffset ) ;
if ( _currentExtent->_freeSpace - (INT32)dmsrecordSize <
(INT32)DMS_MIN_RECORD_SZ &&
_currentExtent->_freeSpace <= (INT32)DMS_RECORD_MAX_SZ )
{
dmsrecordSize = _currentExtent->_freeSpace ;
}
pRecord->setNormal() ;
pRecord->setMyOffset( recordOffset ) ;
pRecord->setSize( dmsrecordSize ) ;
pRecord->setData( recordData ) ;
pRecord->setNextOffset( DMS_INVALID_OFFSET ) ;
pRecord->setPrevOffset( DMS_INVALID_OFFSET ) ;
if ( isAsynchr )
{
_currentExtent->_recCount++ ;
}
_currentExtent->_freeSpace -= dmsrecordSize ;
offset = _currentExtent->_lastRecordOffset ;
if ( DMS_INVALID_OFFSET != offset )
{
pPreRecord = (dmsRecord*)( (const CHAR*)_currentExtent + offset ) ;
pPreRecord->setNextOffset( recordOffset ) ;
pRecord->setPrevOffset( offset ) ;
}
_currentExtent->_lastRecordOffset = recordOffset ;
offset = _currentExtent->_firstRecordOffset ;
if ( DMS_INVALID_OFFSET == offset )
{
_currentExtent->_firstRecordOffset = recordOffset ;
}
}
catch( std::exception &e )
{
PD_LOG( PDERROR, "Occur exception: %s", e.what() ) ;
rc = SDB_SYS ;
goto error ;
}
done:
PD_TRACE_EXITRC ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK, rc );
return rc ;
error:
goto done ;
}
