// PD_TRACE_DECLARE_FUNCTION ( SDB_DMSCOMPRESS, "dmsCompress" )
INT32 dmsCompress ( _pmdEDUCB *cb, _dmsCompressorEntry *compressorEntry,
const BSONObj &obj, const CHAR* pOIDPtr, INT32 oidLen,
const CHAR **ppData, INT32 *pDataSize, UINT8 &ratio )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY( SDB_DMSCOMPRESS ) ;
CHAR *pObjData = NULL ;
INT32 objSize = 0 ;
SDB_ASSERT( compressorEntry, "Compressor entry can't be NULL" ) ;
if ( oidLen && pOIDPtr )
{
UINT32 requestedSize = obj.objsize() + oidLen ;
rc = cb->allocBuff( requestedSize, &pObjData, NULL ) ;
if ( rc )
{
PD_LOG( PDERROR, "Failed to alloc tmp buffer, size: %u",
requestedSize ) ;
goto error ;
}
*(UINT32*)pObjData = oidLen + obj.objsize() ;
ossMemcpy( pObjData + sizeof(UINT32), pOIDPtr, oidLen ) ;
ossMemcpy( pObjData + sizeof(UINT32) + oidLen,
obj.objdata() + sizeof(UINT32),
obj.objsize() - sizeof(UINT32) ) ;
objSize = BSONObj(pObjData).objsize() ;
rc = dmsCompress ( cb, compressorEntry, pObjData,
objSize, ppData, pDataSize, ratio ) ;
}
else
{
objSize = obj.objsize() ;
rc = dmsCompress( cb, compressorEntry, obj.objdata(),
objSize, ppData, pDataSize, ratio ) ;
}
if ( rc )
{
goto error ;
}
done :
if ( pObjData )
{
cb->releaseBuff( pObjData ) ;
}
PD_TRACE_EXITRC( SDB_DMSCOMPRESS, rc ) ;
return rc ;
error :
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB__DMSROUNIT_INSRCD, "_dmsReorgUnit::insertRecord" )
INT32 _dmsReorgUnit::insertRecord ( BSONObj &obj,
_pmdEDUCB *cb, UINT32 attributes )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__DMSROUNIT_INSRCD );
UINT32 dmsrecordSize = 0 ;
ossValuePtr recordPtr = 0 ;
ossValuePtr prevPtr = 0 ;
dmsOffset offset = DMS_INVALID_OFFSET ;
dmsOffset recordOffset = DMS_INVALID_OFFSET ;
dmsExtent *currentExtent = (dmsExtent*)_pCurrentExtent ;
BOOLEAN isCompressed = FALSE ;
const CHAR *compressedData = NULL ;
INT32 compressedDataSize = 0 ;
if ( obj.objsize() + DMS_RECORD_METADATA_SZ >
DMS_RECORD_MAX_SZ )
{
rc = SDB_CORRUPTED_RECORD ;
goto error ;
}
if ( OSS_BIT_TEST ( attributes, DMS_MB_ATTR_COMPRESSED ) )
{
rc = dmsCompress ( cb, obj, NULL, 0, &compressedData,
&compressedDataSize ) ;
PD_RC_CHECK ( rc, PDERROR,
"Failed to compress record, rc = %d: %s",
rc, obj.toString().c_str() ) ;
dmsrecordSize = compressedDataSize + sizeof(INT32) ;
if ( dmsrecordSize > (UINT32)(obj.objsize()) )
{
dmsrecordSize = obj.objsize() ;
}
else
{
isCompressed = TRUE ;
}
}
else
{
dmsrecordSize = obj.objsize() ;
}
dmsrecordSize += DMS_RECORD_METADATA_SZ ;
dmsrecordSize *= DMS_RECORD_OVERFLOW_RATIO ;
dmsrecordSize = OSS_MIN(DMS_RECORD_MAX_SZ, ossAlignX(dmsrecordSize,4)) ;
alloc:
if ( !_pCurrentExtent )
{
rc = _allocateExtent ( dmsrecordSize <<
DMS_RECORDS_PER_EXTENT_SQUARE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
currentExtent = (dmsExtent*)_pCurrentExtent ;
}
if ( dmsrecordSize > (UINT32)currentExtent->_freeSpace )
{
rc = _flushExtent () ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to flush extent, rc = %d", rc ) ;
goto error ;
}
goto alloc ;
}
recordOffset = _currentExtentSize - currentExtent->_freeSpace ;
recordPtr = ((ossValuePtr)currentExtent) + recordOffset ;
if ( currentExtent->_freeSpace - (INT32)dmsrecordSize <
(INT32)DMS_MIN_RECORD_SZ &&
currentExtent->_freeSpace <= (INT32)DMS_RECORD_MAX_SZ )
{
dmsrecordSize = (UINT32)currentExtent->_freeSpace ;
}
DMS_RECORD_SETSTATE ( recordPtr, DMS_RECORD_FLAG_NORMAL ) ;
DMS_RECORD_RESETATTR ( recordPtr ) ;
DMS_RECORD_SETMYOFFSET ( recordPtr, recordOffset ) ;
DMS_RECORD_SETSIZE ( recordPtr, dmsrecordSize ) ;
if ( isCompressed )
{
DMS_RECORD_SETATTR ( recordPtr, DMS_RECORD_FLAG_COMPRESSED ) ;
DMS_RECORD_SETDATA ( recordPtr, compressedData, compressedDataSize ) ;
}
else
{
DMS_RECORD_SETDATA ( recordPtr, obj.objdata(), obj.objsize() ) ;
}
DMS_RECORD_SETNEXTOFFSET ( recordPtr, DMS_INVALID_OFFSET ) ;
DMS_RECORD_SETPREVOFFSET ( recordPtr, DMS_INVALID_OFFSET ) ;
currentExtent->_recCount ++ ;
currentExtent->_freeSpace -= dmsrecordSize ;
offset = currentExtent->_lastRecordOffset ;
if ( DMS_INVALID_OFFSET != offset )
{
prevPtr = ((ossValuePtr)currentExtent) + offset ;
DMS_RECORD_SETNEXTOFFSET ( prevPtr, recordOffset ) ;
DMS_RECORD_SETPREVOFFSET ( recordPtr, offset ) ;
}
currentExtent->_lastRecordOffset = recordOffset ;
offset = currentExtent->_firstRecordOffset ;
if ( DMS_INVALID_OFFSET == offset )
{
currentExtent->_firstRecordOffset = recordOffset ;
}
done :
PD_TRACE_EXITRC ( SDB__DMSROUNIT_INSRCD, rc );
return rc ;
error :
goto done ;
}
// PD_TRACE_DECLARE_FUNCTION ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK, "dmsStorageLoadOp::pushToTempDataBlock" )
INT32 dmsStorageLoadOp::pushToTempDataBlock ( dmsMBContext *mbContext,
pmdEDUCB *cb,
BSONObj &record,
BOOLEAN isLast,
BOOLEAN isAsynchr )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK );
UINT32 dmsrecordSize = 0 ;
dmsRecord *pRecord = NULL ;
dmsRecord *pPreRecord = NULL ;
dmsOffset offset = DMS_INVALID_OFFSET ;
dmsOffset recordOffset = DMS_INVALID_OFFSET ;
_IDToInsert oid ;
idToInsertEle oidEle((CHAR*)(&oid)) ;
CHAR *pNewRecordData = NULL ;
dmsRecordData recordData ;
dmsCompressorEntry *compressorEntry = NULL ;
SDB_ASSERT( mbContext, "mb context can't be NULL" ) ;
compressorEntry = _su->data()->getCompressorEntry( mbContext->mbID() ) ;
/* For concurrency protection with drop CL and set compresor. */
dmsCompressorGuard compGuard( compressorEntry, SHARED ) ;
try
{
recordData.setData( record.objdata(), record.objsize(),
FALSE, TRUE ) ;
/* (0) */
BSONElement ele = record.getField ( DMS_ID_KEY_NAME ) ;
const CHAR *pCheckErr = "" ;
if ( !dmsIsRecordIDValid( ele, TRUE, &pCheckErr ) )
{
PD_LOG( PDERROR, "Record[%s] _id is error: %s",
record.toString().c_str(), pCheckErr ) ;
rc = SDB_INVALIDARG ;
goto error ;
}
if ( ele.eoo() )
{
oid._oid.init() ;
rc = cb->allocBuff( oidEle.size() + record.objsize(),
&pNewRecordData ) ;
if ( rc )
{
PD_LOG( PDERROR, "Alloc memory[size:%u] failed, rc: %d",
oidEle.size() + record.objsize(), rc ) ;
goto error ;
}
*(UINT32*)pNewRecordData = oidEle.size() + record.objsize() ;
ossMemcpy( pNewRecordData + sizeof(UINT32), oidEle.rawdata(),
oidEle.size() ) ;
ossMemcpy( pNewRecordData + sizeof(UINT32) + oidEle.size(),
record.objdata() + sizeof(UINT32),
record.objsize() - sizeof(UINT32) ) ;
recordData.setData( pNewRecordData,
oidEle.size() + record.objsize(),
FALSE, TRUE ) ;
record = BSONObj( pNewRecordData ) ;
}
dmsrecordSize = recordData.len() ;
if ( recordData.len() + DMS_RECORD_METADATA_SZ >
DMS_RECORD_USER_MAX_SZ )
{
rc = SDB_DMS_RECORD_TOO_BIG ;
goto error ;
}
if ( compressorEntry->ready() )
{
const CHAR *compressedData = NULL ;
INT32 compressedDataSize = 0 ;
UINT8 compressRatio = 0 ;
rc = dmsCompress( cb, compressorEntry,
recordData.data(), recordData.len(),
&compressedData, &compressedDataSize,
compressRatio ) ;
if ( SDB_OK == rc &&
compressedDataSize + sizeof(UINT32) < recordData.orgLen() &&
compressRatio < DMS_COMPRESS_RATIO_THRESHOLD )
{
dmsrecordSize = compressedDataSize + sizeof(UINT32) ;
recordData.setData( compressedData, compressedDataSize,
TRUE, FALSE ) ;
}
else if ( rc )
{
if ( SDB_UTIL_COMPRESS_ABORT == rc )
{
PD_LOG( PDINFO, "Record compression aborted. "
"Insert the original data. rc: %d", rc ) ;
}
else
{
PD_LOG( PDWARNING, "Record compression failed. "
"Insert the original data. rc: %d", rc ) ;
}
rc = SDB_OK ;
}
}
/*
* Release the guard to avoid deadlock with truncate/drop collection.
*/
compGuard.release() ;
dmsrecordSize *= DMS_RECORD_OVERFLOW_RATIO ;
dmsrecordSize += DMS_RECORD_METADATA_SZ ;
dmsrecordSize = OSS_MIN( DMS_RECORD_MAX_SZ,
ossAlignX ( dmsrecordSize, 4 ) ) ;
INT32 expandSize = dmsrecordSize << DMS_RECORDS_PER_EXTENT_SQUARE ;
if ( expandSize > DMS_BEST_UP_EXTENT_SZ )
{
expandSize = expandSize < DMS_BEST_UP_EXTENT_SZ ?
DMS_BEST_UP_EXTENT_SZ : expandSize ;
}
if ( !_pCurrentExtent )
{
rc = _allocateExtent ( expandSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
_currentExtent = (dmsExtent*)_pCurrentExtent ;
}
if ( dmsrecordSize > (UINT32)_currentExtent->_freeSpace || isLast )
{
rc = mbContext->mbLock( EXCLUSIVE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to lock collection, rc=%d", rc ) ;
goto error ;
}
if ( !isAsynchr )
{
_currentExtent->_firstRecordOffset = DMS_INVALID_OFFSET ;
_currentExtent->_lastRecordOffset = DMS_INVALID_OFFSET ;
}
rc = _su->loadExtentA( mbContext, _pCurrentExtent,
_currentExtentSize / _pageSize,
TRUE ) ;
mbContext->mbUnlock() ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to load extent, rc = %d", rc ) ;
goto error ;
}
if ( isLast )
{
goto done ;
}
rc = _allocateExtent ( expandSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to allocate new extent in reorg file, "
"rc = %d", rc ) ;
goto error ;
}
}
recordOffset = _currentExtentSize - _currentExtent->_freeSpace ;
pRecord = ( dmsRecord* )( (const CHAR*)_currentExtent + recordOffset ) ;
if ( _currentExtent->_freeSpace - (INT32)dmsrecordSize <
(INT32)DMS_MIN_RECORD_SZ &&
_currentExtent->_freeSpace <= (INT32)DMS_RECORD_MAX_SZ )
{
dmsrecordSize = _currentExtent->_freeSpace ;
}
pRecord->setNormal() ;
pRecord->setMyOffset( recordOffset ) ;
pRecord->setSize( dmsrecordSize ) ;
pRecord->setData( recordData ) ;
pRecord->setNextOffset( DMS_INVALID_OFFSET ) ;
pRecord->setPrevOffset( DMS_INVALID_OFFSET ) ;
if ( isAsynchr )
{
_currentExtent->_recCount++ ;
}
_currentExtent->_freeSpace -= dmsrecordSize ;
offset = _currentExtent->_lastRecordOffset ;
if ( DMS_INVALID_OFFSET != offset )
{
pPreRecord = (dmsRecord*)( (const CHAR*)_currentExtent + offset ) ;
pPreRecord->setNextOffset( recordOffset ) ;
pRecord->setPrevOffset( offset ) ;
}
_currentExtent->_lastRecordOffset = recordOffset ;
offset = _currentExtent->_firstRecordOffset ;
if ( DMS_INVALID_OFFSET == offset )
{
_currentExtent->_firstRecordOffset = recordOffset ;
}
}
catch( std::exception &e )
{
PD_LOG( PDERROR, "Occur exception: %s", e.what() ) ;
rc = SDB_SYS ;
goto error ;
}
done:
PD_TRACE_EXITRC ( SDB__DMSSTORAGELOADEXT__IMPRTBLOCK, rc );
return rc ;
error:
goto done ;
}