// PD_TRACE_DECLARE_FUNCTION ( SDB__MIGLOADJSONPS__INITIALIZE, "migMaster::initialize" )
INT32 migMaster::initialize ( setParameters *pParameters )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY ( SDB__MIGLOADJSONPS__INITIALIZE );
_utilParserParamet parserPara ;
UINT32 startOffset = 0 ;
UINT32 fieldsSize = 0 ;
_pParameters = pParameters ;
_sock = _pParameters->clientSock ;
_fileType = _pParameters->fileType ;
_blockSize = _pParameters->bufferSize/_pParameters->bucketNum ;
_exitSignal = FALSE ;
if ( MIG_PARSER_JSON == _pParameters->fileType )
{
_parser = SDB_OSS_NEW _utilJSONParser() ;
}
else if ( MIG_PARSER_CSV == _pParameters->fileType )
{
_parser = SDB_OSS_NEW _utilCSVParser() ;
}
else
{
rc = SDB_MIG_UNKNOW_FILE_TYPE ;
PD_LOG ( PDERROR, "Unknow file type" ) ;
goto error ;
}
PD_CHECK ( _parser, SDB_OOM, error, PDERROR,
"Failed to new _parser" ) ;
_ppBucket = (_bucket**)SDB_OSS_MALLOC ( sizeof(_bucket*) *
_pParameters->bucketNum ) ;
PD_CHECK ( _ppBucket, SDB_OOM, error, PDERROR,
"Failed to allocate bucket pointer array" ) ;
ossMemset ( _ppBucket, 0, sizeof(_bucket*) * _pParameters->bucketNum ) ;
for ( UINT32 i = 0; i < _pParameters->bucketNum; ++i )
{
_ppBucket[i] = SDB_OSS_NEW _bucket() ;
PD_CHECK ( _ppBucket[i], SDB_OOM, error, PDERROR,
"Failed to allocate bucket pointer array" ) ;
}
_parser->setDel ( _pParameters->delCFR[0],
_pParameters->delCFR[1],
_pParameters->delCFR[2] ) ;
parserPara.fileName = _pParameters->pFileName ;
parserPara.bufferSize = _pParameters->bufferSize ;
parserPara.blockNum = _pParameters->bucketNum ;
rc = _parser->initialize( &parserPara ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to init utilParseJSONs, rc = %d", rc ) ;
if ( SDB_IO == rc || SDB_FNE == rc )
{
sendMsgToClient ( "Failed to open file %s, rc=%d",
_pParameters->pFileName, rc ) ;
}
goto error ;
}
_buffer = _parser->getBuffer() ;
_delChar = _pParameters->delCFR[0] ;
_delField = _pParameters->delCFR[1] ;
_delRecord = _pParameters->delCFR[2] ;
_autoAddField = TRUE ;
_autoCompletion = FALSE ;
_isHeaderline = _pParameters->headerline ;
if ( _isHeaderline )
{
rc = _parser->getNextRecord ( startOffset, fieldsSize ) ;
if ( rc )
{
if ( rc == SDB_EOF )
{
if ( 0 == fieldsSize )
{
goto done ;
}
}
else
{
PD_LOG ( PDERROR, "Failed to _parser getNextRecord, rc=%d", rc ) ;
goto error ;
}
}
}
if ( _pParameters->pFieldArray )
{
_pFields = _pParameters->pFieldArray ;
_fieldsSize = ossStrlen( _pFields ) ;
}
else
{
_pFields = _buffer + startOffset ;
_fieldsSize = fieldsSize ;
}
done:
PD_TRACE_EXITRC ( SDB__MIGLOADJSONPS__INITIALIZE, rc );
return rc ;
error:
goto done ;
}
INT32 _appendValue( CHAR delChar, bson_iterator *pIt,
CHAR **ppBuffer, INT32 *pCSVSize,
BOOLEAN includeBinary,
BOOLEAN includeRegex )
{
INT32 rc = SDB_OK ;
bson_type type = bson_iterator_type( pIt ) ;
INT32 tempSize = 0 ;
INT32 base64Size = 0 ;
INT32 binType = 0 ;
CHAR temp[128] = { 0 } ;
const CHAR *pTemp = NULL ;
CHAR *pBase64 = NULL ;
bson_timestamp_t ts;
time_t timer ;
struct tm psr;
if ( type == BSON_DOUBLE || type == BSON_BOOL ||
type == BSON_NULL || type == BSON_INT ||
type == BSON_LONG )
{
rc = _appendNonString( delChar, pIt, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else
{
rc = _appendString( delChar, &delChar, 1, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
if ( type == BSON_TIMESTAMP )
{
ts = bson_iterator_timestamp( pIt ) ;
timer = (time_t)ts.t;
local_time( &timer, &psr ) ;
tempSize = ossSnprintf ( temp, 64,
"%04d-%02d-%02d-%02d.%02d.%02d.%06d",
psr.tm_year + 1900,
psr.tm_mon + 1,
psr.tm_mday,
psr.tm_hour,
psr.tm_min,
psr.tm_sec,
ts.i ) ;
rc = _appendString( delChar, temp, tempSize, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else if ( type == BSON_DATE )
{
timer = bson_iterator_date( pIt ) / 1000 ;
local_time( &timer, &psr ) ;
tempSize = ossSnprintf ( temp, 64, "%04d-%02d-%02d",
psr.tm_year + 1900,
psr.tm_mon + 1,
psr.tm_mday ) ;
rc = _appendString( delChar, temp, tempSize, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else if ( type == BSON_UNDEFINED )
{
rc = _appendString( delChar, CSV_STR_UNDEFINED,
CSV_STR_UNDEFINED_SIZE,
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else if ( type == BSON_MINKEY )
{
rc = _appendString( delChar, CSV_STR_MINKEY,
CSV_STR_MINKEY_SIZE, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else if ( type == BSON_MAXKEY )
{
rc = _appendString( delChar, CSV_STR_MAXKEY,
CSV_STR_MAXKEY_SIZE, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else if ( type == BSON_CODE )
{
pTemp = bson_iterator_code( pIt ) ;
rc = _appendString( delChar, pTemp, ossStrlen( pTemp ),
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else if ( type == BSON_STRING || type == BSON_SYMBOL )
{
pTemp = bson_iterator_string( pIt ) ;
rc = _appendString( delChar, pTemp, ossStrlen( pTemp ),
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else if ( type == BSON_BINDATA )
{
if( TRUE == includeBinary )
{
rc = _appendString( delChar, CSV_STR_LEFTBRACKET, 1, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
binType = (INT32)bson_iterator_bin_type( pIt ) ;
tempSize = ossSnprintf ( temp, 64, "%d", binType ) ;
rc = _appendString( delChar, &temp, tempSize, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
rc = _appendString( delChar, CSV_STR_RIGHTBRACKET, 1, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
pTemp = bson_iterator_bin_data( pIt ) ;
tempSize = bson_iterator_bin_len ( pIt ) ;
if( tempSize > 0 )
{
base64Size = getEnBase64Size ( tempSize ) ;
pBase64 = (CHAR *)SDB_OSS_MALLOC( base64Size ) ;
if( NULL == pBase64 )
{
rc = SDB_OOM ;
goto error ;
}
ossMemset( pBase64, 0, base64Size ) ;
if ( base64Encode( pTemp, tempSize, pBase64, base64Size ) < 0 )
{
rc = SDB_OOM ;
goto error ;
}
rc = _appendString( delChar, pBase64, base64Size - 1,
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
}
else if ( type == BSON_REGEX )
{
if( TRUE == includeRegex )
{
rc = _appendString( delChar, CSV_STR_BACKSLASH, 1,
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
pTemp = bson_iterator_regex( pIt ) ;
rc = _appendString( delChar, pTemp, ossStrlen( pTemp ),
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
if( TRUE == includeRegex )
{
rc = _appendString( delChar, CSV_STR_BACKSLASH, 1,
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
pTemp = bson_iterator_regex_opts( pIt ) ;
rc = _appendString( delChar, pTemp, ossStrlen( pTemp ),
ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
}
else if ( type == BSON_OID )
{
bson_oid_to_string( bson_iterator_oid( pIt ), temp ) ;
rc = _appendString( delChar, temp, 24, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
else
{
rc = _appendObj( delChar, pIt, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
rc = _appendString( delChar, &delChar, 1, ppBuffer, pCSVSize ) ;
if ( rc )
{
goto error ;
}
}
done:
SAFE_OSS_FREE( pBase64 ) ;
return rc ;
error:
goto done ;
}
_ciCollection() : _next( NULL )
{
ossMemset( _clName, 0, CI_CL_FULLNAME_SIZE + 1 ) ;
ossMemset( _mainClName, 0, CI_CL_FULLNAME_SIZE + 1 ) ;
ossMemcpy( _mainClName, NONE_NAME, ossStrlen( NONE_NAME ) ) ;
}
INT32 restAdaptor::setOPResult( pmdRestSession *pSession,
INT32 result,
const BSONObj &info )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY( SDB__RESTADP_SETOPR ) ;
SDB_ASSERT ( pSession, "pSession is NULL" ) ;
httpConnection *pHttpCon = pSession->getRestConn() ;
if( COM_GETFILE != pHttpCon->_common )
{
std::string str = info.toString( FALSE, TRUE ) ;
INT32 bufferSize = ossStrlen( str.c_str() ) ;
if( TRUE == pHttpCon->_isChunk )
{
if( FALSE == pHttpCon->_isSendHttpHeader )
{
pHttpCon->_isSendHttpHeader = TRUE ;
rc = _setResponseType( pSession ) ;
if( rc )
{
PD_LOG ( PDERROR, "Failed to set respone type, rc=%d",
rc ) ;
goto error ;
}
rc = _sendHttpHeader( pSession, HTTP_OK ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to send http header, rc=%d", rc ) ;
goto error ;
}
}
rc = _sendHttpChunk( pSession, str.c_str(), bufferSize ) ;
if( rc )
{
PD_LOG ( PDERROR, "Failed to send http chunk, rc=%d", rc ) ;
goto error ;
}
}
else
{
CHAR *pBuffer = NULL ;
INT32 tempSize = 0 ;
httpResponse httpRe ;
rc = pSession->allocBuff( bufferSize + 1, &pBuffer, tempSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Unable to allocate %d bytes memory, rc=%d",
bufferSize + 1, rc ) ;
goto error ;
}
ossMemcpy( pBuffer, str.c_str(), bufferSize ) ;
pBuffer[ bufferSize ] = 0 ;
pHttpCon->_firstRecordSize = bufferSize ;
httpRe.pBuffer = pBuffer ;
httpRe.len = bufferSize ;
pHttpCon->_responseBody[0] = httpRe ;
}
}
done:
PD_TRACE_EXITRC( SDB__RESTADP_SETOPR, rc ) ;
return rc ;
error:
goto done ;
}
INT32 restAdaptor::_convertMsg( pmdRestSession *pSession,
HTTP_PARSE_COMMON &common,
CHAR **ppMsg,
INT32 &msgSize )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY( SDB__RESTADP_CONVERTMSG );
SDB_ASSERT ( pSession, "pSession is NULL" ) ;
SDB_ASSERT ( ppMsg, "pMsg is NULL" ) ;
INT32 pathSize = 0 ;
INT32 tempSize = 0 ;
INT32 extenSize = 0 ;
const CHAR *pFileName = NULL ;
const CHAR *pExtension = NULL ;
CHAR *pMsg = NULL ;
httpConnection *pHttpCon = pSession->getRestConn() ;
if( pHttpCon->_pPath )
{
pFileName = _getResourceFileName( pHttpCon->_pPath ) ;
}
if ( pFileName )
{
common = COM_GETFILE ;
pHttpCon->_fileType = HTTP_FILE_UNKNOW ;
pExtension = _getFileExtension( pFileName ) ;
if ( pExtension )
{
extenSize = ossStrlen( pExtension ) ;
for( UINT32 i = 0; i < REST_STRING_FILE_EX_SIZE; ++i )
{
if ( 0 == ossStrncasecmp( pExtension,
fileExtension[i], extenSize ) )
{
pHttpCon->_fileType = (HTTP_FILE_TYPE)i ;
break ;
}
}
}
}
else
{
if ( pHttpCon->_requestQuery.size() > 0 )
{
common = COM_CMD ;
pHttpCon->_fileType = HTTP_FILE_HTML ;
}
else
{
common = COM_GETFILE ;
pHttpCon->_fileType = HTTP_FILE_DEFAULT ;
}
}
if( pHttpCon->_pPath )
{
pathSize = ossStrlen( pHttpCon->_pPath ) ;
rc = pSession->allocBuff( pathSize + 1, &pMsg, tempSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Unable to allocate %d bytes memory, rc=%d",
pathSize, rc ) ;
goto error ;
}
ossMemcpy( pMsg, pHttpCon->_pPath, pathSize ) ;
pMsg[pathSize] = 0 ;
}
else
{
pathSize = 1 ;
rc = pSession->allocBuff( pathSize + 1, &pMsg, tempSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Unable to allocate %d bytes memory, rc=%d",
pathSize, rc ) ;
goto error ;
}
pMsg[0] = '/' ;
pMsg[1] = 0 ;
}
*ppMsg = pMsg ;
msgSize = pathSize ;
done:
PD_TRACE_EXITRC ( SDB__RESTADP_CONVERTMSG, rc ) ;
return rc ;
error:
goto done ;
}
INT32 restAdaptor::appendHttpBody( pmdRestSession *pSession,
const CHAR *pBuffer,
INT32 length,
INT32 number, BOOLEAN isObjBuffer )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY( SDB__RESTADP_APPENDBODY ) ;
SDB_ASSERT ( pSession, "pSession is NULL" ) ;
SDB_ASSERT ( pBuffer, "pBuffer is NULL" ) ;
httpConnection *pHttpCon = pSession->getRestConn() ;
INT32 tempSize = 0 ;
httpResponse httpRe ;
if( TRUE == pHttpCon->_isChunk )
{
if( FALSE == pHttpCon->_isSendHttpHeader )
{
pHttpCon->_isSendHttpHeader = TRUE ;
rc = _setResponseType( pSession ) ;
if( rc )
{
PD_LOG ( PDERROR, "Failed to set respone type, rc=%d",
rc ) ;
goto error ;
}
rc = _sendHttpHeader( pSession, HTTP_OK ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to send http header, rc=%d", rc ) ;
goto error ;
}
}
if( isObjBuffer )
{
INT32 jsonSize = 0 ;
BSONObj record ;
std::string str ;
_rtnObjBuff rtnObj( pBuffer, length, number ) ;
while( TRUE )
{
rc = rtnObj.nextObj( record ) ;
if ( SDB_DMS_EOC == rc )
{
rc = SDB_OK ;
break ;
}
else if ( SDB_SYS == rc )
{
PD_LOG ( PDERROR, "Failed to get nextObj:rc=%d", rc ) ;
goto error ;
}
str = record.toString( FALSE, TRUE ) ;
jsonSize = ossStrlen( str.c_str() ) ;
rc = _sendHttpChunk( pSession, str.c_str(), jsonSize ) ;
if( rc )
{
PD_LOG ( PDERROR, "Failed to send http chunk, rc=%d", rc ) ;
goto error ;
}
}
}
else
{
rc = _sendHttpChunk( pSession, pBuffer, length ) ;
if( rc )
{
PD_LOG ( PDERROR, "Failed to send http chunk, rc=%d", rc ) ;
goto error ;
}
}
}
else
{
if( isObjBuffer )
{
CHAR *pJson = NULL ;
INT32 jsonSize = 0 ;
BSONObj record ;
std::string str ;
_rtnObjBuff rtnObj( pBuffer, length, number ) ;
while( TRUE )
{
rc = rtnObj.nextObj( record ) ;
if ( SDB_DMS_EOC == rc )
{
rc = SDB_OK ;
break ;
}
else if ( SDB_SYS == rc )
{
PD_LOG ( PDERROR, "Failed to get nextObj:rc=%d", rc ) ;
goto error ;
}
str = record.toString( FALSE, TRUE ) ;
jsonSize = ossStrlen( str.c_str() ) ;
rc = pSession->allocBuff( jsonSize + 1, &pJson, tempSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Unable to allocate %d bytes memory, rc=%d",
jsonSize + 1, rc ) ;
goto error ;
}
ossMemcpy( pJson, str.c_str(), jsonSize ) ;
pJson[ jsonSize ] = 0 ;
pHttpCon->_responseSize += jsonSize ;
httpRe.pBuffer = pJson ;
httpRe.len = jsonSize ;
pHttpCon->_responseBody.push_back( httpRe ) ;
pBuffer = NULL ;
jsonSize = 0 ;
}
}
else
{
CHAR *pFileText = NULL ;
rc = pSession->allocBuff( length + 1, &pFileText, tempSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Unable to allocate %d bytes memory, rc=%d",
length + 1, rc ) ;
goto error ;
}
ossMemcpy( pFileText, pBuffer, length ) ;
pFileText[ length ] = 0 ;
pHttpCon->_firstRecordSize = length ;
httpRe.pBuffer = pFileText ;
httpRe.len = length ;
pHttpCon->_responseBody[0] = httpRe ;
}
}
done:
PD_TRACE_EXITRC( SDB__RESTADP_APPENDBODY, rc ) ;
return rc ;
error:
goto done ;
}
INT32 restAdaptor::appendHttpHeader( pmdRestSession *pSession,
const CHAR *pKey,
const CHAR *pValue )
{
INT32 rc = SDB_OK ;
PD_TRACE_ENTRY( SDB__RESTADP_APPENDHEADER ) ;
SDB_ASSERT ( pSession, "pSession is NULL" ) ;
INT32 keySize = ossStrlen( pKey ) ;
INT32 valueSize = ossStrlen( pValue ) ;
INT32 newHeaderSize = keySize + valueSize + 2 ;
INT32 tempSize = 0 ;
CHAR *pNewHeaderBuf = NULL ;
CHAR *pNewKey = NULL ;
CHAR *pNewValue = NULL ;
httpConnection *pHttpCon = pSession->getRestConn() ;
COLNAME_MAP_IT it ;
if( REST_STRING_TRANSFER_SIZE == ossStrlen( pKey ) &&
0 == ossStrncmp( pKey, REST_STRING_TRANSFER,
REST_STRING_TRANSFER_SIZE ) &&
REST_STRING_CHUNKED_SIZE == ossStrlen( pValue ) &&
0 == ossStrncmp( pValue, REST_STRING_CHUNKED,
REST_STRING_CHUNKED_SIZE ) )
{
pHttpCon->_isChunk = TRUE ;
}
it = pHttpCon->_responseHeaders.find( pKey ) ;
if ( it == pHttpCon->_responseHeaders.end() )
{
rc = pSession->allocBuff( newHeaderSize, &pNewHeaderBuf, tempSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Unable to allocate %d bytes memory, rc=%d",
newHeaderSize, rc ) ;
goto error ;
}
pNewKey = pNewHeaderBuf ;
pNewValue = pNewHeaderBuf + keySize + 1 ;
ossStrncpy( pNewKey, pKey, keySize ) ;
ossStrncpy( pNewValue, pValue, valueSize ) ;
pNewKey[ keySize ] = 0 ;
pNewValue[ valueSize ] = 0 ;
pHttpCon->_responseHeaders.insert(
std::make_pair( pNewKey, pNewValue ) ) ;
}
else
{
rc = pSession->allocBuff( valueSize + 1, &pNewValue, tempSize ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Unable to allocate %d bytes memory, rc=%d",
valueSize, rc ) ;
goto error ;
}
ossStrncpy( pNewValue, pValue, valueSize ) ;
pNewValue[ valueSize ] = 0 ;
it->second = pNewValue ;
}
done:
PD_TRACE_EXITRC( SDB__RESTADP_APPENDHEADER, rc ) ;
return rc ;
error:
goto done ;
}
void omClusterNotifier::_getAgentService( string &serviceName )
{
INT32 rc = SDB_OK ;
CHAR conf[OSS_MAX_PATHSIZE + 1] = { 0 } ;
po::options_description desc ( "Config options" ) ;
po::variables_map vm ;
CHAR hostport[OSS_MAX_HOSTNAME + 6] = { 0 } ;
serviceName = boost::lexical_cast<string>( SDBCM_DFT_PORT ) ;
rc = ossGetHostName( hostport, OSS_MAX_HOSTNAME ) ;
if ( rc != SDB_OK )
{
PD_LOG( PDERROR, "get host name failed:rc=%d", rc ) ;
goto error ;
}
ossStrncat ( hostport, SDBCM_CONF_PORT, ossStrlen(SDBCM_CONF_PORT) ) ;
desc.add_options()
(SDBCM_CONF_DFTPORT, po::value<string>(), "sdbcm default "
"listening port")
(hostport, po::value<string>(), "sdbcm specified listening port")
;
rc = ossGetEWD ( conf, OSS_MAX_PATHSIZE ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to get excutable file's working "
"directory" ) ;
goto error ;
}
if ( ( ossStrlen ( conf ) + ossStrlen ( SDBCM_CONF_PATH_FILE ) + 2 ) >
OSS_MAX_PATHSIZE )
{
PD_LOG ( PDERROR, "Working directory too long" ) ;
rc = SDB_INVALIDARG ;
goto error ;
}
ossStrncat( conf, OSS_FILE_SEP, 1 );
ossStrncat( conf, SDBCM_CONF_PATH_FILE,
ossStrlen( SDBCM_CONF_PATH_FILE ) );
rc = utilReadConfigureFile ( conf, desc, vm ) ;
if ( rc )
{
PD_LOG ( PDERROR, "Failed to read configure file, rc = %d", rc ) ;
goto error ;
}
else if ( vm.count( hostport ) )
{
serviceName = vm[hostport].as<string>() ;
}
else if ( vm.count( SDBCM_CONF_DFTPORT ) )
{
serviceName = vm[SDBCM_CONF_DFTPORT].as<string>() ;
}
else
{
serviceName = boost::lexical_cast<string>( SDBCM_DFT_PORT ) ;
}
done:
return ;
error:
goto done ;
}