int post( char *url, char* objectPath ) {
CURLcode res = CURLE_OK;
char errbuf[CURL_ERROR_SIZE];
readData_t readData;
openedDataObjInp_t openedSource;
/* Create readData object */
memset(&readData, 0, sizeof(readData_t));
memset(&openedSource, 0, sizeof(openedDataObjInp_t));
snprintf(readData.path, MAX_NAME_LEN, "%s", objectPath);
readData.desc = 0;
readData.rsComm = rsComm;
// Set up easy handler
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt( curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl, CURLOPT_READDATA, &readData);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, &irodsCurl::my_read_obj);
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-iRODS");
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
// Possibly this timeout is too conservative
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
// CURL call
res = curl_easy_perform(curl);
if ( res != 0 ) {
rodsLog( LOG_ERROR, "irods_http_send_file: curl error (%i): %s", res, errbuf );
return SYS_INTERNAL_NULL_INPUT_ERR;
} else {
long http_code = 0;
curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code != 200) {
rodsLog( LOG_ERROR, "irods_http_send_file: HTTP returned error code %d", http_code );
return SYS_INTERNAL_NULL_INPUT_ERR;
}
}
if (readData.desc) {
openedSource.l1descInx = readData.desc;
int status = rsDataObjClose(rsComm, &openedSource);
if (status < 0) {
rodsLog(LOG_ERROR, "irods_http_send_file: rsDataObjClose failed for %s, status = %d",readData.path, status);
}
}
return 0;
}
int
l3DataGetSingleBuf (rsComm_t *rsComm, int l1descInx,
bytesBuf_t *dataObjOutBBuf, portalOprOut_t **portalOprOut)
{
int status = 0;
int bytesRead;
openedDataObjInp_t dataObjCloseInp;
dataObjInfo_t *dataObjInfo;
/* just malloc an empty portalOprOut */
*portalOprOut = (portalOprOut_t*)malloc (sizeof (portalOprOut_t));
memset (*portalOprOut, 0, sizeof (portalOprOut_t));
dataObjInfo = L1desc[l1descInx].dataObjInfo;
if (dataObjInfo->dataSize > 0) {
dataObjOutBBuf->buf = malloc (dataObjInfo->dataSize);
bytesRead = l3FileGetSingleBuf (rsComm, l1descInx, dataObjOutBBuf);
} else {
bytesRead = 0;
}
#if 0 /* tested in _rsFileGet. don't need to go it again */
if (bytesRead != dataObjInfo->dataSize) {
free (dataObjOutBBuf->buf);
memset (dataObjOutBBuf, 0, sizeof (bytesBuf_t));
if (bytesRead >= 0) {
rodsLog (LOG_NOTICE,
"l3DataGetSingleBuf:Bytes toread %d don't match read %d",
dataObjInfo->dataSize, bytesRead);
bytesRead = SYS_COPY_LEN_ERR - errno;
}
}
#endif
memset (&dataObjCloseInp, 0, sizeof (dataObjCloseInp));
dataObjCloseInp.l1descInx = l1descInx;
status = rsDataObjClose (rsComm, &dataObjCloseInp);
if (status < 0) {
rodsLog (LOG_NOTICE,
"l3DataGetSingleBuf: rsDataObjClose of %d error, status = %d",
l1descInx, status);
}
if (bytesRead < 0)
return (bytesRead);
else
return status;
}
int
l3DataPutSingleBuf( rsComm_t* rsComm,
dataObjInp_t* dataObjInp,
bytesBuf_t* dataObjInpBBuf ) {
int bytesWritten;
int l1descInx;
int status;
openedDataObjInp_t dataObjCloseInp;
std::string resc_name;
/* don't actually physically open the file */
addKeyVal( &dataObjInp->condInput, NO_OPEN_FLAG_KW, "" );
l1descInx = rsDataObjCreate( rsComm, dataObjInp );
if ( l1descInx <= 2 ) {
if ( l1descInx >= 0 ) {
rodsLog( LOG_ERROR,
"l3DataPutSingleBuf: rsDataObjCreate of %s error, status = %d",
dataObjInp->objPath,
l1descInx );
return SYS_FILE_DESC_OUT_OF_RANGE;
}
else {
return l1descInx;
}
}
bytesWritten = _l3DataPutSingleBuf( rsComm, l1descInx, dataObjInp, dataObjInpBBuf );
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
L1desc[l1descInx].oprStatus = bytesWritten;
L1desc[l1descInx].oprType = PUT_OPR;
status = rsDataObjClose( rsComm, &dataObjCloseInp );
if ( status < 0 ) {
rodsLog( LOG_DEBUG,
"l3DataPutSingleBuf: rsDataObjClose of %d error, status = %d",
l1descInx, status );
}
if ( bytesWritten >= 0 ) {
return status;
}
return bytesWritten;
}
int
l3DataGetSingleBuf( rsComm_t *rsComm, int l1descInx,
bytesBuf_t *dataObjOutBBuf, portalOprOut_t **portalOprOut ) {
int status = 0;
int bytesRead;
openedDataObjInp_t dataObjCloseInp;
dataObjInfo_t *dataObjInfo;
/* just malloc an empty portalOprOut */
*portalOprOut = ( portalOprOut_t* )malloc( sizeof( portalOprOut_t ) );
memset( *portalOprOut, 0, sizeof( portalOprOut_t ) );
dataObjInfo = L1desc[l1descInx].dataObjInfo;
if ( dataObjInfo->dataSize > 0 ) {
dataObjOutBBuf->buf = malloc( dataObjInfo->dataSize );
bytesRead = l3FileGetSingleBuf( rsComm, l1descInx, dataObjOutBBuf );
}
else {
bytesRead = 0;
}
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
status = rsDataObjClose( rsComm, &dataObjCloseInp );
if ( status < 0 ) {
rodsLog( LOG_NOTICE,
"l3DataGetSingleBuf: rsDataObjClose of %d error, status = %d",
l1descInx, status );
}
if ( bytesRead < 0 ) {
return bytesRead;
}
else {
return status;
}
}
int
_rsDataObjPut( rsComm_t *rsComm, dataObjInp_t *dataObjInp,
bytesBuf_t *dataObjInpBBuf, portalOprOut_t **portalOprOut ) {
int status;
int l1descInx;
int retval;
openedDataObjInp_t dataObjCloseInp;
int allFlag;
transferStat_t *transStat = NULL;
dataObjInp_t replDataObjInp;
if ( getValByKey( &dataObjInp->condInput, ALL_KW ) != NULL ) {
allFlag = 1;
}
else {
allFlag = 0;
}
if ( getValByKey( &dataObjInp->condInput, DATA_INCLUDED_KW ) != NULL ) {
/* single buffer put */
status = l3DataPutSingleBuf( rsComm, dataObjInp, dataObjInpBBuf );
if ( status >= 0 && allFlag == 1 ) {
/* update the rest of copies */
addKeyVal( &dataObjInp->condInput, UPDATE_REPL_KW, "" );
status = rsDataObjRepl( rsComm, dataObjInp, &transStat );
if ( transStat != NULL ) {
free( transStat );
}
}
if ( status >= 0 ) {
status = applyRuleForPostProcForWrite(
rsComm, dataObjInpBBuf, dataObjInp->objPath );
if ( status >= 0 ) {
status = 0;
}
}
return status;
}
/* get down here. will do parallel I/O */
/* so that mmap will work */
dataObjInp->openFlags |= O_RDWR;
l1descInx = rsDataObjCreate( rsComm, dataObjInp );
if ( l1descInx < 0 ) {
return l1descInx;
}
L1desc[l1descInx].oprType = PUT_OPR;
L1desc[l1descInx].dataSize = dataObjInp->dataSize;
if ( getStructFileType( L1desc[l1descInx].dataObjInfo->specColl ) >= 0 ) { // JMC - backport 4682
*portalOprOut = ( portalOprOut_t * ) malloc( sizeof( portalOprOut_t ) );
bzero( *portalOprOut, sizeof( portalOprOut_t ) );
( *portalOprOut )->l1descInx = l1descInx;
return l1descInx;
}
status = preProcParaPut( rsComm, l1descInx, portalOprOut );
if ( status < 0 ) {
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
L1desc[l1descInx].oprStatus = status;
rsDataObjClose( rsComm, &dataObjCloseInp );
return status;
}
if ( allFlag == 1 ) {
/* need to save dataObjInp. get freed in sendAndRecvBranchMsg */
memset( &replDataObjInp, 0, sizeof( replDataObjInp ) );
rstrcpy( replDataObjInp.objPath, dataObjInp->objPath, MAX_NAME_LEN );
addKeyVal( &replDataObjInp.condInput, UPDATE_REPL_KW, "" );
addKeyVal( &replDataObjInp.condInput, ALL_KW, "" );
}
/* return portalOprOut to the client and wait for the rcOprComplete
* call. That is when the parallel I/O is done */
retval = sendAndRecvBranchMsg( rsComm, rsComm->apiInx, status,
( void * ) * portalOprOut, NULL );
if ( retval < 0 ) {
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
L1desc[l1descInx].oprStatus = retval;
rsDataObjClose( rsComm, &dataObjCloseInp );
if ( allFlag == 1 ) {
clearKeyVal( &replDataObjInp.condInput );
}
}
else if ( allFlag == 1 ) {
status = rsDataObjRepl( rsComm, &replDataObjInp, &transStat );
if ( transStat != NULL ) {
free( transStat );
}
clearKeyVal( &replDataObjInp.condInput );
}
/* already send the client the status */
return SYS_NO_HANDLER_REPLY_MSG;
}
int
bundleAndRegSubFiles( rsComm_t *rsComm, int l1descInx, char *phyBunDir,
char *collection, bunReplCacheHeader_t *bunReplCacheHeader, int chksumFlag ) { // JMC - backport 4528
int status;
openedDataObjInp_t dataObjCloseInp;
bunReplCache_t *tmpBunReplCache, *nextBunReplCache;
regReplica_t regReplicaInp;
dataObjInp_t dataObjUnlinkInp;
keyValPair_t regParam; // JMC - backport 4528
modDataObjMeta_t modDataObjMetaInp; // JMC - backport 4528
int savedStatus = 0;
bzero( &dataObjCloseInp, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
if ( bunReplCacheHeader->numSubFiles == 0 ) {
bzero( &dataObjUnlinkInp, sizeof( dataObjUnlinkInp ) );
rstrcpy( dataObjUnlinkInp.objPath,
L1desc[l1descInx].dataObjInfo->objPath, MAX_NAME_LEN );
dataObjUnlinkS( rsComm, &dataObjUnlinkInp,
L1desc[l1descInx].dataObjInfo );
L1desc[l1descInx].bytesWritten = 0;
rsDataObjClose( rsComm, &dataObjCloseInp );
bzero( bunReplCacheHeader, sizeof( bunReplCacheHeader_t ) );
return 0;
}
status = phyBundle( rsComm, L1desc[l1descInx].dataObjInfo, phyBunDir,
collection, CREATE_TAR_OPR ); // JMC - backport 4643
if ( status < 0 ) {
rodsLog( LOG_ERROR,
"bundleAndRegSubFiles: rsStructFileSync of %s error. stat = %d",
L1desc[l1descInx].dataObjInfo->objPath, status );
rmLinkedFilesInUnixDir( phyBunDir );
rmdir( phyBunDir );
rsDataObjClose( rsComm, &dataObjCloseInp );
tmpBunReplCache = bunReplCacheHeader->bunReplCacheHead;
while ( tmpBunReplCache != NULL ) {
nextBunReplCache = tmpBunReplCache->next;
free( tmpBunReplCache );
tmpBunReplCache = nextBunReplCache; // JMC - backport 4579
}
bzero( bunReplCacheHeader, sizeof( bunReplCacheHeader_t ) );
return status;
}
else {
/* mark it was written so the size would be adjusted */
L1desc[l1descInx].bytesWritten = 1;
}
/* now register a replica for each subfile */
tmpBunReplCache = bunReplCacheHeader->bunReplCacheHead;
if ( tmpBunReplCache == NULL ) {
rmdir( phyBunDir );
bzero( &dataObjUnlinkInp, sizeof( dataObjUnlinkInp ) );
rstrcpy( dataObjUnlinkInp.objPath,
L1desc[l1descInx].dataObjInfo->objPath, MAX_NAME_LEN );
dataObjUnlinkS( rsComm, &dataObjUnlinkInp,
L1desc[l1descInx].dataObjInfo );
L1desc[l1descInx].bytesWritten = 0;
rsDataObjClose( rsComm, &dataObjCloseInp );
bzero( bunReplCacheHeader, sizeof( bunReplCacheHeader_t ) );
return 0;
}
bzero( ®ReplicaInp, sizeof( regReplicaInp ) );
regReplicaInp.srcDataObjInfo = ( dataObjInfo_t* )malloc( sizeof( dataObjInfo_t ) );
regReplicaInp.destDataObjInfo = ( dataObjInfo_t* )malloc( sizeof( dataObjInfo_t ) );
bzero( regReplicaInp.srcDataObjInfo, sizeof( dataObjInfo_t ) );
bzero( regReplicaInp.destDataObjInfo, sizeof( dataObjInfo_t ) );
addKeyVal( ®ReplicaInp.condInput, ADMIN_KW, "" );
rstrcpy( regReplicaInp.destDataObjInfo->rescName, BUNDLE_RESC, NAME_LEN );
rstrcpy( regReplicaInp.destDataObjInfo->filePath,
L1desc[l1descInx].dataObjInfo->objPath, MAX_NAME_LEN );
rstrcpy( regReplicaInp.destDataObjInfo->rescHier,
L1desc[l1descInx].dataObjInfo->rescHier, MAX_NAME_LEN );
// =-=-=-=-=-=-=-
// JMC - backport 4528
if ( chksumFlag != 0 ) {
bzero( &modDataObjMetaInp, sizeof( modDataObjMetaInp ) );
bzero( ®Param, sizeof( regParam ) );
modDataObjMetaInp.dataObjInfo = regReplicaInp.destDataObjInfo;
modDataObjMetaInp.regParam = ®Param;
}
// =-=-=-=-=-=-=-
/* close here because dataObjInfo is still being used */
rsDataObjClose( rsComm, &dataObjCloseInp );
while ( tmpBunReplCache != NULL ) {
char subPhyPath[MAX_NAME_LEN];
nextBunReplCache = tmpBunReplCache->next;
/* rm the hard link here */
snprintf( subPhyPath, MAX_NAME_LEN, "%s/%lld", phyBunDir,
tmpBunReplCache->dataId );
// =-=-=-=-=-=-=-
// JMC - backport 4528
if ( chksumFlag != 0 ) {
status = fileChksum( rsComm, regReplicaInp.destDataObjInfo->filePath,
subPhyPath, regReplicaInp.destDataObjInfo->rescHier, 0, tmpBunReplCache->chksumStr );
if ( status < 0 ) {
savedStatus = status;
rodsLogError( LOG_ERROR, status, "bundleAndRegSubFiles: fileChksum error for %s", tmpBunReplCache->objPath );
}
}
// =-=-=-=-=-=-=-
unlink( subPhyPath );
/* register the replica */
rstrcpy( regReplicaInp.srcDataObjInfo->objPath,
tmpBunReplCache->objPath, MAX_NAME_LEN );
regReplicaInp.srcDataObjInfo->dataId =
regReplicaInp.destDataObjInfo->dataId =
tmpBunReplCache->dataId;
regReplicaInp.srcDataObjInfo->replNum = tmpBunReplCache->srcReplNum;
status = rsRegReplica( rsComm, ®ReplicaInp );
if ( status < 0 ) {
savedStatus = status;
rodsLog( LOG_ERROR,
"bundleAndRegSubFiles: rsRegReplica error for %s. stat = %d",
tmpBunReplCache->objPath, status );
}
// =-=-=-=-=-=-=-
// JMC - backport 4528
if ( chksumFlag != 0 ) {
addKeyVal( ®Param, CHKSUM_KW, tmpBunReplCache->chksumStr );
// avoid triggering file operations
addKeyVal( ®Param, IN_PDMO_KW, "" );
status = rsModDataObjMeta( rsComm, &modDataObjMetaInp );
clearKeyVal( ®Param );
if ( status < 0 ) {
savedStatus = status;
rodsLogError( LOG_ERROR, status, "bundleAndRegSubFiles: rsModDataObjMeta error for %s.", tmpBunReplCache->objPath );
}
}
// =-=-=-=-=-=-=-
free( tmpBunReplCache );
tmpBunReplCache = nextBunReplCache;
}
clearKeyVal( ®ReplicaInp.condInput );
free( regReplicaInp.srcDataObjInfo );
free( regReplicaInp.destDataObjInfo );
bzero( bunReplCacheHeader, sizeof( bunReplCacheHeader_t ) );
rmdir( phyBunDir );
if ( status >= 0 && savedStatus < 0 ) {
return savedStatus;
}
else {
return status;
}
}
int
_rsDataObjCopy( rsComm_t *rsComm, int destL1descInx, int existFlag,
transferStat_t **transStat ) {
dataObjInp_t *srcDataObjInp, *destDataObjInp;
openedDataObjInp_t dataObjCloseInp;
dataObjInfo_t *srcDataObjInfo, *destDataObjInfo;
int srcL1descInx;
int status = 0, status2;
destDataObjInp = L1desc[destL1descInx].dataObjInp;
destDataObjInfo = L1desc[destL1descInx].dataObjInfo;
srcL1descInx = L1desc[destL1descInx].srcL1descInx;
srcDataObjInp = L1desc[srcL1descInx].dataObjInp;
srcDataObjInfo = L1desc[srcL1descInx].dataObjInfo;
if ( destDataObjInp == NULL ) { // JMC cppcheck - null ptr ref
rodsLog( LOG_ERROR, "_rsDataObjCopy: :: destDataObjInp is NULL" );
return -1;
}
if ( destDataObjInfo == NULL ) { // JMC cppcheck - null ptr ref
rodsLog( LOG_ERROR, "_rsDataObjCopy: :: destDataObjInfo is NULL" );
return -1;
}
if ( srcDataObjInp == NULL ) { // JMC cppcheck - null ptr ref
rodsLog( LOG_ERROR, "_rsDataObjCopy: :: srcDataObjInp is NULL" );
return -1;
}
if ( srcDataObjInfo == NULL ) { // JMC cppcheck - null ptr ref
rodsLog( LOG_ERROR, "_rsDataObjCopy: :: srcDataObjInfo is NULL" );
return -1;
}
if ( L1desc[srcL1descInx].l3descInx <= 2 ) {
/* no physical file was opened */
status = l3DataCopySingleBuf( rsComm, destL1descInx );
/* has not been registered yet because of NO_OPEN_FLAG_KW */
if ( status >= 0 &&
existFlag == 0 &&
destDataObjInfo->specColl == NULL &&
L1desc[destL1descInx].remoteZoneHost == NULL ) {
/* If the dest is in remote zone, register in _rsDataObjClose there */
status = svrRegDataObj( rsComm, destDataObjInfo );
if ( status == CAT_UNKNOWN_COLLECTION ) {
/* collection does not exist. make one */
char parColl[MAX_NAME_LEN], child[MAX_NAME_LEN];
splitPathByKey( destDataObjInfo->objPath, parColl, MAX_NAME_LEN, child, MAX_NAME_LEN, '/' );
status = svrRegDataObj( rsComm, destDataObjInfo );
rsMkCollR( rsComm, "/", parColl );
status = svrRegDataObj( rsComm, destDataObjInfo );
}
if ( status < 0 ) {
rodsLog( LOG_NOTICE,
"_rsDataObjCopy: svrRegDataObj for %s failed, status = %d",
destDataObjInfo->objPath, status );
return status;
}
}
}
else {
if ( srcDataObjInfo != NULL ) {
destDataObjInp->numThreads = getNumThreads( rsComm, srcDataObjInfo->dataSize, destDataObjInp->numThreads, NULL,
destDataObjInfo->rescHier, srcDataObjInfo->rescHier, 0 );
}
srcDataObjInp->numThreads = destDataObjInp->numThreads;
status = dataObjCopy( rsComm, destL1descInx );
}
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = destL1descInx;
if ( status >= 0 ) {
*transStat = ( transferStat_t* )malloc( sizeof( transferStat_t ) );
memset( *transStat, 0, sizeof( transferStat_t ) );
( *transStat )->bytesWritten = srcDataObjInfo->dataSize;
( *transStat )->numThreads = destDataObjInp->numThreads;
dataObjCloseInp.bytesWritten = srcDataObjInfo->dataSize;
}
status2 = rsDataObjClose( rsComm, &dataObjCloseInp );
if ( status ) {
return status;
}
return status2;
}
int
l3DataPutSingleBuf( rsComm_t* rsComm,
dataObjInp_t* dataObjInp,
bytesBuf_t* dataObjInpBBuf ) {
int bytesWritten;
int l1descInx;
dataObjInfo_t *myDataObjInfo;
char rescGroupName[NAME_LEN];
rescInfo_t *rescInfo = NULL;
rescGrpInfo_t *myRescGrpInfo = NULL;
rescGrpInfo_t *tmpRescGrpInfo = NULL;
rescInfo_t *tmpRescInfo = NULL;
int status;
openedDataObjInp_t dataObjCloseInp;
/* don't actually physically open the file */
addKeyVal( &dataObjInp->condInput, NO_OPEN_FLAG_KW, "" );
l1descInx = rsDataObjCreate( rsComm, dataObjInp );
if ( l1descInx <= 2 ) {
if ( l1descInx >= 0 ) {
rodsLog( LOG_ERROR,
"l3DataPutSingleBuf: rsDataObjCreate of %s error, status = %d",
dataObjInp->objPath,
l1descInx );
return SYS_FILE_DESC_OUT_OF_RANGE;
}
else {
return l1descInx;
}
}
bytesWritten = _l3DataPutSingleBuf( rsComm, l1descInx, dataObjInp, dataObjInpBBuf );
if ( bytesWritten < 0 ) {
myDataObjInfo = L1desc[l1descInx].dataObjInfo;
if ( getStructFileType( myDataObjInfo->specColl ) < 0 &&
strlen( myDataObjInfo->rescGroupName ) > 0 &&
( L1desc[l1descInx].replStatus & OPEN_EXISTING_COPY ) == 0 ) {
/* getValByKey (&dataObjInp->condInput, FORCE_FLAG_KW) == NULL) { */
/* File not in specColl and resc is a resc group and not
* overwriting existing data. Save resc info in case the put fail
*/
rstrcpy( rescGroupName, myDataObjInfo->rescGroupName, NAME_LEN );
rescInfo = myDataObjInfo->rescInfo;
}
else {
rescGroupName[0] = '\0';
rescInfo = NULL;
}
}
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
L1desc[l1descInx].oprStatus = bytesWritten;
status = rsDataObjClose( rsComm, &dataObjCloseInp );
if ( status < 0 ) {
rodsLog( LOG_DEBUG,
"l3DataPutSingleBuf: rsDataObjClose of %d error, status = %d",
l1descInx, status );
}
if ( bytesWritten >= 0 ) {
return status;
}
else if ( strlen( rescGroupName ) == 0 ) {
return bytesWritten;
}
/* get here when Put failed. and rescGroupName is a valid resc group.
* Try other resc in the resc group */
status = getRescGrpForCreate( rsComm, dataObjInp, &myRescGrpInfo );
if ( status < 0 ) {
return bytesWritten;
}
tmpRescGrpInfo = myRescGrpInfo;
while ( tmpRescGrpInfo != NULL ) {
tmpRescInfo = tmpRescGrpInfo->rescInfo;
if ( rescInfo == tmpRescInfo ) {
/* already tried this resc */
tmpRescGrpInfo = tmpRescGrpInfo->next;
continue;
}
l1descInx = _rsDataObjCreateWithRescInfo( rsComm,
dataObjInp, tmpRescInfo, myRescGrpInfo->rescGroupName );
if ( l1descInx <= 2 ) {
if ( l1descInx >= 0 ) {
rodsLog( LOG_ERROR,
"l3DataPutSingleBuf:_rsDataObjCreateWithRI %s err,stat = %d",
dataObjInp->objPath, l1descInx );
}
}
else {
bytesWritten = _l3DataPutSingleBuf( rsComm, l1descInx, dataObjInp,
dataObjInpBBuf );
dataObjCloseInp.l1descInx = l1descInx;
L1desc[l1descInx].oprStatus = bytesWritten;
status = rsDataObjClose( rsComm, &dataObjCloseInp );
if ( status < 0 ) {
rodsLog( LOG_DEBUG,
"l3DataPutSingleBuf: rsDataObjClose of %d error, status = %d",
l1descInx, status );
}
if ( bytesWritten >= 0 ) {
bytesWritten = status;
break;
}
}
tmpRescGrpInfo = tmpRescGrpInfo->next;
}
delete myRescGrpInfo->rescInfo;
delete myRescGrpInfo;
return bytesWritten;
}
int _rsStructFileBundle( rsComm_t* rsComm,
structFileExtAndRegInp_t* structFileBundleInp ) {
int status;
int handleInx;
char phyBunDir[MAX_NAME_LEN];
char tmpPath[MAX_NAME_LEN];
int l1descInx;
char* dataType = 0; // JMC - backport 4664
openedDataObjInp_t dataObjCloseInp;
// =-=-=-=-=-=-=-
// create an empty data obj
dataObjInp_t dataObjInp;
memset( &dataObjInp, 0, sizeof( dataObjInp ) );
dataObjInp.openFlags = O_WRONLY;
// =-=-=-=-=-=-=-
// get the data type of the structured file
dataType = getValByKey( &structFileBundleInp->condInput, DATA_TYPE_KW );
// =-=-=-=-=-=-=-
// ensure that the file name will end in .zip, if necessary
if ( dataType != NULL && strstr( dataType, ZIP_DT_STR ) != NULL ) {
int len = strlen( structFileBundleInp->objPath );
if ( strcmp( &structFileBundleInp->objPath[len - 4], ".zip" ) != 0 ) {
strcat( structFileBundleInp->objPath, ".zip" );
}
}
// =-=-=-=-=-=-=-
// capture the object path in the data obj struct
rstrcpy( dataObjInp.objPath, structFileBundleInp->objPath, MAX_NAME_LEN );
// =-=-=-=-=-=-=-
// replicate the condInput. may have resource input
replKeyVal( &structFileBundleInp->condInput, &dataObjInp.condInput );
// =-=-=-=-=-=-=-
// open the file if we are in an add operation, otherwise create the new file
if ( ( structFileBundleInp->oprType & ADD_TO_TAR_OPR ) != 0 ) { // JMC - backport 4643
l1descInx = rsDataObjOpen( rsComm, &dataObjInp );
}
else {
l1descInx = rsDataObjCreate( rsComm, &dataObjInp );
}
// =-=-=-=-=-=-=-
// error check create / open
if ( l1descInx < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: rsDataObjCreate of %s error. status = %d",
dataObjInp.objPath, l1descInx );
return l1descInx;
}
// =-=-=-=-=-=-=-
// FIXME :: Why, when we replicate them above?
clearKeyVal( &dataObjInp.condInput ); // JMC - backport 4637
// ???? l3Close (rsComm, l1descInx);
// =-=-=-=-=-=-=-
// zip does not like a zero length file as target
//L1desc[ l1descInx ].l3descInx = 0;
//if( dataType != NULL && strstr( dataType, ZIP_DT_STR ) != NULL ) {
// if( ( structFileBundleInp->oprType & ADD_TO_TAR_OPR) == 0 ) { // JMC - backport 4643
// l3Unlink( rsComm, L1desc[l1descInx].dataObjInfo );
// }
//}
// =-=-=-=-=-=-=-
// check object permissions / stat
chkObjPermAndStat_t chkObjPermAndStatInp;
memset( &chkObjPermAndStatInp, 0, sizeof( chkObjPermAndStatInp ) );
rstrcpy( chkObjPermAndStatInp.objPath, structFileBundleInp->collection, MAX_NAME_LEN );
chkObjPermAndStatInp.flags = CHK_COLL_FOR_BUNDLE_OPR;
addKeyVal( &chkObjPermAndStatInp.condInput, RESC_NAME_KW, L1desc[l1descInx].dataObjInfo->rescName );
// =-=-=-=-=-=-=-
// get the resc hier string
std::string resc_hier;
char* resc_hier_ptr = getValByKey( &structFileBundleInp->condInput, RESC_HIER_STR_KW );
if ( !resc_hier_ptr ) {
rodsLog( LOG_NOTICE, "_rsStructFileBundle :: RESC_HIER_STR_KW is NULL" );
}
else {
addKeyVal( &chkObjPermAndStatInp.condInput, RESC_HIER_STR_KW, resc_hier_ptr );
resc_hier = resc_hier_ptr;
}
status = rsChkObjPermAndStat( rsComm, &chkObjPermAndStatInp );
if ( status < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: rsChkObjPermAndStat of %s error. stat = %d",
chkObjPermAndStatInp.objPath, status );
dataObjCloseInp.l1descInx = l1descInx;
rsDataObjClose( rsComm, &dataObjCloseInp );
return status;
}
clearKeyVal( &chkObjPermAndStatInp.condInput );
// =-=-=-=-=-=-=-
// create the special hidden directory where the bundling happens
createPhyBundleDir( rsComm, L1desc[ l1descInx ].dataObjInfo->filePath, phyBunDir, L1desc[ l1descInx ].dataObjInfo->rescHier );
// =-=-=-=-=-=-=-
// build a collection open input structure
collInp_t collInp;
bzero( &collInp, sizeof( collInp ) );
collInp.flags = RECUR_QUERY_FG | VERY_LONG_METADATA_FG | NO_TRIM_REPL_FG | INCLUDE_CONDINPUT_IN_QUERY;
rstrcpy( collInp.collName, structFileBundleInp->collection, MAX_NAME_LEN );
addKeyVal( &collInp.condInput, RESC_NAME_KW, L1desc[ l1descInx ].dataObjInfo->rescName );
rodsLog( LOG_DEBUG, "rsStructFileBundle: calling rsOpenCollection for [%s]", structFileBundleInp->collection );
// =-=-=-=-=-=-=-
// open the collection from which we will bundle
handleInx = rsOpenCollection( rsComm, &collInp );
if ( handleInx < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: rsOpenCollection of %s error. status = %d",
collInp.collName, handleInx );
rmdir( phyBunDir );
return handleInx;
}
// =-=-=-=-=-=-=-
// preserve the collection path?
int collLen = 0;
if ( ( structFileBundleInp->oprType & PRESERVE_COLL_PATH ) != 0 ) {
// =-=-=-=-=-=-=-
// preserve the last entry of the coll path
char* tmpPtr = collInp.collName;
int tmpLen = 0;
collLen = 0;
// =-=-=-=-=-=-=-
// find length to the last '/'
while ( *tmpPtr != '\0' ) {
if ( *tmpPtr == '/' ) {
collLen = tmpLen;
}
tmpLen++;
tmpPtr++;
}
}
else {
collLen = strlen( collInp.collName );
}
// =-=-=-=-=-=-=-
// preserve the collection path?
collEnt_t* collEnt = NULL;
while ( ( status = rsReadCollection( rsComm, &handleInx, &collEnt ) ) >= 0 ) {
if ( NULL == collEnt ) {
rodsLog(
LOG_ERROR,
"rsStructFileBundle: collEnt is NULL" );
continue;
}
// =-=-=-=-=-=-=-
// entry is a data object
if ( collEnt->objType == DATA_OBJ_T ) {
// =-=-=-=-=-=-=-
// filter out any possible replicas that are not on this resource
if ( resc_hier != collEnt->resc_hier ) {
rodsLog(
LOG_DEBUG,
"_rsStructFileBundle - skipping [%s] on resc [%s]",
collEnt->phyPath,
collEnt->resc_hier );
free( collEnt );
collEnt = NULL;
continue;
}
if ( collEnt->collName[collLen] == '\0' ) {
snprintf(
tmpPath,
MAX_NAME_LEN,
"%s/%s",
phyBunDir,
collEnt->dataName );
}
else {
snprintf(
tmpPath,
MAX_NAME_LEN, "%s/%s/%s",
phyBunDir,
collEnt->collName + collLen + 1,
collEnt->dataName );
status = mkDirForFilePath(
rsComm,
strlen( phyBunDir ),
tmpPath,
collEnt->resc_hier,
getDefDirMode() );
if ( status < 0 ) {
rodsLog(
LOG_ERROR,
"mkDirForFilePath failed in _rsStructFileBundle with status %d",
status );
free( collEnt );
return status;
}
}
// =-=-=-=-=-=-=-
// add a link
status = link( collEnt->phyPath, tmpPath );
if ( status < 0 ) {
rodsLog(
LOG_ERROR,
"rsStructFileBundle: link error %s to %s. errno = %d",
collEnt->phyPath,
tmpPath,
errno );
rmLinkedFilesInUnixDir( phyBunDir );
rmdir( phyBunDir );
free( collEnt );
return UNIX_FILE_LINK_ERR - errno;
}
else {
rodsLog(
LOG_DEBUG,
"_rsStructFileBundle - LINK [%s] on resc [%s]",
collEnt->phyPath,
collEnt->resc_hier );
}
}
else {
// =-=-=-=-=-=-=-
// entry is a collection
if ( ( int ) strlen( collEnt->collName ) + 1 <= collLen ) {
free( collEnt );
collEnt = NULL;
continue;
}
snprintf( tmpPath, MAX_NAME_LEN, "%s/%s", phyBunDir, collEnt->collName + collLen );
status = mkFileDirR( rsComm, strlen( phyBunDir ), tmpPath, resc_hier.c_str(), getDefDirMode() );
if ( status < 0 ) {
rodsLog( LOG_ERROR, "mkFileDirR failed in _rsStructFileBundle with status %d", status );
free( collEnt );
return status;
}
} // else
free( collEnt );
collEnt = NULL;
} // while
// =-=-=-=-=-=-=-
// clean up key vals and close the collection
clearKeyVal( &collInp.condInput );
rsCloseCollection( rsComm, &handleInx );
// =-=-=-=-=-=-=-
// call the helper function to do the actual bundling
status = phyBundle( rsComm, L1desc[l1descInx].dataObjInfo, phyBunDir,
collInp.collName, structFileBundleInp->oprType ); // JMC - backport 4643
int savedStatus = 0;
if ( status < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: phyBundle of %s error. stat = %d",
L1desc[ l1descInx ].dataObjInfo->objPath, status );
L1desc[ l1descInx ].bytesWritten = 0;
savedStatus = status;
}
else {
// mark it was written so the size would be adjusted
L1desc[ l1descInx ].bytesWritten = 1;
}
// =-=-=-=-=-=-=-
// clean up after the bundle directory
rmLinkedFilesInUnixDir( phyBunDir );
rmdir( phyBunDir );
dataObjCloseInp.l1descInx = l1descInx;
status = rsDataObjClose( rsComm, &dataObjCloseInp );
if ( status >= 0 ) {
return savedStatus;
}
return status;
}
int
rsStructFileExtAndReg (rsComm_t *rsComm,
structFileExtAndRegInp_t *structFileExtAndRegInp)
{
int status;
dataObjInp_t dataObjInp;
openedDataObjInp_t dataObjCloseInp;
dataObjInfo_t *dataObjInfo;
int l1descInx;
rescInfo_t *rescInfo;
char *rescGroupName;
int remoteFlag;
rodsServerHost_t *rodsServerHost;
char phyBunDir[MAX_NAME_LEN], *tmpStr;
int flags = 0;
#if 0
dataObjInp_t dirRegInp;
structFileOprInp_t structFileOprInp;
#endif
specCollCache_t *specCollCache = NULL;
resolveLinkedPath (rsComm, structFileExtAndRegInp->objPath, &specCollCache,
&structFileExtAndRegInp->condInput);
resolveLinkedPath (rsComm, structFileExtAndRegInp->collection,
&specCollCache, NULL);
if (!isSameZone (structFileExtAndRegInp->objPath,
structFileExtAndRegInp->collection))
return SYS_CROSS_ZONE_MV_NOT_SUPPORTED;
memset (&dataObjInp, 0, sizeof (dataObjInp));
rstrcpy (dataObjInp.objPath, structFileExtAndRegInp->objPath,
MAX_NAME_LEN);
/* replicate the condInput. may have resource input */
replKeyVal (&structFileExtAndRegInp->condInput, &dataObjInp.condInput);
dataObjInp.openFlags = O_RDONLY;
remoteFlag = getAndConnRemoteZone (rsComm, &dataObjInp, &rodsServerHost,
REMOTE_OPEN);
if (remoteFlag < 0) {
return (remoteFlag);
} else if (remoteFlag == REMOTE_HOST) {
status = rcStructFileExtAndReg (rodsServerHost->conn,
structFileExtAndRegInp);
return status;
}
/* open the structured file */
addKeyVal (&dataObjInp.condInput, NO_OPEN_FLAG_KW, "");
l1descInx = _rsDataObjOpen (rsComm, &dataObjInp);
if (l1descInx < 0) {
rodsLog (LOG_ERROR,
"rsStructFileExtAndReg: _rsDataObjOpen of %s error. status = %d",
dataObjInp.objPath, l1descInx);
return (l1descInx);
}
rescInfo = L1desc[l1descInx].dataObjInfo->rescInfo;
rescGroupName = L1desc[l1descInx].dataObjInfo->rescGroupName;
remoteFlag = resolveHostByRescInfo (rescInfo, &rodsServerHost);
bzero (&dataObjCloseInp, sizeof (dataObjCloseInp));
dataObjCloseInp.l1descInx = l1descInx;
if (remoteFlag == REMOTE_HOST) {
addKeyVal (&structFileExtAndRegInp->condInput, RESC_NAME_KW,
rescInfo->rescName);
if ((status = svrToSvrConnect (rsComm, rodsServerHost)) < 0) {
return status;
}
status = rcStructFileExtAndReg (rodsServerHost->conn,
structFileExtAndRegInp);
rsDataObjClose (rsComm, &dataObjCloseInp);
return status;
}
status = chkCollForExtAndReg (rsComm, structFileExtAndRegInp->collection,
NULL);
if (status < 0) return status;
dataObjInfo = L1desc[l1descInx].dataObjInfo;
createPhyBundleDir (rsComm, dataObjInfo->filePath, phyBunDir);
status = unbunPhyBunFile (rsComm, dataObjInp.objPath, rescInfo,
dataObjInfo->filePath, phyBunDir, dataObjInfo->dataType, 0);
if (status == SYS_DIR_IN_VAULT_NOT_EMPTY) {
/* rename the phyBunDir */
tmpStr = strdup(phyBunDir); // cppcheck - Undefined behavior: same parameter and destination in snprintf().
snprintf (phyBunDir, MAX_NAME_LEN, "%s.%-d", tmpStr, (int) random ());
free(tmpStr);
status = unbunPhyBunFile (rsComm, dataObjInp.objPath, rescInfo,
dataObjInfo->filePath, phyBunDir, dataObjInfo->dataType, 0);
}
if (status < 0) {
rodsLog (LOG_ERROR,
"rsStructFileExtAndReg:unbunPhyBunFile err for %s to dir %s.stat=%d",
dataObjInfo->filePath, phyBunDir, status);
rsDataObjClose (rsComm, &dataObjCloseInp);
return status;
}
if (getValByKey (&structFileExtAndRegInp->condInput, FORCE_FLAG_KW)
!= NULL) {
flags = flags | FORCE_FLAG_FLAG;
}
if (getValByKey (&structFileExtAndRegInp->condInput, BULK_OPR_KW)
!= NULL) {
status = bulkRegUnbunSubfiles (rsComm, rescInfo, rescGroupName,
structFileExtAndRegInp->collection, phyBunDir, flags, NULL);
} else {
status = regUnbunSubfiles (rsComm, rescInfo, rescGroupName,
structFileExtAndRegInp->collection, phyBunDir, flags, NULL);
}
if (status == CAT_NO_ROWS_FOUND) {
/* some subfiles have been deleted. harmless */
status = 0;
} else if (status < 0) {
rodsLog (LOG_ERROR,
"_rsUnbunAndRegPhyBunfile: rsStructFileExtAndReg for dir %s.stat=%d",
phyBunDir, status);
}
rsDataObjClose (rsComm, &dataObjCloseInp);
return status;
}
int
msiLoadMetadataFromXml(msParam_t *targetObj, msParam_t *xmlObj, ruleExecInfo_t *rei)
{
/* for parsing msParams and to open iRODS objects */
dataObjInp_t xmlDataObjInp, *myXmlDataObjInp;
dataObjInp_t targetObjInp, *myTargetObjInp;
int xmlObjID;
/* for getting size of objects to read from */
rodsObjStat_t *rodsObjStatOut = NULL;
/* for reading from iRODS objects */
openedDataObjInp_t openedDataObjInp;
bytesBuf_t *xmlBuf;
/* misc. to avoid repeating rei->rsComm */
rsComm_t *rsComm;
/* for xml parsing */
xmlDocPtr doc;
/* for XPath evaluation */
xmlXPathContextPtr xpathCtx;
xmlXPathObjectPtr xpathObj;
xmlChar xpathExpr[] = "//AVU";
xmlNodeSetPtr nodes;
int avuNbr, i;
/* for new AVU creation */
modAVUMetadataInp_t modAVUMetadataInp;
int max_attr_len = 2700;
char attrStr[max_attr_len];
/********************************* USUAL INIT PROCEDURE **********************************/
/* For testing mode when used with irule --test */
RE_TEST_MACRO (" Calling msiLoadMetadataFromXml")
/* Sanity checks */
if (rei == NULL || rei->rsComm == NULL)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: input rei or rsComm is NULL.");
return (SYS_INTERNAL_NULL_INPUT_ERR);
}
rsComm = rei->rsComm;
/********************************** RETRIEVE INPUT PARAMS **************************************/
/* Get path of target object */
rei->status = parseMspForDataObjInp (targetObj, &targetObjInp, &myTargetObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: input targetObj error. status = %d", rei->status);
return (rei->status);
}
/* Get path of XML document */
rei->status = parseMspForDataObjInp (xmlObj, &xmlDataObjInp, &myXmlDataObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: input xmlObj error. status = %d", rei->status);
return (rei->status);
}
/******************************** OPEN AND READ FROM XML OBJECT ********************************/
/* Open XML file */
if ((xmlObjID = rsDataObjOpen(rsComm, &xmlDataObjInp)) < 0)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Cannot open XML data object. status = %d", xmlObjID);
return (xmlObjID);
}
/* Get size of XML file */
rei->status = rsObjStat (rsComm, &xmlDataObjInp, &rodsObjStatOut);
if (rei->status < 0 || !rodsObjStatOut)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Cannot stat XML data object. status = %d", rei->status);
return (rei->status);
}
/* xmlBuf init */
/* memory for xmlBuf->buf is allocated in rsFileRead() */
xmlBuf = (bytesBuf_t *) malloc (sizeof (bytesBuf_t));
memset (xmlBuf, 0, sizeof (bytesBuf_t));
/* Read XML file */
memset (&openedDataObjInp, 0, sizeof (openedDataObjInp_t));
openedDataObjInp.l1descInx = xmlObjID;
openedDataObjInp.len = (int)rodsObjStatOut->objSize;
rei->status = rsDataObjRead (rsComm, &openedDataObjInp, xmlBuf);
/* Make sure that the result is null terminated */
if (strlen((char*)xmlBuf->buf) > (size_t)openedDataObjInp.len)
{
((char*)xmlBuf->buf)[openedDataObjInp.len-1]='\0';
}
/* Close XML file */
rei->status = rsDataObjClose (rsComm, &openedDataObjInp);
/* cleanup */
freeRodsObjStat (rodsObjStatOut);
/******************************** PARSE XML DOCUMENT ********************************/
xmlSubstituteEntitiesDefault(1);
xmlLoadExtDtdDefaultValue = 1;
/* Parse xmlBuf.buf into an xmlDocPtr */
doc = xmlParseDoc((xmlChar*)xmlBuf->buf);
/* Create xpath evaluation context */
xpathCtx = xmlXPathNewContext(doc);
if(xpathCtx == NULL)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Unable to create new XPath context.");
xmlFreeDoc(doc);
return(-1);
}
/* Evaluate xpath expression */
xpathObj = xmlXPathEvalExpression(xpathExpr, xpathCtx);
if(xpathObj == NULL)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Unable to evaluate XPath expression \"%s\".", xpathExpr);
xmlXPathFreeContext(xpathCtx);
xmlFreeDoc(doc);
return(-1);
}
/* How many AVU nodes did we get? */
if ((nodes = xpathObj->nodesetval) != NULL)
{
avuNbr = nodes->nodeNr;
}
else
{
avuNbr = 0;
}
/******************************** CREATE AVU TRIPLETS ********************************/
/* Add a new AVU for each node. It's ok to process the nodes in forward order since we're not modifying them */
for(i = 0; i < avuNbr; i++)
{
if (nodes->nodeTab[i])
{
// /* temporary: Add index number to avoid duplicating attribute names */
// memset(attrStr, '\0', MAX_NAME_LEN);
// snprintf(attrStr, MAX_NAME_LEN - 1, "%04d: %s", i+1, (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Attribute")) );
/* Truncate if needed. No prefix. */
memset(attrStr, '\0', max_attr_len);
snprintf(attrStr, max_attr_len - 1, "%s", (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Attribute")) );
/* init modAVU input */
memset (&modAVUMetadataInp, 0, sizeof(modAVUMetadataInp_t));
modAVUMetadataInp.arg0 = "add";
modAVUMetadataInp.arg1 = "-d";
/* Use target object if one was provided, otherwise look for it in the XML doc */
if (myTargetObjInp->objPath != NULL && strlen(myTargetObjInp->objPath) > 0)
{
modAVUMetadataInp.arg2 = myTargetObjInp->objPath;
}
else
{
modAVUMetadataInp.arg2 = xmlURIUnescapeString((char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Target")),
MAX_NAME_LEN, NULL);
}
modAVUMetadataInp.arg3 = attrStr;
modAVUMetadataInp.arg4 = (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Value"));
modAVUMetadataInp.arg5 = (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Unit"));
/* invoke rsModAVUMetadata() */
rei->status = rsModAVUMetadata (rsComm, &modAVUMetadataInp);
}
}
/************************************** WE'RE DONE **************************************/
/* cleanup of all xml parsing stuff */
xmlFreeDoc(doc);
xmlCleanupParser();
return (rei->status);
}
/**
* \fn msiXmlDocSchemaValidate(msParam_t *xmlObj, msParam_t *xsdObj, msParam_t *status, ruleExecInfo_t *rei)
*
* \brief This microservice validates an XML file against an XSD schema, both iRODS objects.
*
* \module xml
*
* \since pre-2.1
*
* \author Antoine de Torcy
* \date 2008/05/29
*
* \usage See clients/icommands/test/rules3.0/
*
* \param[in] xmlObj - a msParam of type DataObjInp_MS_T or STR_MS_T which is irods path of the XML object.
* \param[in] xsdObj - a msParam of type DataObjInp_MS_T or STR_MS_T which is irods path of the XSD object.
* \param[out] status - a msParam of type INT_MS_T which is a validation result.
* \param[in,out] rei - The RuleExecInfo structure that is automatically
* handled by the rule engine. The user does not include rei as a
* parameter in the rule invocation.
*
* \DolVarDependence None
* \DolVarModified None
* \iCatAttrDependence None
* \iCatAttrModified None
* \sideeffect None
*
* \return integer
* \retval 0 on success
* \pre None
* \post None
* \sa None
**/
int
msiXmlDocSchemaValidate(msParam_t *xmlObj, msParam_t *xsdObj, msParam_t *status, ruleExecInfo_t *rei)
{
/* for parsing msParams and to open iRODS objects */
dataObjInp_t xmlObjInp, *myXmlObjInp;
dataObjInp_t xsdObjInp, *myXsdObjInp;
int xmlObjID, xsdObjID;
/* for getting size of objects to read from */
rodsObjStat_t *rodsObjStatOut = NULL;
/* for reading from iRODS objects */
openedDataObjInp_t openedDataObjInp;
bytesBuf_t *xmlBuf = NULL;
char *tail;
/* for xml parsing and validating */
xmlDocPtr doc, xsd_doc;
xmlSchemaParserCtxtPtr parser_ctxt;
xmlSchemaPtr schema;
xmlSchemaValidCtxtPtr valid_ctxt;
bytesBuf_t *errBuf;
/* misc. to avoid repeating rei->rsComm */
rsComm_t *rsComm;
/************************************* USUAL INIT PROCEDURE **********************************/
/* For testing mode when used with irule --test */
RE_TEST_MACRO (" Calling msiXmlDocSchemaValidate")
/* Sanity checks */
if (rei == NULL || rei->rsComm == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: input rei or rsComm is NULL.");
return (SYS_INTERNAL_NULL_INPUT_ERR);
}
rsComm = rei->rsComm;
/************************************ ADDITIONAL INIT SETTINGS *********************************/
/* XML constants */
xmlSubstituteEntitiesDefault(1);
xmlLoadExtDtdDefaultValue = 1;
/* allocate memory for output error buffer */
errBuf = (bytesBuf_t *)malloc(sizeof(bytesBuf_t));
errBuf->buf = strdup("");
errBuf->len = strlen((char*)errBuf->buf);
/* Default status is failure, overwrite if success */
fillBufLenInMsParam (status, -1, NULL);
/********************************** RETRIEVE INPUT PARAMS **************************************/
/* Get path of XML document */
rei->status = parseMspForDataObjInp (xmlObj, &xmlObjInp, &myXmlObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: input xmlObj error. status = %d", rei->status);
free(errBuf);
return (rei->status);
}
/* Get path of schema */
rei->status = parseMspForDataObjInp (xsdObj, &xsdObjInp, &myXsdObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: input xsdObj error. status = %d", rei->status);
free(errBuf);
return (rei->status);
}
/******************************** OPEN AND READ FROM XML OBJECT ********************************/
/* Open XML file */
if ((xmlObjID = rsDataObjOpen(rsComm, &xmlObjInp)) < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Cannot open XML data object. status = %d", xmlObjID);
free(errBuf);
return (xmlObjID);
}
/* Get size of XML file */
rei->status = rsObjStat (rsComm, &xmlObjInp, &rodsObjStatOut);
if (rei->status < 0 || !rodsObjStatOut)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Cannot stat XML data object. status = %d", rei->status);
free(errBuf);
return (rei->status);
}
/* xmlBuf init */
/* memory for xmlBuf->buf is allocated in rsFileRead() */
xmlBuf = (bytesBuf_t *) malloc (sizeof (bytesBuf_t));
memset (xmlBuf, 0, sizeof (bytesBuf_t));
/* Read content of XML file */
memset (&openedDataObjInp, 0, sizeof (openedDataObjInp_t));
openedDataObjInp.l1descInx = xmlObjID;
openedDataObjInp.len = (int)rodsObjStatOut->objSize + 1; /* extra byte to add a null char */
rei->status = rsDataObjRead (rsComm, &openedDataObjInp, xmlBuf);
/* add terminating null character */
tail = (char*)xmlBuf->buf;
tail[openedDataObjInp.len - 1] = '\0';
/* Close XML file */
rei->status = rsDataObjClose (rsComm, &openedDataObjInp);
/* cleanup */
freeRodsObjStat (rodsObjStatOut);
/*************************************** PARSE XML DOCUMENT **************************************/
/* Parse xmlBuf.buf into an xmlDocPtr */
doc = xmlParseDoc((xmlChar*)xmlBuf->buf);
clearBBuf(xmlBuf);
if (doc == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: XML document cannot be loaded or is not well-formed.");
free(errBuf);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/******************************** OPEN AND READ FROM XSD OBJECT ********************************/
/* Open schema file */
if ((xsdObjID = rsDataObjOpen(rsComm, &xsdObjInp)) < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Cannot open XSD data object. status = %d", xsdObjID);
free(errBuf);
xmlFreeDoc(doc);
xmlCleanupParser();
return (xsdObjID);
}
/* Get size of schema file */
rei->status = rsObjStat (rsComm, &xsdObjInp, &rodsObjStatOut);
/* Read entire schema file */
memset (&openedDataObjInp, 0, sizeof (openedDataObjInp_t));
openedDataObjInp.l1descInx = xsdObjID;
openedDataObjInp.len = (int)rodsObjStatOut->objSize + 1; /* to add null char */
rei->status = rsDataObjRead (rsComm, &openedDataObjInp, xmlBuf);
/* add terminating null character */
tail = (char*)xmlBuf->buf;
tail[openedDataObjInp.len - 1] = '\0';
/* Close schema file */
rei->status = rsDataObjClose (rsComm, &openedDataObjInp);
/* cleanup */
freeRodsObjStat (rodsObjStatOut);
/*************************************** PARSE XSD DOCUMENT **************************************/
/* Parse xmlBuf.buf into an xmlDocPtr */
xsd_doc = xmlParseDoc((xmlChar*)xmlBuf->buf);
clearBBuf(xmlBuf);
if (xsd_doc == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: XML Schema cannot be loaded or is not well-formed.");
free(errBuf);
xmlFreeDoc(doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/**************************************** VALIDATE DOCUMENT **************************************/
/* Create a parser context */
parser_ctxt = xmlSchemaNewDocParserCtxt(xsd_doc);
if (parser_ctxt == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Unable to create a parser context for the schema.");
free(errBuf);
xmlFreeDoc(xsd_doc);
xmlFreeDoc(doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/* Parse the XML schema */
schema = xmlSchemaParse(parser_ctxt);
if (schema == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Invalid schema.");
free(errBuf);
xmlSchemaFreeParserCtxt(parser_ctxt);
xmlFreeDoc(doc);
xmlFreeDoc(xsd_doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/* Create a validation context */
valid_ctxt = xmlSchemaNewValidCtxt(schema);
if (valid_ctxt == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Unable to create a validation context for the schema.");
free(errBuf);
xmlSchemaFree(schema);
xmlSchemaFreeParserCtxt(parser_ctxt);
xmlFreeDoc(xsd_doc);
xmlFreeDoc(doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/* Set myErrorCallback() as the default handler for error messages and warnings */
xmlSchemaSetValidErrors(valid_ctxt, (xmlSchemaValidityErrorFunc)myErrorCallback, (xmlSchemaValidityWarningFunc)myErrorCallback, errBuf);
/* Validate XML doc */
rei->status = xmlSchemaValidateDoc(valid_ctxt, doc);
/******************************************* WE'RE DONE ******************************************/
/* return both error code and messages through status */
resetMsParam (status);
fillBufLenInMsParam (status, rei->status, errBuf);
/* cleanup of all xml parsing stuff */
xmlSchemaFreeValidCtxt(valid_ctxt);
xmlSchemaFree(schema);
xmlSchemaFreeParserCtxt(parser_ctxt);
xmlFreeDoc(doc);
xmlFreeDoc(xsd_doc);
xmlCleanupParser();
return (rei->status);
}
int
structFileReg (rsComm_t *rsComm, dataObjInp_t *phyPathRegInp)
{
collInp_t collCreateInp;
int status;
dataObjInfo_t *dataObjInfo = NULL;
char *structFilePath = NULL;
dataObjInp_t dataObjInp;
char *collType;
int len;
rodsObjStat_t *rodsObjStatOut = NULL;
specCollCache_t *specCollCache = NULL;
rescInfo_t *rescInfo = NULL;
#if 0 /* fixed */
/* make it a privileged call for now */
if (rsComm->clientUser.authInfo.authFlag < LOCAL_PRIV_USER_AUTH)
return(CAT_INSUFFICIENT_PRIVILEGE_LEVEL);
#endif
if ((structFilePath = getValByKey (&phyPathRegInp->condInput, FILE_PATH_KW))
== NULL) {
rodsLog (LOG_ERROR,
"structFileReg: No structFilePath input for %s",
phyPathRegInp->objPath);
return (SYS_INVALID_FILE_PATH);
}
collType = getValByKey (&phyPathRegInp->condInput, COLLECTION_TYPE_KW);
if (collType == NULL) {
rodsLog (LOG_ERROR,
"structFileReg: Bad COLLECTION_TYPE_KW for structFilePath %s",
dataObjInp.objPath);
return (SYS_INTERNAL_NULL_INPUT_ERR);
}
len = strlen (phyPathRegInp->objPath);
if (strncmp (structFilePath, phyPathRegInp->objPath, len) == 0 &&
(structFilePath[len] == '\0' || structFilePath[len] == '/')) {
rodsLog (LOG_ERROR,
"structFileReg: structFilePath %s inside collection %s",
structFilePath, phyPathRegInp->objPath);
return (SYS_STRUCT_FILE_INMOUNTED_COLL);
}
/* see if the struct file is in spec coll */
if (getSpecCollCache (rsComm, structFilePath, 0, &specCollCache) >= 0) {
rodsLog (LOG_ERROR,
"structFileReg: structFilePath %s is in a mounted path",
structFilePath);
return (SYS_STRUCT_FILE_INMOUNTED_COLL);
}
status = collStat (rsComm, phyPathRegInp, &rodsObjStatOut);
if (status < 0) return status;
if (rodsObjStatOut->specColl != NULL) {
freeRodsObjStat (rodsObjStatOut);
rodsLog (LOG_ERROR,
"structFileReg: %s already mounted", phyPathRegInp->objPath);
return (SYS_MOUNT_MOUNTED_COLL_ERR);
}
freeRodsObjStat (rodsObjStatOut);
if (isCollEmpty (rsComm, phyPathRegInp->objPath) == False) {
rodsLog (LOG_ERROR,
"structFileReg: collection %s not empty", phyPathRegInp->objPath);
return (SYS_COLLECTION_NOT_EMPTY);
}
memset (&dataObjInp, 0, sizeof (dataObjInp));
rstrcpy (dataObjInp.objPath, structFilePath, sizeof (dataObjInp));
/* user need to have write permission */
dataObjInp.openFlags = O_WRONLY;
status = getDataObjInfoIncSpecColl (rsComm, &dataObjInp, &dataObjInfo);
if (status < 0) {
int myStatus;
/* try to make one */
dataObjInp.condInput = phyPathRegInp->condInput;
/* have to remove FILE_PATH_KW because getFullPathName will use it */
rmKeyVal (&dataObjInp.condInput, FILE_PATH_KW);
myStatus = rsDataObjCreate (rsComm, &dataObjInp);
if (myStatus < 0) {
rodsLog (LOG_ERROR,
"structFileReg: Problem with open/create structFilePath %s, status = %d",
dataObjInp.objPath, status);
return (status);
} else {
openedDataObjInp_t dataObjCloseInp;
bzero (&dataObjCloseInp, sizeof (dataObjCloseInp));
rescInfo = L1desc[myStatus].dataObjInfo->rescInfo;
dataObjCloseInp.l1descInx = myStatus;
rsDataObjClose (rsComm, &dataObjCloseInp);
}
} else {
rescInfo = dataObjInfo->rescInfo;
}
if (!structFileSupport (rsComm, phyPathRegInp->objPath,
collType, rescInfo)) {
rodsLog (LOG_ERROR,
"structFileReg: structFileDriver type %s does not exist for %s",
collType, dataObjInp.objPath);
return (SYS_NOT_SUPPORTED);
}
/* mk the collection */
memset (&collCreateInp, 0, sizeof (collCreateInp));
rstrcpy (collCreateInp.collName, phyPathRegInp->objPath, MAX_NAME_LEN);
addKeyVal (&collCreateInp.condInput, COLLECTION_TYPE_KW, collType);
/* have to use dataObjInp.objPath because structFile path was removed */
addKeyVal (&collCreateInp.condInput, COLLECTION_INFO1_KW,
dataObjInp.objPath);
/* try to mod the coll first */
status = rsModColl (rsComm, &collCreateInp);
if (status < 0) { /* try to create it */
status = rsRegColl (rsComm, &collCreateInp);
}
return (status);
}
int _writeString( char *writeId, char *writeStr, ruleExecInfo_t *rei ) {
// =-=-=-=-=-=-=-
// JMC - backport 4619
dataObjInp_t dataObjInp;
openedDataObjInp_t openedDataObjInp;
bytesBuf_t tmpBBuf;
int fd, i;
// =-=-=-=-=-=-=-
if ( writeId != NULL && strcmp( writeId, "serverLog" ) == 0 ) {
rodsLog( LOG_NOTICE, "writeString: inString = %s", writeStr );
return 0;
}
// =-=-=-=-=-=-=-
// JMC - backport 4619
if ( writeId != NULL && writeId[0] == '/' ) {
/* writing to an existing iRODS file */
if ( rei == NULL || rei->rsComm == NULL ) {
rodsLog( LOG_ERROR, "_writeString: input rei or rsComm is NULL" );
return SYS_INTERNAL_NULL_INPUT_ERR;
}
bzero( &dataObjInp, sizeof( dataObjInp ) );
dataObjInp.openFlags = O_RDWR;
snprintf( dataObjInp.objPath, MAX_NAME_LEN, "%s", writeId );
fd = rsDataObjOpen( rei->rsComm, &dataObjInp );
if ( fd < 0 ) {
rodsLog( LOG_ERROR, "_writeString: rsDataObjOpen failed. status = %d", fd );
return fd;
}
bzero( &openedDataObjInp, sizeof( openedDataObjInp ) );
openedDataObjInp.l1descInx = fd;
openedDataObjInp.offset = 0;
openedDataObjInp.whence = SEEK_END;
fileLseekOut_t *dataObjLseekOut = NULL;
i = rsDataObjLseek( rei->rsComm, &openedDataObjInp, &dataObjLseekOut );
free( dataObjLseekOut );
if ( i < 0 ) {
rodsLog( LOG_ERROR, "_writeString: rsDataObjLseek failed. status = %d", i );
return i;
}
bzero( &openedDataObjInp, sizeof( openedDataObjInp ) );
openedDataObjInp.l1descInx = fd;
tmpBBuf.len = openedDataObjInp.len = strlen( writeStr ) + 1;
tmpBBuf.buf = writeStr;
i = rsDataObjWrite( rei->rsComm, &openedDataObjInp, &tmpBBuf );
if ( i < 0 ) {
rodsLog( LOG_ERROR, "_writeString: rsDataObjWrite failed. status = %d", i );
return i;
}
bzero( &openedDataObjInp, sizeof( openedDataObjInp ) );
openedDataObjInp.l1descInx = fd;
i = rsDataObjClose( rei->rsComm, &openedDataObjInp );
return i;
}
// =-=-=-=-=-=-=-
msParam_t * mP = NULL;
msParamArray_t * inMsParamArray = rei->msParamArray;
execCmdOut_t *myExecCmdOut;
if ( ( ( mP = getMsParamByLabel( inMsParamArray, "ruleExecOut" ) ) != NULL ) &&
( mP->inOutStruct != NULL ) ) {
if ( !strcmp( mP->type, STR_MS_T ) ) {
myExecCmdOut = ( execCmdOut_t* )malloc( sizeof( execCmdOut_t ) );
memset( myExecCmdOut, 0, sizeof( execCmdOut_t ) );
mP->inOutStruct = myExecCmdOut;
mP->type = strdup( ExecCmdOut_MS_T );
}
else {
myExecCmdOut = ( execCmdOut_t* )mP->inOutStruct;
}
}
else {
myExecCmdOut = ( execCmdOut_t* )malloc( sizeof( execCmdOut_t ) );
memset( myExecCmdOut, 0, sizeof( execCmdOut_t ) );
if ( mP == NULL ) {
addMsParam( inMsParamArray, "ruleExecOut", ExecCmdOut_MS_T, myExecCmdOut, NULL );
}
else {
mP->inOutStruct = myExecCmdOut;
mP->type = strdup( ExecCmdOut_MS_T );
}
}
/***** Jun 27, 2007
i = replaceVariablesAndMsParams("",writeStr, rei->msParamArray, rei);
if (i < 0)
return i;
****/
if ( writeId != NULL ) {
if ( !strcmp( writeId, "stdout" ) ) {
appendToByteBuf( &( myExecCmdOut->stdoutBuf ), ( char * ) writeStr );
}
else if ( !strcmp( writeId, "stderr" ) ) {
appendToByteBuf( &( myExecCmdOut->stderrBuf ), ( char * ) writeStr );
}
}
return 0;
}
int
_rsStructFileBundle (rsComm_t *rsComm,
structFileExtAndRegInp_t *structFileBundleInp)
{
int status;
dataObjInp_t dataObjInp;
openedDataObjInp_t dataObjCloseInp;
collInp_t collInp;
collEnt_t *collEnt = NULL;
int handleInx;
int collLen;
char phyBunDir[MAX_NAME_LEN];
char tmpPath[MAX_NAME_LEN];
chkObjPermAndStat_t chkObjPermAndStatInp;
int l1descInx;
int savedStatus = 0;
char *dataType;
/* open the structured file */
memset (&dataObjInp, 0, sizeof (dataObjInp));
dataType = getValByKey (&structFileBundleInp->condInput, DATA_TYPE_KW);
if (dataType != NULL && strstr (dataType, ZIP_DT_STR) != NULL) {
/* zipFile type. must end with .zip */
int len = strlen (structFileBundleInp->objPath);
if (strcmp (&structFileBundleInp->objPath[len - 4], ".zip") != 0) {
strcat (structFileBundleInp->objPath, ".zip");
}
}
rstrcpy (dataObjInp.objPath, structFileBundleInp->objPath,
MAX_NAME_LEN);
/* replicate the condInput. may have resource input */
replKeyVal (&structFileBundleInp->condInput, &dataObjInp.condInput);
dataObjInp.openFlags = O_WRONLY;
if ((structFileBundleInp->oprType & ADD_TO_TAR_OPR) != 0) {
l1descInx = rsDataObjOpen (rsComm, &dataObjInp);
} else {
l1descInx = rsDataObjCreate (rsComm, &dataObjInp);
}
if (l1descInx < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: rsDataObjCreate of %s error. status = %d",
dataObjInp.objPath, l1descInx);
return (l1descInx);
}
clearKeyVal (&dataObjInp.condInput);
l3Close (rsComm, l1descInx);
L1desc[l1descInx].l3descInx = 0;
/* zip does not like a zero length file as target */
if ((structFileBundleInp->oprType & ADD_TO_TAR_OPR) == 0)
l3Unlink (rsComm, L1desc[l1descInx].dataObjInfo);
memset (&chkObjPermAndStatInp, 0, sizeof (chkObjPermAndStatInp));
rstrcpy (chkObjPermAndStatInp.objPath,
structFileBundleInp->collection, MAX_NAME_LEN);
chkObjPermAndStatInp.flags = CHK_COLL_FOR_BUNDLE_OPR;
addKeyVal (&chkObjPermAndStatInp.condInput, RESC_NAME_KW,
L1desc[l1descInx].dataObjInfo->rescName);
status = rsChkObjPermAndStat (rsComm, &chkObjPermAndStatInp);
clearKeyVal (&chkObjPermAndStatInp.condInput);
if (status < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: rsChkObjPermAndStat of %s error. stat = %d",
chkObjPermAndStatInp.objPath, status);
dataObjCloseInp.l1descInx = l1descInx;
rsDataObjClose (rsComm, &dataObjCloseInp);
return (status);
}
createPhyBundleDir (rsComm, L1desc[l1descInx].dataObjInfo->filePath,
phyBunDir);
bzero (&collInp, sizeof (collInp));
rstrcpy (collInp.collName, structFileBundleInp->collection, MAX_NAME_LEN);
collInp.flags = RECUR_QUERY_FG | VERY_LONG_METADATA_FG |
NO_TRIM_REPL_FG | INCLUDE_CONDINPUT_IN_QUERY;
addKeyVal (&collInp.condInput, RESC_NAME_KW,
L1desc[l1descInx].dataObjInfo->rescName);
handleInx = rsOpenCollection (rsComm, &collInp);
if (handleInx < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: rsOpenCollection of %s error. status = %d",
collInp.collName, handleInx);
rmdir (phyBunDir);
return (handleInx);
}
if ((structFileBundleInp->oprType & PRESERVE_COLL_PATH) != 0) {
/* preserver the last entry of the coll path */
char *tmpPtr = collInp.collName;
int tmpLen = 0;
collLen = 0;
/* find length to the last '/' */
while (*tmpPtr != '\0') {
if (*tmpPtr == '/') collLen = tmpLen;
tmpLen++;
tmpPtr++;
}
} else {
collLen = strlen (collInp.collName);
}
while ((status = rsReadCollection (rsComm, &handleInx, &collEnt)) >= 0) {
if (collEnt->objType == DATA_OBJ_T) {
if (collEnt->collName[collLen] == '\0') {
snprintf (tmpPath, MAX_NAME_LEN, "%s/%s",
phyBunDir, collEnt->dataName);
} else {
snprintf (tmpPath, MAX_NAME_LEN, "%s/%s/%s",
phyBunDir, collEnt->collName + collLen + 1, collEnt->dataName);
mkDirForFilePath (UNIX_FILE_TYPE, rsComm, phyBunDir,
tmpPath, getDefDirMode ());
}
/* add a link */
status = link (collEnt->phyPath, tmpPath);
if (status < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: link error %s to %s. errno = %d",
collEnt->phyPath, tmpPath, errno);
rmLinkedFilesInUnixDir (phyBunDir);
rmdir (phyBunDir);
return (UNIX_FILE_LINK_ERR - errno);
}
} else { /* a collection */
if ((int) strlen (collEnt->collName) + 1 <= collLen) {
free (collEnt);
continue;
}
snprintf (tmpPath, MAX_NAME_LEN, "%s/%s",
phyBunDir, collEnt->collName + collLen);
mkdirR (phyBunDir, tmpPath, getDefDirMode ());
}
if (collEnt != NULL) {
free (collEnt);
collEnt = NULL;
}
}
clearKeyVal (&collInp.condInput);
rsCloseCollection (rsComm, &handleInx);
status = phyBundle (rsComm, L1desc[l1descInx].dataObjInfo, phyBunDir,
collInp.collName, structFileBundleInp->oprType);
if (status < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: phyBundle of %s error. stat = %d",
L1desc[l1descInx].dataObjInfo->objPath, status);
L1desc[l1descInx].bytesWritten = 0;
savedStatus = status;
} else {
/* mark it was written so the size would be adjusted */
L1desc[l1descInx].bytesWritten = 1;
}
rmLinkedFilesInUnixDir (phyBunDir);
rmdir (phyBunDir);
dataObjCloseInp.l1descInx = l1descInx;
status = rsDataObjClose (rsComm, &dataObjCloseInp);
if (status >= 0)
return savedStatus;
else
return (status);
}
int
_rsDataObjGet( rsComm_t *rsComm, dataObjInp_t *dataObjInp,
portalOprOut_t **portalOprOut, bytesBuf_t *dataObjOutBBuf, int handlerFlag ) {
int status;
dataObjInfo_t *dataObjInfo;
int l1descInx;
char *chksumStr = NULL;
int retval;
openedDataObjInp_t dataObjCloseInp;
/* PHYOPEN_BY_SIZE ask it to check whether "dataInclude" should be done */
addKeyVal( &dataObjInp->condInput, PHYOPEN_BY_SIZE_KW, "" );
l1descInx = _rsDataObjOpen( rsComm, dataObjInp );
if ( l1descInx < 0 ) {
return l1descInx;
}
L1desc[l1descInx].oprType = GET_OPR;
dataObjInfo = L1desc[l1descInx].dataObjInfo;
if ( getStructFileType( dataObjInfo->specColl ) >= 0 && // JMC - backport 4682
L1desc[l1descInx].l3descInx > 0 ) {
/* l3descInx == 0 if included */
*portalOprOut = ( portalOprOut_t * ) malloc( sizeof( portalOprOut_t ) );
bzero( *portalOprOut, sizeof( portalOprOut_t ) );
( *portalOprOut )->l1descInx = l1descInx;
return l1descInx;
}
if ( getValByKey( &dataObjInp->condInput, VERIFY_CHKSUM_KW ) != NULL ) {
if ( strlen( dataObjInfo->chksum ) > 0 ) {
/* a chksum already exists */
chksumStr = strdup( dataObjInfo->chksum );
}
else {
status = dataObjChksumAndReg( rsComm, dataObjInfo, &chksumStr );
if ( status < 0 ) {
return status;
}
rstrcpy( dataObjInfo->chksum, chksumStr, NAME_LEN );
}
}
if ( L1desc[l1descInx].l3descInx <= 2 ) {
/* no physical file was opened */
status = l3DataGetSingleBuf( rsComm, l1descInx, dataObjOutBBuf,
portalOprOut );
if ( status >= 0 ) {
int status2;
/** since the object is read here, we apply post procesing RAJA
* Dec 2 2010 **/
status2 = applyRuleForPostProcForRead( rsComm, dataObjOutBBuf,
dataObjInp->objPath );
if ( status2 >= 0 ) {
status = 0;
}
else {
status = status2;
}
/** since the object is read here, we apply post procesing
* RAJA Dec 2 2010 **/
if ( chksumStr != NULL ) {
rstrcpy( ( *portalOprOut )->chksum, chksumStr, NAME_LEN );
free( chksumStr );
}
}
return status;
}
status = preProcParaGet( rsComm, l1descInx, portalOprOut );
if ( status < 0 ) {
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
rsDataObjClose( rsComm, &dataObjCloseInp );
if ( chksumStr != NULL ) {
free( chksumStr );
}
return status;
}
status = l1descInx; /* means file not included */
if ( chksumStr != NULL ) {
rstrcpy( ( *portalOprOut )->chksum, chksumStr, NAME_LEN );
free( chksumStr );
}
/* return portalOprOut to the client and wait for the rcOprComplete
* call. That is when the parallel I/O is done */
retval = sendAndRecvBranchMsg( rsComm, rsComm->apiInx, status,
( void * ) * portalOprOut, dataObjOutBBuf );
if ( retval < 0 ) {
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = l1descInx;
rsDataObjClose( rsComm, &dataObjCloseInp );
}
if ( handlerFlag & INTERNAL_SVR_CALL ) {
/* internal call. want to know the real status */
return retval;
}
else {
/* already send the client the status */
return SYS_NO_HANDLER_REPLY_MSG;
}
}
int msiobjput_slink(
msParam_t* inMSOPath,
msParam_t* inCacheFilename,
msParam_t* inFileSize,
ruleExecInfo_t* rei ) {
RE_TEST_MACRO( " Calling msiobjput_slink" );
/* check for input parameters */
if ( inMSOPath == NULL ||
strcmp( inMSOPath->type , STR_MS_T ) != 0 ||
inMSOPath->inOutStruct == NULL ) {
return USER_PARAM_TYPE_ERR;
}
if ( inCacheFilename == NULL ||
strcmp( inCacheFilename->type , STR_MS_T ) != 0 ||
inCacheFilename->inOutStruct == NULL ) {
return USER_PARAM_TYPE_ERR;
}
if ( inFileSize == NULL ||
strcmp( inFileSize->type , STR_MS_T ) != 0 ||
inFileSize->inOutStruct == NULL ) {
return USER_PARAM_TYPE_ERR;
}
/* coerce input to local variables */
char * str = strdup( ( char * ) inMSOPath->inOutStruct );
char * reqStr = strstr( str, ":" );
if ( reqStr == NULL ) {
free( str );
return USER_INPUT_FORMAT_ERR;
}
reqStr = reqStr + 1;
dataObjInp_t dataObjInp;
memset( &dataObjInp, 0, sizeof( dataObjInp_t ) );
rstrcpy( dataObjInp.objPath, reqStr, MAX_NAME_LEN );
addKeyVal( &dataObjInp.condInput, FORCE_FLAG_KW, "" );
free( str );
rsComm_t * rsComm = rei->rsComm;
int outDesc = rsDataObjCreate( rsComm, &dataObjInp );
if ( outDesc < 0 ) {
printf( "msiputobj_slink: Unable to open file %s:%i\n", dataObjInp.objPath, outDesc );
return outDesc;
}
/* Read the cache and Do the upload*/
char * cacheFilename = ( char * ) inCacheFilename->inOutStruct;
int srcFd = open( cacheFilename, O_RDONLY, 0 );
if ( srcFd < 0 ) {
int status = UNIX_FILE_OPEN_ERR - errno;
printf( "msiputobj_slink: open error for %s, status = %d\n",
cacheFilename, status );
return status;
}
int single_buff_sz = 0;
irods::error ret = irods::get_advanced_setting<int>(
irods::CFG_MAX_SIZE_FOR_SINGLE_BUFFER,
single_buff_sz );
if( !ret.ok() ) {
irods::log( PASS( ret ) );
return ret.code();
}
single_buff_sz *= 1024 * 1024;
size_t dataSize = atol( ( char * ) inFileSize->inOutStruct );
if ( dataSize > single_buff_sz ) {
dataSize = single_buff_sz;
}
openedDataObjInp_t dataObjWriteInp;
memset( &dataObjWriteInp, 0, sizeof( dataObjWriteInp ) );
dataObjWriteInp.l1descInx = outDesc;
openedDataObjInp_t dataObjCloseInp;
memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
dataObjCloseInp.l1descInx = outDesc;
char * myBuf = ( char * ) malloc( dataSize );
bytesBuf_t writeBuf;
writeBuf.buf = myBuf;
int bytesRead;
for ( bytesRead = read( srcFd, ( void * ) myBuf, dataSize ); bytesRead > 0;
bytesRead = read( srcFd, ( void * ) myBuf, dataSize ) ) {
writeBuf.len = bytesRead;
dataObjWriteInp.len = bytesRead;
int bytesWritten = rsDataObjWrite( rsComm, &dataObjWriteInp, &writeBuf );
if ( bytesWritten != bytesRead ) {
free( myBuf );
close( srcFd );
rsDataObjClose( rsComm, &dataObjCloseInp );
printf( "msiputobj_slink: Write Error: bytesRead %d != bytesWritten %d\n",
bytesRead, bytesWritten );
return SYS_COPY_LEN_ERR;
}
}
free( myBuf );
close( srcFd );
return rsDataObjClose( rsComm, &dataObjCloseInp );
}
static int process_single_obj( rsComm_t *conn, char *parColl, char *fileName,
int required_num_replicas, char *grpRescForReplication, char *emailToNotify ) {
genQueryInp_t genQueryInp;
int i;
genQueryOut_t *genQueryOut = NULL;
sqlResult_t *replNumStruct, *rescStruct, *dataPathStruct, *chkSumStruct;
char *replNum, *rescName, *dataPathName, *chkSum;
int t;
int i1a[10];
int i1b[10];
int i2a[10];
char *condVal[2];
char v1[200], v2[200];
char vault_path[2048];
char *chksum_str = NULL;
dataObjInp_t myDataObjInp;
dataObjInfo_t *myDataObjInfo = NULL;
unregDataObj_t myUnregDataObjInp;
transferStat_t *transStat = NULL;
ReplicaCheckStatusStruct *pReplicaStatus;
int nReplicas;
char tmpstr[1024];
int at_least_one_copy_is_good = 0;
int newN;
int rn;
int validKwFlags;
char *outBadKeyWd;
msParam_t msGrpRescStr; /* it can be a single resc name or a paired values. */
/* fprintf(stderr,"msiAutoReplicateService():process_single_obj()\n"); */
memset( &genQueryInp, 0, sizeof( genQueryInp_t ) );
i1a[0] = COL_DATA_REPL_NUM;
i1b[0] = 0;
i1a[1] = COL_D_RESC_NAME;
i1b[1] = 0;
i1a[2] = COL_D_RESC_GROUP_NAME;
i1b[2] = 0;
i1a[3] = COL_D_DATA_PATH;
i1b[3] = 0;
i1a[4] = COL_D_DATA_CHECKSUM;
i1b[4] = 0;
genQueryInp.selectInp.inx = i1a;
genQueryInp.selectInp.value = i1b;
genQueryInp.selectInp.len = 5;
i2a[0] = COL_COLL_NAME;
i2a[1] = COL_DATA_NAME;
genQueryInp.sqlCondInp.inx = i2a;
sprintf( v1, "='%s'", parColl );
condVal[0] = v1;
sprintf( v2, "='%s'", fileName );
condVal[1] = v2;
genQueryInp.sqlCondInp.value = condVal;
genQueryInp.sqlCondInp.len = 2;
genQueryInp.maxRows = 10;
genQueryInp.continueInx = 0;
t = rsGenQuery( conn, &genQueryInp, &genQueryOut );
if ( t < 0 ) {
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj(): rsGenQuery failed errocode=%d", t );
if ( t == CAT_NO_ROWS_FOUND ) { /* no data is found */
return 0;
}
return( t );
}
if ( genQueryOut->rowCnt <= 0 ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():process_single_obj(): return 0 record from calling rsGenQuery() for objid=%s/%s", parColl, fileName );
return 0;
}
nReplicas = genQueryOut->rowCnt;
pReplicaStatus = ( ReplicaCheckStatusStruct * )calloc( nReplicas, sizeof( ReplicaCheckStatusStruct ) );
for ( i = 0; i < nReplicas; i++ ) {
pReplicaStatus[i].registered = 0;
pReplicaStatus[i].checksum[0] = '\0';
}
for ( i = 0; i < genQueryOut->rowCnt; i++ ) {
replNumStruct = getSqlResultByInx( genQueryOut, COL_DATA_REPL_NUM );
replNum = &replNumStruct->value[replNumStruct->len * i];
pReplicaStatus[i].repl_num = atoi( replNum );
rescStruct = getSqlResultByInx( genQueryOut, COL_D_RESC_NAME );
rescName = &rescStruct->value[rescStruct->len * i];
getSqlResultByInx( genQueryOut, COL_D_RESC_GROUP_NAME );
dataPathStruct = getSqlResultByInx( genQueryOut, COL_D_DATA_PATH );
dataPathName = &dataPathStruct->value[dataPathStruct->len * i];
chkSumStruct = getSqlResultByInx( genQueryOut, COL_D_DATA_CHECKSUM );
chkSum = &chkSumStruct->value[chkSumStruct->len * i];
vault_path[0] = '\0';
t = get_resource_path( conn, rescName, vault_path );
if ( t < 0 ) {
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj():get_resource_path failed, status=%d", t );
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
if ( strncmp( dataPathName, vault_path, strlen( vault_path ) ) != 0 ) {
/* fprintf(stderr,"AB1-> %s/%s, v=%s, is a registered copy.\n", parColl,fileName, replNum); */
pReplicaStatus[i].registered = 1;
}
else {
pReplicaStatus[i].registered = 0;
}
if ( ( chkSum != NULL ) && ( strlen( chkSum ) > 0 ) ) {
strcpy( pReplicaStatus[i].checksum, chkSum );
}
}
freeGenQueryOut( &genQueryOut );
/* check replica status */
at_least_one_copy_is_good = 0;
for ( i = 0; i < nReplicas; i++ ) {
/* check the file existence first */
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
snprintf( myDataObjInp.objPath, MAX_NAME_LEN, "%s/%s", parColl, fileName );
myDataObjInp.openFlags = O_RDONLY;
sprintf( tmpstr, "%d", pReplicaStatus[i].repl_num );
addKeyVal( &myDataObjInp.condInput, REPL_NUM_KW, tmpstr );
rn = pReplicaStatus[i].repl_num;
t = rsDataObjOpen( conn, &myDataObjInp );
if ( t < 0 ) {
/* fprintf(stderr,"%d. %s/%s, %d failed to open and has checksum status=%d\n", i, parColl, fileName, rn, t); */
if ( t == SYS_OUT_OF_FILE_DESC ) {
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
else {
pReplicaStatus[i].chksum_status = t;
}
}
else {
openedDataObjInp_t myDataObjCloseInp;
memset( &myDataObjCloseInp, 0, sizeof( openedDataObjInp_t ) );
myDataObjCloseInp.l1descInx = t;
t = rsDataObjClose( conn, &myDataObjCloseInp );
chksum_str = NULL;
/* compute checksum rsDataObjChksum() */
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
sprintf( myDataObjInp.objPath, "%s/%s", parColl, fileName );
addKeyVal( &myDataObjInp.condInput, FORCE_CHKSUM_KW, "" );
sprintf( tmpstr, "%d", pReplicaStatus[i].repl_num );
addKeyVal( &myDataObjInp.condInput, REPL_NUM_KW, tmpstr );
t = rsDataObjChksum( conn, &myDataObjInp, &chksum_str );
pReplicaStatus[i].chksum_status = t; /* t == USER_CHKSUM_MISMATCH means a bad copy */
/* fprintf(stderr,"%d. %s/%s, %d has checksum status=%d\n", i, parColl, fileName, rn, t); */
if ( t >= 0 ) {
if ( strlen( pReplicaStatus[i].checksum ) > 0 ) {
if ( strcmp( pReplicaStatus[i].checksum, chksum_str ) != 0 ) { /* mismatch */
pReplicaStatus[i].chksum_status = USER_CHKSUM_MISMATCH;
}
else {
at_least_one_copy_is_good = 1;
}
}
else {
at_least_one_copy_is_good = 1;
}
}
}
}
/* if there is none good copies left. In some casee, the checksum failed due
to the fact that the server is down. We leave this case to be taken care of next time. */
if ( at_least_one_copy_is_good == 0 ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():process_single_obj(): Obj='%s/%s': Warning: The system detects that all copies might be corrupted.", parColl, fileName );
#ifndef windows_platform
if ( ( emailToNotify != NULL ) && ( strlen( emailToNotify ) > 0 ) ) {
char msg_sub[1024], msg_body[1024];
strcpy( msg_sub, "iRODS msiAutoReplicateService() error" );
sprintf( msg_body, "msiAutoReplicateService():process_single_obj(): Obj='%s/%s': at least one storage server is down or all copies are corrupted.",
parColl, fileName );
UnixSendEmail( emailToNotify, msg_sub, msg_body );
}
#endif
free( pReplicaStatus ); // JMc cppcheck - leak
return 0;
}
/* since we have at least one copy is good. We delete those bad copies (USER_CHKSUM_MISMATCH). */
newN = nReplicas;
for ( i = 0; i < nReplicas; i++ ) {
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
sprintf( myDataObjInp.objPath, "%s/%s", parColl, fileName );
sprintf( tmpstr, "%d", pReplicaStatus[i].repl_num );
addKeyVal( &myDataObjInp.condInput, REPL_NUM_KW, tmpstr );
rn = pReplicaStatus[i].repl_num;
if ( pReplicaStatus[i].registered == 1 ) {
/* here is the catch. iRODS. */
int adchksum;
/* fprintf(stderr,"CD->%s/%s, v=%d, is a registered copy.\n", parColl, fileName, rn); */
adchksum = ( ( int )( pReplicaStatus[i].chksum_status / 1000 ) ) * 1000;
if ( ( pReplicaStatus[i].chksum_status == USER_CHKSUM_MISMATCH ) || ( pReplicaStatus[i].chksum_status == UNIX_FILE_OPEN_ERR ) || ( adchksum == UNIX_FILE_OPEN_ERR ) )
/* USER_CHKSUM_MISMATCH -> indicates the registered file is changed.
* UNIX_FILE_OPEN_ERR --> indicates the registered file is removed by original owner.
* -510002 is transformed from UNIX open error.
*/
{
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj(): registered copy will be removed: %s, repl=%d", myDataObjInp.objPath, rn );
t = getDataObjInfo( conn, &myDataObjInp, &myDataObjInfo, NULL, 0 );
if ( t >= 0 ) {
myUnregDataObjInp.dataObjInfo = myDataObjInfo;
myUnregDataObjInp.condInput = &myDataObjInp.condInput;
t = rsUnregDataObj( conn, &myUnregDataObjInp );
if ( t >= 0 ) {
newN = newN - 1;
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsUnregDataObj(): failed for %s/%s:%d. erStat=%d", parColl, fileName, rn, t );
return t;
}
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():getDataObjInfo(): failed for %s/%s:%d. erStat=%d", parColl, fileName, rn, t );
return t;
}
}
else {
rodsLog( LOG_ERROR, "%s:%d, the registered copy has errored checksum status=%d.", myDataObjInp.objPath, rn, pReplicaStatus[i].chksum_status );
return t;
}
}
else { /* the data file is in vault */
if ( pReplicaStatus[i].chksum_status == USER_CHKSUM_MISMATCH ) {
t = rsDataObjUnlink( conn, &myDataObjInp );
if ( t >= 0 ) {
newN = newN - 1;
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjUnlink() for %s:%d failed. errStat=%d", myDataObjInp.objPath, rn, t );
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
}
}
}
fillStrInMsParam( &msGrpRescStr, grpRescForReplication );
/* make necessary copies based on the required number of copies */
if ( newN < required_num_replicas ) {
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj(): making necessary %d copies as required.", ( required_num_replicas - newN ) );
for ( i = 0; i < ( required_num_replicas - newN ); i++ ) {
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
snprintf( myDataObjInp.objPath, MAX_NAME_LEN, "%s/%s", parColl, fileName );
/* addKeyVal(&myDataObjInp.condInput, DEST_RESC_NAME_KW, grpRescForReplication); */
validKwFlags = OBJ_PATH_FLAG | DEST_RESC_NAME_FLAG | NUM_THREADS_FLAG |
BACKUP_RESC_NAME_FLAG | RESC_NAME_FLAG | UPDATE_REPL_FLAG |
REPL_NUM_FLAG | ALL_FLAG | ADMIN_FLAG | VERIFY_CHKSUM_FLAG |
RBUDP_TRANSFER_FLAG | RBUDP_SEND_RATE_FLAG | RBUDP_PACK_SIZE_FLAG;
t = parseMsKeyValStrForDataObjInp( &msGrpRescStr, &myDataObjInp, DEST_RESC_NAME_KW, validKwFlags, &outBadKeyWd );
if ( t < 0 ) {
if ( outBadKeyWd != NULL ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjRepl(): input keyWd - %s error. status = %d", outBadKeyWd, t );
free( outBadKeyWd );
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjRepl(): input msKeyValStr error. status = %d", t );
}
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
t = rsDataObjRepl( conn, &myDataObjInp, &transStat );
if ( t < 0 ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjRepl() failed for %s/%s:%d into '%s'. err code=%d.", parColl, fileName, rn, grpRescForReplication, t );
repl_storage_error = 1;
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
if ( transStat != NULL ) {
free( transStat );
}
}
}
free( pReplicaStatus ); // JMC cppcheck - leak
return 0;
}
