static size_t my_read_obj(void *buffer, size_t size, size_t nmemb, void* userp) {
struct readData_t *readData = (struct readData_t *) userp;
dataObjInp_t file;
openedDataObjInp_t openedFile;
bytesBuf_t bytesBuf;
size_t bytesRead;
// Make sure we have something to read from
if (!readData) {
rodsLog(LOG_ERROR, "irods_http_send_file: readData is NULL");
return CURL_READFUNC_ABORT;
}
// Zero fill input structs
memset(&file, 0, sizeof (dataObjInp_t));
memset(&openedFile, 0, sizeof (openedDataObjInp_t));
memset(&bytesBuf, 0, sizeof (bytesBuf_t));
// If this is the first call we need to create our data object before reading from it
if (!readData->desc) {
strncpy(file.objPath, readData->path, MAX_NAME_LEN);
// readData->desc = 5;
readData->desc = rsDataObjOpen(readData->rsComm, &file);
// TODO: <= 2 instead of < 0? Look up rsDataObjOpen return codes
if (readData->desc < 0) {
rodsLog(LOG_ERROR, "irods_http_send_file: Cannot read iRODS object %s Status = %d",readData->path, readData->desc);
return CURL_READFUNC_ABORT;
}
}
// Setup buffer for rsDataObjRead
bytesBuf.len = (int)(size * nmemb);
bytesBuf.buf = buffer;
// Setup input struct for rsDataObjRead
openedFile.l1descInx = readData->desc;
openedFile.len = bytesBuf.len;
// bytesRead = 0;
bytesRead = rsDataObjRead(readData->rsComm, &openedFile, &bytesBuf);
if (bytesRead < 0) {
rodsLog(LOG_ERROR, "irods_http_send_file: Problem reading iRODS object. Status = %d", bytesRead);
return CURL_READFUNC_ABORT;
}
return (bytesRead);
}
int _rsStructFileBundle( rsComm_t* rsComm,
structFileExtAndRegInp_t* structFileBundleInp ) {
int status;
int handleInx;
char phyBunDir[MAX_NAME_LEN];
char tmpPath[MAX_NAME_LEN];
int l1descInx;
char* dataType = 0; // JMC - backport 4664
openedDataObjInp_t dataObjCloseInp;
// =-=-=-=-=-=-=-
// create an empty data obj
dataObjInp_t dataObjInp;
memset( &dataObjInp, 0, sizeof( dataObjInp ) );
dataObjInp.openFlags = O_WRONLY;
// =-=-=-=-=-=-=-
// get the data type of the structured file
dataType = getValByKey( &structFileBundleInp->condInput, DATA_TYPE_KW );
// =-=-=-=-=-=-=-
// ensure that the file name will end in .zip, if necessary
if ( dataType != NULL && strstr( dataType, ZIP_DT_STR ) != NULL ) {
int len = strlen( structFileBundleInp->objPath );
if ( strcmp( &structFileBundleInp->objPath[len - 4], ".zip" ) != 0 ) {
strcat( structFileBundleInp->objPath, ".zip" );
}
}
// =-=-=-=-=-=-=-
// capture the object path in the data obj struct
rstrcpy( dataObjInp.objPath, structFileBundleInp->objPath, MAX_NAME_LEN );
// =-=-=-=-=-=-=-
// replicate the condInput. may have resource input
replKeyVal( &structFileBundleInp->condInput, &dataObjInp.condInput );
// =-=-=-=-=-=-=-
// open the file if we are in an add operation, otherwise create the new file
if ( ( structFileBundleInp->oprType & ADD_TO_TAR_OPR ) != 0 ) { // JMC - backport 4643
l1descInx = rsDataObjOpen( rsComm, &dataObjInp );
}
else {
l1descInx = rsDataObjCreate( rsComm, &dataObjInp );
}
// =-=-=-=-=-=-=-
// error check create / open
if ( l1descInx < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: rsDataObjCreate of %s error. status = %d",
dataObjInp.objPath, l1descInx );
return l1descInx;
}
// =-=-=-=-=-=-=-
// FIXME :: Why, when we replicate them above?
clearKeyVal( &dataObjInp.condInput ); // JMC - backport 4637
// ???? l3Close (rsComm, l1descInx);
// =-=-=-=-=-=-=-
// zip does not like a zero length file as target
//L1desc[ l1descInx ].l3descInx = 0;
//if( dataType != NULL && strstr( dataType, ZIP_DT_STR ) != NULL ) {
// if( ( structFileBundleInp->oprType & ADD_TO_TAR_OPR) == 0 ) { // JMC - backport 4643
// l3Unlink( rsComm, L1desc[l1descInx].dataObjInfo );
// }
//}
// =-=-=-=-=-=-=-
// check object permissions / stat
chkObjPermAndStat_t chkObjPermAndStatInp;
memset( &chkObjPermAndStatInp, 0, sizeof( chkObjPermAndStatInp ) );
rstrcpy( chkObjPermAndStatInp.objPath, structFileBundleInp->collection, MAX_NAME_LEN );
chkObjPermAndStatInp.flags = CHK_COLL_FOR_BUNDLE_OPR;
addKeyVal( &chkObjPermAndStatInp.condInput, RESC_NAME_KW, L1desc[l1descInx].dataObjInfo->rescName );
// =-=-=-=-=-=-=-
// get the resc hier string
std::string resc_hier;
char* resc_hier_ptr = getValByKey( &structFileBundleInp->condInput, RESC_HIER_STR_KW );
if ( !resc_hier_ptr ) {
rodsLog( LOG_NOTICE, "_rsStructFileBundle :: RESC_HIER_STR_KW is NULL" );
}
else {
addKeyVal( &chkObjPermAndStatInp.condInput, RESC_HIER_STR_KW, resc_hier_ptr );
resc_hier = resc_hier_ptr;
}
status = rsChkObjPermAndStat( rsComm, &chkObjPermAndStatInp );
if ( status < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: rsChkObjPermAndStat of %s error. stat = %d",
chkObjPermAndStatInp.objPath, status );
dataObjCloseInp.l1descInx = l1descInx;
rsDataObjClose( rsComm, &dataObjCloseInp );
return status;
}
clearKeyVal( &chkObjPermAndStatInp.condInput );
// =-=-=-=-=-=-=-
// create the special hidden directory where the bundling happens
createPhyBundleDir( rsComm, L1desc[ l1descInx ].dataObjInfo->filePath, phyBunDir, L1desc[ l1descInx ].dataObjInfo->rescHier );
// =-=-=-=-=-=-=-
// build a collection open input structure
collInp_t collInp;
bzero( &collInp, sizeof( collInp ) );
collInp.flags = RECUR_QUERY_FG | VERY_LONG_METADATA_FG | NO_TRIM_REPL_FG | INCLUDE_CONDINPUT_IN_QUERY;
rstrcpy( collInp.collName, structFileBundleInp->collection, MAX_NAME_LEN );
addKeyVal( &collInp.condInput, RESC_NAME_KW, L1desc[ l1descInx ].dataObjInfo->rescName );
rodsLog( LOG_DEBUG, "rsStructFileBundle: calling rsOpenCollection for [%s]", structFileBundleInp->collection );
// =-=-=-=-=-=-=-
// open the collection from which we will bundle
handleInx = rsOpenCollection( rsComm, &collInp );
if ( handleInx < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: rsOpenCollection of %s error. status = %d",
collInp.collName, handleInx );
rmdir( phyBunDir );
return handleInx;
}
// =-=-=-=-=-=-=-
// preserve the collection path?
int collLen = 0;
if ( ( structFileBundleInp->oprType & PRESERVE_COLL_PATH ) != 0 ) {
// =-=-=-=-=-=-=-
// preserve the last entry of the coll path
char* tmpPtr = collInp.collName;
int tmpLen = 0;
collLen = 0;
// =-=-=-=-=-=-=-
// find length to the last '/'
while ( *tmpPtr != '\0' ) {
if ( *tmpPtr == '/' ) {
collLen = tmpLen;
}
tmpLen++;
tmpPtr++;
}
}
else {
collLen = strlen( collInp.collName );
}
// =-=-=-=-=-=-=-
// preserve the collection path?
collEnt_t* collEnt = NULL;
while ( ( status = rsReadCollection( rsComm, &handleInx, &collEnt ) ) >= 0 ) {
if ( NULL == collEnt ) {
rodsLog(
LOG_ERROR,
"rsStructFileBundle: collEnt is NULL" );
continue;
}
// =-=-=-=-=-=-=-
// entry is a data object
if ( collEnt->objType == DATA_OBJ_T ) {
// =-=-=-=-=-=-=-
// filter out any possible replicas that are not on this resource
if ( resc_hier != collEnt->resc_hier ) {
rodsLog(
LOG_DEBUG,
"_rsStructFileBundle - skipping [%s] on resc [%s]",
collEnt->phyPath,
collEnt->resc_hier );
free( collEnt );
collEnt = NULL;
continue;
}
if ( collEnt->collName[collLen] == '\0' ) {
snprintf(
tmpPath,
MAX_NAME_LEN,
"%s/%s",
phyBunDir,
collEnt->dataName );
}
else {
snprintf(
tmpPath,
MAX_NAME_LEN, "%s/%s/%s",
phyBunDir,
collEnt->collName + collLen + 1,
collEnt->dataName );
status = mkDirForFilePath(
rsComm,
strlen( phyBunDir ),
tmpPath,
collEnt->resc_hier,
getDefDirMode() );
if ( status < 0 ) {
rodsLog(
LOG_ERROR,
"mkDirForFilePath failed in _rsStructFileBundle with status %d",
status );
free( collEnt );
return status;
}
}
// =-=-=-=-=-=-=-
// add a link
status = link( collEnt->phyPath, tmpPath );
if ( status < 0 ) {
rodsLog(
LOG_ERROR,
"rsStructFileBundle: link error %s to %s. errno = %d",
collEnt->phyPath,
tmpPath,
errno );
rmLinkedFilesInUnixDir( phyBunDir );
rmdir( phyBunDir );
free( collEnt );
return UNIX_FILE_LINK_ERR - errno;
}
else {
rodsLog(
LOG_DEBUG,
"_rsStructFileBundle - LINK [%s] on resc [%s]",
collEnt->phyPath,
collEnt->resc_hier );
}
}
else {
// =-=-=-=-=-=-=-
// entry is a collection
if ( ( int ) strlen( collEnt->collName ) + 1 <= collLen ) {
free( collEnt );
collEnt = NULL;
continue;
}
snprintf( tmpPath, MAX_NAME_LEN, "%s/%s", phyBunDir, collEnt->collName + collLen );
status = mkFileDirR( rsComm, strlen( phyBunDir ), tmpPath, resc_hier.c_str(), getDefDirMode() );
if ( status < 0 ) {
rodsLog( LOG_ERROR, "mkFileDirR failed in _rsStructFileBundle with status %d", status );
free( collEnt );
return status;
}
} // else
free( collEnt );
collEnt = NULL;
} // while
// =-=-=-=-=-=-=-
// clean up key vals and close the collection
clearKeyVal( &collInp.condInput );
rsCloseCollection( rsComm, &handleInx );
// =-=-=-=-=-=-=-
// call the helper function to do the actual bundling
status = phyBundle( rsComm, L1desc[l1descInx].dataObjInfo, phyBunDir,
collInp.collName, structFileBundleInp->oprType ); // JMC - backport 4643
int savedStatus = 0;
if ( status < 0 ) {
rodsLog( LOG_ERROR, "rsStructFileBundle: phyBundle of %s error. stat = %d",
L1desc[ l1descInx ].dataObjInfo->objPath, status );
L1desc[ l1descInx ].bytesWritten = 0;
savedStatus = status;
}
else {
// mark it was written so the size would be adjusted
L1desc[ l1descInx ].bytesWritten = 1;
}
// =-=-=-=-=-=-=-
// clean up after the bundle directory
rmLinkedFilesInUnixDir( phyBunDir );
rmdir( phyBunDir );
dataObjCloseInp.l1descInx = l1descInx;
status = rsDataObjClose( rsComm, &dataObjCloseInp );
if ( status >= 0 ) {
return savedStatus;
}
return status;
}
int
rsDataObjCopy( rsComm_t *rsComm, dataObjCopyInp_t *dataObjCopyInp,
transferStat_t **transStat ) {
dataObjInp_t *srcDataObjInp, *destDataObjInp;
int srcL1descInx, destL1descInx;
int status;
int existFlag;
uint createMode;
int remoteFlag;
rodsServerHost_t *rodsServerHost;
specCollCache_t *specCollCache = NULL;
srcDataObjInp = &dataObjCopyInp->srcDataObjInp;
destDataObjInp = &dataObjCopyInp->destDataObjInp;
resolveLinkedPath( rsComm, srcDataObjInp->objPath, &specCollCache, &srcDataObjInp->condInput );
resolveLinkedPath( rsComm, destDataObjInp->objPath, &specCollCache, &destDataObjInp->condInput );
remoteFlag = getAndConnRemoteZoneForCopy( rsComm, dataObjCopyInp, &rodsServerHost );
if ( remoteFlag < 0 ) {
return remoteFlag;
}
else if ( remoteFlag == REMOTE_HOST ) {
status = _rcDataObjCopy( rodsServerHost->conn, dataObjCopyInp,
transStat );
return status;
}
if ( strcmp( srcDataObjInp->objPath, destDataObjInp->objPath ) == 0 ) {
rodsLog( LOG_ERROR,
"rsDataObjCopy: same src and dest objPath %s not allowed",
srcDataObjInp->objPath );
return USER_INPUT_PATH_ERR;
}
addKeyVal( &srcDataObjInp->condInput, PHYOPEN_BY_SIZE_KW, "" );
srcL1descInx = rsDataObjOpen( rsComm, srcDataObjInp );
if ( srcL1descInx < 0 ) {
std::stringstream msg;
char* sys_error = NULL;
const char* rods_error = rodsErrorName( srcL1descInx, &sys_error );
msg << __FUNCTION__;
msg << " - Failed to open source object: \"";
msg << srcDataObjInp->objPath;
msg << "\" - ";
msg << rods_error << " " << sys_error;
irods::log( LOG_ERROR, msg.str() );
free( sys_error );
return srcL1descInx;
}
/* have to set L1desc[srcL1descInx].dataSize because open set this to -1 */
destDataObjInp->dataSize = L1desc[srcL1descInx].dataSize =
L1desc[srcL1descInx].dataObjInfo->dataSize;
createMode = atoi( L1desc[srcL1descInx].dataObjInfo->dataMode );
if ( createMode >= 0100 ) {
destDataObjInp->createMode = createMode;
}
L1desc[srcL1descInx].oprType = COPY_SRC;
if ( L1desc[srcL1descInx].l3descInx <= 2 ) {
/* dataSingleBuf */
addKeyVal( &destDataObjInp->condInput, NO_OPEN_FLAG_KW, "" );
}
destL1descInx = rsDataObjCreate( rsComm, destDataObjInp );
if ( destL1descInx == CAT_UNKNOWN_COLLECTION ) {
/* collection does not exist. make one */
char parColl[MAX_NAME_LEN], child[MAX_NAME_LEN];
splitPathByKey( destDataObjInp->objPath, parColl, MAX_NAME_LEN, child, MAX_NAME_LEN, '/' );
rsMkCollR( rsComm, "/", parColl );
destL1descInx = rsDataObjCreate( rsComm, destDataObjInp );
}
if ( destL1descInx < 0 ) {
clearKeyVal( &destDataObjInp->condInput );
std::stringstream msg;
char* sys_error = NULL;
const char* rods_error = rodsErrorName( destL1descInx, &sys_error );
msg << __FUNCTION__;
msg << " - Failed to create destination object: \"";
msg << destDataObjInp->objPath;
msg << "\" - ";
msg << rods_error << " " << sys_error;
irods::log( LOG_ERROR, msg.str() );
free( sys_error );
return destL1descInx;
}
if ( L1desc[destL1descInx].replStatus == NEWLY_CREATED_COPY ) {
existFlag = 0;
}
else {
existFlag = 1;
}
L1desc[destL1descInx].oprType = COPY_DEST;
L1desc[destL1descInx].srcL1descInx = srcL1descInx;
rstrcpy( L1desc[destL1descInx].dataObjInfo->dataType,
L1desc[srcL1descInx].dataObjInfo->dataType, NAME_LEN );
/* set dataSize for verification in _rsDataObjClose */
L1desc[destL1descInx].dataSize =
L1desc[srcL1descInx].dataObjInfo->dataSize;
status = _rsDataObjCopy( rsComm, destL1descInx, existFlag, transStat );
clearKeyVal( &destDataObjInp->condInput );
return status;
}
int
msiLoadMetadataFromXml(msParam_t *targetObj, msParam_t *xmlObj, ruleExecInfo_t *rei)
{
/* for parsing msParams and to open iRODS objects */
dataObjInp_t xmlDataObjInp, *myXmlDataObjInp;
dataObjInp_t targetObjInp, *myTargetObjInp;
int xmlObjID;
/* for getting size of objects to read from */
rodsObjStat_t *rodsObjStatOut = NULL;
/* for reading from iRODS objects */
openedDataObjInp_t openedDataObjInp;
bytesBuf_t *xmlBuf;
/* misc. to avoid repeating rei->rsComm */
rsComm_t *rsComm;
/* for xml parsing */
xmlDocPtr doc;
/* for XPath evaluation */
xmlXPathContextPtr xpathCtx;
xmlXPathObjectPtr xpathObj;
xmlChar xpathExpr[] = "//AVU";
xmlNodeSetPtr nodes;
int avuNbr, i;
/* for new AVU creation */
modAVUMetadataInp_t modAVUMetadataInp;
int max_attr_len = 2700;
char attrStr[max_attr_len];
/********************************* USUAL INIT PROCEDURE **********************************/
/* For testing mode when used with irule --test */
RE_TEST_MACRO (" Calling msiLoadMetadataFromXml")
/* Sanity checks */
if (rei == NULL || rei->rsComm == NULL)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: input rei or rsComm is NULL.");
return (SYS_INTERNAL_NULL_INPUT_ERR);
}
rsComm = rei->rsComm;
/********************************** RETRIEVE INPUT PARAMS **************************************/
/* Get path of target object */
rei->status = parseMspForDataObjInp (targetObj, &targetObjInp, &myTargetObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: input targetObj error. status = %d", rei->status);
return (rei->status);
}
/* Get path of XML document */
rei->status = parseMspForDataObjInp (xmlObj, &xmlDataObjInp, &myXmlDataObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: input xmlObj error. status = %d", rei->status);
return (rei->status);
}
/******************************** OPEN AND READ FROM XML OBJECT ********************************/
/* Open XML file */
if ((xmlObjID = rsDataObjOpen(rsComm, &xmlDataObjInp)) < 0)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Cannot open XML data object. status = %d", xmlObjID);
return (xmlObjID);
}
/* Get size of XML file */
rei->status = rsObjStat (rsComm, &xmlDataObjInp, &rodsObjStatOut);
if (rei->status < 0 || !rodsObjStatOut)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Cannot stat XML data object. status = %d", rei->status);
return (rei->status);
}
/* xmlBuf init */
/* memory for xmlBuf->buf is allocated in rsFileRead() */
xmlBuf = (bytesBuf_t *) malloc (sizeof (bytesBuf_t));
memset (xmlBuf, 0, sizeof (bytesBuf_t));
/* Read XML file */
memset (&openedDataObjInp, 0, sizeof (openedDataObjInp_t));
openedDataObjInp.l1descInx = xmlObjID;
openedDataObjInp.len = (int)rodsObjStatOut->objSize;
rei->status = rsDataObjRead (rsComm, &openedDataObjInp, xmlBuf);
/* Make sure that the result is null terminated */
if (strlen((char*)xmlBuf->buf) > (size_t)openedDataObjInp.len)
{
((char*)xmlBuf->buf)[openedDataObjInp.len-1]='\0';
}
/* Close XML file */
rei->status = rsDataObjClose (rsComm, &openedDataObjInp);
/* cleanup */
freeRodsObjStat (rodsObjStatOut);
/******************************** PARSE XML DOCUMENT ********************************/
xmlSubstituteEntitiesDefault(1);
xmlLoadExtDtdDefaultValue = 1;
/* Parse xmlBuf.buf into an xmlDocPtr */
doc = xmlParseDoc((xmlChar*)xmlBuf->buf);
/* Create xpath evaluation context */
xpathCtx = xmlXPathNewContext(doc);
if(xpathCtx == NULL)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Unable to create new XPath context.");
xmlFreeDoc(doc);
return(-1);
}
/* Evaluate xpath expression */
xpathObj = xmlXPathEvalExpression(xpathExpr, xpathCtx);
if(xpathObj == NULL)
{
rodsLog (LOG_ERROR, "msiLoadMetadataFromXml: Unable to evaluate XPath expression \"%s\".", xpathExpr);
xmlXPathFreeContext(xpathCtx);
xmlFreeDoc(doc);
return(-1);
}
/* How many AVU nodes did we get? */
if ((nodes = xpathObj->nodesetval) != NULL)
{
avuNbr = nodes->nodeNr;
}
else
{
avuNbr = 0;
}
/******************************** CREATE AVU TRIPLETS ********************************/
/* Add a new AVU for each node. It's ok to process the nodes in forward order since we're not modifying them */
for(i = 0; i < avuNbr; i++)
{
if (nodes->nodeTab[i])
{
// /* temporary: Add index number to avoid duplicating attribute names */
// memset(attrStr, '\0', MAX_NAME_LEN);
// snprintf(attrStr, MAX_NAME_LEN - 1, "%04d: %s", i+1, (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Attribute")) );
/* Truncate if needed. No prefix. */
memset(attrStr, '\0', max_attr_len);
snprintf(attrStr, max_attr_len - 1, "%s", (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Attribute")) );
/* init modAVU input */
memset (&modAVUMetadataInp, 0, sizeof(modAVUMetadataInp_t));
modAVUMetadataInp.arg0 = "add";
modAVUMetadataInp.arg1 = "-d";
/* Use target object if one was provided, otherwise look for it in the XML doc */
if (myTargetObjInp->objPath != NULL && strlen(myTargetObjInp->objPath) > 0)
{
modAVUMetadataInp.arg2 = myTargetObjInp->objPath;
}
else
{
modAVUMetadataInp.arg2 = xmlURIUnescapeString((char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Target")),
MAX_NAME_LEN, NULL);
}
modAVUMetadataInp.arg3 = attrStr;
modAVUMetadataInp.arg4 = (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Value"));
modAVUMetadataInp.arg5 = (char*)xmlNodeGetContent(getChildNodeByName(nodes->nodeTab[i], "Unit"));
/* invoke rsModAVUMetadata() */
rei->status = rsModAVUMetadata (rsComm, &modAVUMetadataInp);
}
}
/************************************** WE'RE DONE **************************************/
/* cleanup of all xml parsing stuff */
xmlFreeDoc(doc);
xmlCleanupParser();
return (rei->status);
}
/**
* \fn msiXmlDocSchemaValidate(msParam_t *xmlObj, msParam_t *xsdObj, msParam_t *status, ruleExecInfo_t *rei)
*
* \brief This microservice validates an XML file against an XSD schema, both iRODS objects.
*
* \module xml
*
* \since pre-2.1
*
* \author Antoine de Torcy
* \date 2008/05/29
*
* \usage See clients/icommands/test/rules3.0/
*
* \param[in] xmlObj - a msParam of type DataObjInp_MS_T or STR_MS_T which is irods path of the XML object.
* \param[in] xsdObj - a msParam of type DataObjInp_MS_T or STR_MS_T which is irods path of the XSD object.
* \param[out] status - a msParam of type INT_MS_T which is a validation result.
* \param[in,out] rei - The RuleExecInfo structure that is automatically
* handled by the rule engine. The user does not include rei as a
* parameter in the rule invocation.
*
* \DolVarDependence None
* \DolVarModified None
* \iCatAttrDependence None
* \iCatAttrModified None
* \sideeffect None
*
* \return integer
* \retval 0 on success
* \pre None
* \post None
* \sa None
**/
int
msiXmlDocSchemaValidate(msParam_t *xmlObj, msParam_t *xsdObj, msParam_t *status, ruleExecInfo_t *rei)
{
/* for parsing msParams and to open iRODS objects */
dataObjInp_t xmlObjInp, *myXmlObjInp;
dataObjInp_t xsdObjInp, *myXsdObjInp;
int xmlObjID, xsdObjID;
/* for getting size of objects to read from */
rodsObjStat_t *rodsObjStatOut = NULL;
/* for reading from iRODS objects */
openedDataObjInp_t openedDataObjInp;
bytesBuf_t *xmlBuf = NULL;
char *tail;
/* for xml parsing and validating */
xmlDocPtr doc, xsd_doc;
xmlSchemaParserCtxtPtr parser_ctxt;
xmlSchemaPtr schema;
xmlSchemaValidCtxtPtr valid_ctxt;
bytesBuf_t *errBuf;
/* misc. to avoid repeating rei->rsComm */
rsComm_t *rsComm;
/************************************* USUAL INIT PROCEDURE **********************************/
/* For testing mode when used with irule --test */
RE_TEST_MACRO (" Calling msiXmlDocSchemaValidate")
/* Sanity checks */
if (rei == NULL || rei->rsComm == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: input rei or rsComm is NULL.");
return (SYS_INTERNAL_NULL_INPUT_ERR);
}
rsComm = rei->rsComm;
/************************************ ADDITIONAL INIT SETTINGS *********************************/
/* XML constants */
xmlSubstituteEntitiesDefault(1);
xmlLoadExtDtdDefaultValue = 1;
/* allocate memory for output error buffer */
errBuf = (bytesBuf_t *)malloc(sizeof(bytesBuf_t));
errBuf->buf = strdup("");
errBuf->len = strlen((char*)errBuf->buf);
/* Default status is failure, overwrite if success */
fillBufLenInMsParam (status, -1, NULL);
/********************************** RETRIEVE INPUT PARAMS **************************************/
/* Get path of XML document */
rei->status = parseMspForDataObjInp (xmlObj, &xmlObjInp, &myXmlObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: input xmlObj error. status = %d", rei->status);
free(errBuf);
return (rei->status);
}
/* Get path of schema */
rei->status = parseMspForDataObjInp (xsdObj, &xsdObjInp, &myXsdObjInp, 0);
if (rei->status < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: input xsdObj error. status = %d", rei->status);
free(errBuf);
return (rei->status);
}
/******************************** OPEN AND READ FROM XML OBJECT ********************************/
/* Open XML file */
if ((xmlObjID = rsDataObjOpen(rsComm, &xmlObjInp)) < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Cannot open XML data object. status = %d", xmlObjID);
free(errBuf);
return (xmlObjID);
}
/* Get size of XML file */
rei->status = rsObjStat (rsComm, &xmlObjInp, &rodsObjStatOut);
if (rei->status < 0 || !rodsObjStatOut)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Cannot stat XML data object. status = %d", rei->status);
free(errBuf);
return (rei->status);
}
/* xmlBuf init */
/* memory for xmlBuf->buf is allocated in rsFileRead() */
xmlBuf = (bytesBuf_t *) malloc (sizeof (bytesBuf_t));
memset (xmlBuf, 0, sizeof (bytesBuf_t));
/* Read content of XML file */
memset (&openedDataObjInp, 0, sizeof (openedDataObjInp_t));
openedDataObjInp.l1descInx = xmlObjID;
openedDataObjInp.len = (int)rodsObjStatOut->objSize + 1; /* extra byte to add a null char */
rei->status = rsDataObjRead (rsComm, &openedDataObjInp, xmlBuf);
/* add terminating null character */
tail = (char*)xmlBuf->buf;
tail[openedDataObjInp.len - 1] = '\0';
/* Close XML file */
rei->status = rsDataObjClose (rsComm, &openedDataObjInp);
/* cleanup */
freeRodsObjStat (rodsObjStatOut);
/*************************************** PARSE XML DOCUMENT **************************************/
/* Parse xmlBuf.buf into an xmlDocPtr */
doc = xmlParseDoc((xmlChar*)xmlBuf->buf);
clearBBuf(xmlBuf);
if (doc == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: XML document cannot be loaded or is not well-formed.");
free(errBuf);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/******************************** OPEN AND READ FROM XSD OBJECT ********************************/
/* Open schema file */
if ((xsdObjID = rsDataObjOpen(rsComm, &xsdObjInp)) < 0)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Cannot open XSD data object. status = %d", xsdObjID);
free(errBuf);
xmlFreeDoc(doc);
xmlCleanupParser();
return (xsdObjID);
}
/* Get size of schema file */
rei->status = rsObjStat (rsComm, &xsdObjInp, &rodsObjStatOut);
/* Read entire schema file */
memset (&openedDataObjInp, 0, sizeof (openedDataObjInp_t));
openedDataObjInp.l1descInx = xsdObjID;
openedDataObjInp.len = (int)rodsObjStatOut->objSize + 1; /* to add null char */
rei->status = rsDataObjRead (rsComm, &openedDataObjInp, xmlBuf);
/* add terminating null character */
tail = (char*)xmlBuf->buf;
tail[openedDataObjInp.len - 1] = '\0';
/* Close schema file */
rei->status = rsDataObjClose (rsComm, &openedDataObjInp);
/* cleanup */
freeRodsObjStat (rodsObjStatOut);
/*************************************** PARSE XSD DOCUMENT **************************************/
/* Parse xmlBuf.buf into an xmlDocPtr */
xsd_doc = xmlParseDoc((xmlChar*)xmlBuf->buf);
clearBBuf(xmlBuf);
if (xsd_doc == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: XML Schema cannot be loaded or is not well-formed.");
free(errBuf);
xmlFreeDoc(doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/**************************************** VALIDATE DOCUMENT **************************************/
/* Create a parser context */
parser_ctxt = xmlSchemaNewDocParserCtxt(xsd_doc);
if (parser_ctxt == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Unable to create a parser context for the schema.");
free(errBuf);
xmlFreeDoc(xsd_doc);
xmlFreeDoc(doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/* Parse the XML schema */
schema = xmlSchemaParse(parser_ctxt);
if (schema == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Invalid schema.");
free(errBuf);
xmlSchemaFreeParserCtxt(parser_ctxt);
xmlFreeDoc(doc);
xmlFreeDoc(xsd_doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/* Create a validation context */
valid_ctxt = xmlSchemaNewValidCtxt(schema);
if (valid_ctxt == NULL)
{
rodsLog (LOG_ERROR, "msiXmlDocSchemaValidate: Unable to create a validation context for the schema.");
free(errBuf);
xmlSchemaFree(schema);
xmlSchemaFreeParserCtxt(parser_ctxt);
xmlFreeDoc(xsd_doc);
xmlFreeDoc(doc);
xmlCleanupParser();
return (USER_INPUT_FORMAT_ERR);
}
/* Set myErrorCallback() as the default handler for error messages and warnings */
xmlSchemaSetValidErrors(valid_ctxt, (xmlSchemaValidityErrorFunc)myErrorCallback, (xmlSchemaValidityWarningFunc)myErrorCallback, errBuf);
/* Validate XML doc */
rei->status = xmlSchemaValidateDoc(valid_ctxt, doc);
/******************************************* WE'RE DONE ******************************************/
/* return both error code and messages through status */
resetMsParam (status);
fillBufLenInMsParam (status, rei->status, errBuf);
/* cleanup of all xml parsing stuff */
xmlSchemaFreeValidCtxt(valid_ctxt);
xmlSchemaFree(schema);
xmlSchemaFreeParserCtxt(parser_ctxt);
xmlFreeDoc(doc);
xmlFreeDoc(xsd_doc);
xmlCleanupParser();
return (rei->status);
}
int _writeString( char *writeId, char *writeStr, ruleExecInfo_t *rei ) {
// =-=-=-=-=-=-=-
// JMC - backport 4619
dataObjInp_t dataObjInp;
openedDataObjInp_t openedDataObjInp;
bytesBuf_t tmpBBuf;
int fd, i;
// =-=-=-=-=-=-=-
if ( writeId != NULL && strcmp( writeId, "serverLog" ) == 0 ) {
rodsLog( LOG_NOTICE, "writeString: inString = %s", writeStr );
return 0;
}
// =-=-=-=-=-=-=-
// JMC - backport 4619
if ( writeId != NULL && writeId[0] == '/' ) {
/* writing to an existing iRODS file */
if ( rei == NULL || rei->rsComm == NULL ) {
rodsLog( LOG_ERROR, "_writeString: input rei or rsComm is NULL" );
return SYS_INTERNAL_NULL_INPUT_ERR;
}
bzero( &dataObjInp, sizeof( dataObjInp ) );
dataObjInp.openFlags = O_RDWR;
snprintf( dataObjInp.objPath, MAX_NAME_LEN, "%s", writeId );
fd = rsDataObjOpen( rei->rsComm, &dataObjInp );
if ( fd < 0 ) {
rodsLog( LOG_ERROR, "_writeString: rsDataObjOpen failed. status = %d", fd );
return fd;
}
bzero( &openedDataObjInp, sizeof( openedDataObjInp ) );
openedDataObjInp.l1descInx = fd;
openedDataObjInp.offset = 0;
openedDataObjInp.whence = SEEK_END;
fileLseekOut_t *dataObjLseekOut = NULL;
i = rsDataObjLseek( rei->rsComm, &openedDataObjInp, &dataObjLseekOut );
free( dataObjLseekOut );
if ( i < 0 ) {
rodsLog( LOG_ERROR, "_writeString: rsDataObjLseek failed. status = %d", i );
return i;
}
bzero( &openedDataObjInp, sizeof( openedDataObjInp ) );
openedDataObjInp.l1descInx = fd;
tmpBBuf.len = openedDataObjInp.len = strlen( writeStr ) + 1;
tmpBBuf.buf = writeStr;
i = rsDataObjWrite( rei->rsComm, &openedDataObjInp, &tmpBBuf );
if ( i < 0 ) {
rodsLog( LOG_ERROR, "_writeString: rsDataObjWrite failed. status = %d", i );
return i;
}
bzero( &openedDataObjInp, sizeof( openedDataObjInp ) );
openedDataObjInp.l1descInx = fd;
i = rsDataObjClose( rei->rsComm, &openedDataObjInp );
return i;
}
// =-=-=-=-=-=-=-
msParam_t * mP = NULL;
msParamArray_t * inMsParamArray = rei->msParamArray;
execCmdOut_t *myExecCmdOut;
if ( ( ( mP = getMsParamByLabel( inMsParamArray, "ruleExecOut" ) ) != NULL ) &&
( mP->inOutStruct != NULL ) ) {
if ( !strcmp( mP->type, STR_MS_T ) ) {
myExecCmdOut = ( execCmdOut_t* )malloc( sizeof( execCmdOut_t ) );
memset( myExecCmdOut, 0, sizeof( execCmdOut_t ) );
mP->inOutStruct = myExecCmdOut;
mP->type = strdup( ExecCmdOut_MS_T );
}
else {
myExecCmdOut = ( execCmdOut_t* )mP->inOutStruct;
}
}
else {
myExecCmdOut = ( execCmdOut_t* )malloc( sizeof( execCmdOut_t ) );
memset( myExecCmdOut, 0, sizeof( execCmdOut_t ) );
if ( mP == NULL ) {
addMsParam( inMsParamArray, "ruleExecOut", ExecCmdOut_MS_T, myExecCmdOut, NULL );
}
else {
mP->inOutStruct = myExecCmdOut;
mP->type = strdup( ExecCmdOut_MS_T );
}
}
/***** Jun 27, 2007
i = replaceVariablesAndMsParams("",writeStr, rei->msParamArray, rei);
if (i < 0)
return i;
****/
if ( writeId != NULL ) {
if ( !strcmp( writeId, "stdout" ) ) {
appendToByteBuf( &( myExecCmdOut->stdoutBuf ), ( char * ) writeStr );
}
else if ( !strcmp( writeId, "stderr" ) ) {
appendToByteBuf( &( myExecCmdOut->stderrBuf ), ( char * ) writeStr );
}
}
return 0;
}
int
_rsStructFileBundle (rsComm_t *rsComm,
structFileExtAndRegInp_t *structFileBundleInp)
{
int status;
dataObjInp_t dataObjInp;
openedDataObjInp_t dataObjCloseInp;
collInp_t collInp;
collEnt_t *collEnt = NULL;
int handleInx;
int collLen;
char phyBunDir[MAX_NAME_LEN];
char tmpPath[MAX_NAME_LEN];
chkObjPermAndStat_t chkObjPermAndStatInp;
int l1descInx;
int savedStatus = 0;
char *dataType;
/* open the structured file */
memset (&dataObjInp, 0, sizeof (dataObjInp));
dataType = getValByKey (&structFileBundleInp->condInput, DATA_TYPE_KW);
if (dataType != NULL && strstr (dataType, ZIP_DT_STR) != NULL) {
/* zipFile type. must end with .zip */
int len = strlen (structFileBundleInp->objPath);
if (strcmp (&structFileBundleInp->objPath[len - 4], ".zip") != 0) {
strcat (structFileBundleInp->objPath, ".zip");
}
}
rstrcpy (dataObjInp.objPath, structFileBundleInp->objPath,
MAX_NAME_LEN);
/* replicate the condInput. may have resource input */
replKeyVal (&structFileBundleInp->condInput, &dataObjInp.condInput);
dataObjInp.openFlags = O_WRONLY;
if ((structFileBundleInp->oprType & ADD_TO_TAR_OPR) != 0) {
l1descInx = rsDataObjOpen (rsComm, &dataObjInp);
} else {
l1descInx = rsDataObjCreate (rsComm, &dataObjInp);
}
if (l1descInx < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: rsDataObjCreate of %s error. status = %d",
dataObjInp.objPath, l1descInx);
return (l1descInx);
}
clearKeyVal (&dataObjInp.condInput);
l3Close (rsComm, l1descInx);
L1desc[l1descInx].l3descInx = 0;
/* zip does not like a zero length file as target */
if ((structFileBundleInp->oprType & ADD_TO_TAR_OPR) == 0)
l3Unlink (rsComm, L1desc[l1descInx].dataObjInfo);
memset (&chkObjPermAndStatInp, 0, sizeof (chkObjPermAndStatInp));
rstrcpy (chkObjPermAndStatInp.objPath,
structFileBundleInp->collection, MAX_NAME_LEN);
chkObjPermAndStatInp.flags = CHK_COLL_FOR_BUNDLE_OPR;
addKeyVal (&chkObjPermAndStatInp.condInput, RESC_NAME_KW,
L1desc[l1descInx].dataObjInfo->rescName);
status = rsChkObjPermAndStat (rsComm, &chkObjPermAndStatInp);
clearKeyVal (&chkObjPermAndStatInp.condInput);
if (status < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: rsChkObjPermAndStat of %s error. stat = %d",
chkObjPermAndStatInp.objPath, status);
dataObjCloseInp.l1descInx = l1descInx;
rsDataObjClose (rsComm, &dataObjCloseInp);
return (status);
}
createPhyBundleDir (rsComm, L1desc[l1descInx].dataObjInfo->filePath,
phyBunDir);
bzero (&collInp, sizeof (collInp));
rstrcpy (collInp.collName, structFileBundleInp->collection, MAX_NAME_LEN);
collInp.flags = RECUR_QUERY_FG | VERY_LONG_METADATA_FG |
NO_TRIM_REPL_FG | INCLUDE_CONDINPUT_IN_QUERY;
addKeyVal (&collInp.condInput, RESC_NAME_KW,
L1desc[l1descInx].dataObjInfo->rescName);
handleInx = rsOpenCollection (rsComm, &collInp);
if (handleInx < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: rsOpenCollection of %s error. status = %d",
collInp.collName, handleInx);
rmdir (phyBunDir);
return (handleInx);
}
if ((structFileBundleInp->oprType & PRESERVE_COLL_PATH) != 0) {
/* preserver the last entry of the coll path */
char *tmpPtr = collInp.collName;
int tmpLen = 0;
collLen = 0;
/* find length to the last '/' */
while (*tmpPtr != '\0') {
if (*tmpPtr == '/') collLen = tmpLen;
tmpLen++;
tmpPtr++;
}
} else {
collLen = strlen (collInp.collName);
}
while ((status = rsReadCollection (rsComm, &handleInx, &collEnt)) >= 0) {
if (collEnt->objType == DATA_OBJ_T) {
if (collEnt->collName[collLen] == '\0') {
snprintf (tmpPath, MAX_NAME_LEN, "%s/%s",
phyBunDir, collEnt->dataName);
} else {
snprintf (tmpPath, MAX_NAME_LEN, "%s/%s/%s",
phyBunDir, collEnt->collName + collLen + 1, collEnt->dataName);
mkDirForFilePath (UNIX_FILE_TYPE, rsComm, phyBunDir,
tmpPath, getDefDirMode ());
}
/* add a link */
status = link (collEnt->phyPath, tmpPath);
if (status < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: link error %s to %s. errno = %d",
collEnt->phyPath, tmpPath, errno);
rmLinkedFilesInUnixDir (phyBunDir);
rmdir (phyBunDir);
return (UNIX_FILE_LINK_ERR - errno);
}
} else { /* a collection */
if ((int) strlen (collEnt->collName) + 1 <= collLen) {
free (collEnt);
continue;
}
snprintf (tmpPath, MAX_NAME_LEN, "%s/%s",
phyBunDir, collEnt->collName + collLen);
mkdirR (phyBunDir, tmpPath, getDefDirMode ());
}
if (collEnt != NULL) {
free (collEnt);
collEnt = NULL;
}
}
clearKeyVal (&collInp.condInput);
rsCloseCollection (rsComm, &handleInx);
status = phyBundle (rsComm, L1desc[l1descInx].dataObjInfo, phyBunDir,
collInp.collName, structFileBundleInp->oprType);
if (status < 0) {
rodsLog (LOG_ERROR,
"rsStructFileBundle: phyBundle of %s error. stat = %d",
L1desc[l1descInx].dataObjInfo->objPath, status);
L1desc[l1descInx].bytesWritten = 0;
savedStatus = status;
} else {
/* mark it was written so the size would be adjusted */
L1desc[l1descInx].bytesWritten = 1;
}
rmLinkedFilesInUnixDir (phyBunDir);
rmdir (phyBunDir);
dataObjCloseInp.l1descInx = l1descInx;
status = rsDataObjClose (rsComm, &dataObjCloseInp);
if (status >= 0)
return savedStatus;
else
return (status);
}
static int process_single_obj( rsComm_t *conn, char *parColl, char *fileName,
int required_num_replicas, char *grpRescForReplication, char *emailToNotify ) {
genQueryInp_t genQueryInp;
int i;
genQueryOut_t *genQueryOut = NULL;
sqlResult_t *replNumStruct, *rescStruct, *dataPathStruct, *chkSumStruct;
char *replNum, *rescName, *dataPathName, *chkSum;
int t;
int i1a[10];
int i1b[10];
int i2a[10];
char *condVal[2];
char v1[200], v2[200];
char vault_path[2048];
char *chksum_str = NULL;
dataObjInp_t myDataObjInp;
dataObjInfo_t *myDataObjInfo = NULL;
unregDataObj_t myUnregDataObjInp;
transferStat_t *transStat = NULL;
ReplicaCheckStatusStruct *pReplicaStatus;
int nReplicas;
char tmpstr[1024];
int at_least_one_copy_is_good = 0;
int newN;
int rn;
int validKwFlags;
char *outBadKeyWd;
msParam_t msGrpRescStr; /* it can be a single resc name or a paired values. */
/* fprintf(stderr,"msiAutoReplicateService():process_single_obj()\n"); */
memset( &genQueryInp, 0, sizeof( genQueryInp_t ) );
i1a[0] = COL_DATA_REPL_NUM;
i1b[0] = 0;
i1a[1] = COL_D_RESC_NAME;
i1b[1] = 0;
i1a[2] = COL_D_RESC_GROUP_NAME;
i1b[2] = 0;
i1a[3] = COL_D_DATA_PATH;
i1b[3] = 0;
i1a[4] = COL_D_DATA_CHECKSUM;
i1b[4] = 0;
genQueryInp.selectInp.inx = i1a;
genQueryInp.selectInp.value = i1b;
genQueryInp.selectInp.len = 5;
i2a[0] = COL_COLL_NAME;
i2a[1] = COL_DATA_NAME;
genQueryInp.sqlCondInp.inx = i2a;
sprintf( v1, "='%s'", parColl );
condVal[0] = v1;
sprintf( v2, "='%s'", fileName );
condVal[1] = v2;
genQueryInp.sqlCondInp.value = condVal;
genQueryInp.sqlCondInp.len = 2;
genQueryInp.maxRows = 10;
genQueryInp.continueInx = 0;
t = rsGenQuery( conn, &genQueryInp, &genQueryOut );
if ( t < 0 ) {
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj(): rsGenQuery failed errocode=%d", t );
if ( t == CAT_NO_ROWS_FOUND ) { /* no data is found */
return 0;
}
return( t );
}
if ( genQueryOut->rowCnt <= 0 ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():process_single_obj(): return 0 record from calling rsGenQuery() for objid=%s/%s", parColl, fileName );
return 0;
}
nReplicas = genQueryOut->rowCnt;
pReplicaStatus = ( ReplicaCheckStatusStruct * )calloc( nReplicas, sizeof( ReplicaCheckStatusStruct ) );
for ( i = 0; i < nReplicas; i++ ) {
pReplicaStatus[i].registered = 0;
pReplicaStatus[i].checksum[0] = '\0';
}
for ( i = 0; i < genQueryOut->rowCnt; i++ ) {
replNumStruct = getSqlResultByInx( genQueryOut, COL_DATA_REPL_NUM );
replNum = &replNumStruct->value[replNumStruct->len * i];
pReplicaStatus[i].repl_num = atoi( replNum );
rescStruct = getSqlResultByInx( genQueryOut, COL_D_RESC_NAME );
rescName = &rescStruct->value[rescStruct->len * i];
getSqlResultByInx( genQueryOut, COL_D_RESC_GROUP_NAME );
dataPathStruct = getSqlResultByInx( genQueryOut, COL_D_DATA_PATH );
dataPathName = &dataPathStruct->value[dataPathStruct->len * i];
chkSumStruct = getSqlResultByInx( genQueryOut, COL_D_DATA_CHECKSUM );
chkSum = &chkSumStruct->value[chkSumStruct->len * i];
vault_path[0] = '\0';
t = get_resource_path( conn, rescName, vault_path );
if ( t < 0 ) {
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj():get_resource_path failed, status=%d", t );
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
if ( strncmp( dataPathName, vault_path, strlen( vault_path ) ) != 0 ) {
/* fprintf(stderr,"AB1-> %s/%s, v=%s, is a registered copy.\n", parColl,fileName, replNum); */
pReplicaStatus[i].registered = 1;
}
else {
pReplicaStatus[i].registered = 0;
}
if ( ( chkSum != NULL ) && ( strlen( chkSum ) > 0 ) ) {
strcpy( pReplicaStatus[i].checksum, chkSum );
}
}
freeGenQueryOut( &genQueryOut );
/* check replica status */
at_least_one_copy_is_good = 0;
for ( i = 0; i < nReplicas; i++ ) {
/* check the file existence first */
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
snprintf( myDataObjInp.objPath, MAX_NAME_LEN, "%s/%s", parColl, fileName );
myDataObjInp.openFlags = O_RDONLY;
sprintf( tmpstr, "%d", pReplicaStatus[i].repl_num );
addKeyVal( &myDataObjInp.condInput, REPL_NUM_KW, tmpstr );
rn = pReplicaStatus[i].repl_num;
t = rsDataObjOpen( conn, &myDataObjInp );
if ( t < 0 ) {
/* fprintf(stderr,"%d. %s/%s, %d failed to open and has checksum status=%d\n", i, parColl, fileName, rn, t); */
if ( t == SYS_OUT_OF_FILE_DESC ) {
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
else {
pReplicaStatus[i].chksum_status = t;
}
}
else {
openedDataObjInp_t myDataObjCloseInp;
memset( &myDataObjCloseInp, 0, sizeof( openedDataObjInp_t ) );
myDataObjCloseInp.l1descInx = t;
t = rsDataObjClose( conn, &myDataObjCloseInp );
chksum_str = NULL;
/* compute checksum rsDataObjChksum() */
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
sprintf( myDataObjInp.objPath, "%s/%s", parColl, fileName );
addKeyVal( &myDataObjInp.condInput, FORCE_CHKSUM_KW, "" );
sprintf( tmpstr, "%d", pReplicaStatus[i].repl_num );
addKeyVal( &myDataObjInp.condInput, REPL_NUM_KW, tmpstr );
t = rsDataObjChksum( conn, &myDataObjInp, &chksum_str );
pReplicaStatus[i].chksum_status = t; /* t == USER_CHKSUM_MISMATCH means a bad copy */
/* fprintf(stderr,"%d. %s/%s, %d has checksum status=%d\n", i, parColl, fileName, rn, t); */
if ( t >= 0 ) {
if ( strlen( pReplicaStatus[i].checksum ) > 0 ) {
if ( strcmp( pReplicaStatus[i].checksum, chksum_str ) != 0 ) { /* mismatch */
pReplicaStatus[i].chksum_status = USER_CHKSUM_MISMATCH;
}
else {
at_least_one_copy_is_good = 1;
}
}
else {
at_least_one_copy_is_good = 1;
}
}
}
}
/* if there is none good copies left. In some casee, the checksum failed due
to the fact that the server is down. We leave this case to be taken care of next time. */
if ( at_least_one_copy_is_good == 0 ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():process_single_obj(): Obj='%s/%s': Warning: The system detects that all copies might be corrupted.", parColl, fileName );
#ifndef windows_platform
if ( ( emailToNotify != NULL ) && ( strlen( emailToNotify ) > 0 ) ) {
char msg_sub[1024], msg_body[1024];
strcpy( msg_sub, "iRODS msiAutoReplicateService() error" );
sprintf( msg_body, "msiAutoReplicateService():process_single_obj(): Obj='%s/%s': at least one storage server is down or all copies are corrupted.",
parColl, fileName );
UnixSendEmail( emailToNotify, msg_sub, msg_body );
}
#endif
free( pReplicaStatus ); // JMc cppcheck - leak
return 0;
}
/* since we have at least one copy is good. We delete those bad copies (USER_CHKSUM_MISMATCH). */
newN = nReplicas;
for ( i = 0; i < nReplicas; i++ ) {
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
sprintf( myDataObjInp.objPath, "%s/%s", parColl, fileName );
sprintf( tmpstr, "%d", pReplicaStatus[i].repl_num );
addKeyVal( &myDataObjInp.condInput, REPL_NUM_KW, tmpstr );
rn = pReplicaStatus[i].repl_num;
if ( pReplicaStatus[i].registered == 1 ) {
/* here is the catch. iRODS. */
int adchksum;
/* fprintf(stderr,"CD->%s/%s, v=%d, is a registered copy.\n", parColl, fileName, rn); */
adchksum = ( ( int )( pReplicaStatus[i].chksum_status / 1000 ) ) * 1000;
if ( ( pReplicaStatus[i].chksum_status == USER_CHKSUM_MISMATCH ) || ( pReplicaStatus[i].chksum_status == UNIX_FILE_OPEN_ERR ) || ( adchksum == UNIX_FILE_OPEN_ERR ) )
/* USER_CHKSUM_MISMATCH -> indicates the registered file is changed.
* UNIX_FILE_OPEN_ERR --> indicates the registered file is removed by original owner.
* -510002 is transformed from UNIX open error.
*/
{
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj(): registered copy will be removed: %s, repl=%d", myDataObjInp.objPath, rn );
t = getDataObjInfo( conn, &myDataObjInp, &myDataObjInfo, NULL, 0 );
if ( t >= 0 ) {
myUnregDataObjInp.dataObjInfo = myDataObjInfo;
myUnregDataObjInp.condInput = &myDataObjInp.condInput;
t = rsUnregDataObj( conn, &myUnregDataObjInp );
if ( t >= 0 ) {
newN = newN - 1;
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsUnregDataObj(): failed for %s/%s:%d. erStat=%d", parColl, fileName, rn, t );
return t;
}
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():getDataObjInfo(): failed for %s/%s:%d. erStat=%d", parColl, fileName, rn, t );
return t;
}
}
else {
rodsLog( LOG_ERROR, "%s:%d, the registered copy has errored checksum status=%d.", myDataObjInp.objPath, rn, pReplicaStatus[i].chksum_status );
return t;
}
}
else { /* the data file is in vault */
if ( pReplicaStatus[i].chksum_status == USER_CHKSUM_MISMATCH ) {
t = rsDataObjUnlink( conn, &myDataObjInp );
if ( t >= 0 ) {
newN = newN - 1;
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjUnlink() for %s:%d failed. errStat=%d", myDataObjInp.objPath, rn, t );
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
}
}
}
fillStrInMsParam( &msGrpRescStr, grpRescForReplication );
/* make necessary copies based on the required number of copies */
if ( newN < required_num_replicas ) {
rodsLog( LOG_NOTICE, "msiAutoReplicateService():process_single_obj(): making necessary %d copies as required.", ( required_num_replicas - newN ) );
for ( i = 0; i < ( required_num_replicas - newN ); i++ ) {
memset( &myDataObjInp, 0, sizeof( dataObjInp_t ) );
snprintf( myDataObjInp.objPath, MAX_NAME_LEN, "%s/%s", parColl, fileName );
/* addKeyVal(&myDataObjInp.condInput, DEST_RESC_NAME_KW, grpRescForReplication); */
validKwFlags = OBJ_PATH_FLAG | DEST_RESC_NAME_FLAG | NUM_THREADS_FLAG |
BACKUP_RESC_NAME_FLAG | RESC_NAME_FLAG | UPDATE_REPL_FLAG |
REPL_NUM_FLAG | ALL_FLAG | ADMIN_FLAG | VERIFY_CHKSUM_FLAG |
RBUDP_TRANSFER_FLAG | RBUDP_SEND_RATE_FLAG | RBUDP_PACK_SIZE_FLAG;
t = parseMsKeyValStrForDataObjInp( &msGrpRescStr, &myDataObjInp, DEST_RESC_NAME_KW, validKwFlags, &outBadKeyWd );
if ( t < 0 ) {
if ( outBadKeyWd != NULL ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjRepl(): input keyWd - %s error. status = %d", outBadKeyWd, t );
free( outBadKeyWd );
}
else {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjRepl(): input msKeyValStr error. status = %d", t );
}
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
t = rsDataObjRepl( conn, &myDataObjInp, &transStat );
if ( t < 0 ) {
rodsLog( LOG_ERROR, "msiAutoReplicateService():rsDataObjRepl() failed for %s/%s:%d into '%s'. err code=%d.", parColl, fileName, rn, grpRescForReplication, t );
repl_storage_error = 1;
free( pReplicaStatus ); // JMc cppcheck - leak
return t;
}
if ( transStat != NULL ) {
free( transStat );
}
}
}
free( pReplicaStatus ); // JMC cppcheck - leak
return 0;
}
