int cpCollUtil( rcComm_t *conn, char *srcColl, char *targColl, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjCopyInp_t *dataObjCopyInp, rodsRestart_t *rodsRestart ) { int status = 0; int savedStatus = 0; char parPath[MAX_NAME_LEN], childPath[MAX_NAME_LEN]; collHandle_t collHandle; collEnt_t collEnt; char srcChildPath[MAX_NAME_LEN], targChildPath[MAX_NAME_LEN]; dataObjCopyInp_t childDataObjCopyInp; if ( srcColl == NULL || targColl == NULL ) { rodsLog( LOG_ERROR, "cpCollUtil: NULL srcColl or targColl in cp" ); return USER__NULL_INPUT_ERR; } if ( rodsArgs->recursive != True ) { rodsLog( LOG_ERROR, "cpCollUtil: -r option must be used for copying %s directory", srcColl ); return USER_INPUT_OPTION_ERR; } status = rclOpenCollection( conn, srcColl, 0, &collHandle ); if ( status < 0 ) { rodsLog( LOG_ERROR, "cpCollUtil: rclOpenCollection of %s error. status = %d", srcColl, status ); return status; } while ( ( status = rclReadCollection( conn, &collHandle, &collEnt ) ) >= 0 ) { if ( collEnt.objType == DATA_OBJ_T ) { snprintf( srcChildPath, MAX_NAME_LEN, "%s/%s", collEnt.collName, collEnt.dataName ); snprintf( targChildPath, MAX_NAME_LEN, "%s/%s", targColl, collEnt.dataName ); status = chkStateForResume( conn, rodsRestart, targChildPath, rodsArgs, DATA_OBJ_T, &dataObjCopyInp->destDataObjInp.condInput, 1 ); if ( status < 0 ) { /* restart failed */ break; } else if ( status == 0 ) { continue; } status = cpFileUtil( conn, srcChildPath, targChildPath, collEnt.dataSize, rodsArgs, dataObjCopyInp ); if ( status < 0 ) { rodsLogError( LOG_ERROR, status, "getCollUtil: getDataObjUtil failed for %s. status = %d", srcChildPath, status ); if ( rodsRestart->fd > 0 ) { break; } else { savedStatus = status; } } else { status = procAndWrriteRestartFile( rodsRestart, targChildPath ); } } else if ( collEnt.objType == COLL_OBJ_T ) { if ( ( status = splitPathByKey( collEnt.collName, parPath, MAX_NAME_LEN, childPath, MAX_NAME_LEN, '/' ) ) < 0 ) { rodsLogError( LOG_ERROR, status, "cpCollUtil:: splitPathByKey for %s error, status = %d", collEnt.collName, status ); return status; } snprintf( targChildPath, MAX_NAME_LEN, "%s/%s", targColl, childPath ); mkCollR( conn, targColl, targChildPath ); if ( rodsArgs->verbose == True ) { fprintf( stdout, "C- %s:\n", targChildPath ); } /* the child is a spec coll. need to drill down */ childDataObjCopyInp = *dataObjCopyInp; if ( collEnt.specColl.collClass != NO_SPEC_COLL ) childDataObjCopyInp.srcDataObjInp.specColl = &collEnt.specColl; status = cpCollUtil( conn, collEnt.collName, targChildPath, myRodsEnv, rodsArgs, &childDataObjCopyInp, rodsRestart ); if ( status < 0 && status != CAT_NO_ROWS_FOUND && status != SYS_SPEC_COLL_OBJ_NOT_EXIST ) { savedStatus = status; } } } rclCloseCollection( &collHandle ); if ( savedStatus < 0 ) { return savedStatus; } else if ( status == CAT_NO_ROWS_FOUND || status == SYS_SPEC_COLL_OBJ_NOT_EXIST ) { return 0; } else { return status; } }
int rsyncDirToCollUtil (rcComm_t *conn, rodsPath_t *srcPath, rodsPath_t *targPath, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp) { int status = 0; int savedStatus = 0; #ifndef USE_BOOST_FS DIR *dirPtr; struct dirent *myDirent; #ifndef windows_platform struct stat statbuf; #else struct irodsntstat statbuf; #endif #endif /* #ifndef USE_BOOST_FS */ char *srcDir, *targColl; rodsPath_t mySrcPath, myTargPath; if (srcPath == NULL || targPath == NULL) { rodsLog (LOG_ERROR, "rsyncDirToCollUtil: NULL srcPath or targPath input"); return (USER__NULL_INPUT_ERR); } srcDir = srcPath->outPath; targColl = targPath->outPath; if (isPathSymlink (rodsArgs, srcDir) > 0) return 0; if (rodsArgs->recursive != True) { rodsLog (LOG_ERROR, "rsyncDirToCollUtil: -r option must be used for putting %s directory", srcDir); return (USER_INPUT_OPTION_ERR); } #ifdef USE_BOOST_FS path srcDirPath (srcDir); if (!exists(srcDirPath) || !is_directory(srcDirPath)) { #else dirPtr = opendir (srcDir); if (dirPtr == NULL) { #endif /* USE_BOOST_FS */ rodsLog (LOG_ERROR, "rsyncDirToCollUtil: opendir local dir error for %s, errno = %d\n", srcDir, errno); return (USER_INPUT_PATH_ERR); } if (rodsArgs->verbose == True) { fprintf (stdout, "C- %s:\n", targColl); } memset (&mySrcPath, 0, sizeof (mySrcPath)); memset (&myTargPath, 0, sizeof (myTargPath)); myTargPath.objType = DATA_OBJ_T; mySrcPath.objType = LOCAL_FILE_T; #ifdef USE_BOOST_FS directory_iterator end_itr; // default construction yields past-the-end for (directory_iterator itr(srcDirPath); itr != end_itr;++itr) { path p = itr->path(); snprintf (mySrcPath.outPath, MAX_NAME_LEN, "%s", p.c_str ()); #else while ((myDirent = readdir (dirPtr)) != NULL) { if (strcmp (myDirent->d_name, ".") == 0 || strcmp (myDirent->d_name, "..") == 0) { continue; } snprintf (mySrcPath.outPath, MAX_NAME_LEN, "%s/%s", srcDir, myDirent->d_name); #endif /* USE_BOOST_FS */ if (isPathSymlink (rodsArgs, mySrcPath.outPath) > 0) continue; #ifdef USE_BOOST_FS #if 0 path p (mySrcPath.outPath); #endif if (!exists(p)) { #else #ifndef windows_platform status = stat (mySrcPath.outPath, &statbuf); #else status = iRODSNt_stat(mySrcPath.outPath, &statbuf); #endif if (status != 0) { closedir (dirPtr); #endif /* USE_BOOST_FS */ rodsLog (LOG_ERROR, "rsyncDirToCollUtil: stat error for %s, errno = %d\n", mySrcPath.outPath, errno); return (USER_INPUT_PATH_ERR); } #ifndef USE_BOOST_FS if ((statbuf.st_mode & S_IFREG) != 0 && rodsArgs->age == True) { if (ageExceeded (rodsArgs->agevalue, statbuf.st_mtime, rodsArgs->verbose, mySrcPath.outPath, statbuf.st_size)) continue; } #endif /* #ifndef USE_BOOST_FS */ #ifdef FILESYSTEM_META getFileMetaFromPath(mySrcPath.outPath, &dataObjOprInp->condInput); #endif bzero (&myTargPath, sizeof (myTargPath)); #ifdef USE_BOOST_FS path childPath = p.filename(); snprintf (myTargPath.outPath, MAX_NAME_LEN, "%s/%s", targColl, childPath.c_str()); if (is_symlink (p)) { path cp = read_symlink (p); snprintf (mySrcPath.outPath, MAX_NAME_LEN, "%s/%s", srcDir, cp.c_str ()); p = path (mySrcPath.outPath); } dataObjOprInp->createMode = getPathStMode (p); if (is_regular_file(p)) { #else snprintf (myTargPath.outPath, MAX_NAME_LEN, "%s/%s", targColl, myDirent->d_name); dataObjOprInp->createMode = statbuf.st_mode; if ((statbuf.st_mode & S_IFREG) != 0) { /* a file */ #endif myTargPath.objType = DATA_OBJ_T; mySrcPath.objType = LOCAL_FILE_T; mySrcPath.objState = EXIST_ST; #ifdef USE_BOOST_FS mySrcPath.size = file_size (p); #else mySrcPath.size = statbuf.st_size; #endif getRodsObjType (conn, &myTargPath); status = rsyncFileToDataUtil (conn, &mySrcPath, &myTargPath, myRodsEnv, rodsArgs, dataObjOprInp); /* fix a big mem leak */ if (myTargPath.rodsObjStat != NULL) { freeRodsObjStat (myTargPath.rodsObjStat); myTargPath.rodsObjStat = NULL; } #if 0 if (myTargPath.objState != NOT_EXIST_ST) freeRodsObjStat (myTargPath.rodsObjStat); #endif #ifdef USE_BOOST_FS } else if (is_directory(p)) { #else } else if ((statbuf.st_mode & S_IFDIR) != 0) { /* a directory */ #endif status = 0; /* only do the sync if no -l option specified */ if ( rodsArgs->longOption != True ) { #ifdef FILESYSTEM_META status = mkCollRWithDirMeta (conn, targColl, myTargPath.outPath, mySrcPath.outPath); #else status = mkCollR (conn, targColl, myTargPath.outPath); #endif } if (status < 0) { rodsLogError (LOG_ERROR, status, "rsyncDirToCollUtil: mkCollR error for %s", myTargPath.outPath); } else { myTargPath.objType = COLL_OBJ_T; mySrcPath.objType = LOCAL_DIR_T; mySrcPath.objState = myTargPath.objState = EXIST_ST; getRodsObjType (conn, &myTargPath); status = rsyncDirToCollUtil (conn, &mySrcPath, &myTargPath, myRodsEnv, rodsArgs, dataObjOprInp); /* fix a big mem leak */ if (myTargPath.rodsObjStat != NULL) { freeRodsObjStat (myTargPath.rodsObjStat); myTargPath.rodsObjStat = NULL; } #if 0 if (myTargPath.objState != NOT_EXIST_ST) freeRodsObjStat (myTargPath.rodsObjStat); #endif } } else { rodsLog (LOG_ERROR, #ifdef USE_BOOST_FS "rsyncDirToCollUtil: unknown local path %s", mySrcPath.outPath); #else "rsyncDirToCollUtil: unknown local path type %d for %s", statbuf.st_mode, mySrcPath.outPath); #endif /* USE_BOOST_FS */ status = USER_INPUT_PATH_ERR; } if (status < 0) { savedStatus = status; rodsLogError (LOG_ERROR, status, "rsyncDirToCollUtil: put %s failed. status = %d", mySrcPath.outPath, status); } } #ifndef USE_BOOST_FS closedir (dirPtr); #endif if (savedStatus < 0) { return (savedStatus); } else if (status == CAT_NO_ROWS_FOUND || status == SYS_SPEC_COLL_OBJ_NOT_EXIST) { return (0); } else { return (status); } } int rsyncCollToCollUtil (rcComm_t *conn, rodsPath_t *srcPath, rodsPath_t *targPath, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjCopyInp_t *dataObjCopyInp) { int status = 0; int savedStatus = 0; char *srcColl, *targColl; char targChildPath[MAX_NAME_LEN]; #if 0 int collLen; #else char parPath[MAX_NAME_LEN], childPath[MAX_NAME_LEN]; #endif rodsPath_t mySrcPath, myTargPath; collHandle_t collHandle; collEnt_t collEnt; dataObjCopyInp_t childDataObjCopyInp; dataObjInp_t *dataObjOprInp = &collHandle.dataObjInp; if (srcPath == NULL || targPath == NULL) { rodsLog (LOG_ERROR, "rsyncCollToCollUtil: NULL srcPath or targPath input"); return (USER__NULL_INPUT_ERR); } srcColl = srcPath->outPath; targColl = targPath->outPath; if (rodsArgs->recursive != True) { rodsLog (LOG_ERROR, "rsyncCollToCollUtil: -r option must be used for collection %s", targColl); return (USER_INPUT_OPTION_ERR); } #if 0 status = rclOpenCollection (conn, srcColl, RECUR_QUERY_FG | VERY_LONG_METADATA_FG, &collHandle); #else status = rclOpenCollection (conn, srcColl, VERY_LONG_METADATA_FG, &collHandle); #endif if (status < 0) { rodsLog (LOG_ERROR, "getCollUtil: rclOpenCollection of %s error. status = %d", srcColl, status); return status; } if (dataObjOprInp->specColl != NULL) { if (rodsArgs->verbose == True) { if (rodsArgs->verbose == True) { char objType[NAME_LEN]; status = getSpecCollTypeStr (dataObjOprInp->specColl, objType); if (status < 0) fprintf (stdout, "C- %s :\n", targColl); else fprintf (stdout, "C- %s %-5.5s :\n", targColl, objType); } } } memset (&mySrcPath, 0, sizeof (mySrcPath)); memset (&myTargPath, 0, sizeof (myTargPath)); myTargPath.objType = DATA_OBJ_T; mySrcPath.objType = DATA_OBJ_T; #if 0 collLen = strlen (srcColl); collLen = getOpenedCollLen (&collHandle); #endif while ((status = rclReadCollection (conn, &collHandle, &collEnt)) >= 0) { if (collEnt.objType == DATA_OBJ_T) { if (rodsArgs->age == True) { if (ageExceeded (rodsArgs->agevalue, atoi (collEnt.modifyTime), rodsArgs->verbose, collEnt.dataName, collEnt.dataSize)) continue; } #if 0 snprintf (myTargPath.outPath, MAX_NAME_LEN, "%s%s/%s", targColl, collEnt.collName + collLen, collEnt.dataName); #else snprintf (myTargPath.outPath, MAX_NAME_LEN, "%s/%s", targColl, collEnt.dataName); #endif snprintf (mySrcPath.outPath, MAX_NAME_LEN, "%s/%s", collEnt.collName, collEnt.dataName); /* fill in some info for mySrcPath */ if (strlen (mySrcPath.dataId) == 0) rstrcpy (mySrcPath.dataId, collEnt.dataId, NAME_LEN); mySrcPath.size = collEnt.dataSize; rstrcpy (mySrcPath.chksum, collEnt.chksum, CHKSUM_LEN); mySrcPath.objState = EXIST_ST; #if 0 getFileType (&myTargPath); #else getRodsObjType (conn, &myTargPath); #endif status = rsyncDataToDataUtil (conn, &mySrcPath, &myTargPath, myRodsEnv, rodsArgs, dataObjCopyInp); if (myTargPath.rodsObjStat != NULL) { freeRodsObjStat (myTargPath.rodsObjStat); myTargPath.rodsObjStat = NULL; } if (status < 0) { rodsLogError (LOG_ERROR, status, "rsyncCollToCollUtil: rsyncDataToDataUtil failed for %s.stat=%d", myTargPath.outPath, status); /* need to set global error here */ savedStatus = status; status = 0; } } else if (collEnt.objType == COLL_OBJ_T) { #if 0 if (strlen (collEnt.collName) <= collLen) continue; snprintf (targChildPath, MAX_NAME_LEN, "%s%s", targColl, collEnt.collName + collLen); #else if ((status = splitPathByKey ( collEnt.collName, parPath, childPath, '/')) < 0) { rodsLogError (LOG_ERROR, status, "rsyncCollToCollUtil:: splitPathByKey for %s error, status = %d", collEnt.collName, status); return (status); } snprintf (targChildPath, MAX_NAME_LEN, "%s/%s", targColl, childPath); #endif if ( rodsArgs->longOption != True ) { /* only do the sync if no -l option specified */ #ifdef FILESYSTEM_META mkCollWithSrcCollMeta (conn, targChildPath, collEnt.collName); #else mkColl (conn, targChildPath); #endif } #if 0 if (collEnt.specColl.collClass != NO_SPEC_COLL) { #endif /* the child is a spec coll. need to drill down */ childDataObjCopyInp = *dataObjCopyInp; if (collEnt.specColl.collClass != NO_SPEC_COLL) childDataObjCopyInp.srcDataObjInp.specColl = &collEnt.specColl; rstrcpy (myTargPath.outPath, targChildPath, MAX_NAME_LEN); rstrcpy (mySrcPath.outPath, collEnt.collName, MAX_NAME_LEN); status = rsyncCollToCollUtil (conn, &mySrcPath, &myTargPath, myRodsEnv, rodsArgs, &childDataObjCopyInp); if (status < 0 && status != CAT_NO_ROWS_FOUND) { return (status); } #if 0 } #endif } } rclCloseCollection (&collHandle); if (savedStatus < 0) { return (savedStatus); } else if (status == CAT_NO_ROWS_FOUND || status == SYS_SPEC_COLL_OBJ_NOT_EXIST) { return (0); } else { return (status); } } int initCondForRsync (rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjInp) { char *myResc = NULL; char tmpStr[NAME_LEN]; if (dataObjInp == NULL) { rodsLog (LOG_ERROR, "initCondForRsync: NULL dataObjOprInp input"); return (USER__NULL_INPUT_ERR); } memset (dataObjInp, 0, sizeof (dataObjInp_t)); if (rodsArgs == NULL) { return (0); } /* always turn on the force flag */ addKeyVal (&dataObjInp->condInput, FORCE_FLAG_KW, ""); if (rodsArgs->sizeFlag == True) { addKeyVal (&dataObjInp->condInput, VERIFY_BY_SIZE_KW, ""); } if (rodsArgs->all == True) { addKeyVal (&dataObjInp->condInput, ALL_KW, ""); } #ifdef windows_platform dataObjInp->numThreads = NO_THREADING; #else if (rodsArgs->number == True) { if (rodsArgs->numberValue == 0) { dataObjInp->numThreads = NO_THREADING; } else { dataObjInp->numThreads = rodsArgs->numberValue; } } #endif if (rodsArgs->resource == True) { if (rodsArgs->resourceString == NULL) { rodsLog (LOG_ERROR, "initCondForRepl: NULL resourceString error"); return (USER__NULL_INPUT_ERR); } else { myResc = rodsArgs->resourceString; addKeyVal (&dataObjInp->condInput, DEST_RESC_NAME_KW, rodsArgs->resourceString); } } else if (myRodsEnv != NULL && strlen (myRodsEnv->rodsDefResource) > 0) { myResc = myRodsEnv->rodsDefResource; addKeyVal (&dataObjInp->condInput, DEST_RESC_NAME_KW, myRodsEnv->rodsDefResource); } if (rodsArgs->age == True) { snprintf (tmpStr, NAME_LEN, "%d", rodsArgs->agevalue); addKeyVal (&dataObjInp->condInput, AGE_KW, tmpStr); } return (0); }