int getCollUtil (rcComm_t **myConn, char *srcColl, char *targDir, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp, rodsRestart_t *rodsRestart) { int status = 0; int savedStatus = 0; char srcChildPath[MAX_NAME_LEN], targChildPath[MAX_NAME_LEN]; #if 0 int collLen; #else char parPath[MAX_NAME_LEN], childPath[MAX_NAME_LEN]; #endif collHandle_t collHandle; collEnt_t collEnt; dataObjInp_t childDataObjInp; rcComm_t *conn; if (srcColl == NULL || targDir == NULL) { rodsLog (LOG_ERROR, "getCollUtil: NULL srcColl or targDir input"); return (USER__NULL_INPUT_ERR); } if (rodsArgs->recursive != True) { rodsLog (LOG_ERROR, "getCollUtil: -r option must be used for getting %s collection", targDir); return (USER_INPUT_OPTION_ERR); } if (rodsArgs->redirectConn == True) { int reconnFlag; if (rodsArgs->reconnect == True) { reconnFlag = RECONN_TIMEOUT; } else { reconnFlag = NO_RECONN; } /* reconnect to the resource server */ rstrcpy (dataObjOprInp->objPath, srcColl, MAX_NAME_LEN); redirectConnToRescSvr (myConn, dataObjOprInp, myRodsEnv, reconnFlag); rodsArgs->redirectConn = 0; /* only do it once */ } conn = *myConn; printCollOrDir (targDir, LOCAL_DIR_T, rodsArgs, dataObjOprInp->specColl); #if 0 status = rclOpenCollection (conn, srcColl, RECUR_QUERY_FG, &collHandle); #else status = rclOpenCollection (conn, srcColl, 0, &collHandle); #endif if (status < 0) { rodsLog (LOG_ERROR, "getCollUtil: rclOpenCollection of %s error. status = %d", srcColl, status); return status; } #if 0 collLen = strlen (srcColl); collLen = getOpenedCollLen (&collHandle); #endif while ((status = rclReadCollection (conn, &collHandle, &collEnt)) >= 0) { if (collEnt.objType == DATA_OBJ_T) { rodsLong_t mySize; mySize = collEnt.dataSize; /* have to save it. May be freed */ #if 0 snprintf (targChildPath, MAX_NAME_LEN, "%s%s/%s", targDir, collEnt.collName + collLen, collEnt.dataName); #else snprintf (targChildPath, MAX_NAME_LEN, "%s/%s", targDir, collEnt.dataName); #endif snprintf (srcChildPath, MAX_NAME_LEN, "%s/%s", collEnt.collName, collEnt.dataName); status = chkStateForResume (conn, rodsRestart, targChildPath, rodsArgs, LOCAL_FILE_T, &dataObjOprInp->condInput, 1); if (status < 0) { /* restart failed */ break; } else if (status == 0) { continue; } status = getDataObjUtil (conn, srcChildPath, targChildPath, mySize, collEnt.dataMode, myRodsEnv, rodsArgs, dataObjOprInp); if (status < 0) { rodsLogError (LOG_ERROR, status, "getCollUtil: getDataObjUtil failed for %s. status = %d", srcChildPath, status); savedStatus = status; if (rodsRestart->fd > 0) break; } else { status = procAndWrriteRestartFile (rodsRestart, targChildPath); } } else if (collEnt.objType == COLL_OBJ_T) { if ((status = splitPathByKey ( collEnt.collName, parPath, childPath, '/')) < 0) { rodsLogError (LOG_ERROR, status, "getCollUtil:: splitPathByKey for %s error, status = %d", collEnt.collName, status); return (status); } snprintf (targChildPath, MAX_NAME_LEN, "%s/%s", targDir, childPath); mkdirR (targDir, targChildPath, 0750); /* the child is a spec coll. need to drill down */ childDataObjInp = *dataObjOprInp; if (collEnt.specColl.collClass != NO_SPEC_COLL) childDataObjInp.specColl = &collEnt.specColl; else childDataObjInp.specColl = NULL; status = getCollUtil (myConn, collEnt.collName, targChildPath, myRodsEnv, rodsArgs, &childDataObjInp, rodsRestart); if (status < 0 && status != CAT_NO_ROWS_FOUND) { rodsLogError (LOG_ERROR, status, "getCollUtil: getCollUtil failed for %s. status = %d", collEnt.collName, status); savedStatus = status; if (rodsRestart->fd > 0) break; } } } rclCloseCollection (&collHandle); if (savedStatus < 0) { return (savedStatus); } else if (status == CAT_NO_ROWS_FOUND || status == SYS_SPEC_COLL_OBJ_NOT_EXIST) { return (0); } else { return (status); } }
int putDirUtil( rcComm_t **myConn, char *srcDir, char *targColl, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp, bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart, bulkOprInfo_t *bulkOprInfo ) { char srcChildPath[MAX_NAME_LEN], targChildPath[MAX_NAME_LEN]; if ( srcDir == NULL || targColl == NULL ) { rodsLog( LOG_ERROR, "putDirUtil: NULL srcDir or targColl input" ); return USER__NULL_INPUT_ERR; } if ( isPathSymlink( rodsArgs, srcDir ) > 0 ) { return 0; } if ( rodsArgs->recursive != True ) { rodsLog( LOG_ERROR, "putDirUtil: -r option must be used for putting %s directory", srcDir ); return USER_INPUT_OPTION_ERR; } if ( rodsArgs->redirectConn == True && rodsArgs->force != True ) { int reconnFlag; if ( rodsArgs->reconnect == True ) { reconnFlag = RECONN_TIMEOUT; } else { reconnFlag = NO_RECONN; } /* reconnect to the resource server */ rstrcpy( dataObjOprInp->objPath, targColl, MAX_NAME_LEN ); redirectConnToRescSvr( myConn, dataObjOprInp, myRodsEnv, reconnFlag ); rodsArgs->redirectConn = 0; /* only do it once */ } rcComm_t *conn = *myConn; boost::filesystem::path srcDirPath( srcDir ); if ( !exists( srcDirPath ) || !is_directory( srcDirPath ) ) { rodsLog( LOG_ERROR, "putDirUtil: opendir local dir error for %s, errno = %d\n", srcDir, errno ); return USER_INPUT_PATH_ERR; } if ( rodsArgs->verbose == True ) { fprintf( stdout, "C- %s:\n", targColl ); } int bulkFlag = NON_BULK_OPR; if ( bulkOprInfo != NULL ) { bulkFlag = bulkOprInfo->flags; } int savedStatus = 0; boost::filesystem::directory_iterator end_itr; // default construction yields past-the-end for ( boost::filesystem::directory_iterator itr( srcDirPath ); itr != end_itr; ++itr ) { boost::filesystem::path p = itr->path(); snprintf( srcChildPath, MAX_NAME_LEN, "%s", p.c_str() ); if ( isPathSymlink( rodsArgs, srcChildPath ) > 0 ) { continue; } if ( !exists( p ) ) { rodsLog( LOG_ERROR, "putDirUtil: stat error for %s, errno = %d\n", srcChildPath, errno ); return USER_INPUT_PATH_ERR; } if ( is_symlink( p ) ) { boost::filesystem::path cp = read_symlink( p ); snprintf( srcChildPath, MAX_NAME_LEN, "%s/%s", srcDir, cp.c_str() ); p = boost::filesystem::path( srcChildPath ); } rodsLong_t dataSize = 0; dataObjOprInp->createMode = getPathStMode( p.c_str() ); objType_t childObjType; if ( is_regular_file( p ) ) { childObjType = DATA_OBJ_T; dataSize = file_size( p ); } else if ( is_directory( p ) ) { childObjType = COLL_OBJ_T; } else { rodsLog( LOG_ERROR, "putDirUtil: unknown local path type for %s", srcChildPath ); savedStatus = USER_INPUT_PATH_ERR; continue; } boost::filesystem::path childPath = p.filename(); snprintf( targChildPath, MAX_NAME_LEN, "%s/%s", targColl, childPath.c_str() ); if ( childObjType == DATA_OBJ_T ) { if ( bulkFlag == BULK_OPR_SMALL_FILES && file_size( p ) > MAX_BULK_OPR_FILE_SIZE ) { continue; } else if ( bulkFlag == BULK_OPR_LARGE_FILES && file_size( p ) <= MAX_BULK_OPR_FILE_SIZE ) { continue; } } int status = chkStateForResume( conn, rodsRestart, targChildPath, rodsArgs, childObjType, &dataObjOprInp->condInput, 1 ); if ( status < 0 ) { /* restart failed */ return status; } else if ( status == 0 ) { if ( bulkFlag == BULK_OPR_SMALL_FILES && ( rodsRestart->restartState & LAST_PATH_MATCHED ) != 0 ) { /* enable foreFlag one time */ setForceFlagForRestart( bulkOprInp, bulkOprInfo ); } continue; } if ( childObjType == DATA_OBJ_T ) { /* a file */ if ( bulkFlag == BULK_OPR_SMALL_FILES ) { status = bulkPutFileUtil( conn, srcChildPath, targChildPath, dataSize, dataObjOprInp->createMode, rodsArgs, bulkOprInp, bulkOprInfo ); } else { /* normal put */ status = putFileUtil( conn, srcChildPath, targChildPath, dataSize, rodsArgs, dataObjOprInp ); } if ( rodsRestart->fd > 0 ) { if ( status >= 0 ) { if ( bulkFlag == BULK_OPR_SMALL_FILES ) { if ( status > 0 ) { /* status is the number of files bulk loaded */ rodsRestart->curCnt += status; status = writeRestartFile( rodsRestart, targChildPath ); } } else { /* write the restart file */ rodsRestart->curCnt ++; status = writeRestartFile( rodsRestart, targChildPath ); } } } } else { /* a directory */ status = mkColl( conn, targChildPath ); if ( status < 0 ) { rodsLogError( LOG_ERROR, status, "putDirUtil: mkColl error for %s", targChildPath ); } status = putDirUtil( myConn, srcChildPath, targChildPath, myRodsEnv, rodsArgs, dataObjOprInp, bulkOprInp, rodsRestart, bulkOprInfo ); } if ( status < 0 && status != CAT_NO_ROWS_FOUND ) { rodsLogError( LOG_ERROR, status, "putDirUtil: put %s failed. status = %d", srcChildPath, status ); savedStatus = status; if ( rodsRestart->fd > 0 ) { break; } } } return savedStatus; }
int cpCollUtil( rcComm_t *conn, char *srcColl, char *targColl, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjCopyInp_t *dataObjCopyInp, rodsRestart_t *rodsRestart ) { int status = 0; int savedStatus = 0; char parPath[MAX_NAME_LEN], childPath[MAX_NAME_LEN]; collHandle_t collHandle; collEnt_t collEnt; char srcChildPath[MAX_NAME_LEN], targChildPath[MAX_NAME_LEN]; dataObjCopyInp_t childDataObjCopyInp; if ( srcColl == NULL || targColl == NULL ) { rodsLog( LOG_ERROR, "cpCollUtil: NULL srcColl or targColl in cp" ); return USER__NULL_INPUT_ERR; } if ( rodsArgs->recursive != True ) { rodsLog( LOG_ERROR, "cpCollUtil: -r option must be used for copying %s directory", srcColl ); return USER_INPUT_OPTION_ERR; } status = rclOpenCollection( conn, srcColl, 0, &collHandle ); if ( status < 0 ) { rodsLog( LOG_ERROR, "cpCollUtil: rclOpenCollection of %s error. status = %d", srcColl, status ); return status; } while ( ( status = rclReadCollection( conn, &collHandle, &collEnt ) ) >= 0 ) { if ( collEnt.objType == DATA_OBJ_T ) { snprintf( srcChildPath, MAX_NAME_LEN, "%s/%s", collEnt.collName, collEnt.dataName ); snprintf( targChildPath, MAX_NAME_LEN, "%s/%s", targColl, collEnt.dataName ); status = chkStateForResume( conn, rodsRestart, targChildPath, rodsArgs, DATA_OBJ_T, &dataObjCopyInp->destDataObjInp.condInput, 1 ); if ( status < 0 ) { /* restart failed */ break; } else if ( status == 0 ) { continue; } status = cpFileUtil( conn, srcChildPath, targChildPath, collEnt.dataSize, rodsArgs, dataObjCopyInp ); if ( status < 0 ) { rodsLogError( LOG_ERROR, status, "getCollUtil: getDataObjUtil failed for %s. status = %d", srcChildPath, status ); if ( rodsRestart->fd > 0 ) { break; } else { savedStatus = status; } } else { status = procAndWrriteRestartFile( rodsRestart, targChildPath ); } } else if ( collEnt.objType == COLL_OBJ_T ) { if ( ( status = splitPathByKey( collEnt.collName, parPath, MAX_NAME_LEN, childPath, MAX_NAME_LEN, '/' ) ) < 0 ) { rodsLogError( LOG_ERROR, status, "cpCollUtil:: splitPathByKey for %s error, status = %d", collEnt.collName, status ); return status; } snprintf( targChildPath, MAX_NAME_LEN, "%s/%s", targColl, childPath ); mkCollR( conn, targColl, targChildPath ); if ( rodsArgs->verbose == True ) { fprintf( stdout, "C- %s:\n", targChildPath ); } /* the child is a spec coll. need to drill down */ childDataObjCopyInp = *dataObjCopyInp; if ( collEnt.specColl.collClass != NO_SPEC_COLL ) childDataObjCopyInp.srcDataObjInp.specColl = &collEnt.specColl; status = cpCollUtil( conn, collEnt.collName, targChildPath, myRodsEnv, rodsArgs, &childDataObjCopyInp, rodsRestart ); if ( status < 0 && status != CAT_NO_ROWS_FOUND && status != SYS_SPEC_COLL_OBJ_NOT_EXIST ) { savedStatus = status; } } } rclCloseCollection( &collHandle ); if ( savedStatus < 0 ) { return savedStatus; } else if ( status == CAT_NO_ROWS_FOUND || status == SYS_SPEC_COLL_OBJ_NOT_EXIST ) { return 0; } else { return status; } }
int replCollUtil( rcComm_t *conn, char *srcColl, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjInp, rodsRestart_t *rodsRestart ) { int status; int savedStatus = 0; collHandle_t collHandle; collEnt_t collEnt; char srcChildPath[MAX_NAME_LEN]; if ( srcColl == NULL ) { rodsLog( LOG_ERROR, "replCollUtil: NULL srcColl input" ); return ( USER__NULL_INPUT_ERR ); } if ( rodsArgs->recursive != True ) { rodsLog( LOG_ERROR, "replCollUtil: -r option must be used for getting %s collection", srcColl ); return ( USER_INPUT_OPTION_ERR ); } if ( rodsArgs->verbose == True ) { fprintf( stdout, "C- %s:\n", srcColl ); } bzero( &collHandle, sizeof( collHandle ) ); replKeyVal( &dataObjInp->condInput, &collHandle.dataObjInp.condInput ); status = rclOpenCollection( conn, srcColl, INCLUDE_CONDINPUT_IN_QUERY, &collHandle ); if ( status < 0 ) { rodsLog( LOG_ERROR, "replCollUtil: rclOpenCollection of %s error. status = %d", srcColl, status ); return status; } if ( collHandle.rodsObjStat->specColl != NULL && collHandle.rodsObjStat->specColl->collClass != LINKED_COLL ) { /* no repl for mounted coll */ rclCloseCollection( &collHandle ); return 0; } while ( ( status = rclReadCollection( conn, &collHandle, &collEnt ) ) >= 0 ) { if ( collEnt.objType == DATA_OBJ_T ) { snprintf( srcChildPath, MAX_NAME_LEN, "%s/%s", collEnt.collName, collEnt.dataName ); status = chkStateForResume( conn, rodsRestart, srcChildPath, rodsArgs, DATA_OBJ_T, &dataObjInp->condInput, 0 ); if ( status < 0 ) { /* restart failed */ break; } else if ( status == 0 ) { continue; } status = replDataObjUtil( conn, srcChildPath, collEnt.dataSize, myRodsEnv, rodsArgs, dataObjInp ); if ( status == SYS_COPY_ALREADY_IN_RESC ) { if ( rodsArgs->verbose == True ) { printf( "copy of %s already exists. Probably OK\n", srcChildPath ); } status = 0; } if ( status < 0 ) { rodsLogError( LOG_ERROR, status, "replCollUtil: replDataObjUtil failed for %s. status = %d", srcChildPath, status ); if ( rodsRestart->fd > 0 ) { break; } else { savedStatus = status; } } else { status = procAndWrriteRestartFile( rodsRestart, srcChildPath ); } } else if ( collEnt.objType == COLL_OBJ_T ) { dataObjInp_t childDataObjInp; childDataObjInp = *dataObjInp; if ( collEnt.specColl.collClass != NO_SPEC_COLL ) { childDataObjInp.specColl = &collEnt.specColl; } else { childDataObjInp.specColl = NULL; } status = replCollUtil( conn, collEnt.collName, myRodsEnv, rodsArgs, &childDataObjInp, rodsRestart ); if ( status < 0 && status != CAT_NO_ROWS_FOUND ) { savedStatus = status; } } } rclCloseCollection( &collHandle ); if ( savedStatus < 0 ) { return ( savedStatus ); } else if ( status == CAT_NO_ROWS_FOUND ) { return ( 0 ); } else { return ( status ); } }