int
bulkPutDirUtil( rcComm_t **myConn, char *srcDir, char *targColl,
rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp,
bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart ) {
int status;
bulkOprInfo_t bulkOprInfo;
/* do large files first */
bzero( &bulkOprInfo, sizeof( bulkOprInfo ) );
bulkOprInfo.flags = BULK_OPR_LARGE_FILES;
status = putDirUtil( myConn, srcDir, targColl, myRodsEnv, rodsArgs,
dataObjOprInp, bulkOprInp, rodsRestart, &bulkOprInfo );
if ( status < 0 ) {
rodsLogError( LOG_ERROR, status,
"bulkPutDirUtil: Large files bulkPut error for %s", srcDir );
return status;
}
/* now bulk put the small files */
bzero( &bulkOprInfo, sizeof( bulkOprInfo ) );
bulkOprInfo.flags = BULK_OPR_SMALL_FILES;
rstrcpy( dataObjOprInp->objPath, targColl, MAX_NAME_LEN );
rstrcpy( bulkOprInp->objPath, targColl, MAX_NAME_LEN );
bulkOprInfo.bytesBuf.len = 0;
bulkOprInfo.bytesBuf.buf = malloc( BULK_OPR_BUF_SIZE );
status = putDirUtil( myConn, srcDir, targColl, myRodsEnv, rodsArgs,
dataObjOprInp, bulkOprInp, rodsRestart, &bulkOprInfo );
if ( status < 0 ) {
rodsLogError( LOG_ERROR, status,
"bulkPutDirUtil: Small files bulkPut error for %s", srcDir );
return status;
}
if ( bulkOprInfo.count > 0 ) {
status = sendBulkPut( *myConn, bulkOprInp, &bulkOprInfo, rodsArgs );
if ( status >= 0 ) {
if ( rodsRestart->fd > 0 ) {
rodsRestart->curCnt += bulkOprInfo.count;
/* last entry. */
writeRestartFile( rodsRestart, bulkOprInfo.cachedTargPath );
}
}
else {
rodsLogError( LOG_ERROR, status,
"bulkPutDirUtil: tarAndBulkPut error for %s",
bulkOprInfo.phyBunDir );
}
clearBulkOprInfo( &bulkOprInfo );
if ( bulkOprInfo.bytesBuf.buf != NULL ) {
free( bulkOprInfo.bytesBuf.buf );
bulkOprInfo.bytesBuf.buf = NULL;
}
}
return status;
}
void AbstractEngine::buildFinalSummary() {
int completed = 0;
int failed = 0;
int cancelled = 0;
int invalid = 0;
int total = taskMap.size();
map<int,GreasyTask*>::iterator it;
GreasyTask* task;
unsigned long usedTime = 0;
float rup = 0;
log->record(GreasyLog::devel, "AbstractEngine::buildFinalSummary", "Entering...");
// Compute final stats
for (it=taskMap.begin();it!=taskMap.end(); it++) {
task = it->second;
usedTime += task->getElapsedTimeAcc();
switch(task->getTaskState()) {
case GreasyTask::invalid:
invalid++;
break;
case GreasyTask::completed:
completed++;
break;
case GreasyTask::failed:
failed++;
break;
case GreasyTask::cancelled:
cancelled++;
break;
}
}
// Compute the resource utilization %
if (globalTimer.secsElapsed()>0&&nworkers>0) {
int aux = usedTime*10000 / (globalTimer.secsElapsed()*nworkers);
rup = (float)aux/(float)100;
}
log->record(GreasyLog::info,"Summary of " + toString(total) + " tasks: " + toString(completed) +
" OK, "+toString(failed) + " FAILED, " + toString(cancelled) +
" CANCELLED, " + toString(invalid) + " INVALID.");
log->record(GreasyLog::info,"Total time: " + globalTimer.getElapsed());
log->record(GreasyLog::info,"Resource Utilization: " + toString(rup) +"%" );
// Write a restart if we find not completed tasks
if (failed + cancelled + invalid > 0) writeRestartFile();
log->record(GreasyLog::devel, "AbstractEngine::buildFinalSummary", "Exiting...");
}
int
putDirUtil( rcComm_t **myConn, char *srcDir, char *targColl,
rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp,
bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart,
bulkOprInfo_t *bulkOprInfo ) {
char srcChildPath[MAX_NAME_LEN], targChildPath[MAX_NAME_LEN];
if ( srcDir == NULL || targColl == NULL ) {
rodsLog( LOG_ERROR,
"putDirUtil: NULL srcDir or targColl input" );
return USER__NULL_INPUT_ERR;
}
if ( isPathSymlink( rodsArgs, srcDir ) > 0 ) {
return 0;
}
if ( rodsArgs->recursive != True ) {
rodsLog( LOG_ERROR,
"putDirUtil: -r option must be used for putting %s directory",
srcDir );
return USER_INPUT_OPTION_ERR;
}
if ( rodsArgs->redirectConn == True && rodsArgs->force != True ) {
int reconnFlag;
if ( rodsArgs->reconnect == True ) {
reconnFlag = RECONN_TIMEOUT;
}
else {
reconnFlag = NO_RECONN;
}
/* reconnect to the resource server */
rstrcpy( dataObjOprInp->objPath, targColl, MAX_NAME_LEN );
redirectConnToRescSvr( myConn, dataObjOprInp, myRodsEnv, reconnFlag );
rodsArgs->redirectConn = 0; /* only do it once */
}
rcComm_t *conn = *myConn;
boost::filesystem::path srcDirPath( srcDir );
if ( !exists( srcDirPath ) || !is_directory( srcDirPath ) ) {
rodsLog( LOG_ERROR,
"putDirUtil: opendir local dir error for %s, errno = %d\n",
srcDir, errno );
return USER_INPUT_PATH_ERR;
}
if ( rodsArgs->verbose == True ) {
fprintf( stdout, "C- %s:\n", targColl );
}
int bulkFlag = NON_BULK_OPR;
if ( bulkOprInfo != NULL ) {
bulkFlag = bulkOprInfo->flags;
}
int savedStatus = 0;
boost::filesystem::directory_iterator end_itr; // default construction yields past-the-end
for ( boost::filesystem::directory_iterator itr( srcDirPath ); itr != end_itr; ++itr ) {
boost::filesystem::path p = itr->path();
snprintf( srcChildPath, MAX_NAME_LEN, "%s",
p.c_str() );
if ( isPathSymlink( rodsArgs, srcChildPath ) > 0 ) {
continue;
}
if ( !exists( p ) ) {
rodsLog( LOG_ERROR,
"putDirUtil: stat error for %s, errno = %d\n",
srcChildPath, errno );
return USER_INPUT_PATH_ERR;
}
if ( is_symlink( p ) ) {
boost::filesystem::path cp = read_symlink( p );
snprintf( srcChildPath, MAX_NAME_LEN, "%s/%s",
srcDir, cp.c_str() );
p = boost::filesystem::path( srcChildPath );
}
rodsLong_t dataSize = 0;
dataObjOprInp->createMode = getPathStMode( p.c_str() );
objType_t childObjType;
if ( is_regular_file( p ) ) {
childObjType = DATA_OBJ_T;
dataSize = file_size( p );
}
else if ( is_directory( p ) ) {
childObjType = COLL_OBJ_T;
}
else {
rodsLog( LOG_ERROR,
"putDirUtil: unknown local path type for %s",
srcChildPath );
savedStatus = USER_INPUT_PATH_ERR;
continue;
}
boost::filesystem::path childPath = p.filename();
snprintf( targChildPath, MAX_NAME_LEN, "%s/%s",
targColl, childPath.c_str() );
if ( childObjType == DATA_OBJ_T ) {
if ( bulkFlag == BULK_OPR_SMALL_FILES &&
file_size( p ) > MAX_BULK_OPR_FILE_SIZE ) {
continue;
}
else if ( bulkFlag == BULK_OPR_LARGE_FILES &&
file_size( p ) <= MAX_BULK_OPR_FILE_SIZE ) {
continue;
}
}
int status = chkStateForResume( conn, rodsRestart, targChildPath,
rodsArgs, childObjType, &dataObjOprInp->condInput, 1 );
if ( status < 0 ) {
/* restart failed */
return status;
}
else if ( status == 0 ) {
if ( bulkFlag == BULK_OPR_SMALL_FILES &&
( rodsRestart->restartState & LAST_PATH_MATCHED ) != 0 ) {
/* enable foreFlag one time */
setForceFlagForRestart( bulkOprInp, bulkOprInfo );
}
continue;
}
if ( childObjType == DATA_OBJ_T ) { /* a file */
if ( bulkFlag == BULK_OPR_SMALL_FILES ) {
status = bulkPutFileUtil( conn, srcChildPath, targChildPath,
dataSize, dataObjOprInp->createMode, rodsArgs,
bulkOprInp, bulkOprInfo );
}
else {
/* normal put */
status = putFileUtil( conn, srcChildPath, targChildPath,
dataSize, rodsArgs, dataObjOprInp );
}
if ( rodsRestart->fd > 0 ) {
if ( status >= 0 ) {
if ( bulkFlag == BULK_OPR_SMALL_FILES ) {
if ( status > 0 ) {
/* status is the number of files bulk loaded */
rodsRestart->curCnt += status;
status = writeRestartFile( rodsRestart,
targChildPath );
}
}
else {
/* write the restart file */
rodsRestart->curCnt ++;
status = writeRestartFile( rodsRestart, targChildPath );
}
}
}
}
else { /* a directory */
status = mkColl( conn, targChildPath );
if ( status < 0 ) {
rodsLogError( LOG_ERROR, status,
"putDirUtil: mkColl error for %s", targChildPath );
}
status = putDirUtil( myConn, srcChildPath, targChildPath,
myRodsEnv, rodsArgs, dataObjOprInp, bulkOprInp,
rodsRestart, bulkOprInfo );
}
if ( status < 0 &&
status != CAT_NO_ROWS_FOUND ) {
rodsLogError( LOG_ERROR, status,
"putDirUtil: put %s failed. status = %d",
srcChildPath, status );
savedStatus = status;
if ( rodsRestart->fd > 0 ) {
break;
}
}
}
return savedStatus;
}
