void ADIOI_GRIDFTP_Open(ADIO_File fd, int *error_code)
{
static char myname[]="ADIOI_GRIDFTP_Open";
int myrank, nprocs, keyfound;
char hintval[MPI_MAX_INFO_VAL+1];
globus_ftp_client_handleattr_t hattr;
globus_result_t result;
MPI_Comm_size(fd->comm, &nprocs);
MPI_Comm_rank(fd->comm, &myrank);
/* activate Globus ftp client module -- can be called multiple times, so
it's safest to call once per file/connection */
globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
fd->fd_sys = num_gridftp_handles;
/* No shared file pointers for now */
fd->shared_fp_fname = NULL;
*error_code = MPI_SUCCESS;
/* Access modes here mean something very different here than they
would on a "real" filesystem... As a result, the amode and hint
processing here is intermingled and a little weird because many
of them have to do with the connection rather than the file itself.
The thing that sucks about this is that read and write ops will
have to check themselves if the file is being accessed rdonly, rdwr,
or wronly.
*/
result=globus_ftp_client_handleattr_init(&hattr);
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_handleattr_init",
myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
result = globus_ftp_client_operationattr_init(&(oattr[fd->fd_sys]));
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_operationattr_init",
myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* Always use connection caching unless told otherwise */
result=globus_ftp_client_handleattr_set_cache_all(&hattr,GLOBUS_TRUE);
if ( result !=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_handleattr_set_cache_all",myname,result);
/* Assume that it's safe to cache a file if it's read-only */
if ( (fd->access_mode&ADIO_RDONLY) &&
(result=globus_ftp_client_handleattr_add_cached_url(&hattr,fd->filename))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_handleattr_add_cached_url",myname,result);
/* Since we're (almost by definition) doing things that FTP S (stream)
control mode can't handle, default to E (extended block) control mode
for gsiftp:// URLs. ftp:// URLs use standard stream control mode
by default. This behavior can be overridden by the ftp_control_mode
hint. */
/*
if ( !strncmp(fd->filename,"gsiftp:",7) &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strncmp(fd->filename,"ftp:",4) &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_STREAM))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
*/
/* Set append mode if necessary */
if ( (fd->access_mode&ADIO_APPEND) &&
((result=globus_ftp_client_operationattr_set_append(&(oattr[fd->fd_sys]),GLOBUS_TRUE))!=GLOBUS_SUCCESS) )
globus_err_handler("globus_ftp_client_operationattr_set_append",myname,result);
/* Other hint and amode processing that would affect hattr and/or
oattr[] (eg. parallelism, striping, etc.) goes here */
if ( fd->info!=MPI_INFO_NULL )
{
ADIOI_Info_get(fd->info,"ftp_control_mode",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
if ( ( !strcmp(hintval,"extended") || !strcmp(hintval,"extended_block") ) &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strcmp(hintval,"block") &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_BLOCK))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strcmp(hintval,"compressed") &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_COMPRESSED))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strcmp(hintval,"stream") &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_STREAM))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
}
ADIOI_Info_get(fd->info,"parallelism",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
int nftpthreads;
if ( sscanf(hintval,"%d",&nftpthreads)==1 )
{
globus_ftp_control_parallelism_t parallelism;
parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED;
parallelism.fixed.size = nftpthreads;
if ( (result=globus_ftp_client_operationattr_set_parallelism(&(oattr[fd->fd_sys]),
¶llelism))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_parallelism",myname,result);
}
}
ADIOI_Info_get(fd->info,"striped_ftp",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
/* if set to "true" or "enable", set up round-robin block layout */
if ( !strncmp("true",hintval,4) || !strncmp("TRUE",hintval,4) ||
!strncmp("enable",hintval,4) || !strncmp("ENABLE",hintval,4) )
{
ADIOI_Info_get(fd->info,"striping_factor",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
int striping_factor;
if ( sscanf(hintval,"%d",&striping_factor)==1 )
{
globus_ftp_control_layout_t layout;
layout.mode = GLOBUS_FTP_CONTROL_STRIPING_BLOCKED_ROUND_ROBIN;
layout.round_robin.block_size = striping_factor;
if ( (result=globus_ftp_client_operationattr_set_layout(&(oattr[fd->fd_sys]),
&layout))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_layout",
myname,result);
}
}
}
}
ADIOI_Info_get(fd->info,"tcp_buffer",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
/* set tcp buffer size */
int buffer_size;
if ( sscanf(hintval,"%d",&buffer_size)==1 )
{
globus_ftp_control_tcpbuffer_t tcpbuf;
tcpbuf.mode = GLOBUS_FTP_CONTROL_TCPBUFFER_FIXED;
tcpbuf.fixed.size = buffer_size;
if ( (result=globus_ftp_client_operationattr_set_tcp_buffer(&(oattr[fd->fd_sys]),
&tcpbuf))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_tcp_buffer",myname,result);
}
}
ADIOI_Info_get(fd->info,"transfer_type",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
globus_ftp_control_type_t filetype;
/* set transfer type (i.e. ASCII or binary) */
if ( !strcmp("ascii",hintval) || !strcmp("ASCII",hintval) )
{
filetype=GLOBUS_FTP_CONTROL_TYPE_ASCII;
}
else
{
filetype=GLOBUS_FTP_CONTROL_TYPE_IMAGE;
}
if ( (result=globus_ftp_client_operationattr_set_type(&(oattr[fd->fd_sys]),filetype))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_type",myname,result);
}
}
else
FPRINTF(stderr,"no MPI_Info object associated with %s\n",fd->filename);
/* Create the ftp handle */
result=globus_ftp_client_handle_init(&(gridftp_fh[fd->fd_sys]),&hattr);
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_handle_init",myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* Check for existence of the file */
globus_mutex_init(&lock, GLOBUS_NULL);
globus_cond_init(&cond, GLOBUS_NULL);
file_exists=GLOBUS_FALSE;
exists_done=GLOBUS_FALSE;
if ( myrank==0 )
{
if ( (result=globus_ftp_client_exists(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
exists_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_exists",myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* wait till the callback completes */
globus_mutex_lock(&lock);
while ( exists_done!=GLOBUS_TRUE )
globus_cond_wait(&cond,&lock);
globus_mutex_unlock(&lock);
}
MPI_Barrier(fd->comm);
MPI_Bcast(&file_exists,1,MPI_INT,0,fd->comm);
/* It turns out that this is handled by MPI_File_open() directly */
if ( (file_exists!=GLOBUS_TRUE) && (fd->access_mode&ADIO_CREATE) &&
!(fd->access_mode&ADIO_EXCL) && !(fd->access_mode&ADIO_RDONLY) )
{
if ( myrank==0 )
{
/* if the file doesn't exist, write a single NULL to it */
globus_byte_t touchbuf=(globus_byte_t)'\0';
touch_ctl_done=GLOBUS_FALSE;
if ( (result=globus_ftp_client_put(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
GLOBUS_NULL,
touch_ctl_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_put",myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
(globus_byte_t *)&touchbuf, 0,
(globus_off_t)0, GLOBUS_TRUE,
touch_data_cb, GLOBUS_NULL);
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_register_write",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
globus_mutex_lock(&lock);
while ( touch_ctl_done!=GLOBUS_TRUE )
globus_cond_wait(&cond,&lock);
globus_mutex_unlock(&lock);
}
MPI_Barrier(fd->comm);
}
else if ( (fd->access_mode&ADIO_EXCL) && (file_exists==GLOBUS_TRUE) )
{
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", 0);
return;
}
else if ( (fd->access_mode&ADIO_RDONLY) && (file_exists!=GLOBUS_TRUE) )
{
if ( myrank==0 )
{
FPRINTF(stderr,"WARNING: read-only file %s does not exist!\n",fd->filename);
}
}
num_gridftp_handles++;
}
int gsiftp_put(char *filename, FILE **gsiftpfile, int num_streams)
{
int i;
char gsiurl[MAXLEN];
globus_ftp_client_handle_t handle;
globus_ftp_client_operationattr_t attr;
globus_ftp_client_handleattr_t handle_attr;
globus_ftp_control_parallelism_t parallelism;
globus_ftp_control_layout_t layout;
globus_byte_t * buffer;
globus_size_t buffer_length = sizeof(buffer);
globus_result_t result;
globus_ftp_client_restart_marker_t restart;
globus_ftp_control_type_t filetype;
globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
globus_mutex_init(&lock, GLOBUS_NULL);
globus_cond_init(&cond, GLOBUS_NULL);
globus_ftp_client_handle_init(&handle, GLOBUS_NULL);
globus_ftp_client_handleattr_init(&handle_attr);
globus_ftp_client_operationattr_init(&attr);
layout.mode = GLOBUS_FTP_CONTROL_STRIPING_NONE;
globus_ftp_client_restart_marker_init(&restart);
globus_ftp_client_operationattr_set_mode(
&attr,
GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK);
if (num_streams >= 1)
{
parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED;
parallelism.fixed.size = num_streams;
globus_ftp_client_operationattr_set_parallelism(
&attr,
¶llelism);
}
globus_ftp_client_operationattr_set_layout(&attr,
&layout);
filetype = GLOBUS_FTP_CONTROL_TYPE_IMAGE;
globus_ftp_client_operationattr_set_type (&attr,
filetype);
globus_ftp_client_handle_init(&handle, &handle_attr);
done = GLOBUS_FALSE;
strcpy(gsiurl,"gsiftp://");
if (strlen(gsiurl)+strlen(filename) > MAXLEN-1)
{
ffpmsg("file name too long (gsiftp_put)");
return (FILE_NOT_OPENED);
}
strcat(gsiurl,filename);
*gsiftpfile = fopen(gsiftp_tmpfile,"r");
if (!*gsiftpfile) {
ffpmsg("Unable to open temporary file!");
return (FILE_NOT_OPENED);
}
result = globus_ftp_client_put(&handle,
gsiurl,
&attr,
&restart,
done_cb,
0);
if(result != GLOBUS_SUCCESS) {
globus_object_t * err;
err = globus_error_get(result);
fprintf(stderr, "%s", globus_object_printable_to_string(err));
done = GLOBUS_TRUE;
}
else {
int rc;
int curr_offset;
for (i = 0; i< 2 * num_streams && feof(*gsiftpfile) == 0; i++)
{
buffer = malloc(MAX_BUFFER_SIZE_W);
globus_mutex_lock(&lock);
curr_offset = global_offset;
rc = fread(buffer, 1, MAX_BUFFER_SIZE_W, *gsiftpfile);
global_offset += rc;
globus_mutex_unlock(&lock);
globus_ftp_client_register_write(
&handle,
buffer,
rc,
curr_offset,
feof(*gsiftpfile) != 0,
data_cb_write,
(void*) *gsiftpfile);
}
}
globus_mutex_lock(&lock);
while(!done) {
globus_cond_wait(&cond, &lock);
}
globus_mutex_unlock(&lock);
globus_ftp_client_handle_destroy(&handle);
globus_module_deactivate_all();
return 0;
}
int main(int argc,
char *argv[])
{
globus_ftp_client_handle_t handle;
globus_ftp_client_operationattr_t attr;
globus_byte_t buffer[SIZE];
globus_size_t buffer_length = sizeof(buffer);
globus_result_t result;
char * src;
char * dst;
globus_ftp_client_handleattr_t handle_attr;
globus_ftp_control_layout_t layout;
globus_ftp_control_parallelism_t parallelism;
int i;
globus_ftp_client_restart_marker_t restart;
LTDL_SET_PRELOADED_SYMBOLS();
globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
globus_ftp_client_handleattr_init(&handle_attr);
globus_ftp_client_operationattr_init(&attr);
layout.mode = GLOBUS_FTP_CONTROL_STRIPING_NONE;
parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_NONE;
globus_ftp_client_restart_marker_init(&restart);
/* Parse local arguments */
for(i = 1; i < argc; i++)
{
if(strcmp(argv[i], "-S") == 0 && i < argc)
{
layout.mode =
GLOBUS_FTP_CONTROL_STRIPING_BLOCKED_ROUND_ROBIN;
layout.round_robin.block_size = 15;
test_remove_arg(&argc, argv, &i, 0);
}
else if(strcmp(argv[i], "-P") == 0 && i + 1 < argc)
{
parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED;
parallelism.fixed.size = atoi(argv[i+1]);
test_remove_arg(&argc, argv, &i, 1);
}
else if(strcmp(argv[i], "-R") == 0 && i+1 < argc)
{
char * p;
globus_off_t offset, end;
int bytes;
p = argv[i+1];
while((*p) && (sscanf(p,
"%"GLOBUS_OFF_T_FORMAT
"-%"GLOBUS_OFF_T_FORMAT"%n",
&offset,
&end,
&bytes) >= 2))
{
globus_ftp_client_restart_marker_insert_range(
&restart,
offset, end);
p += bytes;
if(*p && *p == ',') p++;
}
test_remove_arg(&argc, argv, &i, 1);
}
}
test_parse_args(argc,
argv,
&handle_attr,
&attr,
&src,
&dst);
globus_mutex_init(&lock, GLOBUS_NULL);
globus_cond_init(&cond, GLOBUS_NULL);
globus_ftp_client_operationattr_set_mode(
&attr,
GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK);
globus_ftp_client_operationattr_set_layout(&attr,
&layout);
globus_ftp_client_operationattr_set_parallelism(&attr,
¶llelism);
globus_ftp_client_handle_init(&handle, &handle_attr);
done = GLOBUS_FALSE;
result = globus_ftp_client_get(&handle,
src,
&attr,
&restart,
done_cb,
0);
if(result != GLOBUS_SUCCESS)
{
fprintf(stderr, "%s", globus_object_printable_to_string(globus_error_get(result)));
done = GLOBUS_TRUE;
error = GLOBUS_TRUE;
}
else
{
globus_ftp_client_register_read(
&handle,
buffer,
buffer_length,
data_cb,
0);
}
globus_mutex_lock(&lock);
while(!done)
{
globus_cond_wait(&cond, &lock);
}
globus_mutex_unlock(&lock);
globus_ftp_client_handle_destroy(&handle);
globus_module_deactivate_all();
if(test_abort_count && error)
{
return 0;
}
return error;
}
int main(int argc, char **argv)
{
globus_ftp_client_handle_t handle;
globus_ftp_client_operationattr_t attr;
globus_byte_t * buffer;
globus_result_t result;
globus_ftp_client_handleattr_t handle_attr;
char * src;
char * dst;
int i;
globus_size_t parallelism_level = 1;
globus_ftp_control_parallelism_t parallelism;
globus_ftp_control_layout_t layout;
LTDL_SET_PRELOADED_SYMBOLS();
globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
globus_ftp_client_handleattr_init(&handle_attr);
globus_ftp_client_operationattr_init(&attr);
/* Parse local arguments */
for(i = 1; i < argc; i++)
{
if(strcmp(argv[i], "-P") == 0 && i + 1 < argc)
{
parallelism_level = atoi(argv[i+1]);
test_remove_arg(&argc, argv, &i, 1);
}
}
test_parse_args(argc,
argv,
&handle_attr,
&attr,
&src,
&dst);
if(parallelism_level < 1)
{
parallelism_level = 1;
}
parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED;
parallelism.fixed.size = parallelism_level;
layout.mode = GLOBUS_FTP_CONTROL_STRIPING_BLOCKED_ROUND_ROBIN;
layout.round_robin.block_size = 64*1024;
globus_mutex_init(&lock, GLOBUS_NULL);
globus_cond_init(&cond, GLOBUS_NULL);
globus_ftp_client_operationattr_set_mode(
&attr,
GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK);
globus_ftp_client_operationattr_set_parallelism(&attr,
¶llelism);
globus_ftp_client_operationattr_set_layout(&attr,
&layout);
globus_ftp_client_handle_init(&handle, &handle_attr);
done = GLOBUS_FALSE;
result = globus_ftp_client_put(&handle,
dst,
&attr,
GLOBUS_NULL,
done_cb,
0);
globus_mutex_lock(&lock);
if(result != GLOBUS_SUCCESS)
{
globus_object_t * err;
err = globus_error_get(result);
fprintf(stderr, "%s", globus_object_printable_to_string(err));
done = GLOBUS_TRUE;
}
else
{
int rc=1;
int i;
for(i = 0; i < parallelism_level && rc != 0; i++)
{
buffer = malloc(MYSIZE);
rc = read(0, buffer, MYSIZE);
globus_ftp_client_register_write(
&handle,
buffer,
rc,
global_offset,
rc == 0,
data_cb,
0);
global_offset += rc;
}
}
while(!done)
{
globus_cond_wait(&cond, &lock);
}
globus_mutex_unlock(&lock);
globus_ftp_client_handle_destroy(&handle);
globus_module_deactivate_all();
if(test_abort_count && error)
{
return 0;
}
return error;
}
