void ADIOI_GRIDFTP_Open(ADIO_File fd, int *error_code)
{
static char myname[]="ADIOI_GRIDFTP_Open";
int myrank, nprocs, keyfound;
char hintval[MPI_MAX_INFO_VAL+1];
globus_ftp_client_handleattr_t hattr;
globus_result_t result;
MPI_Comm_size(fd->comm, &nprocs);
MPI_Comm_rank(fd->comm, &myrank);
/* activate Globus ftp client module -- can be called multiple times, so
it's safest to call once per file/connection */
globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
fd->fd_sys = num_gridftp_handles;
/* No shared file pointers for now */
fd->shared_fp_fname = NULL;
*error_code = MPI_SUCCESS;
/* Access modes here mean something very different here than they
would on a "real" filesystem... As a result, the amode and hint
processing here is intermingled and a little weird because many
of them have to do with the connection rather than the file itself.
The thing that sucks about this is that read and write ops will
have to check themselves if the file is being accessed rdonly, rdwr,
or wronly.
*/
result=globus_ftp_client_handleattr_init(&hattr);
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_handleattr_init",
myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
result = globus_ftp_client_operationattr_init(&(oattr[fd->fd_sys]));
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_operationattr_init",
myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* Always use connection caching unless told otherwise */
result=globus_ftp_client_handleattr_set_cache_all(&hattr,GLOBUS_TRUE);
if ( result !=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_handleattr_set_cache_all",myname,result);
/* Assume that it's safe to cache a file if it's read-only */
if ( (fd->access_mode&ADIO_RDONLY) &&
(result=globus_ftp_client_handleattr_add_cached_url(&hattr,fd->filename))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_handleattr_add_cached_url",myname,result);
/* Since we're (almost by definition) doing things that FTP S (stream)
control mode can't handle, default to E (extended block) control mode
for gsiftp:// URLs. ftp:// URLs use standard stream control mode
by default. This behavior can be overridden by the ftp_control_mode
hint. */
/*
if ( !strncmp(fd->filename,"gsiftp:",7) &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strncmp(fd->filename,"ftp:",4) &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_STREAM))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
*/
/* Set append mode if necessary */
if ( (fd->access_mode&ADIO_APPEND) &&
((result=globus_ftp_client_operationattr_set_append(&(oattr[fd->fd_sys]),GLOBUS_TRUE))!=GLOBUS_SUCCESS) )
globus_err_handler("globus_ftp_client_operationattr_set_append",myname,result);
/* Other hint and amode processing that would affect hattr and/or
oattr[] (eg. parallelism, striping, etc.) goes here */
if ( fd->info!=MPI_INFO_NULL )
{
ADIOI_Info_get(fd->info,"ftp_control_mode",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
if ( ( !strcmp(hintval,"extended") || !strcmp(hintval,"extended_block") ) &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strcmp(hintval,"block") &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_BLOCK))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strcmp(hintval,"compressed") &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_COMPRESSED))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
else if ( !strcmp(hintval,"stream") &&
(result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_STREAM))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
}
ADIOI_Info_get(fd->info,"parallelism",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
int nftpthreads;
if ( sscanf(hintval,"%d",&nftpthreads)==1 )
{
globus_ftp_control_parallelism_t parallelism;
parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED;
parallelism.fixed.size = nftpthreads;
if ( (result=globus_ftp_client_operationattr_set_parallelism(&(oattr[fd->fd_sys]),
¶llelism))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_parallelism",myname,result);
}
}
ADIOI_Info_get(fd->info,"striped_ftp",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
/* if set to "true" or "enable", set up round-robin block layout */
if ( !strncmp("true",hintval,4) || !strncmp("TRUE",hintval,4) ||
!strncmp("enable",hintval,4) || !strncmp("ENABLE",hintval,4) )
{
ADIOI_Info_get(fd->info,"striping_factor",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
int striping_factor;
if ( sscanf(hintval,"%d",&striping_factor)==1 )
{
globus_ftp_control_layout_t layout;
layout.mode = GLOBUS_FTP_CONTROL_STRIPING_BLOCKED_ROUND_ROBIN;
layout.round_robin.block_size = striping_factor;
if ( (result=globus_ftp_client_operationattr_set_layout(&(oattr[fd->fd_sys]),
&layout))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_layout",
myname,result);
}
}
}
}
ADIOI_Info_get(fd->info,"tcp_buffer",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
/* set tcp buffer size */
int buffer_size;
if ( sscanf(hintval,"%d",&buffer_size)==1 )
{
globus_ftp_control_tcpbuffer_t tcpbuf;
tcpbuf.mode = GLOBUS_FTP_CONTROL_TCPBUFFER_FIXED;
tcpbuf.fixed.size = buffer_size;
if ( (result=globus_ftp_client_operationattr_set_tcp_buffer(&(oattr[fd->fd_sys]),
&tcpbuf))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_tcp_buffer",myname,result);
}
}
ADIOI_Info_get(fd->info,"transfer_type",MPI_MAX_INFO_VAL,hintval,&keyfound);
if ( keyfound )
{
globus_ftp_control_type_t filetype;
/* set transfer type (i.e. ASCII or binary) */
if ( !strcmp("ascii",hintval) || !strcmp("ASCII",hintval) )
{
filetype=GLOBUS_FTP_CONTROL_TYPE_ASCII;
}
else
{
filetype=GLOBUS_FTP_CONTROL_TYPE_IMAGE;
}
if ( (result=globus_ftp_client_operationattr_set_type(&(oattr[fd->fd_sys]),filetype))!=GLOBUS_SUCCESS )
globus_err_handler("globus_ftp_client_operationattr_set_type",myname,result);
}
}
else
FPRINTF(stderr,"no MPI_Info object associated with %s\n",fd->filename);
/* Create the ftp handle */
result=globus_ftp_client_handle_init(&(gridftp_fh[fd->fd_sys]),&hattr);
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_handle_init",myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* Check for existence of the file */
globus_mutex_init(&lock, GLOBUS_NULL);
globus_cond_init(&cond, GLOBUS_NULL);
file_exists=GLOBUS_FALSE;
exists_done=GLOBUS_FALSE;
if ( myrank==0 )
{
if ( (result=globus_ftp_client_exists(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
exists_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_exists",myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* wait till the callback completes */
globus_mutex_lock(&lock);
while ( exists_done!=GLOBUS_TRUE )
globus_cond_wait(&cond,&lock);
globus_mutex_unlock(&lock);
}
MPI_Barrier(fd->comm);
MPI_Bcast(&file_exists,1,MPI_INT,0,fd->comm);
/* It turns out that this is handled by MPI_File_open() directly */
if ( (file_exists!=GLOBUS_TRUE) && (fd->access_mode&ADIO_CREATE) &&
!(fd->access_mode&ADIO_EXCL) && !(fd->access_mode&ADIO_RDONLY) )
{
if ( myrank==0 )
{
/* if the file doesn't exist, write a single NULL to it */
globus_byte_t touchbuf=(globus_byte_t)'\0';
touch_ctl_done=GLOBUS_FALSE;
if ( (result=globus_ftp_client_put(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
GLOBUS_NULL,
touch_ctl_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_put",myname,result);
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
(globus_byte_t *)&touchbuf, 0,
(globus_off_t)0, GLOBUS_TRUE,
touch_data_cb, GLOBUS_NULL);
if ( result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_register_write",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
globus_mutex_lock(&lock);
while ( touch_ctl_done!=GLOBUS_TRUE )
globus_cond_wait(&cond,&lock);
globus_mutex_unlock(&lock);
}
MPI_Barrier(fd->comm);
}
else if ( (fd->access_mode&ADIO_EXCL) && (file_exists==GLOBUS_TRUE) )
{
fd->fd_sys = -1;
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", 0);
return;
}
else if ( (fd->access_mode&ADIO_RDONLY) && (file_exists!=GLOBUS_TRUE) )
{
if ( myrank==0 )
{
FPRINTF(stderr,"WARNING: read-only file %s does not exist!\n",fd->filename);
}
}
num_gridftp_handles++;
}
void ADIOI_GRIDFTP_WriteDiscontig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status, int
*error_code)
{
char myname[]="ADIOI_GRIDFTP_WriteDiscontig";
int myrank,nprocs;
MPI_Aint btype_size,btype_extent;
MPI_Aint ftype_size,ftype_extent;
MPI_Aint etype_size;
MPI_Aint extent;
ADIOI_Flatlist_node *flat_file;
int buf_contig,boff,i,nblks;
globus_off_t start,end,goff;
globus_size_t bytes_written;
globus_result_t result;
MPI_Comm_rank(fd->comm,&myrank);
MPI_Comm_size(fd->comm,&nprocs);
etype_size=fd->etype_size;
MPI_Type_size(fd->filetype,&ftype_size);
MPI_Type_extent(fd->filetype,&ftype_extent);
/* This is arguably unnecessary, as this routine assumes that the
buffer in memory is contiguous */
MPI_Type_size(datatype,&btype_size);
MPI_Type_extent(datatype,&btype_extent);
ADIOI_Datatype_iscontig(datatype,&buf_contig);
if ( ( btype_extent!=btype_size ) || ( ! buf_contig ) )
{
FPRINTF(stderr,"[%d/%d] %s called with discontigous memory buffer\n",
myrank,nprocs,myname);
fflush(stderr);
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* from here we can assume btype_extent==btype_size */
/* Flatten out fd->filetype so we know which blocks to skip */
ADIOI_Flatten_datatype(fd->filetype);
flat_file = ADIOI_Flatlist;
while (flat_file->type != fd->filetype && flat_file->next!=NULL)
flat_file = flat_file->next;
/* Figure out how big the area to write is */
/* ASSUMPTION: ftype_size is an integer multiple of btype_size or vice versa. */
start=(globus_off_t)(offset*etype_size);
goff=start;
boff=0;
extent=0;
nblks=0;
while ( boff < (count*btype_size) )
{
int blklen;
for (i=0;i<flat_file->count;i++)
{
if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
blklen=flat_file->blocklens[i];
else
blklen=(count*btype_size)-boff;
boff+=blklen;
extent=MAX(extent,nblks*ftype_extent+flat_file->indices[i]+blklen);
if ( boff>=(count*btype_size) )
break;
}
nblks++;
}
if ( extent < count*btype_size )
{
FPRINTF(stderr,"[%d/%d] %s error in computing extent -- extent %d is smaller than total bytes requested %d!\n",
myrank,nprocs,myname,extent,count*btype_size);
fflush(stderr);
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
end=start+(globus_off_t)extent;
FPRINTF(stderr,"[%d/%d] %s writing %d bytes into extent of %d bytes starting at offset %Ld\n",
myrank,nprocs,myname,count*btype_size,extent,(long long)start);
fflush(stderr);
/* start up the globus partial write */
globus_mutex_init(&writediscontig_ctl_lock, GLOBUS_NULL);
globus_cond_init(&writediscontig_ctl_cond, GLOBUS_NULL);
writediscontig_ctl_done=GLOBUS_FALSE;
if ( (result=globus_ftp_client_partial_put(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
GLOBUS_NULL,
start,
end,
writediscontig_ctl_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_partial_get",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* Do all the actual I/Os */
boff=0;
nblks=0;
while ( boff < (count*btype_size) )
{
int i,blklen;
for (i=0;i<flat_file->count;i++)
{
if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
blklen=flat_file->blocklens[i];
else
blklen=(count*btype_size)-boff;
if ( blklen > 0 )
{
goff=start+nblks*ftype_extent+((globus_off_t)flat_file->indices[i]);
/*
FPRINTF(stderr,"[%d/%d] %s writing %d bytes from boff=%d at goff=%Ld\n",myrank,nprocs,myname,blklen,boff,goff);
*/
if ( (result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
((globus_byte_t *)buf)+boff,
(globus_size_t)blklen,
goff,
GLOBUS_TRUE,
writediscontig_data_cb,
(void *)(&bytes_written)))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_register_write",myname,result);
*error_code=MPI_ERR_IO;
ADIOI_Error(fd,*error_code,myname);
return;
}
boff+=blklen;
if ( boff>=(count*btype_size) )
break;
}
}
nblks++;
}
/* The ctl callback won't start till the data callbacks complete, so it's
safe to wait on just the ctl callback */
globus_mutex_lock(&writediscontig_ctl_lock);
while ( writediscontig_ctl_done!=GLOBUS_TRUE )
globus_cond_wait(&writediscontig_ctl_cond,&writediscontig_ctl_lock);
globus_mutex_unlock(&writediscontig_ctl_lock);
globus_mutex_destroy(&writediscontig_ctl_lock);
globus_cond_destroy(&writediscontig_ctl_cond);
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, bytes_written);
#endif
if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
{
fd->fp_ind += extent;
fd->fp_sys_posn = fd->fp_ind;
}
else {
fd->fp_sys_posn = offset + extent;
}
}
void ADIOI_GRIDFTP_WriteContig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status, int
*error_code)
{
char myname[]="ADIOI_GRIDFTP_WriteContig";
int myrank, nprocs, datatype_size;
globus_size_t len,bytes_written=0;
globus_off_t goff;
globus_result_t result;
if ( fd->access_mode&ADIO_RDONLY )
{
*error_code=MPI_ERR_AMODE;
return;
}
*error_code = MPI_SUCCESS;
MPI_Comm_size(fd->comm, &nprocs);
MPI_Comm_rank(fd->comm, &myrank);
MPI_Type_size(datatype, &datatype_size);
if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
{
offset = fd->fp_ind;
}
/* Do the gridftp I/O transfer */
goff = (globus_off_t)offset;
len = ((globus_size_t)datatype_size)*((globus_size_t)count);
globus_mutex_init(&writecontig_ctl_lock, GLOBUS_NULL);
globus_cond_init(&writecontig_ctl_cond, GLOBUS_NULL);
writecontig_ctl_done=GLOBUS_FALSE;
if ( (result=globus_ftp_client_partial_put(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
GLOBUS_NULL,
goff,
goff+(globus_off_t)len,
writecontig_ctl_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_partial_put",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
if ( (result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
(globus_byte_t *)buf,
len,
goff,
GLOBUS_TRUE,
writecontig_data_cb,
(void *)(&bytes_written)))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_register_write",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(globus_error_get(result)));
return;
}
/* The ctl callback won't start till the data callbacks complete, so it's
safe to wait on just the ctl callback */
globus_mutex_lock(&writecontig_ctl_lock);
while ( writecontig_ctl_done!=GLOBUS_TRUE )
globus_cond_wait(&writecontig_ctl_cond,&writecontig_ctl_lock);
globus_mutex_unlock(&writecontig_ctl_lock);
globus_mutex_destroy(&writecontig_ctl_lock);
globus_cond_destroy(&writecontig_ctl_cond);
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, bytes_written);
#endif
if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
{
offset = fd->fp_ind;
fd->fp_ind += bytes_written;
fd->fp_sys_posn = fd->fp_ind;
}
else {
fd->fp_sys_posn = offset + bytes_written;
}
}
void ADIOI_GRIDFTP_Delete(char *filename, int *error_code)
{
char myname[]="ADIOI_GRIDFTP_Delete";
int myrank, nprocs;
globus_ftp_client_handle_t handle;
globus_result_t result;
*error_code = MPI_SUCCESS;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
result=globus_ftp_client_handle_init(&handle,GLOBUS_NULL);
if (result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_handle_init",myname,result);
*error_code= MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
delete_done=GLOBUS_FALSE;
delete_success=GLOBUS_FALSE;
result=globus_ftp_client_delete(&handle,filename,GLOBUS_NULL,delete_cb,GLOBUS_NULL);
if (result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_delete",myname,result);
*error_code= MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
globus_mutex_lock(&lock);
while ( delete_done!=GLOBUS_TRUE )
globus_cond_wait(&cond,&lock);
globus_mutex_unlock(&lock);
result=globus_ftp_client_handle_destroy(&handle);
if (result != GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_handle_destroy",myname,result);
*error_code= MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
if ( delete_success!=GLOBUS_TRUE )
{
*error_code= MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
}
}
void ADIOI_GRIDFTP_ReadDiscontig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status, int
*error_code)
{
char myname[]="ADIOI_GRIDFTP_ReadDiscontig";
int myrank,nprocs;
/* size and extent of buffer in memory */
MPI_Aint btype_size,btype_extent;
/* size and extent of file record layout */
MPI_Aint ftype_size,ftype_extent;
/* size of file elemental type; seeks are done in units of this */
MPI_Aint etype_size;
MPI_Aint extent;
ADIOI_Flatlist_node *flat_file;
int i,buf_contig,boff,nblks;
globus_off_t start,end,goff;
globus_size_t bytes_read;
globus_result_t result;
globus_byte_t *tmp;
if ( fd->access_mode&MPI_MODE_WRONLY )
{
*error_code=MPIR_ERR_MODE_WRONLY;
return;
}
*error_code=MPI_SUCCESS;
MPI_Comm_rank(fd->comm,&myrank);
MPI_Comm_size(fd->comm,&nprocs);
etype_size=fd->etype_size;
MPI_Type_size(fd->filetype,&ftype_size);
MPI_Type_extent(fd->filetype,&ftype_extent);
/* This is arguably unnecessary, as this routine assumes that the
buffer in memory is contiguous */
MPI_Type_size(datatype,&btype_size);
MPI_Type_extent(datatype,&btype_extent);
ADIOI_Datatype_iscontig(datatype,&buf_contig);
if ( ( btype_extent!=btype_size ) || ( ! buf_contig ) )
{
FPRINTF(stderr,"[%d/%d] %s called with discontigous memory buffer\n",
myrank,nprocs,myname);
fflush(stderr);
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, myname, __LINE__,
MPI_ERR_IO, "**io", 0 );
return;
}
/* from here we can assume btype_extent==btype_size */
/* Flatten out fd->filetype so we know which blocks to skip */
ADIOI_Flatten_datatype(fd->filetype);
flat_file = ADIOI_Flatlist;
while (flat_file->type != fd->filetype && flat_file->next!=NULL)
flat_file = flat_file->next;
/* Figure out how big the area to read is */
start=(globus_off_t)(offset*etype_size);
goff=start;
boff=0;
extent=0;
nblks=0;
while ( boff < (count*btype_size) )
{
int blklen=0;
for (i=0;i<flat_file->count;i++)
{
/* find the length of the next block */
if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
blklen=flat_file->blocklens[i];
else
blklen=(count*btype_size)-boff;
/* increment buffer size to be used */
boff+=blklen;
/* compute extent -- the nblks*ftype_extent bit is
there so we remember how many ftypes we've already
been through */
extent=MAX(extent,nblks*ftype_extent+flat_file->indices[i]+blklen);
if ( boff>=(count*btype_size) )
break;
}
nblks++;
}
if ( extent < count*btype_size )
{
fprintf(stderr,"[%d/%d] %s error in computing extent -- extent %d is smaller than total bytes requested %d!\n",
myrank,nprocs,myname,extent,count*btype_size);
fflush(stderr);
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, myanem, __LINE__,
MPI_ERR_IO, "**io", 0);
return;
}
end=start+(globus_off_t)extent;
tmp=(globus_byte_t *)malloc((size_t)extent*sizeof(globus_byte_t));
/* start up the globus partial read */
globus_mutex_init(&readdiscontig_ctl_lock, GLOBUS_NULL);
globus_cond_init(&readdiscontig_ctl_cond, GLOBUS_NULL);
readdiscontig_ctl_done=GLOBUS_FALSE;
if ( (result=globus_ftp_client_partial_get(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
GLOBUS_NULL,
start,
end,
readdiscontig_ctl_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_partial_get",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, myanem, __LINE__,
MPI_ERR_IO, "**io", "**io %s",
globus_object_printable_to_string(result));
return;
}
/* Do all the actual I/Os */
/* Since globus_ftp_client_register_read() is brain-dead and doesn't
let you specify an offset, we have to slurp the entire extent into
memory and then parse out the pieces we want... Sucks, doesn't it?
This should probably be done in chunks (preferably of a size
set using a file hint), but that'll have to come later.
--TB */
if ( (result=globus_ftp_client_register_read(&(gridftp_fh[fd->fd_sys]),
tmp,
(globus_size_t)extent,
readdiscontig_data_cb,
(void *)(&bytes_read)))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_register_read",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCESS, MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io",
"**io %s", globus_object_printable_to_string(result));
return;
}
/* The ctl callback won't start till the data callbacks complete, so it's
safe to wait on just the ctl callback */
globus_mutex_lock(&readdiscontig_ctl_lock);
while ( readdiscontig_ctl_done!=GLOBUS_TRUE )
globus_cond_wait(&readdiscontig_ctl_cond,&readdiscontig_ctl_lock);
globus_mutex_unlock(&readdiscontig_ctl_lock);
globus_mutex_destroy(&readdiscontig_ctl_lock);
globus_cond_destroy(&readdiscontig_ctl_cond);
boff=0;
nblks=0;
goff=0;
while ( boff < (count*btype_size) )
{
int i,blklen;
for (i=0;i<flat_file->count;i++)
{
if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
blklen=flat_file->blocklens[i];
else
blklen=(count*btype_size)-boff;
if ( blklen > 0 )
{
goff=nblks*ftype_extent+flat_file->indices[i];
memcpy((globus_byte_t *)buf+boff,tmp+goff,(size_t)blklen);
boff+=blklen;
if ( boff>=(count*btype_size) )
break;
}
}
nblks++;
}
free(tmp);
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, bytes_read);
#endif
if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
{
fd->fp_ind += extent;
fd->fp_sys_posn = fd->fp_ind;
}
else {
fd->fp_sys_posn = offset + extent;
}
}
void ADIOI_GRIDFTP_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
int *error_code)
{
MPI_Datatype copy_etype, copy_filetype;
int combiner, i, j, k, filetype_is_contig, err;
ADIOI_Flatlist_node *flat_file;
char myname[]="ADIOI_GRIDFTP_Fcntl";
int myrank, nprocs;
*error_code = MPI_SUCCESS;
MPI_Comm_size(fd->comm, &nprocs);
MPI_Comm_rank(fd->comm, &myrank);
switch(flag) {
case ADIO_FCNTL_GET_FSIZE:
{
globus_result_t result;
globus_off_t fsize=0;
globus_mutex_init(&fcntl_size_lock,GLOBUS_NULL);
globus_cond_init(&fcntl_size_cond,GLOBUS_NULL);
fcntl_size_done=GLOBUS_FALSE;
if ( (result=globus_ftp_client_size(&(gridftp_fh[fd->fd_sys]),
fd->filename,
&(oattr[fd->fd_sys]),
&(fsize),
fcntl_size_cb,
GLOBUS_NULL))!=GLOBUS_SUCCESS )
{
globus_err_handler("globus_ftp_client_size",myname,result);
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__, MPI_ERR_IO,
"**io", "**io %s",
globus_object_printable_to_string(globus_error_get(result)));
return;
}
globus_mutex_lock(&fcntl_size_lock);
while ( fcntl_size_done!=GLOBUS_TRUE )
globus_cond_wait(&fcntl_size_lock,&fcntl_size_cond);
globus_mutex_unlock(&fcntl_size_lock);
globus_mutex_destroy(&fcntl_size_lock);
globus_cond_destroy(&fcntl_size_cond);
fcntl_struct->fsize=fsize;
}
*error_code = MPI_SUCCESS;
break;
case ADIO_FCNTL_SET_DISKSPACE:
ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
break;
case ADIO_FCNTL_SET_ATOMICITY:
default:
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
MPI_ERR_ARG,
"**flag", "**flag %d", flag);
}
}