/* read 'count' bytes from a (unix or pvfs2) file 'src', placing the result in
* 'buffer' */
size_t generic_read(file_object *src, char *buffer,
int64_t offset, size_t count, PVFS_credentials *credentials)
{
PVFS_Request mem_req, file_req;
PVFS_sysresp_io resp_io;
int ret;
if(src->fs_type == UNIX_FILE)
return (read(src->u.ufs.fd, buffer, count));
else
{
file_req = PVFS_BYTE;
ret = PVFS_Request_contiguous(count, PVFS_BYTE, &mem_req);
if (ret < 0)
{
fprintf(stderr, "Error: PVFS_Request_contiguous failure\n");
return (ret);
}
ret = PVFS_sys_read(src->u.pvfs2.ref, file_req, offset,
buffer, mem_req, credentials, &resp_io, hints);
if (ret == 0)
{
PVFS_Request_free(&mem_req);
return (resp_io.total_completed);
}
else
PVFS_perror("PVFS_sys_read", ret);
}
return (ret);
}
/* write 'count' bytes from 'buffer' into (unix or pvfs2) file 'dest' */
size_t generic_write(file_object *dest, char *buffer,
int64_t offset, size_t count, PVFS_credentials *credentials)
{
PVFS_Request mem_req, file_req;
PVFS_sysresp_io resp_io;
int ret;
if (dest->fs_type == UNIX_FILE)
return(write(dest->u.ufs.fd, buffer, count));
else
{
file_req = PVFS_BYTE;
ret = PVFS_Request_contiguous(count, PVFS_BYTE, &mem_req);
if (ret < 0)
{
PVFS_perror("PVFS_Request_contiguous", ret);
return(ret);
}
ret = PVFS_sys_write(dest->u.pvfs2.ref, file_req, offset,
buffer, mem_req, credentials, &resp_io, hints);
if (ret == 0)
{
PVFS_Request_free(&mem_req);
return(resp_io.total_completed);
}
else
PVFS_perror("PVFS_sys_write", ret);
}
return ret;
}
/**
* PVFSIOSHandle::Pwrite. positional write wrapper.
*
* @param buf the buffer to write from
* @param count the number of bytes to write
* @param offset the offset to write from
* @param bytes_written return bytes that have been written(0, size count)
* @return PLFS_SUCCESS or PLFS_E*
*/
plfs_error_t PVFSIOSHandle::Pwrite(const void* buf, size_t count,
off_t offset, ssize_t *bytes_written) {
PVFS_Request mem_req, file_req;
PVFS_sysresp_io resp_io;
int pev;
file_req = PVFS_BYTE; /* reading bytes from the file ... */
/* ... into a contig buffer of size count */
pev = PVFS_Request_contiguous(count, PVFS_BYTE, &mem_req);
if (pev < 0) {
return errno_to_plfs_error(-get_err(pev));
}
pev = PVFS_sys_write(this->ref, file_req, offset, (void*)buf, mem_req,
&this->creds, &resp_io);
PVFS_Request_free(&mem_req); /* XXX: see comment in Pread */
if (pev < 0) {
/* XXX: don't need to free mem_req in this case? */
return errno_to_plfs_error(-get_err(pev));
}
*bytes_written = resp_io.total_completed;
return PLFS_SUCCESS;
}
/**
* PVFSIOSHandle::Pread: A wrapper around the PVFS read call
*
* @param buf the buffer to read into
* @param count the number of bytes to read
* @param offset the offset to read from
* @param bytes_read return bytes that have been read(0, size count)
* @return PLFS_SUCCESS or PLFS_E*
*/
plfs_error_t PVFSIOSHandle::Pread(void* buf, size_t count,
off_t offset, ssize_t *bytes_read) {
PVFS_Request mem_req, file_req;
PVFS_sysresp_io resp_io;
int pev;
file_req = PVFS_BYTE; /* reading bytes from the file ... */
/* ... into a contig buffer of size count */
pev = PVFS_Request_contiguous(count, PVFS_BYTE, &mem_req);
if (pev < 0) {
return errno_to_plfs_error(-get_err(pev));
}
pev = PVFS_sys_read(this->ref, file_req, offset, buf, mem_req,
&this->creds, &resp_io);
/*
* pvfs2fuse doesn't PVFS_Request_free on error, this seem like a
* memory leak bug to me, since mem_req is a pointer that gets
* malloc'd and set in PVFS_Request_contiguous()... you still
* gotta free it even if PVFS_sys_real fails.
*/
PVFS_Request_free(&mem_req);
if (pev < 0) {
/* XXX: don't need to free mem_req in this case? */
return errno_to_plfs_error(-get_err(pev));
}
*bytes_read = resp_io.total_completed;
return PLFS_SUCCESS;
}
/**
* PVFSIOSHandle::GetDataBuf: load some data into buffers. PVFS
* doesn't support mmap, so we will malloc/free the buffer.
*
* @param bufp allocated buffer pointer put here
* @param length length of the data we want
* @return PLFS_SUCCESS or PLFS_E*
*/
plfs_error_t PVFSIOSHandle::GetDataBuf(void **bufp, size_t length) {
size_t bytes_read;
char *buffer;
PVFS_Request mem_req, file_req;
PVFS_sysresp_io resp_io;
int pev, nev, goteof;
/* init and allocate a buffer */
bytes_read = 0;
buffer = (char *)malloc(length);
if (!buffer) {
return(PLFS_ENOMEM);
}
pev = goteof = 0;
while (bytes_read < length) {
/*
* describe the format of the file and the buffer we are loading
* the data in. in this case it is simple: all contiguous.
*/
file_req = PVFS_BYTE; /* reading bytes from the file ... */
/* ... into a contig buffer of size length-bytes_read */
pev = PVFS_Request_contiguous(length-bytes_read, PVFS_BYTE, &mem_req);
if (pev < 0) {
break;
}
pev = PVFS_sys_read(this->ref, file_req, bytes_read /*offset*/,
buffer+bytes_read, mem_req, &this->creds, &resp_io);
PVFS_Request_free(&mem_req); /* XXX: see comment in Pread */
if (pev < 0) {
break;
}
if (resp_io.total_completed == 0) {
goteof++;
break;
}
bytes_read += resp_io.total_completed;
}
if (pev < 0 || goteof) {
free(buffer);
nev = (goteof) ? -EWOULDBLOCK : get_err(pev);
return errno_to_plfs_error(-nev);
}
*bufp = buffer;
return PLFS_SUCCESS;
}
int main(int argc, char **argv)
{
int i;
PINT_Request *r1;
PINT_Request *r2;
PINT_Request_state *rs1;
PINT_Request_state *rs2;
PINT_request_file_data rf1;
PINT_Request_result seg1;
/* PVFS_Process_request arguments */
int retval;
int32_t blksz[] = {4};
PVFS_size disps[] = {32};
/* set up file type request */
PVFS_Request_hindexed(1, blksz, disps, PVFS_INT, &r1);
rs1 = PINT_new_request_state(r1);
/* set up memory request */
PVFS_Request_contiguous(96, PVFS_BYTE, &r2);
rs2 = PINT_new_request_state(r2);
/* set up file data for request */
PINT_dist_initialize(NULL);
rf1.server_nr = 0;
rf1.server_ct = 4;
rf1.fsize = 6000;
rf1.dist = PINT_dist_create("simple_stripe");
rf1.extend_flag = 0;
PINT_dist_lookup(rf1.dist);
/* set up result struct */
seg1.offset_array = (int64_t *)malloc(SEGMAX * sizeof(int64_t));
seg1.size_array = (int64_t *)malloc(SEGMAX * sizeof(int64_t));
seg1.bytemax = BYTEMAX;
seg1.segmax = SEGMAX;
seg1.bytes = 0;
seg1.segs = 0;
/* skip into the file datatype */
/*PINT_REQUEST_STATE_SET_TARGET(rs1, 500);*/
PINT_REQUEST_STATE_SET_FINAL(rs1,96);
/* Turn on debugging */
// gossip_enable_stderr();
// gossip_set_debug_mask(1,GOSSIP_REQUEST_DEBUG);
/* skipping logical bytes */
// PINT_REQUEST_STATE_SET_TARGET(rs1,(3 * 1024) + 512);
// PINT_REQUEST_STATE_SET_FINAL(rs1,(6 * 1024) + 512);
printf("\n************************************\n");
printf("One request in CLIENT mode server 0 of 4\n");
printf("Simple stripe, default stripe size (64K)\n");
printf("Offset 0, file size 6000, no extend flag\n");
printf("MemReq size 96 coniguous\n");
printf("\n************************************\n");
PINT_REQUEST_STATE_RESET(rs1);
PINT_REQUEST_STATE_RESET(rs2);
do
{
int r = 0;
seg1.bytes = 0;
seg1.segs = 0;
/* process request */
retval = PINT_process_request(rs1, rs2, &rf1, &seg1, PINT_CLIENT);
if(retval >= 0)
{
printf("results of PINT_process_request():\n");
printf("%d segments with %lld bytes\n", seg1.segs, lld(seg1.bytes));
for(i=0; i<seg1.segs; i++, r++)
{
printf(" segment %d: offset: %d size: %d\n",
i, (int)seg1.offset_array[i], (int)seg1.size_array[i]);
}
}
} while(!PINT_REQUEST_DONE(rs1) && retval >= 0);
if(retval < 0)
{
fprintf(stderr, "Error: PINT_process_request() failure.\n");
return(-1);
}
if(PINT_REQUEST_DONE(rs1))
{
printf("**** request done.\n");
}
printf("\n************************************\n");
printf("One request in SERVER mode server 0 of 4\n");
printf("Simple stripe, default stripe size (64K)\n");
printf("Offset 32, file size 6000, no extend flag\n");
printf("MemReq size 96 coniguous\n");
printf("\n************************************\n");
PINT_REQUEST_STATE_RESET(rs1);
do
{
int r = 0;
seg1.bytes = 0;
seg1.segs = 0;
/* process request */
retval = PINT_process_request(rs1, NULL, &rf1, &seg1, PINT_SERVER);
if(retval >= 0)
{
printf("results of PINT_process_request():\n");
printf("%d segments with %lld bytes\n", seg1.segs, lld(seg1.bytes));
for(i=0; i<seg1.segs; i++, r++)
{
printf(" segment %d: offset: %d size: %d\n",
i, (int)seg1.offset_array[i], (int)seg1.size_array[i]);
}
}
} while(!PINT_REQUEST_DONE(rs1) && retval >= 0);
if(retval < 0)
{
fprintf(stderr, "Error: PINT_process_request() failure.\n");
return(-1);
}
if(PINT_REQUEST_DONE(rs1))
{
printf("**** request done.\n");
}
return 0;
}
int main(int argc, char * argv[])
{
FILE * f;
int ret;
PVFS_fs_id curfs;
PVFS_Request file_req;
PVFS_Request mem_req;
int count;
char line[255];
int size;
PVFS_offset offset=0;
PVFS_credentials creds;
PVFS_sysresp_create create_resp;
PVFS_sysresp_io io_resp;
PVFS_sysresp_lookup lookup_resp;
PVFS_sys_attr attr;
const char * filename = "test-accesses-file";
int j = 0, i = 0;
char * membuff;
char errormsg[255];
if(argc < 2)
{
fprintf(stderr, "test-accesses <sizes file>\n");
exit(1);
}
f = fopen(argv[1], "r");
if(!f)
{
fprintf(stderr, "error opening file\n");
return errno;
}
if(fgets(line, 255, f) == NULL)
{
fprintf(stderr, "error in file\n");
exit(1);
}
if(sscanf(line, "%d", &count) < 1)
{
fprintf(stderr, "error in file\n");
exit(1);
}
ret = PVFS_util_init_defaults();
if(ret < 0) goto error;
ret = PVFS_util_get_default_fsid(&curfs);
if(ret < 0) goto error;
ret = PVFS_sys_lookup(curfs, "/", &creds, &lookup_resp, 0, NULL);
if(ret < 0) goto error;
PVFS_util_gen_credentials(&creds);
attr.mask = PVFS_ATTR_SYS_ALL_SETABLE;
attr.owner = creds.uid;
attr.group = creds.gid;
attr.perms = 0644;
attr.atime = attr.ctime = attr.mtime = time(NULL);
ret = PVFS_sys_create(
(char*)filename,
lookup_resp.ref, attr, &creds, NULL, &create_resp, NULL, NULL);
if(ret < 0) goto error;
for(; i < count; ++i)
{
if(fgets(line, 255, f) == NULL)
{
fprintf(stderr, "error in file\n");
exit(1);
}
if(sscanf(line, "%d", &size) < 1)
{
fprintf(stderr, "error in file\n");
exit(1);
}
membuff = malloc(size);
assert(membuff);
for(j = 0; j < size; ++j)
{
membuff[j] = j;
}
ret = PVFS_Request_contiguous(
size, PVFS_BYTE, &file_req);
if(ret < 0) goto error;
ret = PVFS_Request_contiguous(
size, PVFS_BYTE, &mem_req);
if(ret < 0) goto error;
printf("Performing Write: offset: %llu, size: %d\n",
llu(offset), size);
ret = PVFS_sys_io(
create_resp.ref, file_req, offset, membuff, mem_req,
&creds, &io_resp, PVFS_IO_WRITE, NULL);
if(ret < 0) goto error;
printf("Write response: size: %llu\n", llu(io_resp.total_completed));
offset += size;
PVFS_Request_free(&mem_req);
PVFS_Request_free(&file_req);
free(membuff);
}
return 0;
error:
fclose(f);
PVFS_sys_remove(
(char*)filename,
lookup_resp.ref,
&creds,
NULL);
PVFS_perror_gossip(errormsg, ret);
fprintf(stderr, "%s\n", errormsg);
return PVFS_get_errno_mapping(ret);
}
int convert_named(MPI_Datatype *mpi_dtype,
PVFS_Request *pvfs_dtype, int combiner)
{
int ret = -1;
#ifdef DEBUG_DTYPE
fprintf(stderr, "NAMED");
#endif
if (MPI_CHAR == *mpi_dtype)
{
ret = PVFS_Request_contiguous(1, PVFS_CHAR, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_CHAR\n");
#endif
}
else if ( MPI_BYTE == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_BYTE, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_BYTE\n");
#endif
}
else if ( MPI_SHORT == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_SHORT, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_SHORT\n");
#endif
}
else if ( MPI_INT == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_INT, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_INT\n");
#endif
}
else if ( MPI_LONG == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_LONG, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_LONG\n");
#endif
}
else if ( MPI_FLOAT == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_FLOAT, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_FLOAT\n");
#endif
}
else if ( MPI_DOUBLE == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_DOUBLE, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_DOUBLE\n");
#endif
}
else if ( MPI_UNSIGNED_CHAR == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED_CHAR, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_UNSIGNED_CHAR\n");
#endif
}
else if ( MPI_UNSIGNED_SHORT == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_UNSIGNED_SHORT\n");
#endif
}
else if ( MPI_UNSIGNED == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_SHORT\n");
#endif
}
else if ( MPI_UNSIGNED_LONG == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED_LONG, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_UNSIGNED_LONG\n");
#endif
}
else if ( MPI_LONG_DOUBLE == *mpi_dtype )
{
ret = PVFS_Request_contiguous(1, PVFS_LONG_DOUBLE, pvfs_dtype);
#ifdef DEBUG_DTYPE
fprintf(stderr, "-MPI_LONG_DOUBLE\n");
#endif
}
else
{
fprintf(stderr, "convert_named: predefined type not found");
return -1;
}
if (ret != 0)
fprintf(stderr, "convert_named: Datatype creation failed\n");
return ret;
}
int convert_mpi_pvfs2_dtype(MPI_Datatype *mpi_dtype,
PVFS_Request *pvfs_dtype)
{
int num_int = -1, num_addr = -1, num_dtype = -1,
combiner = -1, i = -1, ret = -1, leaf = -1;
int *arr_int = NULL;
MPI_Aint *arr_addr = NULL;
MPI_Datatype *arr_dtype = NULL;
PVFS_Request *old_pvfs_dtype = NULL;
PVFS_Request *old_pvfs_dtype_arr = NULL;
int arr_count = -1;
PVFS_size *pvfs_arr_disp = NULL;
int *pvfs_arr_len = NULL;
MPI_Type_get_envelope(*mpi_dtype,
&num_int,
&num_addr,
&num_dtype,
&combiner);
/* Depending on type of datatype do the following
* operations */
if (combiner == MPI_COMBINER_NAMED)
{
convert_named(mpi_dtype, pvfs_dtype, combiner);
return 1;
}
/* Allocate space for the arrays necessary for
* MPI_Type_get_contents */
if ((arr_int = ADIOI_Malloc(sizeof(int)*num_int)) == NULL)
{
fprintf(stderr, "Failed to allocate array_int\n");
return -1;
}
if ((arr_addr = ADIOI_Malloc(sizeof(int)*num_addr)) == NULL)
{
ADIOI_Free(arr_int);
fprintf(stderr, "Failed to allocate array_addr\n");
return -1;
}
if ((arr_dtype = ADIOI_Malloc(sizeof(MPI_Datatype)*num_dtype)) == NULL)
{
ADIOI_Free(arr_int);
ADIOI_Free(arr_addr);
fprintf(stderr, "Failed to allocate array_dtypes\n");
return -1;
}
MPI_Type_get_contents(*mpi_dtype,
num_int,
num_addr,
num_dtype,
arr_int,
arr_addr,
arr_dtype);
/* If it's not a predefined datatype, it is either a
* derived datatype or a structured datatype */
if (combiner != MPI_COMBINER_STRUCT)
{
if ((old_pvfs_dtype = ADIOI_Malloc(sizeof(PVFS_Request))) == NULL)
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"Failed to allocate PVFS_Request\n");
switch (combiner)
{
case MPI_COMBINER_CONTIGUOUS:
leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
ret = PVFS_Request_contiguous(arr_int[0],
*old_pvfs_dtype, pvfs_dtype);
break;
case MPI_COMBINER_VECTOR:
leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
ret = PVFS_Request_vector(arr_int[0], arr_int[1],
arr_int[2], *old_pvfs_dtype,
pvfs_dtype);
break;
case MPI_COMBINER_HVECTOR:
leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
ret = PVFS_Request_hvector(arr_int[0], arr_int[1],
arr_addr[0], *old_pvfs_dtype,
pvfs_dtype);
break;
/* Both INDEXED and HINDEXED types require PVFS_size
* address arrays. Therefore, we need to copy and
* convert the data from MPI_get_contents() into
* a PVFS_size buffer */
case MPI_COMBINER_INDEXED:
leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
if ((pvfs_arr_disp =
ADIOI_Malloc(arr_int[0]*sizeof(PVFS_size))) == 0)
{
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"Failed to allocate pvfs_arr_disp\n");
}
for (i = 0; i < arr_int[0]; i++)
{
pvfs_arr_disp[i] =
(PVFS_size) arr_int[arr_int[0]+1+i];
}
ret = PVFS_Request_indexed(arr_int[0], &arr_int[1],
pvfs_arr_disp,
*old_pvfs_dtype, pvfs_dtype);
ADIOI_Free(pvfs_arr_disp);
break;
case MPI_COMBINER_HINDEXED:
leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
if ((pvfs_arr_disp =
ADIOI_Malloc(arr_int[0]*sizeof(PVFS_size))) == 0)
{
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"Failed to allocate pvfs_arr_disp\n");
}
for (i = 0; i < arr_int[0]; i++)
{
pvfs_arr_disp[i] =
(PVFS_size) arr_addr[i];
}
ret = PVFS_Request_hindexed(arr_int[0], &arr_int[1],
(int64_t *)&arr_addr[0],
*old_pvfs_dtype, pvfs_dtype);
ADIOI_Free(pvfs_arr_disp);
break;
case MPI_COMBINER_DUP:
leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
ret = PVFS_Request_contiguous(1,
*old_pvfs_dtype, pvfs_dtype);
break;
case MPI_COMBINER_INDEXED_BLOCK:
/* No native PVFS2 support for this operation currently */
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"INDEXED_BLOCK is unsupported\n");
break;
case MPI_COMBINER_HINDEXED_BLOCK:
/* No native PVFS2 support for this operation currently */
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"HINDEXED_BLOCK is unsupported\n");
break;
case MPI_COMBINER_HINDEXED_INTEGER:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"HINDEXED_INTEGER is unsupported\n");
break;
case MPI_COMBINER_STRUCT_INTEGER:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"STRUCT_INTEGER is unsupported\n");
break;
case MPI_COMBINER_SUBARRAY:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"SUBARRAY is unsupported\n");
break;
case MPI_COMBINER_DARRAY:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"DARRAY is unsupported\n");
break;
case MPI_COMBINER_F90_REAL:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"F90_REAL is unsupported\n");
break;
case MPI_COMBINER_F90_COMPLEX:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"F90_COMPLEX is unsupported\n");
break;
case MPI_COMBINER_F90_INTEGER:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"F90_INTEGER is unsupported\n");
break;
case MPI_COMBINER_RESIZED:
ADIOI_Free(old_pvfs_dtype);
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"RESIZED is unsupported\n");
break;
default:
break;
}
if (ret != 0)
fprintf(stderr, "Error in PVFS_Request_* "
"for a derived datatype\n");
#ifdef DEBUG_DTYPE
print_dtype_info(combiner,
num_int,
num_addr,
num_dtype,
arr_int,
arr_addr,
arr_dtype);
#endif
if (leaf != 1 && combiner != MPI_COMBINER_DUP)
MPI_Type_free(&arr_dtype[0]);
ADIOI_Free(arr_int);
ADIOI_Free(arr_addr);
ADIOI_Free(arr_dtype);
PVFS_Request_free(old_pvfs_dtype);
ADIOI_Free(old_pvfs_dtype);
return ret;
}
else /* MPI_COMBINER_STRUCT */
{
MPI_Aint mpi_lb = -1, mpi_extent = -1;
PVFS_offset pvfs_lb = -1;
PVFS_size pvfs_extent = -1;
int has_lb_ub = 0;
/* When converting into a PVFS_Request_struct, we no longer
* can use MPI_LB and MPI_UB. Therfore, we have to do the
* following.
* We simply ignore all the MPI_LB and MPI_UB types and
* get the lb and extent and pass it on through a
* PVFS resized_req */
arr_count = 0;
for (i = 0; i < arr_int[0]; i++)
{
if (arr_dtype[i] != MPI_LB &&
arr_dtype[i] != MPI_UB)
{
arr_count++;
}
}
if (arr_int[0] != arr_count)
{
MPI_Type_get_extent(*mpi_dtype, &mpi_lb, &mpi_extent);
pvfs_lb = mpi_lb;
pvfs_extent = mpi_extent;
if ((pvfs_arr_len = ADIOI_Malloc(arr_count*sizeof(int)))
== NULL)
{
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"Failed to allocate pvfs_arr_len\n");
}
has_lb_ub = 1;
}
if ((old_pvfs_dtype_arr
= ADIOI_Malloc(arr_count*sizeof(PVFS_Request))) == NULL)
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"Failed to allocate PVFS_Requests\n");
if ((pvfs_arr_disp = ADIOI_Malloc(arr_count*sizeof(PVFS_size)))
== NULL)
{
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"Failed to allocate pvfs_arr_disp\n");
}
arr_count = 0;
for (i = 0; i < arr_int[0]; i++)
{
if (arr_dtype[i] != MPI_LB &&
arr_dtype[i] != MPI_UB)
{
leaf = convert_mpi_pvfs2_dtype(
&arr_dtype[i], &old_pvfs_dtype_arr[arr_count]);
if (leaf != 1)
MPI_Type_free(&arr_dtype[i]);
pvfs_arr_disp[arr_count] =
(PVFS_size) arr_addr[i];
if (has_lb_ub)
{
pvfs_arr_len[arr_count] =
arr_int[i+1];
}
arr_count++;
}
}
/* If a MPI_UB or MPI_LB did exist, we have to
* resize the datatype */
if (has_lb_ub)
{
PVFS_Request *tmp_pvfs_dtype = NULL;
if ((tmp_pvfs_dtype = ADIOI_Malloc(sizeof(PVFS_Request))) == NULL)
fprintf(stderr, "convert_mpi_pvfs2_dtype: "
"Failed to allocate PVFS_Request\n");
ret = PVFS_Request_struct(arr_count, pvfs_arr_len,
pvfs_arr_disp,
old_pvfs_dtype_arr, tmp_pvfs_dtype);
if (ret != 0)
fprintf(stderr, "Error in PVFS_Request_struct\n");
arr_count = 0;
for (i = 0; i < arr_int[0]; i++)
{
if (arr_dtype[i] != MPI_LB &&
arr_dtype[i] != MPI_UB)
{
PVFS_Request_free(&old_pvfs_dtype_arr[arr_count]);
arr_count++;
}
}
#ifdef DEBUG_DTYPE
fprintf(stderr, "STRUCT(WITHOUT %d LB or UB)(%d,[",
arr_int[0] - arr_count, arr_count);
for (i = 0; i < arr_count; i++)
fprintf(stderr, "(%d,%Ld) ",
pvfs_arr_len[i],
pvfs_arr_disp[i]);
fprintf(stderr, "]\n");
fprintf(stderr, "RESIZED(LB = %Ld, EXTENT = %Ld)\n",
pvfs_lb, pvfs_extent);
#endif
ret = PVFS_Request_resized(*tmp_pvfs_dtype,
pvfs_lb, pvfs_extent, pvfs_dtype);
if (ret != 0)
fprintf(stderr, "Error in PVFS_Request_resize\n");
PVFS_Request_free(tmp_pvfs_dtype);
ADIOI_Free(tmp_pvfs_dtype);
}
else /* No MPI_LB or MPI_UB datatypes */
{
ret = PVFS_Request_struct(arr_int[0], &arr_int[1],
pvfs_arr_disp,
old_pvfs_dtype_arr, pvfs_dtype);
if (ret != 0)
fprintf(stderr, "Error in PVFS_Request_struct\n");
for (i = 0; i < arr_int[0]; i++)
{
if (arr_dtype[i] != MPI_LB &&
arr_dtype[i] != MPI_UB)
PVFS_Request_free(&old_pvfs_dtype_arr[i]);
}
#ifdef DEBUG_DTYPE
print_dtype_info(combiner,
num_int,
num_addr,
num_dtype,
arr_int,
arr_addr,
arr_dtype);
#endif
}
ADIOI_Free(arr_int);
ADIOI_Free(arr_addr);
ADIOI_Free(arr_dtype);
ADIOI_Free(old_pvfs_dtype_arr);
ADIOI_Free(pvfs_arr_disp);
ADIOI_Free(pvfs_arr_len);
return ret;
}
/* Shouldn't have gotten here */
fprintf(stderr, "convert_mpi_pvfs2_dtype: SERIOUS ERROR\n");
return -1;
}
int main(
int argc,
char **argv)
{
int ret = -1;
int outcount = 0;
void *mybuffer;
PVFS_BMI_addr_t server_addr;
bmi_op_id_t op;
bmi_error_code_t error_code;
flow_descriptor *flow_d = NULL;
int i = 0;
bmi_size_t actual_size;
double time1, time2;
PINT_Request *req;
bmi_context_id context;
/*************************************************************/
/* initialization stuff */
/* set debugging level */
gossip_enable_stderr();
gossip_set_debug_mask(
0, (GOSSIP_FLOW_PROTO_DEBUG | GOSSIP_BMI_DEBUG_TCP));
/* Dist init */
PINT_dist_initialize(NULL);
/* start up BMI */
ret = BMI_initialize("bmi_tcp", NULL, 0);
if (ret < 0)
{
fprintf(stderr, "BMI init failure.\n");
return (-1);
}
ret = BMI_open_context(&context);
if (ret < 0)
{
fprintf(stderr, "BMI_open_context() failure.\n");
return (-1);
}
/* initialize the flow interface */
ret = PINT_flow_initialize("flowproto_multiqueue", 0);
if (ret < 0)
{
fprintf(stderr, "flow init failure.\n");
return (-1);
}
/* send some random crap to the other side to start up communication */
ret = BMI_addr_lookup(&server_addr, "tcp://localhost:3335");
if (ret < 0)
{
fprintf(stderr, "BMI lookup failure.\n");
return (-1);
}
ret = BMI_post_sendunexpected(&op, server_addr, &mybuffer, 1,
BMI_EXT_ALLOC, 0, NULL, context, NULL);
if (ret < 0)
{
fprintf(stderr, "BMI_post_sendunexpected failure.\n");
return (-1);
}
if (ret == 0)
{
/* turning this into a blocking call for testing :) */
/* check for completion of request */
do
{
ret = BMI_test(op, &outcount, &error_code, &actual_size,
NULL, 10, context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0)
{
fprintf(stderr, "Request send failed.\n");
if (ret < 0)
{
errno = -ret;
perror("BMI_test");
}
return (-1);
}
}
/******************************************************/
/* setup request/dist stuff */
/* request description */
/* just want one contiguous region */
ret = PVFS_Request_contiguous(TEST_SIZE, PVFS_BYTE, &req);
if (ret < 0)
{
fprintf(stderr, "PVFS_Request_contiguous() failure.\n");
return (-1);
}
/******************************************************/
/* setup communicaton stuff */
/* memory buffer to xfer */
mybuffer = (void *) malloc(TEST_SIZE);
if (!mybuffer)
{
fprintf(stderr, "mem.\n");
return (-1);
}
/* mark it so that we can check correctness */
for (i = 0; i < (TEST_SIZE / (sizeof(int))); i++)
{
((int *) mybuffer)[i] = i;
}
/* create a flow descriptor */
flow_d = PINT_flow_alloc();
if (!flow_d)
{
fprintf(stderr, "mem.\n");
return (-1);
}
/* file data */
flow_d->file_data.fsize = TEST_SIZE;
flow_d->file_data.server_nr = 0;
flow_d->file_data.server_ct = 1;
flow_d->file_data.extend_flag = 0;
flow_d->file_data.dist = PINT_dist_create("basic_dist");
if (!flow_d->file_data.dist)
{
fprintf(stderr, "Error: failed to create dist.\n");
return (-1);
}
ret = PINT_dist_lookup(flow_d->file_data.dist);
if (ret != 0)
{
fprintf(stderr, "Error: failed to lookup dist.\n");
return (-1);
}
flow_d->file_req = req;
flow_d->tag = 0;
flow_d->user_ptr = NULL;
flow_d->aggregate_size = TEST_SIZE;
/* fill in flow details */
flow_d->src.endpoint_id = MEM_ENDPOINT;
flow_d->src.u.mem.buffer = mybuffer;
flow_d->dest.endpoint_id = BMI_ENDPOINT;
flow_d->dest.u.bmi.address = server_addr;
/***************************************************
* test memory to bmi (analogous to client side write)
*/
time1 = Wtime();
ret = block_on_flow(flow_d);
if (ret < 0)
{
return (-1);
}
time2 = Wtime();
/*******************************************************/
/* final cleanup and output */
#if 0
printf("Client bw (send): %f MB/sec\n",
((TEST_SIZE) / ((time2 - time1) * 1000000.0)));
#endif
PINT_flow_free(flow_d);
/* shut down flow interface */
ret = PINT_flow_finalize();
if (ret < 0)
{
fprintf(stderr, "flow finalize failure.\n");
return (-1);
}
/* shut down BMI */
BMI_close_context(context);
BMI_finalize();
free(mybuffer);
gossip_disable();
return (0);
}
void ADIOI_PVFS2_WriteContig(ADIO_File fd, const void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status * status, int *error_code)
{
int ret;
MPI_Count datatype_size, len;
PVFS_Request file_req, mem_req;
PVFS_sysresp_io resp_io;
ADIOI_PVFS2_fs *pvfs_fs;
static char myname[] = "ADIOI_PVFS2_WRITECONTIG";
pvfs_fs = (ADIOI_PVFS2_fs *) fd->fs_ptr;
MPI_Type_size_x(datatype, &datatype_size);
len = datatype_size * count;
ret = PVFS_Request_contiguous(len, PVFS_BYTE, &mem_req);
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in PVFS_Request_contiguous (memory)", 0);
return;
}
/* --END ERROR HANDLING-- */
ret = PVFS_Request_contiguous(len, PVFS_BYTE, &file_req);
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in PVFS_Request_contiguous (file)", 0);
return;
}
/* --END ERROR HANDLING-- */
if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_write_a, 0, NULL);
#endif
ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, offset, (void *) buf,
mem_req, &(pvfs_fs->credentials), &resp_io);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_write_b, 0, NULL);
#endif
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in PVFS_sys_write", 0);
goto fn_exit;
}
/* --END ERROR HANDLING-- */
fd->fp_sys_posn = offset + (int) resp_io.total_completed;
} else {
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_write_a, 0, NULL);
#endif
ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, fd->fp_ind, (void *) buf,
mem_req, &(pvfs_fs->credentials), &resp_io);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event(ADIOI_MPE_write_b, 0, NULL);
#endif
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in PVFS_sys_write", 0);
goto fn_exit;
}
/* --END ERROR HANDLING-- */
fd->fp_ind += (int) resp_io.total_completed;
fd->fp_sys_posn = fd->fp_ind;
}
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, resp_io.total_completed);
#endif
*error_code = MPI_SUCCESS;
fn_exit:
PVFS_Request_free(&file_req);
PVFS_Request_free(&mem_req);
return;
}
void ADIOI_PVFS2_ReadContig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status,
int *error_code)
{
int ret, datatype_size, len;
PVFS_Request file_req, mem_req;
PVFS_sysresp_io resp_io;
ADIOI_PVFS2_fs *pvfs_fs;
static char myname[] = "ADIOI_PVFS2_READCONTIG";
pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
MPI_Type_size(datatype, &datatype_size);
len = datatype_size * count;
ret = PVFS_Request_contiguous(len, PVFS_BYTE, &mem_req);
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in pvfs_request_contig (memory)", 0);
return;
}
/* --END ERROR HANDLING-- */
ret = PVFS_Request_contiguous(len, PVFS_BYTE, &file_req);
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in pvfs_request_contig (file)", 0);
return;
}
/* --END ERROR HANDLING-- */
if (file_ptr_type == ADIO_INDIVIDUAL) {
/* copy individual file pointer into offset variable, continue */
offset = fd->fp_ind;
}
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
#endif
ret = PVFS_sys_read(pvfs_fs->object_ref, file_req, offset, buf,
mem_req, &(pvfs_fs->credentials), &resp_io);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
#endif
/* --BEGIN ERROR HANDLING-- */
if (ret != 0 ) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in PVFS_sys_read", 0);
goto fn_exit;
}
/* --END ERROR HANDLING-- */
if (file_ptr_type == ADIO_INDIVIDUAL) {
fd->fp_ind += (int) resp_io.total_completed;
/* TODO: WHY THE INT CAST? */
}
fd->fp_sys_posn = offset + (int)resp_io.total_completed;
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, (int)resp_io.total_completed);
#endif
*error_code = MPI_SUCCESS;
fn_exit:
PVFS_Request_free(&mem_req);
PVFS_Request_free(&file_req);
return;
}
void ADIOI_PVFS2_OldWriteStrided(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status,
int *error_code)
{
/* as with all the other WriteStrided functions, offset is in units of
* etype relative to the filetype */
/* Since PVFS2 does not support file locking, can't do buffered writes
as on Unix */
ADIOI_Flatlist_node *flat_buf, *flat_file;
int i, j, k, bwr_size, fwr_size=0, st_index=0;
int bufsize, sum, n_etypes_in_filetype, size_in_filetype;
int n_filetypes, etype_in_filetype;
ADIO_Offset abs_off_in_filetype=0;
int filetype_size, etype_size, buftype_size;
MPI_Aint filetype_extent, buftype_extent;
int buf_count, buftype_is_contig, filetype_is_contig;
ADIO_Offset off, disp, start_off, initial_off;
int flag, st_fwr_size, st_n_filetypes;
int err_flag=0;
int mem_list_count, file_list_count;
PVFS_size * mem_offsets;
int64_t *file_offsets;
int *mem_lengths;
int32_t *file_lengths;
int total_blks_to_write;
int max_mem_list, max_file_list;
int b_blks_wrote;
int f_data_wrote;
int size_wrote=0, n_write_lists, extra_blks;
int end_bwr_size, end_fwr_size;
int start_k, start_j, new_file_write, new_buffer_write;
int start_mem_offset;
PVFS_Request mem_req, file_req;
ADIOI_PVFS2_fs * pvfs_fs;
PVFS_sysresp_io resp_io;
MPI_Offset total_bytes_written=0;
static char myname[] = "ADIOI_PVFS2_WRITESTRIDED";
/* note: don't increase this: several parts of PVFS2 now
* assume this limit*/
#define MAX_ARRAY_SIZE 64
/* --BEGIN ERROR HANDLING-- */
if (fd->atomicity) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
MPI_ERR_ARG,
"Atomic noncontiguous writes are not supported by PVFS2", 0);
return;
}
/* --END ERROR HANDLING-- */
ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
/* the HDF5 tests showed a bug in this list processing code (see many many
* lines down below). We added a workaround, but common HDF5 file types
* are actually contiguous and do not need the expensive workarond */
if (!filetype_is_contig) {
flat_file = ADIOI_Flatlist;
while (flat_file->type != fd->filetype) flat_file = flat_file->next;
if (flat_file->count == 1 && !buftype_is_contig)
filetype_is_contig = 1;
}
MPI_Type_size(fd->filetype, &filetype_size);
if ( ! filetype_size ) {
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, 0);
#endif
*error_code = MPI_SUCCESS;
return;
}
MPI_Type_extent(fd->filetype, &filetype_extent);
MPI_Type_size(datatype, &buftype_size);
MPI_Type_extent(datatype, &buftype_extent);
etype_size = fd->etype_size;
bufsize = buftype_size * count;
pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
if (!buftype_is_contig && filetype_is_contig) {
/* noncontiguous in memory, contiguous in file. */
int64_t file_offsets;
int32_t file_lengths;
ADIOI_Flatten_datatype(datatype);
flat_buf = ADIOI_Flatlist;
while (flat_buf->type != datatype) flat_buf = flat_buf->next;
if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
off = fd->disp + etype_size * offset;
}
else off = fd->fp_ind;
file_list_count = 1;
file_offsets = off;
file_lengths = 0;
total_blks_to_write = count*flat_buf->count;
b_blks_wrote = 0;
/* allocate arrays according to max usage */
if (total_blks_to_write > MAX_ARRAY_SIZE)
mem_list_count = MAX_ARRAY_SIZE;
else mem_list_count = total_blks_to_write;
mem_offsets = (PVFS_size*)ADIOI_Malloc(mem_list_count*sizeof(PVFS_size));
mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
j = 0;
/* step through each block in memory, filling memory arrays */
while (b_blks_wrote < total_blks_to_write) {
for (i=0; i<flat_buf->count; i++) {
mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] =
/* TODO: fix this warning by casting to an integer that's
* the same size as a char * and /then/ casting to
* PVFS_size */
((PVFS_size)buf + j*buftype_extent + flat_buf->indices[i]);
mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] =
flat_buf->blocklens[i];
file_lengths += flat_buf->blocklens[i];
b_blks_wrote++;
if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
(b_blks_wrote == total_blks_to_write)) {
/* in the case of the last write list call,
adjust mem_list_count */
if (b_blks_wrote == total_blks_to_write) {
mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
/* in case last write list call fills max arrays */
if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
}
err_flag = PVFS_Request_hindexed(mem_list_count,
mem_lengths, mem_offsets,
PVFS_BYTE, &mem_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_hindexed (memory)", 0);
break;
}
/* --END ERROR HANDLING-- */
err_flag = PVFS_Request_contiguous(file_lengths,
PVFS_BYTE, &file_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_contiguous (file)", 0);
break;
}
/* --END ERROR HANDLING-- */
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
#endif
err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req,
file_offsets, PVFS_BOTTOM,
mem_req,
&(pvfs_fs->credentials),
&resp_io);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
#endif
total_bytes_written += resp_io.total_completed;
/* in the case of error or the last write list call,
* leave here */
/* --BEGIN ERROR HANDLING-- */
if (err_flag) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_sys_write", 0);
break;
}
/* --END ERROR HANDLING-- */
if (b_blks_wrote == total_blks_to_write) break;
file_offsets += file_lengths;
file_lengths = 0;
PVFS_Request_free(&mem_req);
PVFS_Request_free(&file_req);
}
} /* for (i=0; i<flat_buf->count; i++) */
j++;
} /* while (b_blks_wrote < total_blks_to_write) */
ADIOI_Free(mem_offsets);
ADIOI_Free(mem_lengths);
if (file_ptr_type == ADIO_INDIVIDUAL)
fd->fp_ind += total_bytes_written;
if (!err_flag) *error_code = MPI_SUCCESS;
fd->fp_sys_posn = -1; /* clear this. */
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, bufsize);
/* This is a temporary way of filling in status. The right way is to
keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
#endif
ADIOI_Delete_flattened(datatype);
return;
} /* if (!buftype_is_contig && filetype_is_contig) */
/* already know that file is noncontiguous from above */
/* noncontiguous in file */
/* filetype already flattened in ADIO_Open */
flat_file = ADIOI_Flatlist;
while (flat_file->type != fd->filetype) flat_file = flat_file->next;
disp = fd->disp;
initial_off = offset;
/* for each case - ADIO_Individual pointer or explicit, find offset
(file offset in bytes), n_filetypes (how many filetypes into file
to start), fwr_size (remaining amount of data in present file
block), and st_index (start point in terms of blocks in starting
filetype) */
if (file_ptr_type == ADIO_INDIVIDUAL) {
offset = fd->fp_ind; /* in bytes */
n_filetypes = -1;
flag = 0;
while (!flag) {
n_filetypes++;
for (i=0; i<flat_file->count; i++) {
if (disp + flat_file->indices[i] +
((ADIO_Offset) n_filetypes)*filetype_extent +
flat_file->blocklens[i] >= offset) {
st_index = i;
fwr_size = disp + flat_file->indices[i] +
((ADIO_Offset) n_filetypes)*filetype_extent
+ flat_file->blocklens[i] - offset;
flag = 1;
break;
}
}
} /* while (!flag) */
} /* if (file_ptr_type == ADIO_INDIVIDUAL) */
else {
n_etypes_in_filetype = filetype_size/etype_size;
n_filetypes = (int) (offset / n_etypes_in_filetype);
etype_in_filetype = (int) (offset % n_etypes_in_filetype);
size_in_filetype = etype_in_filetype * etype_size;
sum = 0;
for (i=0; i<flat_file->count; i++) {
sum += flat_file->blocklens[i];
if (sum > size_in_filetype) {
st_index = i;
fwr_size = sum - size_in_filetype;
abs_off_in_filetype = flat_file->indices[i] +
size_in_filetype - (sum - flat_file->blocklens[i]);
break;
}
}
/* abs. offset in bytes in the file */
offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
abs_off_in_filetype;
} /* else [file_ptr_type != ADIO_INDIVIDUAL] */
start_off = offset;
st_fwr_size = fwr_size;
st_n_filetypes = n_filetypes;
if (buftype_is_contig && !filetype_is_contig) {
/* contiguous in memory, noncontiguous in file. should be the most
common case. */
int mem_lengths;
char *mem_offsets;
i = 0;
j = st_index;
off = offset;
n_filetypes = st_n_filetypes;
mem_list_count = 1;
/* determine how many blocks in file to write */
f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
total_blks_to_write = 1;
if (j < (flat_file->count -1)) j++;
else {
j = 0;
n_filetypes++;
}
while (f_data_wrote < bufsize) {
f_data_wrote += flat_file->blocklens[j];
total_blks_to_write++;
if (j<(flat_file->count-1)) j++;
else j = 0;
}
j = st_index;
n_filetypes = st_n_filetypes;
n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
mem_offsets = buf;
mem_lengths = 0;
/* if at least one full writelist, allocate file arrays
at max array size and don't free until very end */
if (n_write_lists) {
file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
sizeof(int64_t));
file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
sizeof(int32_t));
}
/* if there's no full writelist allocate file arrays according
to needed size (extra_blks) */
else {
file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
sizeof(int64_t));
file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
sizeof(int32_t));
}
/* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
for (i=0; i<n_write_lists; i++) {
file_list_count = MAX_ARRAY_SIZE;
if(!i) {
file_offsets[0] = offset;
file_lengths[0] = st_fwr_size;
mem_lengths = st_fwr_size;
}
for (k=0; k<MAX_ARRAY_SIZE; k++) {
if (i || k) {
file_offsets[k] = disp +
((ADIO_Offset)n_filetypes)*filetype_extent
+ flat_file->indices[j];
file_lengths[k] = flat_file->blocklens[j];
mem_lengths += file_lengths[k];
}
if (j<(flat_file->count - 1)) j++;
else {
j = 0;
n_filetypes++;
}
} /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
err_flag = PVFS_Request_contiguous(mem_lengths,
PVFS_BYTE, &mem_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_contiguous (memory)", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
err_flag = PVFS_Request_hindexed(file_list_count, file_lengths,
file_offsets, PVFS_BYTE,
&file_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_hindexed (file)", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
/* PVFS_Request_hindexed already expresses the offsets into the
* file, so we should not pass in an offset if we are using
* hindexed for the file type */
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
#endif
err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
mem_offsets, mem_req,
&(pvfs_fs->credentials), &resp_io);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
#endif
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_sys_write", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
total_bytes_written += resp_io.total_completed;
mem_offsets += mem_lengths;
mem_lengths = 0;
PVFS_Request_free(&file_req);
PVFS_Request_free(&mem_req);
} /* for (i=0; i<n_write_lists; i++) */
/* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
if (extra_blks) {
file_list_count = extra_blks;
if(!i) {
file_offsets[0] = offset;
file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
}
for (k=0; k<extra_blks; k++) {
if(i || k) {
file_offsets[k] = disp +
((ADIO_Offset)n_filetypes)*filetype_extent +
flat_file->indices[j];
if (k == (extra_blks - 1)) {
file_lengths[k] = bufsize - (int32_t) mem_lengths
- (int32_t) mem_offsets + (int32_t) buf;
}
else file_lengths[k] = flat_file->blocklens[j];
} /* if(i || k) */
mem_lengths += file_lengths[k];
if (j<(flat_file->count - 1)) j++;
else {
j = 0;
n_filetypes++;
}
} /* for (k=0; k<extra_blks; k++) */
err_flag = PVFS_Request_contiguous(mem_lengths,
PVFS_BYTE, &mem_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_contiguous (memory)", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
err_flag = PVFS_Request_hindexed(file_list_count, file_lengths,
file_offsets, PVFS_BYTE,
&file_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_hindexed(file)", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
/* as above, use 0 for 'offset' when using hindexed file type*/
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
#endif
err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
mem_offsets, mem_req,
&(pvfs_fs->credentials), &resp_io);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
#endif
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_sys_write", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
total_bytes_written += resp_io.total_completed;
PVFS_Request_free(&mem_req);
PVFS_Request_free(&file_req);
}
}
else {
/* noncontiguous in memory as well as in file */
ADIOI_Flatten_datatype(datatype);
flat_buf = ADIOI_Flatlist;
while (flat_buf->type != datatype) flat_buf = flat_buf->next;
size_wrote = 0;
n_filetypes = st_n_filetypes;
fwr_size = st_fwr_size;
bwr_size = flat_buf->blocklens[0];
buf_count = 0;
start_mem_offset = 0;
start_k = k = 0;
start_j = st_index;
max_mem_list = 0;
max_file_list = 0;
/* run through and file max_file_list and max_mem_list so that you
can allocate the file and memory arrays less than MAX_ARRAY_SIZE
if possible */
while (size_wrote < bufsize) {
k = start_k;
new_buffer_write = 0;
mem_list_count = 0;
while ((mem_list_count < MAX_ARRAY_SIZE) &&
(new_buffer_write < bufsize-size_wrote)) {
/* find mem_list_count and file_list_count such that both are
less than MAX_ARRAY_SIZE, the sum of their lengths are
equal, and the sum of all the data written and data to be
written in the next immediate write list is less than
bufsize */
if(mem_list_count) {
if((new_buffer_write + flat_buf->blocklens[k] +
size_wrote) > bufsize) {
end_bwr_size = new_buffer_write +
flat_buf->blocklens[k] - (bufsize - size_wrote);
new_buffer_write = bufsize - size_wrote;
}
else {
new_buffer_write += flat_buf->blocklens[k];
end_bwr_size = flat_buf->blocklens[k];
}
}
else {
if (bwr_size > (bufsize - size_wrote)) {
new_buffer_write = bufsize - size_wrote;
bwr_size = new_buffer_write;
}
else new_buffer_write = bwr_size;
}
mem_list_count++;
k = (k + 1)%flat_buf->count;
} /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
(new_buffer_write < bufsize-size_wrote)) */
j = start_j;
new_file_write = 0;
file_list_count = 0;
while ((file_list_count < MAX_ARRAY_SIZE) &&
(new_file_write < new_buffer_write)) {
if(file_list_count) {
if((new_file_write + flat_file->blocklens[j]) >
new_buffer_write) {
end_fwr_size = new_buffer_write - new_file_write;
new_file_write = new_buffer_write;
j--;
}
else {
new_file_write += flat_file->blocklens[j];
end_fwr_size = flat_file->blocklens[j];
}
}
else {
if (fwr_size > new_buffer_write) {
new_file_write = new_buffer_write;
fwr_size = new_file_write;
}
else new_file_write = fwr_size;
}
file_list_count++;
if (j < (flat_file->count - 1)) j++;
else j = 0;
k = start_k;
if ((new_file_write < new_buffer_write) &&
(file_list_count == MAX_ARRAY_SIZE)) {
new_buffer_write = 0;
mem_list_count = 0;
while (new_buffer_write < new_file_write) {
if(mem_list_count) {
if((new_buffer_write + flat_buf->blocklens[k]) >
new_file_write) {
end_bwr_size = new_file_write -
new_buffer_write;
new_buffer_write = new_file_write;
k--;
}
else {
new_buffer_write += flat_buf->blocklens[k];
end_bwr_size = flat_buf->blocklens[k];
}
}
else {
new_buffer_write = bwr_size;
if (bwr_size > (bufsize - size_wrote)) {
new_buffer_write = bufsize - size_wrote;
bwr_size = new_buffer_write;
}
}
mem_list_count++;
k = (k + 1)%flat_buf->count;
} /* while (new_buffer_write < new_file_write) */
} /* if ((new_file_write < new_buffer_write) &&
(file_list_count == MAX_ARRAY_SIZE)) */
} /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
(new_buffer_write < bufsize-size_wrote)) */
/* fakes filling the writelist arrays of lengths found above */
k = start_k;
j = start_j;
for (i=0; i<mem_list_count; i++) {
if(i) {
if (i == (mem_list_count - 1)) {
if (flat_buf->blocklens[k] == end_bwr_size)
bwr_size = flat_buf->blocklens[(k+1)%
flat_buf->count];
else {
bwr_size = flat_buf->blocklens[k] - end_bwr_size;
k--;
buf_count--;
}
}
}
buf_count++;
k = (k + 1)%flat_buf->count;
} /* for (i=0; i<mem_list_count; i++) */
for (i=0; i<file_list_count; i++) {
if (i) {
if (i == (file_list_count - 1)) {
if (flat_file->blocklens[j] == end_fwr_size)
fwr_size = flat_file->blocklens[(j+1)%
flat_file->count];
else {
fwr_size = flat_file->blocklens[j] - end_fwr_size;
j--;
}
}
}
if (j < flat_file->count - 1) j++;
else {
j = 0;
n_filetypes++;
}
} /* for (i=0; i<file_list_count; i++) */
size_wrote += new_buffer_write;
start_k = k;
start_j = j;
if (max_mem_list < mem_list_count)
max_mem_list = mem_list_count;
if (max_file_list < file_list_count)
max_file_list = file_list_count;
} /* while (size_wrote < bufsize) */
/* one last check before we actually carry out the operation:
* this code has hard-to-fix bugs when a noncontiguous file type has
* such large pieces that the sum of the lengths of the memory type is
* not larger than one of those pieces (and vice versa for large memory
* types and many pices of file types. In these cases, give up and
* fall back to naive reads and writes. The testphdf5 test created a
* type with two very large memory regions and 600 very small file
* regions. The same test also created a type with one very large file
* region and many (700) very small memory regions. both cases caused
* problems for this code */
if ( ( (file_list_count == 1) &&
(new_file_write < flat_file->blocklens[0] ) ) ||
((mem_list_count == 1) &&
(new_buffer_write < flat_buf->blocklens[0]) ) ||
((file_list_count == MAX_ARRAY_SIZE) &&
(new_file_write < flat_buf->blocklens[0]) ) ||
( (mem_list_count == MAX_ARRAY_SIZE) &&
(new_buffer_write < flat_file->blocklens[0])) )
{
ADIOI_Delete_flattened(datatype);
ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
file_ptr_type, initial_off, status, error_code);
return;
}
mem_offsets = (PVFS_size*)ADIOI_Malloc(max_mem_list*sizeof(PVFS_size));
mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
size_wrote = 0;
n_filetypes = st_n_filetypes;
fwr_size = st_fwr_size;
bwr_size = flat_buf->blocklens[0];
buf_count = 0;
start_mem_offset = 0;
start_k = k = 0;
start_j = st_index;
/* this section calculates mem_list_count and file_list_count
and also finds the possibly odd sized last array elements
in new_fwr_size and new_bwr_size */
while (size_wrote < bufsize) {
k = start_k;
new_buffer_write = 0;
mem_list_count = 0;
while ((mem_list_count < MAX_ARRAY_SIZE) &&
(new_buffer_write < bufsize-size_wrote)) {
/* find mem_list_count and file_list_count such that both are
less than MAX_ARRAY_SIZE, the sum of their lengths are
equal, and the sum of all the data written and data to be
written in the next immediate write list is less than
bufsize */
if(mem_list_count) {
if((new_buffer_write + flat_buf->blocklens[k] +
size_wrote) > bufsize) {
end_bwr_size = new_buffer_write +
flat_buf->blocklens[k] - (bufsize - size_wrote);
new_buffer_write = bufsize - size_wrote;
}
else {
new_buffer_write += flat_buf->blocklens[k];
end_bwr_size = flat_buf->blocklens[k];
}
}
else {
if (bwr_size > (bufsize - size_wrote)) {
new_buffer_write = bufsize - size_wrote;
bwr_size = new_buffer_write;
}
else new_buffer_write = bwr_size;
}
mem_list_count++;
k = (k + 1)%flat_buf->count;
} /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
(new_buffer_write < bufsize-size_wrote)) */
j = start_j;
new_file_write = 0;
file_list_count = 0;
while ((file_list_count < MAX_ARRAY_SIZE) &&
(new_file_write < new_buffer_write)) {
if(file_list_count) {
if((new_file_write + flat_file->blocklens[j]) >
new_buffer_write) {
end_fwr_size = new_buffer_write - new_file_write;
new_file_write = new_buffer_write;
j--;
}
else {
new_file_write += flat_file->blocklens[j];
end_fwr_size = flat_file->blocklens[j];
}
}
else {
if (fwr_size > new_buffer_write) {
new_file_write = new_buffer_write;
fwr_size = new_file_write;
}
else new_file_write = fwr_size;
}
file_list_count++;
if (j < (flat_file->count - 1)) j++;
else j = 0;
k = start_k;
if ((new_file_write < new_buffer_write) &&
(file_list_count == MAX_ARRAY_SIZE)) {
new_buffer_write = 0;
mem_list_count = 0;
while (new_buffer_write < new_file_write) {
if(mem_list_count) {
if((new_buffer_write + flat_buf->blocklens[k]) >
new_file_write) {
end_bwr_size = new_file_write -
new_buffer_write;
new_buffer_write = new_file_write;
k--;
}
else {
new_buffer_write += flat_buf->blocklens[k];
end_bwr_size = flat_buf->blocklens[k];
}
}
else {
new_buffer_write = bwr_size;
if (bwr_size > (bufsize - size_wrote)) {
new_buffer_write = bufsize - size_wrote;
bwr_size = new_buffer_write;
}
}
mem_list_count++;
k = (k + 1)%flat_buf->count;
} /* while (new_buffer_write < new_file_write) */
} /* if ((new_file_write < new_buffer_write) &&
(file_list_count == MAX_ARRAY_SIZE)) */
} /* while ((mem_list_count < MAX_ARRAY_SIZE) &&
(new_buffer_write < bufsize-size_wrote)) */
/* fills the allocated writelist arrays */
k = start_k;
j = start_j;
for (i=0; i<mem_list_count; i++) {
/* TODO: fix this warning by casting to an integer that's the
* same size as a char * and /then/ casting to PVFS_size */
mem_offsets[i] = ((PVFS_size)buf + buftype_extent*
(buf_count/flat_buf->count) +
(int)flat_buf->indices[k]);
if(!i) {
mem_lengths[0] = bwr_size;
mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
}
else {
if (i == (mem_list_count - 1)) {
mem_lengths[i] = end_bwr_size;
if (flat_buf->blocklens[k] == end_bwr_size)
bwr_size = flat_buf->blocklens[(k+1)%
flat_buf->count];
else {
bwr_size = flat_buf->blocklens[k] - end_bwr_size;
k--;
buf_count--;
}
}
else {
mem_lengths[i] = flat_buf->blocklens[k];
}
}
buf_count++;
k = (k + 1)%flat_buf->count;
} /* for (i=0; i<mem_list_count; i++) */
for (i=0; i<file_list_count; i++) {
file_offsets[i] = disp + flat_file->indices[j] +
((ADIO_Offset)n_filetypes) * filetype_extent;
if (!i) {
file_lengths[0] = fwr_size;
file_offsets[0] += flat_file->blocklens[j] - fwr_size;
}
else {
if (i == (file_list_count - 1)) {
file_lengths[i] = end_fwr_size;
if (flat_file->blocklens[j] == end_fwr_size)
fwr_size = flat_file->blocklens[(j+1)%
flat_file->count];
else {
fwr_size = flat_file->blocklens[j] - end_fwr_size;
j--;
}
}
else file_lengths[i] = flat_file->blocklens[j];
}
if (j < flat_file->count - 1) j++;
else {
j = 0;
n_filetypes++;
}
} /* for (i=0; i<file_list_count; i++) */
err_flag = PVFS_Request_hindexed(mem_list_count, mem_lengths,
mem_offsets, PVFS_BYTE, &mem_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0 ) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_hindexed (memory)", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
err_flag = PVFS_Request_hindexed(file_list_count, file_lengths,
file_offsets, PVFS_BYTE,
&file_req);
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_Request_hindexed", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
/* offset will be expressed in memory and file datatypes */
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
#endif
err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
PVFS_BOTTOM, mem_req,
&(pvfs_fs->credentials), &resp_io);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
#endif
/* --BEGIN ERROR HANDLING-- */
if (err_flag != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(err_flag),
"Error in PVFS_sys_write", 0);
goto error_state;
}
/* --END ERROR HANDLING-- */
size_wrote += new_buffer_write;
total_bytes_written += resp_io.total_completed;
start_k = k;
start_j = j;
PVFS_Request_free(&mem_req);
PVFS_Request_free(&file_req);
} /* while (size_wrote < bufsize) */
ADIOI_Free(mem_offsets);
ADIOI_Free(mem_lengths);
}
/* when incrementing fp_ind, need to also take into account the file type:
* consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
* if we wrote N elements, offset needs to point at beginning of type, not
* at empty region at offset N+1).
*
* As we discussed on mpich-discuss in may/june 2009, the code below might
* look wierd, but by putting fp_ind at the last byte written, the next
* time we run through the strided code we'll update the fp_ind to the
* right location. */
if (file_ptr_type == ADIO_INDIVIDUAL) {
fd->fp_ind = file_offsets[file_list_count-1]+
file_lengths[file_list_count-1];
}
ADIOI_Free(file_offsets);
ADIOI_Free(file_lengths);
*error_code = MPI_SUCCESS;
error_state:
fd->fp_sys_posn = -1; /* set it to null. */
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, bufsize);
/* This is a temporary way of filling in status. The right way is to
keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
#endif
if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
}
int main(
int argc,
char **argv)
{
int ret = -1;
int outcount = 0, count;
struct BMI_unexpected_info request_info;
flow_descriptor *flow_d = NULL;
double time1, time2;
int i;
PINT_Request *req;
char path_name[PATH_SIZE];
TROVE_op_id op_id;
TROVE_coll_id coll_id;
TROVE_handle file_handle, parent_handle;
TROVE_ds_state state;
char *file_name;
TROVE_keyval_s key, val;
bmi_context_id context;
TROVE_context_id trove_context;
PVFS_handle_extent cur_extent;
PVFS_handle_extent_array extent_array;
/*************************************************************/
/* initialization stuff */
/* set debugging level */
gossip_enable_stderr();
gossip_set_debug_mask(0, GOSSIP_FLOW_PROTO_DEBUG | GOSSIP_BMI_DEBUG_TCP );
/* start up BMI */
ret = BMI_initialize("bmi_tcp", "tcp://NULL:3335", BMI_INIT_SERVER);
if (ret < 0)
{
fprintf(stderr, "BMI init failure.\n");
return (-1);
}
ret = BMI_open_context(&context);
if (ret < 0)
{
fprintf(stderr, "BMI_open_context() failure.\n");
return (-1);
}
ret = trove_initialize(
TROVE_METHOD_DBPF, NULL, storage_space, 0);
if (ret < 0)
{
fprintf(stderr, "initialize failed: run trove-mkfs first.\n");
return -1;
}
/* initialize the flow interface. protocol specific */
ret = PINT_flow_initialize("flowproto_bmi_cache", 0);
if (ret < 0)
{
fprintf(stderr, "flow init failure.\n");
return (-1);
}
/* try to look up collection used to store file system */
ret = trove_collection_lookup(
TROVE_METHOD_DBPF, file_system, &coll_id, NULL, &op_id);
if (ret < 0)
{
fprintf(stderr, "collection lookup failed.\n");
return -1;
}
ret = trove_open_context(coll_id, &trove_context);
if (ret < 0)
{
fprintf(stderr, "TROVE_open_context() failure.\n");
return (-1);
}
/* find the parent directory name */
strcpy(path_name, path_to_file);
for (i = strlen(path_name); i >= 0; i--)
{
if (path_name[i] != '/')
path_name[i] = '\0';
else
break;
}
file_name = path_to_file + strlen(path_name);
printf("path is %s\n", path_name);
printf("file is %s\n", file_name);
/* find the parent directory handle */
ret = path_lookup(coll_id, trove_context, path_name, &parent_handle);
if (ret < 0)
{
return -1;
}
file_handle = 0;
cur_extent.first = cur_extent.last = requested_file_handle;
extent_array.extent_count = 1;
extent_array.extent_array = &cur_extent;
ret = trove_dspace_create(coll_id,
&extent_array,
&file_handle,
TROVE_TEST_FILE,
NULL,
TROVE_FORCE_REQUESTED_HANDLE,
NULL, trove_context, &op_id, NULL);
while (ret == 0)
ret =
trove_dspace_test(coll_id, op_id, trove_context, &count, NULL, NULL,
&state, TROVE_DEFAULT_TEST_TIMEOUT);
if (ret < 0)
{
fprintf(stderr, "dspace create failed.\n");
return -1;
}
/* TODO: set attributes of file? */
/* add new file name/handle pair to parent directory */
key.buffer = file_name;
key.buffer_sz = strlen(file_name) + 1;
val.buffer = &file_handle;
val.buffer_sz = sizeof(file_handle);
ret =
trove_keyval_write(coll_id, parent_handle, &key, &val, 0, NULL, NULL,
trove_context, &op_id, NULL);
while (ret == 0)
ret =
trove_dspace_test(coll_id, op_id, trove_context, &count, NULL, NULL,
&state, TROVE_DEFAULT_TEST_TIMEOUT);
if (ret < 0)
{
fprintf(stderr, "keyval write failed.\n");
return -1;
}
/* wait for an initial communication via BMI */
/* we don't give a crap about that message except that it tells us
* where to find the client
*/
do
{
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0 || request_info.error_code != 0)
{
fprintf(stderr, "waitunexpected failure.\n");
return (-1);
}
BMI_unexpected_free(request_info.addr, request_info.buffer);
/******************************************************/
/* setup request/dist stuff */
/* request description */
/* just want one contiguous region */
ret = PVFS_Request_contiguous(TEST_SIZE, PVFS_BYTE, &req);
if (ret < 0)
{
fprintf(stderr, "PVFS_Request_contiguous() failure.\n");
return (-1);
}
/******************************************************/
/* setup communicaton stuff */
/* create a flow descriptor */
flow_d = PINT_flow_alloc();
if (!flow_d)
{
fprintf(stderr, "flow_alloc failed.\n");
return (-1);
}
/* file data */
flow_d->file_data.fsize = TEST_SIZE;
flow_d->file_data.server_nr = 0;
flow_d->file_data.server_ct = 1;
flow_d->file_data.extend_flag = 1;
flow_d->file_data.dist = PINT_dist_create("basic_dist");
if (!flow_d->file_data.dist)
{
fprintf(stderr, "Error: failed to create dist.\n");
return (-1);
}
ret = PINT_dist_lookup(flow_d->file_data.dist);
if (ret != 0)
{
fprintf(stderr, "Error: failed to lookup dist.\n");
return (-1);
}
flow_d->file_req = req;
flow_d->tag = 0;
flow_d->user_ptr = NULL;
flow_d->aggregate_size = TEST_SIZE;
/* fill in flow details */
flow_d->src.endpoint_id = BMI_ENDPOINT;
flow_d->src.u.bmi.address = request_info.addr;
flow_d->dest.endpoint_id = TROVE_ENDPOINT;
flow_d->dest.u.trove.handle = file_handle;
flow_d->dest.u.trove.coll_id = coll_id;
/***************************************************
* test bmi to file (analogous to a client side write)
*/
time1 = Wtime();
ret = block_on_flow(flow_d);
if (ret < 0)
{
return (-1);
}
time2 = Wtime();
#if 0
printf("Server bw (recv): %f MB/sec\n",
((TEST_SIZE) / ((time2 - time1) * 1000000.0)));
#endif
/*******************************************************/
/* final cleanup and output */
PINT_flow_free(flow_d);
/* shut down flow interface */
ret = PINT_flow_finalize();
if (ret < 0)
{
fprintf(stderr, "flow finalize failure.\n");
return (-1);
}
/* shut down BMI */
BMI_close_context(context);
BMI_finalize();
trove_close_context(coll_id, trove_context);
trove_finalize(TROVE_METHOD_DBPF);
gossip_disable();
return (0);
}
int ADIOI_PVFS2_StridedDtypeIO(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status, int
*error_code,
int rw_type)
{
int ret = -1, filetype_is_contig = -1;
MPI_Count filetype_size = -1;
int num_filetypes = 0, cur_flat_file_reg_off = 0;
PVFS_Request tmp_mem_req, mem_req, tmp_file_req, file_req;
PVFS_sysresp_io resp_io;
ADIO_Offset off = -1, bytes_into_filetype = 0;
MPI_Aint filetype_extent = -1;
int i = -1;
MPI_Count etype_size;
PVFS_size pvfs_disp = -1;
ADIOI_Flatlist_node *flat_file_p = ADIOI_Flatlist;
/* Use for offseting the PVFS2 filetype */
int pvfs_blk = 1;
ADIOI_PVFS2_fs *pvfs_fs;
static char myname[] = "ADIOI_PVFS2_STRIDED_DTYPE";
memset(&tmp_mem_req, 0, sizeof(PVFS_Request));
memset(&mem_req, 0, sizeof(PVFS_Request));
memset(&tmp_file_req, 0, sizeof(PVFS_Request));
memset(&file_req, 0, sizeof(PVFS_Request));
pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
/* changed below if error */
*error_code = MPI_SUCCESS;
/* datatype is the memory type
* fd->filetype is the file type */
MPI_Type_size_x(fd->filetype, &filetype_size);
if (filetype_size == 0) {
*error_code = MPI_SUCCESS;
return -1;
}
MPI_Type_extent(fd->filetype, &filetype_extent);
MPI_Type_size_x(fd->etype, &etype_size);
if (filetype_size == 0) {
*error_code = MPI_SUCCESS;
return -1;
}
/* offset is in units of etype relative to the filetype. We
* convert this to off in terms of actual data bytes (the offset
* minus the number of bytes that are not used). We are allowed
* to do this since PVFS2 handles offsets with respect to a
* file_req in bytes, otherwise we would have to convert into a
* pure byte offset as is done in other methods. Explicit offset
* case is handled by using fd->disp and byte-converted off. */
pvfs_disp = fd->disp;
if (file_ptr_type == ADIO_INDIVIDUAL)
{
if (filetype_is_contig)
{
off = fd->fp_ind - fd->disp;
}
else
{
int flag = 0;
/* Should have already been flattened in ADIO_Open*/
while (flat_file_p->type != fd->filetype)
{
flat_file_p = flat_file_p->next;
}
num_filetypes = -1;
while (!flag)
{
num_filetypes++;
for (i = 0; i < flat_file_p->count; i++)
{
/* Start on a non zero-length region */
if (flat_file_p->blocklens[i])
{
if (fd->disp + flat_file_p->indices[i] +
(num_filetypes * filetype_extent) +
flat_file_p->blocklens[i] > fd->fp_ind &&
fd->disp + flat_file_p->indices[i] <=
fd->fp_ind)
{
cur_flat_file_reg_off = fd->fp_ind -
(fd->disp + flat_file_p->indices[i] +
(num_filetypes * filetype_extent));
flag = 1;
break;
}
else
bytes_into_filetype += flat_file_p->blocklens[i];
}
}
}
/* Impossible that we don't find it in this datatype */
assert(i != flat_file_p->count);
off = bytes_into_filetype + cur_flat_file_reg_off;
}
}
else /* ADIO_EXPLICIT */
{
off = etype_size * offset;
}
#ifdef DEBUG_DTYPE
fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: (fd->fp_ind=%Ld,fd->disp=%Ld,"
" offset=%Ld),(pvfs_disp=%Ld,off=%Ld)\n",
fd->fp_ind, fd->disp, offset, pvfs_disp, off);
#endif
/* Convert the MPI memory and file datatypes into
* PVFS2 datatypes */
ret = convert_mpi_pvfs2_dtype(&datatype, &tmp_mem_req);
if (ret < 0)
{
goto error_state;
}
ret = convert_mpi_pvfs2_dtype(&(fd->filetype), &tmp_file_req);
if (ret < 0)
{
goto error_state;
}
ret = PVFS_Request_contiguous(count, tmp_mem_req, &mem_req);
if (ret != 0) /* TODO: convert this to MPIO error handling */
fprintf(stderr, "ADIOI_PVFS2_stridedDtypeIO: error in final"
" CONTIG memory type\n");
PVFS_Request_free(&tmp_mem_req);
/* pvfs_disp is used to offset the filetype */
ret = PVFS_Request_hindexed(1, &pvfs_blk, &pvfs_disp,
tmp_file_req, &file_req);
if (ret != 0)
fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: error in final"
" HINDEXED file type\n");
PVFS_Request_free(&tmp_file_req);
if (rw_type == READ)
ret = PVFS_sys_read(pvfs_fs->object_ref, file_req, off, buf,
mem_req, &(pvfs_fs->credentials), &resp_io);
else
ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, off, buf,
mem_req, &(pvfs_fs->credentials), &resp_io);
if (ret != 0) {
fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: Warning - PVFS_sys_"
"read/write returned %d and completed %Ld bytes.\n",
ret, (long long)resp_io.total_completed);
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in PVFS_sys_io \n", 0);
goto error_state;
}
if (file_ptr_type == ADIO_INDIVIDUAL)
{
fd->fp_ind = off += resp_io.total_completed;
}
error_state:
fd->fp_sys_posn = -1; /* set it to null. */
PVFS_Request_free(&mem_req);
PVFS_Request_free(&file_req);
#ifdef DEBUG_DTYPE
fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: "
"resp_io.total_completed=%Ld,ret=%d\n",
resp_io.total_completed, ret);
#endif
#ifdef HAVE_STATUS_SET_BYTES
MPIR_Status_set_bytes(status, datatype, resp_io.total_completed);
/* This is a temporary way of filling in status. The right way is to
* keep track of how much data was actually acccessed by
* ADIOI_BUFFERED operations */
#endif
return ret;
}
void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, MPI_Request *request,
int flag, int *error_code)
{
int ret;
MPI_Count datatype_size, len;
ADIOI_PVFS2_fs *pvfs_fs;
ADIOI_AIO_Request *aio_req;
static char myname[] = "ADIOI_PVFS2_AIO_contig";
pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
aio_req = (ADIOI_AIO_Request*)ADIOI_Calloc(sizeof(ADIOI_AIO_Request), 1);
MPI_Type_size_x(datatype, &datatype_size);
len = datatype_size * count;
ret = PVFS_Request_contiguous(len, PVFS_BYTE, &(aio_req->mem_req));
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in pvfs_request_contig (memory)", 0);
return;
}
/* --END ERROR HANDLING-- */
ret = PVFS_Request_contiguous(len, PVFS_BYTE, &(aio_req->file_req));
/* --BEGIN ERROR HANDLING-- */
if (ret != 0) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in pvfs_request_contig (file)", 0);
return;
}
/* --END ERROR HANDLING-- */
if (file_ptr_type == ADIO_INDIVIDUAL) {
/* copy individual file pointer into offset variable, continue */
offset = fd->fp_ind;
}
if (flag == READ) {
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_iread_a, 0, NULL );
#endif
ret = PVFS_isys_read(pvfs_fs->object_ref, aio_req->file_req, offset,
buf, aio_req->mem_req, &(pvfs_fs->credentials),
&(aio_req->resp_io), &(aio_req->op_id), NULL);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_iread_b, 0, NULL );
#endif
} else if (flag == WRITE) {
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_iwrite_a, 0, NULL );
#endif
ret = PVFS_isys_write(pvfs_fs->object_ref, aio_req->file_req, offset,
buf, aio_req->mem_req, &(pvfs_fs->credentials),
&(aio_req->resp_io), &(aio_req->op_id), NULL);
#ifdef ADIOI_MPE_LOGGING
MPE_Log_event( ADIOI_MPE_iwrite_b, 0, NULL );
#endif
}
/* --BEGIN ERROR HANDLING-- */
if (ret < 0 ) {
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
myname, __LINE__,
ADIOI_PVFS2_error_convert(ret),
"Error in PVFS_isys_io", 0);
goto fn_exit;
}
/* --END ERROR HANDLING-- */
#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
/* posted. defered completion */
if (ret == 0) {
if (ADIOI_PVFS2_greq_class == 0) {
MPIX_Grequest_class_create(ADIOI_GEN_aio_query_fn,
ADIOI_PVFS2_aio_free_fn, MPIU_Greq_cancel_fn,
ADIOI_PVFS2_aio_poll_fn, ADIOI_PVFS2_aio_wait_fn,
&ADIOI_PVFS2_greq_class);
}
MPIX_Grequest_class_allocate(ADIOI_PVFS2_greq_class, aio_req, request);
memcpy(&(aio_req->req), request, sizeof(*request));
}
#else
/* if generalized request extensions not available, we will have to process
* this operation right here */
int error;
ret = PVFS_sys_wait(aio_req->op_id, "ADIOI_PVFS2_AIO_Contig", &error);
if (ret == 0) {
MPIO_Completed_request_create(&fd, len, error_code, request);
}
#endif
/* immediate completion */
if (ret == 1) {
MPIO_Completed_request_create(&fd, len, error_code, request);
}
if (file_ptr_type == ADIO_INDIVIDUAL) {
fd->fp_ind += len;
}
fd->fp_sys_posn = offset + len;
*error_code = MPI_SUCCESS;
fn_exit:
return;
}
int main(int argc, char **argv)
{
PVFS_sysresp_lookup resp_lk;
PVFS_sysresp_create resp_cr;
PVFS_sysresp_io resp_io;
char *filename = NULL;
int ret = -1, io_size = DEFAULT_IO_SIZE;
int *io_buffer = NULL;
int i, errors, buffer_size;
PVFS_fs_id fs_id;
char name[512] = {0};
char *entry_name = NULL;
PVFS_credentials credentials;
PVFS_object_ref parent_refn;
PVFS_sys_attr attr;
PVFS_object_ref pinode_refn;
PVFS_Request file_req;
PVFS_Request mem_req;
void *buffer = NULL;
PVFS_sysresp_getattr resp_getattr;
PVFS_handle *dfile_array = NULL;
if (argc != 2)
{
fprintf(stderr, "Usage: %s <file name>\n", argv[0]);
return (-1);
}
/* create a buffer for running I/O on */
io_buffer = (int *) malloc(io_size * sizeof(int));
if (!io_buffer)
{
return (-1);
}
/* put some data in the buffer so we can verify */
for (i = 0; i < io_size; i++)
{
io_buffer[i] = i;
}
ret = PVFS_util_init_defaults();
if (ret < 0)
{
PVFS_perror("PVFS_util_init_defaults", ret);
return (-1);
}
ret = PVFS_util_get_default_fsid(&fs_id);
if (ret < 0)
{
PVFS_perror("PVFS_util_get_default_fsid", ret);
return (-1);
}
if (argv[1][0] == '/')
{
snprintf(name, 512, "%s", argv[1]);
}
else
{
snprintf(name, 512, "/%s", argv[1]);
}
PVFS_util_gen_credentials(&credentials);
ret = PVFS_sys_lookup(fs_id, name, &credentials,
&resp_lk, PVFS2_LOOKUP_LINK_FOLLOW, NULL);
if (ret == -PVFS_ENOENT)
{
PVFS_sysresp_getparent gp_resp;
printf("IO-TEST: lookup failed; creating new file.\n");
memset(&gp_resp, 0, sizeof(PVFS_sysresp_getparent));
ret = PVFS_sys_getparent(fs_id, name, &credentials, &gp_resp, NULL);
if (ret < 0)
{
PVFS_perror("PVFS_sys_getparent failed", ret);
return ret;
}
attr.owner = credentials.uid;
attr.group = credentials.gid;
attr.perms = PVFS_U_WRITE | PVFS_U_READ;
attr.atime = attr.ctime = attr.mtime = time(NULL);
attr.mask = PVFS_ATTR_SYS_ALL_SETABLE;
parent_refn = gp_resp.parent_ref;
entry_name = rindex(name, (int)'/');
assert(entry_name);
entry_name++;
assert(entry_name);
ret = PVFS_sys_create(entry_name, parent_refn, attr,
&credentials, NULL, &resp_cr, NULL, NULL);
if (ret < 0)
{
PVFS_perror("PVFS_sys_create() failure", ret);
return (-1);
}
pinode_refn.fs_id = fs_id;
pinode_refn.handle = resp_cr.ref.handle;
}
else
{
printf("IO-TEST: lookup succeeded; performing I/O on "
"existing file.\n");
pinode_refn.fs_id = fs_id;
pinode_refn.handle = resp_lk.ref.handle;
}
/**************************************************************
* carry out I/O operation
*/
printf("IO-TEST: performing write on handle: %ld, fs: %d\n",
(long) pinode_refn.handle, (int) pinode_refn.fs_id);
buffer = io_buffer;
buffer_size = io_size * sizeof(int);
/*
file datatype is tiled, so we can get away with a trivial type
here
*/
file_req = PVFS_BYTE;
ret = PVFS_Request_contiguous(io_size * sizeof(int),
PVFS_BYTE, &(mem_req));
if (ret < 0)
{
PVFS_perror("PVFS_request_contiguous failure", ret);
return (-1);
}
ret = PVFS_sys_write(pinode_refn, file_req, 0, buffer, mem_req,
&credentials, &resp_io, NULL);
if (ret < 0)
{
PVFS_perror("PVFS_sys_write failure", ret);
return (-1);
}
printf("IO-TEST: wrote %d bytes.\n", (int) resp_io.total_completed);
/* uncomment and try out the readback-and-verify stuff that follows
* once reading back actually works */
memset(io_buffer, 0, io_size * sizeof(int));
/* verify */
printf("IO-TEST: performing read on handle: %ld, fs: %d\n",
(long) pinode_refn.handle, (int) pinode_refn.fs_id);
ret = PVFS_sys_read(pinode_refn, file_req, 0, buffer, mem_req,
&credentials, &resp_io, NULL);
if (ret < 0)
{
PVFS_perror("PVFS_sys_read failure", ret);
return (-1);
}
printf("IO-TEST: read %d bytes.\n", (int) resp_io.total_completed);
if ((io_size * sizeof(int)) != resp_io.total_completed)
{
fprintf(stderr, "Error: SHORT READ! skipping verification...\n");
}
else
{
errors = 0;
for (i = 0; i < io_size; i++)
{
if (i != io_buffer[i])
{
fprintf(stderr,
"error: element %d differs: should be %d, is %d\n", i,
i, io_buffer[i]);
errors++;
}
}
if (errors != 0)
{
fprintf(stderr, "ERROR: found %d errors\n", errors);
}
else
{
printf("IO-TEST: no errors found.\n");
}
}
/* test out some of the mgmt functionality */
ret = PVFS_sys_getattr(pinode_refn, PVFS_ATTR_SYS_ALL_NOSIZE,
&credentials, &resp_getattr, NULL);
if (ret < 0)
{
PVFS_perror("PVFS_sys_getattr", ret);
return (-1);
}
printf("Target file had %d datafiles:\n", resp_getattr.attr.dfile_count);
dfile_array = (PVFS_handle *) malloc(resp_getattr.attr.dfile_count
* sizeof(PVFS_handle));
if (!dfile_array)
{
perror("malloc");
return (-1);
}
ret = PVFS_mgmt_get_dfile_array(pinode_refn, &credentials,
dfile_array, resp_getattr.attr.dfile_count, NULL);
if (ret < 0)
{
PVFS_perror("PVFS_mgmt_get_dfile_array", ret);
return (-1);
}
for (i = 0; i < resp_getattr.attr.dfile_count; i++)
{
printf("%llu\n", llu(dfile_array[i]));
}
/**************************************************************
* shut down pending interfaces
*/
ret = PVFS_sys_finalize();
if (ret < 0)
{
fprintf(stderr, "Error: PVFS_sys_finalize() failed with errcode = %d\n",
ret);
return (-1);
}
free(filename);
free(io_buffer);
return (0);
}