int CreateStriped( const char * path, const stripe_info_t * old_stripe, int overwrite )
{
int rc;
/* try to restripe using previous pool name */
if ( !EMPTY_STRING( old_stripe->pool_name ) )
rc = llapi_file_create_pool( path, old_stripe->stripe_size,
-1, old_stripe->stripe_count, 0,
(char *)old_stripe->pool_name );
else
rc = llapi_file_create( path, old_stripe->stripe_size,
-1, old_stripe->stripe_count, 0 );
if ((rc == -EEXIST) && overwrite)
{
if (unlink(path)) {
rc = -errno;
DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Can't remove previous entry %s: %s",
path, strerror(-rc));
return rc;
}
return CreateStriped(path, old_stripe, false /*target not expected to exist*/);
}
else if ( rc != 0 && rc != -EEXIST )
{
DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Error %d creating '%s' with stripe.",
rc, path );
}
return rc;
}
int File_CreateSetStripe( const char * path, const stripe_info_t * old_stripe )
{
int rc;
/* try to restripe using previous pool name */
if ( !EMPTY_STRING( old_stripe->pool_name ) )
{
rc = llapi_file_create_pool( path, old_stripe->stripe_size,
-1, old_stripe->stripe_count, 0,
(char *)old_stripe->pool_name );
if ( rc == 0 || rc == -EEXIST )
return rc;
else
{
DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Error %d creating '%s' in pool '%s': %s",
rc, path, old_stripe->pool_name, strerror(-rc) );
DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Trying to create it without pool information..." );
}
}
rc = llapi_file_create( path, old_stripe->stripe_size,
-1, old_stripe->stripe_count, 0 );
if ( rc != 0 || rc == -EEXIST )
DisplayLog( LVL_MAJOR, TAG_CR_STRIPE,
"Error %d creating '%s' with stripe. Trying to create it without specific stripe...",
rc, path );
return rc;
}
int t3pio_asklustre(MPI_Comm comm, int myProc, const char* path)
{
int ierr;
int stripes = 1;
#ifdef HAVE_LUSTRE
const char * dir = path2dir(path);
if (myProc == 0 && t3pio_usingLustreFS(dir))
{
char line[MAXLINE];
int stripe_size = 0;
int stripe_offset = -1;
int stripe_pattern = 0;
int stripes_max = -1;
int flags = (O_WRONLY | O_CREAT | O_EXCL);
// Create a Lustre Striped Test File
char fn[MAXLINE];
sprintf(&fn[0], "%s/foo_%d.bar", dir, getpid());
int rc = llapi_file_create(fn, stripe_size, stripe_offset, stripes_max,
stripe_pattern);
if (rc == 0)
{
flags |= O_LOV_DELAY_CREATE;
flags &= ~O_EXCL;
int fd = open(fn, flags, RW_USER);
FILE* fp = fdopen(fd,"w");
fprintf(fp, "foo.bar\n");
fclose(fp);
sprintf(&line[0],"lfs getstripe -q %s",fn);
if ( (fp = popen(line, "r")) == NULL)
{
fprintf(stderr,"unable to popen\n");
abort();
}
int count = 0;
while (fgets(line, MAXLINE, fp) != NULL)
{
size_t k = strspn(&line[0], " \t");
if (isdigit(line[k]))
count++;
}
stripes = count;
fclose(fp);
}
unlink(fn);
}
ierr = MPI_Bcast(&stripes, 1, MPI_INTEGER, 0, comm);
#endif
return stripes;
}
/*-------------------------------------------------------------------------*/
int FTI_InitMpiICP(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec,
FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt,
FTIT_dataset* FTI_Data)
{
int res;
FTI_Print("I/O mode: MPI-IO.", FTI_DBUG);
char str[FTI_BUFS], mpi_err[FTI_BUFS];
// enable collective buffer optimization
MPI_Info info;
MPI_Info_create(&info);
MPI_Info_set(info, "romio_cb_write", "enable");
/*
* update ckpt file name (neccessary for the restart!)
* not very nice TODO we should think about another mechanism
*/
snprintf(FTI_Exec->meta[0].ckptFile, FTI_BUFS,
"Ckpt%d-Rank%d.fti", FTI_Exec->ckptID, FTI_Topo->myRank);
// TODO enable to set stripping unit in the config file (Maybe also other hints)
// set stripping unit to 4MB
MPI_Info_set(info, "stripping_unit", "4194304");
char gfn[FTI_BUFS], ckptFile[FTI_BUFS];
snprintf(ckptFile, FTI_BUFS, "Ckpt%d-mpiio.fti", FTI_Exec->ckptID);
snprintf(gfn, FTI_BUFS, "%s/%s", FTI_Conf->gTmpDir, ckptFile);
// open parallel file (collective call)
MPI_File pfh;
#ifdef LUSTRE
if (FTI_Topo->splitRank == 0) {
res = llapi_file_create(gfn, FTI_Conf->stripeUnit, FTI_Conf->stripeOffset, FTI_Conf->stripeFactor, 0);
if (res) {
char error_msg[FTI_BUFS];
error_msg[0] = 0;
strerror_r(-res, error_msg, FTI_BUFS);
snprintf(str, FTI_BUFS, "[Lustre] %s.", error_msg);
FTI_Print(str, FTI_WARN);
} else {
snprintf(str, FTI_BUFS, "[LUSTRE] file:%s striping_unit:%i striping_factor:%i striping_offset:%i",
ckptFile, FTI_Conf->stripeUnit, FTI_Conf->stripeFactor, FTI_Conf->stripeOffset);
FTI_Print(str, FTI_DBUG);
}
}
#endif
res = MPI_File_open(FTI_COMM_WORLD, gfn, MPI_MODE_WRONLY|MPI_MODE_CREATE, info, &pfh);
// check if successful
if (res != 0) {
errno = 0;
int reslen;
MPI_Error_string(res, mpi_err, &reslen);
snprintf(str, FTI_BUFS, "unable to create file %s [MPI ERROR - %i] %s", gfn, res, mpi_err);
FTI_Print(str, FTI_EROR);
return FTI_NSCS;
}
MPI_Offset chunkSize = FTI_Exec->ckptSize;
// collect chunksizes of other ranks
MPI_Offset* chunkSizes = talloc(MPI_Offset, FTI_Topo->nbApprocs * FTI_Topo->nbNodes);
MPI_Allgather(&chunkSize, 1, MPI_OFFSET, chunkSizes, 1, MPI_OFFSET, FTI_COMM_WORLD);
// set file offset
MPI_Offset offset = 0;
int i;
for (i = 0; i < FTI_Topo->splitRank; i++) {
offset += chunkSizes[i];
}
free(chunkSizes);
FTI_Exec->iCPInfo.offset = offset;
memcpy( FTI_Exec->iCPInfo.fh, &pfh, sizeof(FTI_MI_FH) );
MPI_Info_free(&info);
return FTI_SCES;
}
int t55(char *name)
{
char path[MAX_PATH_LENGTH] = "";
char file[MAX_PATH_LENGTH] = "";
struct lov_user_md *lum = NULL;
struct lov_user_ost_data *lo = NULL;
int index, fd, buflen, rc;
ENTER("setstripe/getstripe");
snprintf(path, MAX_PATH_LENGTH, "%s/test_t55", lustre_path);
snprintf(file, MAX_PATH_LENGTH, "%s/test_t55/file_t55", lustre_path);
buflen = sizeof(struct lov_user_md);
buflen += STRIPE_COUNT * sizeof(struct lov_user_ost_data);
lum = (struct lov_user_md *)malloc(buflen);
if (!lum) {
printf("out of memory!\n");
return -1;
}
memset(lum, 0, buflen);
t_mkdir(path);
rc = llapi_file_create(path, STRIPE_SIZE, STRIPE_OFFSET,
STRIPE_COUNT, LOV_PATTERN_RAID0);
if (rc) {
printf("llapi_file_create failed: rc = %d (%s) \n",
rc, strerror(-rc));
t_rmdir(path);
free(lum);
return -1;
}
fd = open(file, O_CREAT | O_RDWR, 0644);
if (fd < 0) {
printf("open file(%s) failed: rc = %d (%s) \n)",
file, fd, strerror(errno));
t_rmdir(path);
free(lum);
return -1;
}
lum->lmm_magic = LOV_USER_MAGIC;
lum->lmm_stripe_count = STRIPE_COUNT;
rc = ioctl(fd, LL_IOC_LOV_GETSTRIPE, lum);
if (rc) {
printf("dir:ioctl(LL_IOC_LOV_GETSTRIPE) failed: rc = %d(%s)\n",
rc, strerror(errno));
close(fd);
t_unlink(file);
t_rmdir(path);
free(lum);
return -1;
}
close(fd);
if (opt_verbose) {
printf("lmm_magic: 0x%08X\n", lum->lmm_magic);
printf("lmm_object_id: "LPX64"\n", lum->lmm_object_id);
printf("lmm_object_seq: "LPX64"\n", lum->lmm_object_seq);
printf("lmm_stripe_count: %u\n", (int)lum->lmm_stripe_count);
printf("lmm_stripe_size: %u\n", lum->lmm_stripe_size);
printf("lmm_stripe_pattern: %x\n", lum->lmm_pattern);
for (index = 0; index < lum->lmm_stripe_count; index++) {
lo = lum->lmm_objects + index;
printf("object %d:\n", index);
printf("\tobject_seq: "LPX64"\n", lo->l_object_seq);
printf("\tobject_id: "LPX64"\n", lo->l_object_id);
printf("\tost_gen: %#x\n", lo->l_ost_gen);
printf("\tost_idx: %u\n", lo->l_ost_idx);
}
}
if (lum->lmm_magic != LOV_USER_MAGIC ||
lum->lmm_pattern != LOV_PATTERN_RAID0 ||
lum->lmm_stripe_size != STRIPE_SIZE ||
lum->lmm_objects[0].l_ost_idx != STRIPE_OFFSET ||
lum->lmm_stripe_count != STRIPE_COUNT) {
printf("incorrect striping information!\n");
t_unlink(file);
t_rmdir(path);
free(lum);
return -1;
}
t_unlink(file);
/* setstripe on regular file */
rc = llapi_file_create(file, STRIPE_SIZE, STRIPE_OFFSET,
STRIPE_COUNT, LOV_PATTERN_RAID0);
if (rc) {
printf("llapi_file_create failed: rc = %d (%s) \n",
rc, strerror(-rc));
t_unlink(file);
t_rmdir(path);
free(lum);
return -1;
}
fd = open(file, O_RDWR, 0644);
if (fd < 0) {
printf("failed to open(%s): rc = %d (%s)\n",
file, fd, strerror(errno));
t_unlink(file);
t_rmdir(path);
free(lum);
return -1;
}
lum->lmm_magic = LOV_USER_MAGIC;
lum->lmm_stripe_count = STRIPE_COUNT;
rc = ioctl(fd, LL_IOC_LOV_GETSTRIPE, lum);
if (rc) {
printf("file:ioctl(LL_IOC_LOV_GETSTRIPE) failed: rc = %d(%s)\n",
rc, strerror(errno));
close(fd);
t_unlink(file);
t_rmdir(path);
free(lum);
return -1;
}
close(fd);
if (opt_verbose) {
printf("lmm_magic: 0x%08X\n", lum->lmm_magic);
printf("lmm_object_id: "LPX64"\n", lum->lmm_object_id);
printf("lmm_object_seq: "LPX64"\n", lum->lmm_object_seq);
printf("lmm_stripe_count: %u\n", (int)lum->lmm_stripe_count);
printf("lmm_stripe_size: %u\n", lum->lmm_stripe_size);
printf("lmm_stripe_pattern: %x\n", lum->lmm_pattern);
for (index = 0; index < lum->lmm_stripe_count; index++) {
lo = lum->lmm_objects + index;
printf("object %d:\n", index);
printf("\tobject_seq: "LPX64"\n", lo->l_object_seq);
printf("\tobject_id: "LPX64"\n", lo->l_object_id);
printf("\tost_gen: %#x\n", lo->l_ost_gen);
printf("\tost_idx: %u\n", lo->l_ost_idx);
}
}
if (lum->lmm_magic != LOV_USER_MAGIC ||
lum->lmm_pattern != LOV_PATTERN_RAID0 ||
lum->lmm_stripe_size != STRIPE_SIZE ||
lum->lmm_objects[0].l_ost_idx != STRIPE_OFFSET ||
lum->lmm_stripe_count != STRIPE_COUNT) {
printf("incorrect striping information!\n");
t_unlink(file);
t_rmdir(path);
free(lum);
return -1;
}
t_unlink(file);
t_rmdir(path);
free(lum);
LEAVE();
}
int
mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
char* filename,
int access_mode,
struct ompi_info_t *info,
mca_io_ompio_file_t *fh)
{
int amode;
int old_mask, perm;
int rc;
int flag;
int fs_lustre_stripe_size = -1;
int fs_lustre_stripe_width = -1;
char char_stripe[MPI_MAX_INFO_KEY];
struct lov_user_md *lump=NULL;
if (fh->f_perm == OMPIO_PERM_NULL) {
old_mask = umask(022);
umask(old_mask);
perm = old_mask ^ 0666;
}
else {
perm = fh->f_perm;
}
amode = 0;
if (access_mode & MPI_MODE_CREATE)
amode = amode | O_CREAT;
if (access_mode & MPI_MODE_RDONLY)
amode = amode | O_RDONLY;
if (access_mode & MPI_MODE_WRONLY)
amode = amode | O_WRONLY;
if (access_mode & MPI_MODE_RDWR)
amode = amode | O_RDWR;
if (access_mode & MPI_MODE_EXCL)
amode = amode | O_EXCL;
ompi_info_get (info, "stripe_size", MPI_MAX_INFO_VAL, char_stripe, &flag);
if ( flag ) {
sscanf ( char_stripe, "%d", &fs_lustre_stripe_size );
}
ompi_info_get (info, "stripe_width", MPI_MAX_INFO_VAL, char_stripe, &flag);
if ( flag ) {
sscanf ( char_stripe, "%d", &fs_lustre_stripe_width );
}
if (fs_lustre_stripe_size < 0) {
fs_lustre_stripe_size = mca_fs_lustre_stripe_size;
}
if (fs_lustre_stripe_width < 0) {
fs_lustre_stripe_width = mca_fs_lustre_stripe_width;
}
if ( (fs_lustre_stripe_size>0 || fs_lustre_stripe_width>0) &&
(amode&O_CREAT) && (amode&O_RDWR)) {
if (0 == fh->f_rank) {
llapi_file_create(filename,
fs_lustre_stripe_size,
-1, /* MSC need to change that */
fs_lustre_stripe_width,
0); /* MSC need to change that */
fh->fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE, perm);
if (fh->fd < 0) {
fprintf(stderr, "Can't open %s file: %d (%s)\n",
filename, errno, strerror(errno));
return OMPI_ERROR;
}
close (fh->fd);
}
fh->f_comm->c_coll.coll_barrier (fh->f_comm,
fh->f_comm->c_coll.coll_barrier_module);
}
fh->fd = open (filename, amode, perm);
if (fh->fd < 0) {
opal_output(1, "error opening file %s\n", filename);
return OMPI_ERROR;
}
if (mca_fs_lustre_stripe_size > 0) {
fh->f_stripe_size = mca_fs_lustre_stripe_size;
}
else {
lump = alloc_lum();
if (NULL == lump ){
fprintf(stderr,"Cannot allocate memory for extracting stripe size\n");
return OMPI_ERROR;
}
rc = llapi_file_get_stripe(filename, lump);
if (rc != 0) {
opal_output(1, "get_stripe failed: %d (%s)\n", errno, strerror(errno));
return OMPI_ERROR;
}
fh->f_stripe_size = lump->lmm_stripe_size;
// if ( NULL != lump ) {
// free ( lump );
// }
}
return OMPI_SUCCESS;
}
