int CreateStriped( const char * path, const stripe_info_t * old_stripe, int overwrite ) { int rc; /* try to restripe using previous pool name */ if ( !EMPTY_STRING( old_stripe->pool_name ) ) rc = llapi_file_create_pool( path, old_stripe->stripe_size, -1, old_stripe->stripe_count, 0, (char *)old_stripe->pool_name ); else rc = llapi_file_create( path, old_stripe->stripe_size, -1, old_stripe->stripe_count, 0 ); if ((rc == -EEXIST) && overwrite) { if (unlink(path)) { rc = -errno; DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Can't remove previous entry %s: %s", path, strerror(-rc)); return rc; } return CreateStriped(path, old_stripe, false /*target not expected to exist*/); } else if ( rc != 0 && rc != -EEXIST ) { DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Error %d creating '%s' with stripe.", rc, path ); } return rc; }
int File_CreateSetStripe( const char * path, const stripe_info_t * old_stripe ) { int rc; /* try to restripe using previous pool name */ if ( !EMPTY_STRING( old_stripe->pool_name ) ) { rc = llapi_file_create_pool( path, old_stripe->stripe_size, -1, old_stripe->stripe_count, 0, (char *)old_stripe->pool_name ); if ( rc == 0 || rc == -EEXIST ) return rc; else { DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Error %d creating '%s' in pool '%s': %s", rc, path, old_stripe->pool_name, strerror(-rc) ); DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Trying to create it without pool information..." ); } } rc = llapi_file_create( path, old_stripe->stripe_size, -1, old_stripe->stripe_count, 0 ); if ( rc != 0 || rc == -EEXIST ) DisplayLog( LVL_MAJOR, TAG_CR_STRIPE, "Error %d creating '%s' with stripe. Trying to create it without specific stripe...", rc, path ); return rc; }
int t3pio_asklustre(MPI_Comm comm, int myProc, const char* path) { int ierr; int stripes = 1; #ifdef HAVE_LUSTRE const char * dir = path2dir(path); if (myProc == 0 && t3pio_usingLustreFS(dir)) { char line[MAXLINE]; int stripe_size = 0; int stripe_offset = -1; int stripe_pattern = 0; int stripes_max = -1; int flags = (O_WRONLY | O_CREAT | O_EXCL); // Create a Lustre Striped Test File char fn[MAXLINE]; sprintf(&fn[0], "%s/foo_%d.bar", dir, getpid()); int rc = llapi_file_create(fn, stripe_size, stripe_offset, stripes_max, stripe_pattern); if (rc == 0) { flags |= O_LOV_DELAY_CREATE; flags &= ~O_EXCL; int fd = open(fn, flags, RW_USER); FILE* fp = fdopen(fd,"w"); fprintf(fp, "foo.bar\n"); fclose(fp); sprintf(&line[0],"lfs getstripe -q %s",fn); if ( (fp = popen(line, "r")) == NULL) { fprintf(stderr,"unable to popen\n"); abort(); } int count = 0; while (fgets(line, MAXLINE, fp) != NULL) { size_t k = strspn(&line[0], " \t"); if (isdigit(line[k])) count++; } stripes = count; fclose(fp); } unlink(fn); } ierr = MPI_Bcast(&stripes, 1, MPI_INTEGER, 0, comm); #endif return stripes; }
/*-------------------------------------------------------------------------*/ int FTI_InitMpiICP(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, FTIT_dataset* FTI_Data) { int res; FTI_Print("I/O mode: MPI-IO.", FTI_DBUG); char str[FTI_BUFS], mpi_err[FTI_BUFS]; // enable collective buffer optimization MPI_Info info; MPI_Info_create(&info); MPI_Info_set(info, "romio_cb_write", "enable"); /* * update ckpt file name (neccessary for the restart!) * not very nice TODO we should think about another mechanism */ snprintf(FTI_Exec->meta[0].ckptFile, FTI_BUFS, "Ckpt%d-Rank%d.fti", FTI_Exec->ckptID, FTI_Topo->myRank); // TODO enable to set stripping unit in the config file (Maybe also other hints) // set stripping unit to 4MB MPI_Info_set(info, "stripping_unit", "4194304"); char gfn[FTI_BUFS], ckptFile[FTI_BUFS]; snprintf(ckptFile, FTI_BUFS, "Ckpt%d-mpiio.fti", FTI_Exec->ckptID); snprintf(gfn, FTI_BUFS, "%s/%s", FTI_Conf->gTmpDir, ckptFile); // open parallel file (collective call) MPI_File pfh; #ifdef LUSTRE if (FTI_Topo->splitRank == 0) { res = llapi_file_create(gfn, FTI_Conf->stripeUnit, FTI_Conf->stripeOffset, FTI_Conf->stripeFactor, 0); if (res) { char error_msg[FTI_BUFS]; error_msg[0] = 0; strerror_r(-res, error_msg, FTI_BUFS); snprintf(str, FTI_BUFS, "[Lustre] %s.", error_msg); FTI_Print(str, FTI_WARN); } else { snprintf(str, FTI_BUFS, "[LUSTRE] file:%s striping_unit:%i striping_factor:%i striping_offset:%i", ckptFile, FTI_Conf->stripeUnit, FTI_Conf->stripeFactor, FTI_Conf->stripeOffset); FTI_Print(str, FTI_DBUG); } } #endif res = MPI_File_open(FTI_COMM_WORLD, gfn, MPI_MODE_WRONLY|MPI_MODE_CREATE, info, &pfh); // check if successful if (res != 0) { errno = 0; int reslen; MPI_Error_string(res, mpi_err, &reslen); snprintf(str, FTI_BUFS, "unable to create file %s [MPI ERROR - %i] %s", gfn, res, mpi_err); FTI_Print(str, FTI_EROR); return FTI_NSCS; } MPI_Offset chunkSize = FTI_Exec->ckptSize; // collect chunksizes of other ranks MPI_Offset* chunkSizes = talloc(MPI_Offset, FTI_Topo->nbApprocs * FTI_Topo->nbNodes); MPI_Allgather(&chunkSize, 1, MPI_OFFSET, chunkSizes, 1, MPI_OFFSET, FTI_COMM_WORLD); // set file offset MPI_Offset offset = 0; int i; for (i = 0; i < FTI_Topo->splitRank; i++) { offset += chunkSizes[i]; } free(chunkSizes); FTI_Exec->iCPInfo.offset = offset; memcpy( FTI_Exec->iCPInfo.fh, &pfh, sizeof(FTI_MI_FH) ); MPI_Info_free(&info); return FTI_SCES; }
int t55(char *name) { char path[MAX_PATH_LENGTH] = ""; char file[MAX_PATH_LENGTH] = ""; struct lov_user_md *lum = NULL; struct lov_user_ost_data *lo = NULL; int index, fd, buflen, rc; ENTER("setstripe/getstripe"); snprintf(path, MAX_PATH_LENGTH, "%s/test_t55", lustre_path); snprintf(file, MAX_PATH_LENGTH, "%s/test_t55/file_t55", lustre_path); buflen = sizeof(struct lov_user_md); buflen += STRIPE_COUNT * sizeof(struct lov_user_ost_data); lum = (struct lov_user_md *)malloc(buflen); if (!lum) { printf("out of memory!\n"); return -1; } memset(lum, 0, buflen); t_mkdir(path); rc = llapi_file_create(path, STRIPE_SIZE, STRIPE_OFFSET, STRIPE_COUNT, LOV_PATTERN_RAID0); if (rc) { printf("llapi_file_create failed: rc = %d (%s) \n", rc, strerror(-rc)); t_rmdir(path); free(lum); return -1; } fd = open(file, O_CREAT | O_RDWR, 0644); if (fd < 0) { printf("open file(%s) failed: rc = %d (%s) \n)", file, fd, strerror(errno)); t_rmdir(path); free(lum); return -1; } lum->lmm_magic = LOV_USER_MAGIC; lum->lmm_stripe_count = STRIPE_COUNT; rc = ioctl(fd, LL_IOC_LOV_GETSTRIPE, lum); if (rc) { printf("dir:ioctl(LL_IOC_LOV_GETSTRIPE) failed: rc = %d(%s)\n", rc, strerror(errno)); close(fd); t_unlink(file); t_rmdir(path); free(lum); return -1; } close(fd); if (opt_verbose) { printf("lmm_magic: 0x%08X\n", lum->lmm_magic); printf("lmm_object_id: "LPX64"\n", lum->lmm_object_id); printf("lmm_object_seq: "LPX64"\n", lum->lmm_object_seq); printf("lmm_stripe_count: %u\n", (int)lum->lmm_stripe_count); printf("lmm_stripe_size: %u\n", lum->lmm_stripe_size); printf("lmm_stripe_pattern: %x\n", lum->lmm_pattern); for (index = 0; index < lum->lmm_stripe_count; index++) { lo = lum->lmm_objects + index; printf("object %d:\n", index); printf("\tobject_seq: "LPX64"\n", lo->l_object_seq); printf("\tobject_id: "LPX64"\n", lo->l_object_id); printf("\tost_gen: %#x\n", lo->l_ost_gen); printf("\tost_idx: %u\n", lo->l_ost_idx); } } if (lum->lmm_magic != LOV_USER_MAGIC || lum->lmm_pattern != LOV_PATTERN_RAID0 || lum->lmm_stripe_size != STRIPE_SIZE || lum->lmm_objects[0].l_ost_idx != STRIPE_OFFSET || lum->lmm_stripe_count != STRIPE_COUNT) { printf("incorrect striping information!\n"); t_unlink(file); t_rmdir(path); free(lum); return -1; } t_unlink(file); /* setstripe on regular file */ rc = llapi_file_create(file, STRIPE_SIZE, STRIPE_OFFSET, STRIPE_COUNT, LOV_PATTERN_RAID0); if (rc) { printf("llapi_file_create failed: rc = %d (%s) \n", rc, strerror(-rc)); t_unlink(file); t_rmdir(path); free(lum); return -1; } fd = open(file, O_RDWR, 0644); if (fd < 0) { printf("failed to open(%s): rc = %d (%s)\n", file, fd, strerror(errno)); t_unlink(file); t_rmdir(path); free(lum); return -1; } lum->lmm_magic = LOV_USER_MAGIC; lum->lmm_stripe_count = STRIPE_COUNT; rc = ioctl(fd, LL_IOC_LOV_GETSTRIPE, lum); if (rc) { printf("file:ioctl(LL_IOC_LOV_GETSTRIPE) failed: rc = %d(%s)\n", rc, strerror(errno)); close(fd); t_unlink(file); t_rmdir(path); free(lum); return -1; } close(fd); if (opt_verbose) { printf("lmm_magic: 0x%08X\n", lum->lmm_magic); printf("lmm_object_id: "LPX64"\n", lum->lmm_object_id); printf("lmm_object_seq: "LPX64"\n", lum->lmm_object_seq); printf("lmm_stripe_count: %u\n", (int)lum->lmm_stripe_count); printf("lmm_stripe_size: %u\n", lum->lmm_stripe_size); printf("lmm_stripe_pattern: %x\n", lum->lmm_pattern); for (index = 0; index < lum->lmm_stripe_count; index++) { lo = lum->lmm_objects + index; printf("object %d:\n", index); printf("\tobject_seq: "LPX64"\n", lo->l_object_seq); printf("\tobject_id: "LPX64"\n", lo->l_object_id); printf("\tost_gen: %#x\n", lo->l_ost_gen); printf("\tost_idx: %u\n", lo->l_ost_idx); } } if (lum->lmm_magic != LOV_USER_MAGIC || lum->lmm_pattern != LOV_PATTERN_RAID0 || lum->lmm_stripe_size != STRIPE_SIZE || lum->lmm_objects[0].l_ost_idx != STRIPE_OFFSET || lum->lmm_stripe_count != STRIPE_COUNT) { printf("incorrect striping information!\n"); t_unlink(file); t_rmdir(path); free(lum); return -1; } t_unlink(file); t_rmdir(path); free(lum); LEAVE(); }
int mca_fs_lustre_file_open (struct ompi_communicator_t *comm, char* filename, int access_mode, struct ompi_info_t *info, mca_io_ompio_file_t *fh) { int amode; int old_mask, perm; int rc; int flag; int fs_lustre_stripe_size = -1; int fs_lustre_stripe_width = -1; char char_stripe[MPI_MAX_INFO_KEY]; struct lov_user_md *lump=NULL; if (fh->f_perm == OMPIO_PERM_NULL) { old_mask = umask(022); umask(old_mask); perm = old_mask ^ 0666; } else { perm = fh->f_perm; } amode = 0; if (access_mode & MPI_MODE_CREATE) amode = amode | O_CREAT; if (access_mode & MPI_MODE_RDONLY) amode = amode | O_RDONLY; if (access_mode & MPI_MODE_WRONLY) amode = amode | O_WRONLY; if (access_mode & MPI_MODE_RDWR) amode = amode | O_RDWR; if (access_mode & MPI_MODE_EXCL) amode = amode | O_EXCL; ompi_info_get (info, "stripe_size", MPI_MAX_INFO_VAL, char_stripe, &flag); if ( flag ) { sscanf ( char_stripe, "%d", &fs_lustre_stripe_size ); } ompi_info_get (info, "stripe_width", MPI_MAX_INFO_VAL, char_stripe, &flag); if ( flag ) { sscanf ( char_stripe, "%d", &fs_lustre_stripe_width ); } if (fs_lustre_stripe_size < 0) { fs_lustre_stripe_size = mca_fs_lustre_stripe_size; } if (fs_lustre_stripe_width < 0) { fs_lustre_stripe_width = mca_fs_lustre_stripe_width; } if ( (fs_lustre_stripe_size>0 || fs_lustre_stripe_width>0) && (amode&O_CREAT) && (amode&O_RDWR)) { if (0 == fh->f_rank) { llapi_file_create(filename, fs_lustre_stripe_size, -1, /* MSC need to change that */ fs_lustre_stripe_width, 0); /* MSC need to change that */ fh->fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE, perm); if (fh->fd < 0) { fprintf(stderr, "Can't open %s file: %d (%s)\n", filename, errno, strerror(errno)); return OMPI_ERROR; } close (fh->fd); } fh->f_comm->c_coll.coll_barrier (fh->f_comm, fh->f_comm->c_coll.coll_barrier_module); } fh->fd = open (filename, amode, perm); if (fh->fd < 0) { opal_output(1, "error opening file %s\n", filename); return OMPI_ERROR; } if (mca_fs_lustre_stripe_size > 0) { fh->f_stripe_size = mca_fs_lustre_stripe_size; } else { lump = alloc_lum(); if (NULL == lump ){ fprintf(stderr,"Cannot allocate memory for extracting stripe size\n"); return OMPI_ERROR; } rc = llapi_file_get_stripe(filename, lump); if (rc != 0) { opal_output(1, "get_stripe failed: %d (%s)\n", errno, strerror(errno)); return OMPI_ERROR; } fh->f_stripe_size = lump->lmm_stripe_size; // if ( NULL != lump ) { // free ( lump ); // } } return OMPI_SUCCESS; }