err_t sys_lustre_get_stripe_size(fd_t fd, int64_t* size_out) { struct lov_user_md_v1 *lum; size_t lum_size = sizeof(*lum) + LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1); int rc = 0; err_t err = 0; lum = qio_calloc(lum_size, 1); lum->lmm_magic = LOV_USER_MAGIC_V1; lum->lmm_stripe_count = LOV_MAX_STRIPE_COUNT; STARTING_SLOW_SYSCALL; rc = ioctl(fd, LL_IOC_LOV_GETSTRIPE, lum); *size_out = lum->lmm_stripe_size; if (rc < 0) { *size_out = 0; err = errno; } DONE_SLOW_SYSCALL; qio_free(lum); return err; }
qioerr hdfs_get_owners_for_bytes(qio_file_t* file, hdfs_block_byte_map_t** locs, int* out_num_blocks, char** locale_array, int num_locales, off_t start_byte, off_t len) { int i; int j = 0; int k; qioerr err = 0; char* tmp; int rnd; int block_count = 0; hdfs_block_byte_map_t* loc = NULL; char*** info = NULL; hdfsFileInfo* f_info = hdfsGetPathInfo(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm); if (start_byte == 0 && len == -1) // We want the whole thing info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, f_info->mSize); else info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, start_byte + len); while(info[block_count] != NULL) { // Get the number of blocks that we have block_count++; } loc = (hdfs_block_byte_map_t*)qio_calloc(sizeof(hdfs_block_byte_map_t), block_count); CREATE_ERROR((!info), err, EREMOTEIO, "Unable to get host for HDFS", end); for (i = 0; info[i] != NULL; i++) { // Assign block owners rnd = rand() % f_info->mReplication; // pick an owner if (info[i][rnd]) {// Valid access tmp = get_locale_name(info[i][rnd]); // strip off .___ for (k = 0; k < num_locales; k++) { // Now find the owner if (strcmp(tmp, locale_array[k]) == 0) { loc[i].locale_id = k; // return locale ID for that name break; } } loc[i].start_byte = (off_t)(i*f_info->mBlockSize); loc[i].len = (off_t)(f_info->mBlockSize); j++; } else { QIO_GET_CONSTANT_ERROR(err, EINVAL, "Unable to find address for blocks in hdfs_get_owners_for_bytes"); qio_free(loc); *locs = NULL; *out_num_blocks = 0; goto end; } } *locs = loc; *out_num_blocks = j; end: return err; }
err_t qbytes_create_generic(qbytes_t** out, void* give_data, int64_t len, qbytes_free_t free_function) { qbytes_t* ret = NULL; ret = (qbytes_t*) qio_calloc(1, sizeof(qbytes_t)); if( ! ret ) return ENOMEM; _qbytes_init_generic(ret, give_data, len, free_function); *out = ret; return 0; }
// On return, the ref count is 1. qioerr qbytes_create_generic(qbytes_t** out, void* give_data, int64_t len, qbytes_free_t free_function) { qbytes_t* ret = NULL; ret = (qbytes_t*) qio_calloc(1, sizeof(qbytes_t)); if( ! ret ) return QIO_ENOMEM; // On return the ref count is 1. _qbytes_init_generic(ret, give_data, len, free_function); *out = ret; return 0; }
err_t qbytes_create_calloc(qbytes_t** out, int64_t len) { qbytes_t* ret = NULL; void* data; ret = (qbytes_t*) qio_calloc(1, sizeof(qbytes_t) + len); if( ! ret ) return ENOMEM; data = ret + 1; // ie ret + sizeof(qbytes_t) _qbytes_init_generic(ret, data, len, qbytes_free_null); *out = ret; return 0; }
static int curl_seekable(void* file, double* length_out) { struct str_t buf; int ret = 0; // We're on HTTP/HTTPS so we should look for byte ranges to see if we can request // them if (startWith(to_curl_handle(file)->pathnm, "http://") || startWith(to_curl_handle(file)->pathnm, "https://")) { // The size doesn't really matter, we just want a place on the heap. This will get // expanded in curl_write_string. // Headers tend to be ~800, although they can grow much larger than this. If it is // larger than this, we'll take care of it in chpl_curl_write_string. buf.mem = (char*)qio_calloc(800, 1); buf.size = 0; curl_easy_setopt(to_curl_handle(file)->curl, CURLOPT_WRITEFUNCTION, chpl_curl_write_string); curl_easy_setopt(to_curl_handle(file)->curl, CURLOPT_HEADERDATA, &buf); curl_easy_setopt(to_curl_handle(file)->curl, CURLOPT_NOBODY, 1L); curl_easy_perform(to_curl_handle(file)->curl); curl_easy_setopt(to_curl_handle(file)->curl, CURLOPT_NOBODY, 0L); curl_easy_setopt(to_curl_handle(file)->curl, CURLOPT_WRITEFUNCTION, NULL); curl_easy_setopt(to_curl_handle(file)->curl, CURLOPT_HEADERDATA, NULL); // Does this URL accept range requests? if (strstr(buf.mem, "Accept-Ranges") == NULL) ret = 0; else ret = 1; qio_free(buf.mem); } // This works for things other than http curl_easy_getinfo(to_curl_handle(file)->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, length_out); // We can always "seek" on ftp (with the REST command/RESUME_FROM_LARGE) if (startWith(to_curl_handle(file)->pathnm, "ftp://")) ret = 1; return ret; }
// The ref count returned in 'out' is initially 1. // The caller is responsible for calling qbytes_release on it. qioerr qbytes_create_calloc(qbytes_t** out, int64_t len) { qbytes_t* ret = NULL; void* data; ret = (qbytes_t*) qio_calloc(1, sizeof(qbytes_t) + len); if( ! ret ) { *out = NULL; return QIO_ENOMEM; } data = ret + 1; // ie ret + sizeof(qbytes_t) // On return, the ref count in ret is 1. _qbytes_init_generic(ret, data, len, qbytes_free_null); *out = ret; return 0; }
err_t qbytes_create_iobuf(qbytes_t** out) { qbytes_t* ret = NULL; err_t err; ret = (qbytes_t*) qio_calloc(1, sizeof(qbytes_t)); if( ! ret ) return ENOMEM; err = _qbytes_init_iobuf(ret); if( err ) { qio_free(ret); *out = NULL; return err; } *out = ret; return 0; }
qioerr hdfs_open(void** fd, const char* path, int* flags, mode_t mode, qio_hint_t iohints, void* fs) { qioerr err_out = 0; int rc; hdfs_file* fl = (hdfs_file*)qio_calloc(sizeof(hdfs_file), 1); STARTING_SLOW_SYSCALL; DO_RETAIN(((hdfs_fs*)fs)); // assert that we connected CREATE_ERROR((to_hdfs_fs(fs)->hfs == NULL), err_out, ECONNREFUSED,"Unable to open HDFS file", error); fl->file = hdfsOpenFile(to_hdfs_fs(fs)->hfs, path, *flags, 0, 0, 0); // Assert that we opened the file if (fl->file == NULL) { err_out = qio_mkerror_errno(); goto error; } DONE_SLOW_SYSCALL; fl->pathnm = path; rc = *flags | ~O_ACCMODE; rc &= O_ACCMODE; if( rc == O_RDONLY ) { *flags |= QIO_FDFLAG_READABLE; } else if( rc == O_WRONLY ) { *flags |= QIO_FDFLAG_WRITEABLE; } else if( rc == O_RDWR ) { *flags |= QIO_FDFLAG_READABLE; *flags |= QIO_FDFLAG_WRITEABLE; } *fd = fl; // Set fd to fl and return return err_out; error: qio_free(fl); return err_out; }
qioerr qbytes_create_iobuf(qbytes_t** out) { qbytes_t* ret = NULL; qioerr err; ret = (qbytes_t*) qio_calloc(1, sizeof(qbytes_t)); if( ! ret ) { *out = NULL; return QIO_ENOMEM; } err = _qbytes_init_iobuf(ret); if( err ) { qio_free(ret); *out = NULL; return err; } *out = ret; return 0; }
qioerr hdfs_connect(void** fs_out, const char* pathname, int port) { qioerr err_out = 0; hdfsFS fs; hdfs_fs* ret = (hdfs_fs*)qio_calloc(sizeof(hdfs_fs), 1); STARTING_SLOW_SYSCALL; fs = hdfsConnect(pathname, port); CREATE_ERROR((fs == NULL), err_out, ECONNREFUSED,"Unable to connect HDFS", error); DONE_SLOW_SYSCALL; ret->hfs = fs; DO_INIT_REFCNT(ret); ret->fs_name = pathname; ret->fs_port = port; *fs_out = ret; error: return err_out; }
static qioerr curl_open(void** fd, const char* path, int* flags, mode_t mode, qio_hint_t iohints, void* fs) { qioerr err_out = 0; int rc = 0; // Curl expects (NEEDS) this to be a double double filelength; curl_handle* fl = (curl_handle*)qio_calloc(sizeof(curl_handle), 1); STARTING_SLOW_SYSCALL; to_curl_handle(fl)->curl = curl_easy_init(); // Assert that we opened the file if (to_curl_handle(fl)->curl == NULL) { QIO_GET_CONSTANT_ERROR(err_out, ECONNREFUSED, "Unable to connect with Curl"); goto error; } // set URL curl_easy_setopt(to_curl_handle(fl)->curl, CURLOPT_URL, path); to_curl_handle(fl)->pathnm = path; to_curl_handle(fl)->current_offset = 0; // Read the header in order to get the length of the thing we are reading // If we are writing, we can't really get this information (even if we try to do a // 0 length read). if (*flags & O_WRONLY) { to_curl_handle(fl)->length = -1; to_curl_handle(fl)->seekable = 0; } else { to_curl_handle(fl)->seekable = curl_seekable(fl, &filelength); to_curl_handle(fl)->length = (ssize_t)filelength; } DONE_SLOW_SYSCALL; // Not seekable unless we specify otherwise *flags &= ~QIO_FDFLAG_SEEKABLE; rc = *flags | ~O_ACCMODE; rc &= O_ACCMODE; if( rc == O_RDONLY ) { *flags |= QIO_FDFLAG_READABLE; } else if( rc == O_WRONLY ) { *flags |= QIO_FDFLAG_WRITEABLE; } else if( rc == O_RDWR ) { *flags |= QIO_FDFLAG_READABLE; *flags |= QIO_FDFLAG_WRITEABLE; } // We can seek if (to_curl_handle(fl)->seekable) *flags |= QIO_FDFLAG_SEEKABLE; *fd = fl; return err_out; error: qio_free(fl); return err_out; }