// Note: libcurl really doesn't support seeking. See: // http://curl.haxx.se/mail/lib-2009-05/0085.html // So instead, we keep track of where we need to seek to, in order to replicate the // correct behaviour. We return ESPIPE in the case that we can't get the length // since we can't get the length of the file (which we expect in this case in qio). // Instead, in this case we fall back to using readv instead of preadv. In the case // where we can get the length of the file we call preadv. // Note: We could get rid of this requirement if we don't have getlength return // an error message in the case that we don't have the length (but this can cause // weird user level behaviour). static qioerr curl_seek(void* fl, off_t offset, int whence, off_t* offset_out, void* fs) { curl_handle* curl_local = to_curl_handle(fl); if (curl_local->length == -1) QIO_RETURN_CONSTANT_ERROR(ESPIPE, "Unable to seek: URL does not support byte ranges"); switch (whence) { case SEEK_CUR: curl_local->current_offset = curl_local->current_offset + offset; break; case SEEK_END: if (curl_local->length != -1) // we have the length curl_local->current_offset= curl_local->length + offset; else QIO_RETURN_CONSTANT_ERROR(ESPIPE, "Unable to SEEK_END for path with unkown length"); break; case SEEK_SET: curl_local->current_offset = offset; break; } *offset_out = curl_local->current_offset + offset; return 0; }
qioerr hdfs_locales_for_range(void* file, off_t start_byte, off_t end_byte, const char*** loc_names_out, int* num_locs_out, void* fs) { int i = 0; int j = 0; char*** info = NULL; info = hdfsGetHosts(to_hdfs_fs(fs)->hfs, to_hdfs_file(file)->pathnm, start_byte, end_byte); // unable to get hosts for this byte range if (!info || !info[0]) { *num_locs_out = 0; hdfsFreeHosts(info); QIO_RETURN_CONSTANT_ERROR(EREMOTEIO, "Unable to get owners for byterange"); } while(info[0][i]) { info[0][i] = get_locale_name(info[0][i]); i++; } *num_locs_out = i - 1; *loc_names_out = (const char**)info[0]; // Free the other hosts that we don't need for (i = 1; info[i]; i++) { for (j = 0; info[i][j]; j++) qio_free(info[i][j]); qio_free(info[i]); } return 0; }
qioerr hdfs_getlength(void* fl, int64_t* len_out, void* fs) { hdfsFileInfo* f_info = NULL; f_info = hdfsGetPathInfo(to_hdfs_fs(fs)->hfs, to_hdfs_file(fl)->pathnm); if (f_info == NULL) QIO_RETURN_CONSTANT_ERROR(EREMOTEIO, "Unable to get length of file in HDFS"); *len_out = f_info->mSize; return 0; }
qioerr qio_openproc(const char** argv, const char** envp, const char* executable, int* stdin_fd, int* stdout_fd, int* stderr_fd, int64_t *pid_out) { // runs qio_do_openproc in a pthread in order // to avoid issues where a Chapel task is allocated // from memory with MAP_SHARED. // // If such a thread is the thread running fork(), // after the fork() occurs, there will be 2 threads // sharing the same stack. // // If it mattered, we could do this extra step // only for configurations where the Chapel heap // has this problem (GASNet with SEGMENT=fast,large // and possibly others). int rc; pthread_t thread; struct openproc_args_s s; s.argv = argv; s.envp = envp; s.executable = executable; s.stdin_fd = stdin_fd; s.stdout_fd = stdout_fd; s.stderr_fd = stderr_fd; s.pid_out = pid_out; s.err = 0; rc = pthread_create(&thread, NULL, qio_openproc_wrapper, &s); if (rc) QIO_RETURN_CONSTANT_ERROR(EAGAIN, "failed pthread_create in qio_openproc"); rc = pthread_join(thread, NULL); if (rc) QIO_RETURN_CONSTANT_ERROR(EAGAIN, "failed pthread_join in qio_openproc"); return s.err; }
static qioerr curl_getlength(void* fl, int64_t* len_out, void* fs) { if (to_curl_handle(fl)->length == -1) { // This will set initial length to 0 in QIO *len_out = 0; QIO_RETURN_CONSTANT_ERROR(ENOTSUP, "Unable to get length of curl URL"); } *len_out = to_curl_handle(fl)->length; return 0; }
qioerr qbuffer_flatten(qbuffer_t* buf, qbuffer_iter_t start, qbuffer_iter_t end, qbytes_t** bytes_out) { int64_t num_bytes = qbuffer_iter_num_bytes(start, end); ssize_t num_parts = qbuffer_iter_num_parts(start, end); struct iovec* iov = NULL; size_t iovcnt; size_t i,j; qbytes_t* ret; MAYBE_STACK_SPACE(struct iovec, iov_onstack); qioerr err; if( num_bytes < 0 || num_parts < 0 || start.offset < buf->offset_start || end.offset > buf->offset_end ) { *bytes_out = 0; QIO_RETURN_CONSTANT_ERROR(EINVAL, "range outside of buffer"); } err = qbytes_create_calloc(&ret, num_bytes); if( err ) { *bytes_out = 0; return err; } MAYBE_STACK_ALLOC(struct iovec, num_parts, iov, iov_onstack); if( ! iov ) { // The buffer was successfully allocated, so we have to release it here. qbytes_release(ret); *bytes_out = 0; return QIO_ENOMEM; } err = qbuffer_to_iov(buf, start, end, num_parts, iov, NULL, &iovcnt); if( err ) { MAYBE_STACK_FREE(iov, iov_onstack); // The buffer was successfully allocated, so we have to release it here. qbytes_release(ret); *bytes_out = 0; return err; } j = 0; for( i = 0; i < iovcnt; i++ ) { qio_memcpy(PTR_ADDBYTES(ret->data, j), iov[i].iov_base, iov[i].iov_len); j += iov[i].iov_len; } MAYBE_STACK_FREE(iov, iov_onstack); *bytes_out = ret; return 0; }
qioerr qbuffer_init_part(qbuffer_part_t* p, qbytes_t* bytes, int64_t skip_bytes, int64_t len_bytes, int64_t end_offset) { if( len_bytes < 0 || skip_bytes < 0 || skip_bytes + len_bytes > bytes->len ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "range outside of buffer"); qbytes_retain(bytes); p->bytes = bytes; p->skip_bytes = skip_bytes; p->len_bytes = len_bytes; p->end_offset = end_offset; p->flags = QB_PART_FLAGS_NONE; if( skip_bytes == 0 && len_bytes == bytes->len ) p->flags = QB_PART_FLAGS_EXTENDABLE_TO_ENTIRE_BYTES; return 0; }
qioerr bulk_put_buffer(int64_t dst_locale, void* dst_addr, int64_t dst_len, qbuffer_t* buf, qbuffer_iter_t start, qbuffer_iter_t end) { int64_t num_bytes = qbuffer_iter_num_bytes(start, end); ssize_t num_parts = qbuffer_iter_num_parts(start, end); struct iovec* iov = NULL; size_t iovcnt; size_t i,j; MAYBE_STACK_SPACE(struct iovec, iov_onstack); qioerr err; if( num_bytes < 0 || num_parts < 0 || start.offset < buf->offset_start || end.offset > buf->offset_end ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "range outside of buffer"); MAYBE_STACK_ALLOC(struct iovec, num_parts, iov, iov_onstack); if( ! iov ) return QIO_ENOMEM; err = qbuffer_to_iov(buf, start, end, num_parts, iov, NULL, &iovcnt); if( err ) goto error; j = 0; for( i = 0; i < iovcnt; i++ ) { if( j + iov[i].iov_len > dst_len ) goto error_nospace; //chpl_memcpy(PTR_ADDBYTES(ptr, j), iov[i].iov_base, iov[i].iov_len); // TODO -- note -- technically, this should be gasnet_put_bulk, // since we don't want to require src/dst to have a particular alignment. chpl_gen_comm_put( iov[i].iov_base, dst_locale, PTR_ADDBYTES(dst_addr, j), sizeof(uint8_t), CHPL_TYPE_uint8_t, iov[i].iov_len, -1, "<internal>" ); j += iov[i].iov_len; } MAYBE_STACK_FREE(iov, iov_onstack); return 0; error_nospace: QIO_RETURN_CONSTANT_ERROR(EMSGSIZE, "no space in buffer"); error: MAYBE_STACK_FREE(iov, iov_onstack); return err; }
qioerr qbuffer_pop_front(qbuffer_t* buf) { qbytes_t* bytes; int64_t skip; int64_t len; qbuffer_iter_t chunk; if ( qbuffer_num_parts(buf) == 0 ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "cannot pop from empty buffer"); chunk = qbuffer_begin(buf); qbuffer_iter_get(chunk, qbuffer_end(buf), &bytes, &skip, &len); deque_pop_front(sizeof(qbuffer_part_t), &buf->deque); buf->offset_start += len; return 0; }
qioerr qbuffer_copyin_buffer(qbuffer_t* dst, qbuffer_iter_t dst_start, qbuffer_iter_t dst_end, qbuffer_t* src, qbuffer_iter_t src_start, qbuffer_iter_t src_end) { int64_t dst_num_bytes = qbuffer_iter_num_bytes(dst_start, dst_end); ssize_t dst_num_parts = qbuffer_iter_num_parts(dst_start, dst_end); int64_t src_num_bytes = qbuffer_iter_num_bytes(src_start, src_end); ssize_t src_num_parts = qbuffer_iter_num_parts(src_start, src_end); struct iovec* iov = NULL; size_t iovcnt; size_t i; MAYBE_STACK_SPACE(struct iovec, iov_onstack); qioerr err; qbuffer_iter_t dst_cur, dst_cur_end; if( dst == src ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "cannot copy a buffer to itself"); if( dst_num_bytes < 0 || dst_num_parts < 0 || dst_start.offset < dst->offset_start || dst_end.offset > dst->offset_end ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "dst range outside of buffer"); if( src_num_bytes < 0 || src_num_parts < 0 || src_start.offset < src->offset_start || src_end.offset > src->offset_end ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "src range outside of buffer"); MAYBE_STACK_ALLOC(struct iovec, src_num_parts, iov, iov_onstack); if( ! iov ) return QIO_ENOMEM; err = qbuffer_to_iov(src, src_start, src_end, src_num_parts, iov, NULL, &iovcnt); if( err ) goto error; dst_cur = dst_start; for( i = 0; i < iovcnt; i++ ) { dst_cur_end = dst_cur; qbuffer_iter_advance(dst, &dst_cur_end, iov[i].iov_len); err = qbuffer_copyin(dst, dst_cur, dst_cur_end, iov[i].iov_base, iov[i].iov_len); if( err ) goto error; dst_cur = dst_cur_end; } MAYBE_STACK_FREE(iov, iov_onstack); return 0; error: MAYBE_STACK_FREE(iov, iov_onstack); return err; }
qioerr hdfs_seek(void* fl, off_t offset, int whence, off_t* offset_out, void* fs) { off_t got; qioerr err_out = 0; // We cannot seek unless we are in read mode! (HDFS restriction) if (to_hdfs_file(fl)->file->type != INPUT) QIO_RETURN_CONSTANT_ERROR(ENOSYS, "Seeking is not supported in write mode in HDFS"); STARTING_SLOW_SYSCALL; got = (off_t)hdfsSeek(to_hdfs_fs(fs)->hfs, to_hdfs_file(fl)->file, offset); if( got != (off_t) -1) { *offset_out = got; } else { *offset_out = got; } DONE_SLOW_SYSCALL; return err_out; }
qioerr qbuffer_append_buffer(qbuffer_t* buf, qbuffer_t* src, qbuffer_iter_t src_start, qbuffer_iter_t src_end) { qbuffer_iter_t src_cur = src_start; qbytes_t* bytes; int64_t skip; int64_t len; qioerr err; if( buf == src ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "cannot append a buffer to itself"); while( qbuffer_iter_num_bytes(src_cur, src_end) > 0 ) { qbuffer_iter_get(src_cur, src_end, &bytes, &skip, &len); err = qbuffer_append(buf, bytes, skip, len); if( err ) return err; qbuffer_iter_next_part(src, &src_cur); } return 0; }
qioerr qbuffer_copyin(qbuffer_t* buf, qbuffer_iter_t start, qbuffer_iter_t end, const void* ptr, size_t ret_len) { int64_t num_bytes = qbuffer_iter_num_bytes(start, end); ssize_t num_parts = qbuffer_iter_num_parts(start, end); struct iovec* iov = NULL; size_t iovcnt; size_t i,j; MAYBE_STACK_SPACE(struct iovec, iov_onstack); qioerr err; if( num_bytes < 0 || num_parts < 0 || start.offset < buf->offset_start || end.offset > buf->offset_end ) { QIO_RETURN_CONSTANT_ERROR(EINVAL, "range outside of buffer"); } MAYBE_STACK_ALLOC(struct iovec, num_parts, iov, iov_onstack); if( ! iov ) return QIO_ENOMEM; err = qbuffer_to_iov(buf, start, end, num_parts, iov, NULL, &iovcnt); if( err ) goto error; j = 0; for( i = 0; i < iovcnt; i++ ) { if( j + iov[i].iov_len > ret_len ) goto error_nospace; qio_memcpy(iov[i].iov_base, PTR_ADDBYTES(ptr, j), iov[i].iov_len); j += iov[i].iov_len; } MAYBE_STACK_FREE(iov, iov_onstack); return 0; error_nospace: QIO_GET_CONSTANT_ERROR(err, EMSGSIZE, "no space in buffer"); error: MAYBE_STACK_FREE(iov, iov_onstack); return err; }
qioerr qbuffer_to_iov(qbuffer_t* buf, qbuffer_iter_t start, qbuffer_iter_t end, size_t max_iov, struct iovec *iov_out, qbytes_t** bytes_out /* can be NULL */, size_t *iovcnt_out) { deque_iterator_t d_end = deque_end(& buf->deque); deque_iterator_t iter; qbuffer_part_t* qbp; size_t i = 0; iter = start.iter; // invalid range! if( start.offset > end.offset ) { *iovcnt_out = 0; QIO_RETURN_CONSTANT_ERROR(EINVAL, "invalid range"); } if( deque_it_equals(iter, d_end) ) { // start is actually pointing to the end of the deque. no data. *iovcnt_out = 0; return 0; } if( deque_it_equals(iter, end.iter) ) { // we're only pointing to a single block. qbp = (qbuffer_part_t*) deque_it_get_cur_ptr(sizeof(qbuffer_part_t), iter); if( i >= max_iov ) goto error_nospace; iov_out[i].iov_base = PTR_ADDBYTES(qbp->bytes->data, qbp->skip_bytes + (start.offset - (qbp->end_offset - qbp->len_bytes))); iov_out[i].iov_len = end.offset - start.offset; if( bytes_out ) bytes_out[i] = qbp->bytes; if( iov_out[i].iov_len > 0 ) i++; } else { // otherwise, there's a possibly partial block in start. qbp = (qbuffer_part_t*) deque_it_get_cur_ptr(sizeof(qbuffer_part_t), iter); if( i >= max_iov ) goto error_nospace; iov_out[i].iov_base = PTR_ADDBYTES(qbp->bytes->data, qbp->skip_bytes + (start.offset - (qbp->end_offset - qbp->len_bytes))); iov_out[i].iov_len = qbp->end_offset - start.offset; if( bytes_out ) bytes_out[i] = qbp->bytes; // store it if we had any data there. if( iov_out[i].iov_len > 0 ) i++; // Now, on to the next. deque_it_forward_one(sizeof(qbuffer_part_t), &iter); // until we get to the same block as end, we need to store full blocks. while( ! deque_it_equals( iter, end.iter ) ) { if( deque_it_equals( iter, d_end ) ) { // error: end is not in deque. *iovcnt_out = 0; QIO_RETURN_CONSTANT_ERROR(EINVAL, "end is not in deque"); } qbp = (qbuffer_part_t*) deque_it_get_cur_ptr(sizeof(qbuffer_part_t), iter); if( i >= max_iov ) goto error_nospace; iov_out[i].iov_base = PTR_ADDBYTES(qbp->bytes->data, qbp->skip_bytes); iov_out[i].iov_len = qbp->len_bytes; if( bytes_out ) bytes_out[i] = qbp->bytes; // store it if we had any data there. if( iov_out[i].iov_len > 0 ) i++; // Now, on to the next. deque_it_forward_one(sizeof(qbuffer_part_t), &iter); } // at the end of the loop // is there any data in end? if( deque_it_equals(iter, d_end) ) { // we're currently pointing to the end; no need to add more. } else { qbp = (qbuffer_part_t*) deque_it_get_cur_ptr(sizeof(qbuffer_part_t), iter); // add a partial end block. We know it's different from // start since we handled that above. if( i >= max_iov ) goto error_nospace; iov_out[i].iov_base = PTR_ADDBYTES(qbp->bytes->data, qbp->skip_bytes); iov_out[i].iov_len = end.offset - (qbp->end_offset - qbp->len_bytes); if( bytes_out ) bytes_out[i] = qbp->bytes; if( iov_out[i].iov_len > 0 ) i++; } } *iovcnt_out = i; return 0; error_nospace: *iovcnt_out = 0; // EOVERFLOW or ENOBUFS would make sense too QIO_RETURN_CONSTANT_ERROR(EMSGSIZE, "no space in buffer"); }
qioerr qbuffer_memset(qbuffer_t* buf, qbuffer_iter_t start, qbuffer_iter_t end, unsigned char byte) { int64_t num_bytes = qbuffer_iter_num_bytes(start, end); ssize_t num_parts = qbuffer_iter_num_parts(start, end); struct iovec* iov = NULL; size_t iovcnt; size_t i; MAYBE_STACK_SPACE(struct iovec, iov_onstack); qioerr err; if( num_bytes < 0 || num_parts < 0 || start.offset < buf->offset_start || end.offset > buf->offset_end ) QIO_RETURN_CONSTANT_ERROR(EINVAL, "range outside of buffer"); MAYBE_STACK_ALLOC(struct iovec, num_parts, iov, iov_onstack); if( ! iov ) return QIO_ENOMEM; err = qbuffer_to_iov(buf, start, end, num_parts, iov, NULL, &iovcnt); if( err ) goto error; for( i = 0; i < iovcnt; i++ ) { memset(iov[i].iov_base, byte, iov[i].iov_len); } MAYBE_STACK_FREE(iov, iov_onstack); return 0; error: MAYBE_STACK_FREE(iov, iov_onstack); return err; }