qioerr hdfs_pwritev(void* fd, const struct iovec* iov, int iovcnt, off_t see_to_offset, ssize_t* num_read_out, void* fs)
{
qioerr t = 0;
QIO_GET_CONSTANT_ERROR(t, ENOSYS, "pwrites in HDFS are not supported");
fprintf(stderr, "positional writes in HDFS are not supported!!\n");
return t;
}
qioerr hdfs_get_owners_for_bytes(qio_file_t* file, hdfs_block_byte_map_t** locs, int* out_num_blocks, char** locale_array, int num_locales, off_t start_byte, off_t len)
{
int i;
int j = 0;
int k;
qioerr err = 0;
char* tmp;
int rnd;
int block_count = 0;
hdfs_block_byte_map_t* loc = NULL;
char*** info = NULL;
hdfsFileInfo* f_info = hdfsGetPathInfo(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm);
if (start_byte == 0 && len == -1) // We want the whole thing
info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, f_info->mSize);
else info = hdfsGetHosts(to_hdfs_fs(file->fs_info)->hfs, to_hdfs_file(file->file_info)->pathnm, start_byte, start_byte + len);
while(info[block_count] != NULL) { // Get the number of blocks that we have
block_count++;
}
loc = (hdfs_block_byte_map_t*)qio_calloc(sizeof(hdfs_block_byte_map_t), block_count);
CREATE_ERROR((!info), err, EREMOTEIO, "Unable to get host for HDFS", end);
for (i = 0; info[i] != NULL; i++) { // Assign block owners
rnd = rand() % f_info->mReplication; // pick an owner
if (info[i][rnd]) {// Valid access
tmp = get_locale_name(info[i][rnd]); // strip off .___
for (k = 0; k < num_locales; k++) { // Now find the owner
if (strcmp(tmp, locale_array[k]) == 0) {
loc[i].locale_id = k; // return locale ID for that name
break;
}
}
loc[i].start_byte = (off_t)(i*f_info->mBlockSize);
loc[i].len = (off_t)(f_info->mBlockSize);
j++;
} else {
QIO_GET_CONSTANT_ERROR(err, EINVAL, "Unable to find address for blocks in hdfs_get_owners_for_bytes");
qio_free(loc);
*locs = NULL;
*out_num_blocks = 0;
goto end;
}
}
*locs = loc;
*out_num_blocks = j;
end:
return err;
}
qioerr hdfs_getcwd(void* file, const char** path_out, void* fs)
{
int sz = 128;
char* buf = (char*) qio_malloc(sz);
qioerr err = 0;
if ( !buf )
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getcwd");
// hdfsGetWorkingDirectory will return 0 if buf[] is not large enough
// If this happens, grow the buffer and try again
while (err == 0 && hdfsGetWorkingDirectory(to_hdfs_fs(fs)->hfs, buf, sz) == 0) {
if (errno == ERANGE) {
int newSz = 2 * sz;
char* newBuf = (char*) qio_realloc(buf, newSz);
if (newBuf == 0) {
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getcwd");
} else {
sz = newSz;
buf = newBuf;
}
} else {
// Other error, stop.
QIO_GET_CONSTANT_ERROR(err, EREMOTEIO, "Unable to get path to file in HDFS");
}
}
if (err != 0) {
qio_free(buf);
buf = 0;
}
*path_out = buf;
return err;
}
qioerr hdfs_getpath(void* file, const char** string_out, void* fs) {
// Speculatively allocate 128 bytes for the string
int sz = 128;
int left = 0;
char* buf;
char* got;
qioerr err = 0;
const char* host = to_hdfs_fs(fs)->fs_name;
int port = to_hdfs_fs(fs)->fs_port;
const char* path = to_hdfs_file(file)->pathnm;
buf = (char*) qio_malloc(sz);
if( !buf )
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getpath");
while (1) {
left = snprintf(buf, sz, "hdfs://%s:%d/%s", host, port, path);
if (left > -1 && left < sz) {
break;
} else {
// keep looping but with bigger buffer.
// We know the size that we need now if n > -1
sz = left > -1 ? left + 1 : 2*sz;
got = (char*) qio_realloc(buf, sz);
if( ! got ) {
qio_free(buf);
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getpath");
}
}
}
*string_out = buf;
return err;
}
qioerr qbuffer_copyin(qbuffer_t* buf, qbuffer_iter_t start, qbuffer_iter_t end, const void* ptr, size_t ret_len)
{
int64_t num_bytes = qbuffer_iter_num_bytes(start, end);
ssize_t num_parts = qbuffer_iter_num_parts(start, end);
struct iovec* iov = NULL;
size_t iovcnt;
size_t i,j;
MAYBE_STACK_SPACE(struct iovec, iov_onstack);
qioerr err;
if( num_bytes < 0 || num_parts < 0 ||
start.offset < buf->offset_start || end.offset > buf->offset_end ) {
QIO_RETURN_CONSTANT_ERROR(EINVAL, "range outside of buffer");
}
MAYBE_STACK_ALLOC(struct iovec, num_parts, iov, iov_onstack);
if( ! iov ) return QIO_ENOMEM;
err = qbuffer_to_iov(buf, start, end, num_parts, iov, NULL, &iovcnt);
if( err ) goto error;
j = 0;
for( i = 0; i < iovcnt; i++ ) {
if( j + iov[i].iov_len > ret_len ) goto error_nospace;
qio_memcpy(iov[i].iov_base, PTR_ADDBYTES(ptr, j), iov[i].iov_len);
j += iov[i].iov_len;
}
MAYBE_STACK_FREE(iov, iov_onstack);
return 0;
error_nospace:
QIO_GET_CONSTANT_ERROR(err, EMSGSIZE, "no space in buffer");
error:
MAYBE_STACK_FREE(iov, iov_onstack);
return err;
}
static
qioerr curl_open(void** fd, const char* path, int* flags, mode_t mode, qio_hint_t iohints, void* fs)
{
qioerr err_out = 0;
int rc = 0;
// Curl expects (NEEDS) this to be a double
double filelength;
curl_handle* fl = (curl_handle*)qio_calloc(sizeof(curl_handle), 1);
STARTING_SLOW_SYSCALL;
to_curl_handle(fl)->curl = curl_easy_init();
// Assert that we opened the file
if (to_curl_handle(fl)->curl == NULL) {
QIO_GET_CONSTANT_ERROR(err_out, ECONNREFUSED, "Unable to connect with Curl");
goto error;
}
// set URL
curl_easy_setopt(to_curl_handle(fl)->curl, CURLOPT_URL, path);
to_curl_handle(fl)->pathnm = path;
to_curl_handle(fl)->current_offset = 0;
// Read the header in order to get the length of the thing we are reading
// If we are writing, we can't really get this information (even if we try to do a
// 0 length read).
if (*flags & O_WRONLY) {
to_curl_handle(fl)->length = -1;
to_curl_handle(fl)->seekable = 0;
} else {
to_curl_handle(fl)->seekable = curl_seekable(fl, &filelength);
to_curl_handle(fl)->length = (ssize_t)filelength;
}
DONE_SLOW_SYSCALL;
// Not seekable unless we specify otherwise
*flags &= ~QIO_FDFLAG_SEEKABLE;
rc = *flags | ~O_ACCMODE;
rc &= O_ACCMODE;
if( rc == O_RDONLY ) {
*flags |= QIO_FDFLAG_READABLE;
} else if( rc == O_WRONLY ) {
*flags |= QIO_FDFLAG_WRITEABLE;
} else if( rc == O_RDWR ) {
*flags |= QIO_FDFLAG_READABLE;
*flags |= QIO_FDFLAG_WRITEABLE;
}
// We can seek
if (to_curl_handle(fl)->seekable)
*flags |= QIO_FDFLAG_SEEKABLE;
*fd = fl;
return err_out;
error:
qio_free(fl);
return err_out;
}
