qioerr qbuffer_create(qbuffer_t** out)
{
qbuffer_t* ret = NULL;
qioerr err;
ret = (qbuffer_t*) qio_malloc(sizeof(qbuffer_t));
if( ! ret ) {
*out = NULL;
return QIO_ENOMEM;
}
err = qbuffer_init(ret);
if( err ) {
qio_free(ret);
*out = NULL;
return err;
}
*out = ret;
return 0;
}
qioerr hdfs_getcwd(void* file, const char** path_out, void* fs)
{
int sz = 128;
char* buf = (char*) qio_malloc(sz);
qioerr err = 0;
if ( !buf )
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getcwd");
// hdfsGetWorkingDirectory will return 0 if buf[] is not large enough
// If this happens, grow the buffer and try again
while (err == 0 && hdfsGetWorkingDirectory(to_hdfs_fs(fs)->hfs, buf, sz) == 0) {
if (errno == ERANGE) {
int newSz = 2 * sz;
char* newBuf = (char*) qio_realloc(buf, newSz);
if (newBuf == 0) {
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getcwd");
} else {
sz = newSz;
buf = newBuf;
}
} else {
// Other error, stop.
QIO_GET_CONSTANT_ERROR(err, EREMOTEIO, "Unable to get path to file in HDFS");
}
}
if (err != 0) {
qio_free(buf);
buf = 0;
}
*path_out = buf;
return err;
}
qioerr hdfs_getpath(void* file, const char** string_out, void* fs) {
// Speculatively allocate 128 bytes for the string
int sz = 128;
int left = 0;
char* buf;
char* got;
qioerr err = 0;
const char* host = to_hdfs_fs(fs)->fs_name;
int port = to_hdfs_fs(fs)->fs_port;
const char* path = to_hdfs_file(file)->pathnm;
buf = (char*) qio_malloc(sz);
if( !buf )
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getpath");
while (1) {
left = snprintf(buf, sz, "hdfs://%s:%d/%s", host, port, path);
if (left > -1 && left < sz) {
break;
} else {
// keep looping but with bigger buffer.
// We know the size that we need now if n > -1
sz = left > -1 ? left + 1 : 2*sz;
got = (char*) qio_realloc(buf, sz);
if( ! got ) {
qio_free(buf);
QIO_GET_CONSTANT_ERROR(err, ENOMEM, "Out of memory in hdfs_getpath");
}
}
}
*string_out = buf;
return err;
}
static
qioerr curl_open(void** fd, const char* path, int* flags, mode_t mode, qio_hint_t iohints, void* fs)
{
qioerr err_out = 0;
int rc = 0;
// Curl expects (NEEDS) this to be a double
double filelength;
curl_handle* fl = (curl_handle*)qio_calloc(sizeof(curl_handle), 1);
STARTING_SLOW_SYSCALL;
to_curl_handle(fl)->curl = curl_easy_init();
// Assert that we opened the file
if (to_curl_handle(fl)->curl == NULL) {
QIO_GET_CONSTANT_ERROR(err_out, ECONNREFUSED, "Unable to connect with Curl");
goto error;
}
// set URL
curl_easy_setopt(to_curl_handle(fl)->curl, CURLOPT_URL, path);
to_curl_handle(fl)->pathnm = path;
to_curl_handle(fl)->current_offset = 0;
// Read the header in order to get the length of the thing we are reading
// If we are writing, we can't really get this information (even if we try to do a
// 0 length read).
if (*flags & O_WRONLY) {
to_curl_handle(fl)->length = -1;
to_curl_handle(fl)->seekable = 0;
} else {
to_curl_handle(fl)->seekable = curl_seekable(fl, &filelength);
to_curl_handle(fl)->length = (ssize_t)filelength;
}
DONE_SLOW_SYSCALL;
// Not seekable unless we specify otherwise
*flags &= ~QIO_FDFLAG_SEEKABLE;
rc = *flags | ~O_ACCMODE;
rc &= O_ACCMODE;
if( rc == O_RDONLY ) {
*flags |= QIO_FDFLAG_READABLE;
} else if( rc == O_WRONLY ) {
*flags |= QIO_FDFLAG_WRITEABLE;
} else if( rc == O_RDWR ) {
*flags |= QIO_FDFLAG_READABLE;
*flags |= QIO_FDFLAG_WRITEABLE;
}
// We can seek
if (to_curl_handle(fl)->seekable)
*flags |= QIO_FDFLAG_SEEKABLE;
*fd = fl;
return err_out;
error:
qio_free(fl);
return err_out;
}
void check_write_read_pat(int width, int num, int pat, qio_chtype_t type, qio_hint_t file_hints, qio_hint_t ch_hints, char reopen)
{
qio_file_t* f;
qio_channel_t* writing;
qio_channel_t* reading;
qioerr err;
int memory;
char* chhints;
char* fhints;
char filename[128];
int fd = -1;
uint64_t one = 1;
uint64_t mask = (one << width) - 1;
if( width == 64 ) mask = -1;
strcpy(filename,"/tmp/qio_bits_testXXXXXX");
ch_hints = (ch_hints & ~ QIO_CHTYPEMASK) | type;
memory = 0;
if( (file_hints & QIO_METHODMASK) == QIO_METHOD_MEMORY ||
(ch_hints & QIO_METHODMASK) == QIO_METHOD_MEMORY ) {
memory = 1;
}
if( memory ) {
file_hints = (file_hints & ~ QIO_METHODMASK ) | QIO_METHOD_MEMORY;
ch_hints = (ch_hints & ~ QIO_METHODMASK ) | QIO_METHOD_MEMORY;
}
if( memory && type == QIO_CH_ALWAYS_UNBUFFERED ) return;
if( memory && reopen ) return;
if( (ch_hints & QIO_METHODMASK) == QIO_METHOD_FREADFWRITE ) {
if( (file_hints & QIO_METHODMASK) != QIO_METHOD_FREADFWRITE ) return;
}
if( (ch_hints & QIO_METHODMASK) == QIO_METHOD_MMAP ) {
if( (file_hints & QIO_METHODMASK) != QIO_METHOD_MMAP ) return;
}
fhints = qio_hints_to_string(file_hints);
chhints = qio_hints_to_string(ch_hints);
if( verbose )
printf("check_write_read_pat(width=%i, num=%i, pat=%i, type=%i, file_hints=%s %i, ch_hints=%s %i, reopen=%i)\n",
width, num, pat, type,
fhints, (int) file_hints, chhints, (int) ch_hints, (int) reopen);
qio_free(fhints);
qio_free(chhints);
if( memory ) {
err = qio_file_open_mem_ext(&f, NULL, QIO_FDFLAG_READABLE|QIO_FDFLAG_WRITEABLE|QIO_FDFLAG_SEEKABLE, file_hints, NULL);
assert(!err);
} else {
if( reopen ) {
fd = mkstemp(filename);
close(fd);
err = qio_file_open_access(&f, filename, "w", file_hints, NULL);
assert(!err);
} else {
err = qio_file_open_tmp(&f, 0, NULL);
assert(!err);
}
}
err = qio_channel_create(&writing, f, ch_hints, 0, 1, 0, INT64_MAX, NULL);
assert(!err);
for( int i = 0; i < num; i++ ) {
uint64_t x;
if( pat == 0 ) {
if( i & 1 ) x = mask;
else x = 0;
} else {
x = ((uint64_t) i) & mask;
}
{ // check offset
int64_t off = qio_channel_offset_unlocked(writing);
assert( off == (width * i + 7) / 8 );
}
err = qio_channel_write_bits(false, writing, x, width);
assert(!err);
}
qio_channel_release(writing);
// Reopen the file if we're doing reopen
if( reopen ) {
// Close the file.
qio_file_release(f);
err = qio_file_open_access(&f, filename, "r", file_hints, NULL);
assert(!err);
}
// Rewind the file
if( !memory ) {
off_t off;
sys_lseek(f->fd, 0, SEEK_SET, &off);
assert(!err);
}
err = qio_channel_create(&reading, f, ch_hints, 1, 0, 0, INT64_MAX, NULL);
assert(!err);
for( int i = 0; i < num; i++ ) {
uint64_t got = 0;
uint64_t x;
if( pat == 0 ) {
if( i & 1 ) x = mask;
else x = 0;
} else {
x = ((uint64_t) i) & mask;
}
{ // check offset
int64_t off = qio_channel_offset_unlocked(reading);
assert( off == (width * i + 7) / 8 );
}
//printf("Reading at %lli\n", (long long int) qio_channel_offset_unlocked(reading));
err = qio_channel_read_bits(false, reading, &got, width);
assert(!err);
//printf("Got %lli\n", (long long int) got);
if( got != x ) {
printf("Fails (%i %i %i) at %i got=%llx expect=%llx\n", width, num, pat, i, (unsigned long long int) got, (unsigned long long int) x);
assert(got == x);
break;
}
}
qio_channel_release(reading);
// Close the file.
qio_file_release(f);
if( reopen ) {
unlink(filename);
}
}
// allocates and returns an error string in *string_out
// which must be freed.
static
err_t sys_strerror_internal(err_t error, char** string_out, size_t extra_space)
{
// normal errors are in normal places.
// EAI_AGAIN... etc are at 10000 + num.
int buf_sz = 248 + extra_space;
char* buf = NULL;
char* newbuf;
const char* errmsg;
int got;
err_t err_out;
err_out = 0;
if( error == 0 ||
(EXTEND_ERROR_OFFSET <= error
&& error < EXTEND_ERROR_OFFSET+EXTEND_ERROR_NUM) ) {
if( error == 0 ) errmsg = error_string_no_error;
else errmsg = extended_errors[error - EXTEND_ERROR_OFFSET];
buf_sz = strlen(errmsg) + 1;
buf = (char*) qio_malloc(buf_sz + extra_space);
if( ! buf ) return ENOMEM;
strcpy(buf, errmsg);
*string_out = buf;
return 0;
}
while( 1 ) {
newbuf = (char*) qio_realloc(buf, buf_sz + extra_space);
if( ! newbuf ) {
qio_free(buf);
return ENOMEM;
}
buf = newbuf;
got = sys_xsi_strerror_r(error, buf, buf_sz);
if( got == 0 ) break;
if( got == -1 && errno != ERANGE ) {
err_out = errno;
break;
}
buf_sz *= 2; // try again with a bigger buffer.
}
// maybe it's a EAI/gai error, which we add GAI_ERROR_OFFSET to.
#ifdef HAS_GETADDRINFO
if( got == -1 && err_out == EINVAL ) {
const char* gai_str;
int len;
gai_str = gai_strerror(error - GAI_ERROR_OFFSET);
if( ! gai_str ) {
err_out = errno;
} else {
len = strlen(gai_str);
if( len + 1 > buf_sz ) {
newbuf = (char*) qio_realloc(buf, len + 1 + extra_space);
if( ! newbuf ) {
qio_free(buf);
return ENOMEM;
}
buf = newbuf;
}
strcpy(buf, gai_str);
}
}
#endif
*string_out = buf;
return err_out;
}
err_t sys_getnameinfo(const sys_sockaddr_t* addr, char** host_out, char** serv_out, int flags)
{
char* host_buf=0;
char* new_host_buf;
char* serv_buf=0;
char* new_serv_buf;
int host_buf_sz;
int serv_buf_sz;
int got;
err_t err_out;
#ifdef NI_MAXHOST
host_buf_sz = NI_MAXHOST;
#else
host_buf_sz = 1025;
#endif
#ifdef NI_MAXSERV
serv_buf_sz = NI_MAXSERV;
#else
serv_buf_sz = 32;
#endif
STARTING_SLOW_SYSCALL;
while( 1 ) {
new_host_buf = (char*) qio_realloc(host_buf, host_buf_sz);
new_serv_buf = (char*) qio_realloc(serv_buf, serv_buf_sz);
if( ! new_host_buf || ! new_serv_buf ) {
qio_free(host_buf);
qio_free(serv_buf);
err_out = ENOMEM;
goto error;
}
host_buf = new_host_buf;
serv_buf = new_serv_buf;
got = getnameinfo((const struct sockaddr*) & addr->addr, addr->len,
host_buf, host_buf_sz,
serv_buf, serv_buf_sz,
flags);
#ifndef EAI_OVERFLOW
break; // oddly enough... old Mac OS X does not have EAI_OVERFLOW.
#else
if( got != EAI_OVERFLOW ) break;
host_buf_sz *= 2;
serv_buf_sz *= 2;
#endif
}
if( got == 0 ) {
*host_out = NULL;
*serv_out = NULL;
err_out = 0;
} else {
*host_out = host_buf;
*serv_out = serv_buf;
if( got == EAI_SYSTEM ) err_out = errno;
else err_out = GAI_ERROR_OFFSET + got;
}
error:
DONE_SLOW_SYSCALL;
return err_out;
}
void qbytes_free_qio_free(qbytes_t* b) {
qio_free(b->data);
_qbytes_free_qbytes(b);
}