hFILE *hfile_init(size_t struct_size, const char *mode, size_t capacity)
{
hFILE *fp = (hFILE *) malloc(struct_size);
if (fp == NULL) goto error;
// TODO: In the case that we are initializing a hFILE_mem, do we really
// need to allocate a buffer of this size to wrap the original memory buffer?
if (capacity == 0) capacity = 32768;
// FIXME For now, clamp input buffer sizes so mpileup doesn't eat memory
if (strchr(mode, 'r') && capacity > 32768) capacity = 32768;
fp->buffer = (char *) malloc(capacity);
if (fp->buffer == NULL) goto error;
fp->begin = fp->end = fp->buffer;
fp->limit = &fp->buffer[capacity];
fp->offset = 0;
fp->at_eof = 0;
fp->has_errno = 0;
return fp;
error:
hfile_destroy(fp);
return NULL;
}
hFILE *hfile_init(size_t struct_size, const char *mode, size_t capacity)
{
hFILE *fp = (hFILE *) malloc(struct_size);
if (fp == NULL) goto error;
if (capacity == 0) capacity = 32768;
// FIXME For now, clamp input buffer sizes so mpileup doesn't eat memory
if (strchr(mode, 'r') && capacity > 32768) capacity = 32768;
fp->buffer = (char *) malloc(capacity);
if (fp->buffer == NULL) goto error;
fp->begin = fp->end = fp->buffer;
fp->limit = &fp->buffer[capacity];
fp->offset = 0;
fp->at_eof = 0;
fp->mobile = 1;
fp->readonly = (strchr(mode, 'r') && ! strchr(mode, '+'));
fp->has_errno = 0;
return fp;
error:
hfile_destroy(fp);
return NULL;
}
void hclose_abruptly(hFILE *fp)
{
int save = errno;
if (fp->backend->close(fp) < 0) { /* Ignore subsequent errors */ }
hfile_destroy(fp);
errno = save;
}
int hclose(hFILE *fp)
{
int err = fp->has_errno;
if (writebuffer_is_nonempty(fp) && hflush(fp) < 0) err = fp->has_errno;
if (fp->backend->close(fp) < 0) err = errno;
hfile_destroy(fp);
if (err) {
errno = err;
return EOF;
}
else return 0;
}
void hfilelist_destroy(hfilelist_t *hl)
{
hfile_t *fp, *next_fp;
if (hl == NULL)
return;
for (fp = hl->first; fp != NULL; ) {
next_fp = fp->next;
hfile_destroy(fp);
fp = next_fp;
}
hfilelist_free(hl);
}
hfile_t *hfile_load(const int8_t *filename)
{
int32_t fd;
hfile_t *fp;
meta_t *o;
htail_t *ht;
hinfo_t *hi;
if (!filename)
return(NULL);
if ((fp = hfile_new()) == NULL)
return(NULL);
memcpy(fp->name,filename,strlen(filename));
if ((fd = open(fp->name,O_RDONLY)) < 0)
__ERROR_LOG(H_ERROR);
fp->trailer = htail_load(fd,-sizeof(htail_t),sizeof(htail_t));
if (fp->trailer == NULL)
__ERROR_LOG(H_ERROR);
else
ht = fp->trailer;
fp->fileinfo = hinfo_load(fd,ht->infooffset,ht->infosize);
if (fp->fileinfo == NULL)
__ERROR_LOG(H_ERROR);
else
hi = fp->fileinfo;
fp->metas = index_load(fd,ht->indexoffset,ht->indexsize);
if (fp->metas == NULL)
__ERROR_LOG(H_ERROR);
fp->bloom = bloom_load(fd,ht->bloomoffset,ht->bloomsize);
if (fp->bloom == NULL)
__ERROR_LOG(H_ERROR);
fp->blocks = malloc(hi->blockcount*sizeof(block_t *));
if (fp->blocks == NULL)
__ERROR_LOG(H_ERROR);
memset(fp->blocks,0,hi->blockcount*sizeof(block_t *));
close(fd);
return(fp);
H_ERROR:
close(fd);
hfile_destroy(fp);
return(NULL);
}
hFILE *hopen_net(const char *filename, const char *mode)
{
hFILE_net *fp;
// Do any networking initialisation if this is the first use.
if (! net_inited) { if (net_init() < 0) return NULL; }
fp = (hFILE_net *) hfile_init(sizeof (hFILE_net), mode, 0);
if (fp == NULL) return NULL;
fp->netfp = knet_open(filename, mode);
if (fp->netfp == NULL) { hfile_destroy((hFILE *) fp); return NULL; }
fp->base.backend = &net_backend;
return &fp->base;
}
static hFILE *hopen_fd(const char *filename, const char *mode)
{
hFILE_fd *fp = NULL;
int fd = open(filename, hfile_oflags(mode), 0666);
if (fd < 0) goto error;
fp = (hFILE_fd *) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));
if (fp == NULL) goto error;
fp->fd = fd;
fp->is_socket = 0;
fp->base.backend = &fd_backend;
return &fp->base;
error:
if (fd >= 0) { int save = errno; (void) close(fd); errno = save; }
hfile_destroy((hFILE *) fp);
return NULL;
}
hFILE *hopen_irods(const char *filename, const char *mode)
{
hFILE_irods *fp;
rodsPath_t path;
dataObjInp_t args;
int ret;
// Initialise the iRODS connection if this is the first use.
if (irods.conn == NULL) { if (irods_init() < 0) return NULL; }
if (strncmp(filename, "irods:", 6) == 0) filename += 6;
else { errno = EINVAL; return NULL; }
fp = (hFILE_irods *) hfile_init(sizeof (hFILE_irods), mode, 0);
if (fp == NULL) return NULL;
strncpy(path.inPath, filename, MAX_NAME_LEN-1);
path.inPath[MAX_NAME_LEN-1] = '\0';
ret = parseRodsPath(&path, &irods.env);
if (ret < 0) goto error;
memset(&args, 0, sizeof args);
strcpy(args.objPath, path.outPath);
args.openFlags = hfile_oflags(mode);
if (args.openFlags & O_CREAT) {
args.createMode = 0666;
addKeyVal(&args.condInput, DEST_RESC_NAME_KW,irods.env.rodsDefResource);
}
ret = rcDataObjOpen(irods.conn, &args);
if (ret < 0) goto error;
fp->descriptor = ret;
fp->base.backend = &irods_backend;
return &fp->base;
error:
hfile_destroy((hFILE *) fp);
set_errno(ret);
return NULL;
}
static hFILE *hopen_fd(const char *filename, const char *mode)
{
hFILE_fd *fp = NULL;
int fd = open(filename, hfile_oflags(mode), 0666);
if (fd < 0) goto error;
off_t currentPos = lseek(fd, (size_t)0, SEEK_CUR);
// Get the file size
off_t fileSize = lseek(fd, (size_t)0, SEEK_END);
// Seek back to the begining of file
lseek(fd, currentPos, SEEK_SET);
size_t lenstr = strlen(filename);
// For small files that are not indexes and given rb mode,
// we are just going to load them into memory to avoid
// repeatedly fetching the same data from disk.
if ((fileSize < (1 << 17)) &&
strcmp(mode, "rb") == 0 &&
lenstr > 4 &&
(strcmp(filename + lenstr - 4, ".pbi") != 0 && strcmp(filename + lenstr - 4, ".bai") != 0) ) {
char* buffer = malloc(fileSize);
ssize_t n = read(fd, buffer, fileSize);
if (n < 0) goto error;
close(fd);
return hopen_mem(buffer, mode, fileSize, 1);
} else {
fp = (hFILE_fd *) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));
if (fp == NULL) goto error;
fp->fd = fd;
fp->is_socket = 0;
fp->base.backend = &fd_backend;
return &fp->base;
}
error:
if (fd >= 0) { int save = errno; (void) close(fd); errno = save; }
hfile_destroy((hFILE *) fp);
return NULL;
}
hFILE *hopen_libcurl(const char *url, const char *modes)
{
hFILE_libcurl *fp;
char mode;
const char *s;
CURLcode err;
CURLMcode errm;
int save;
if ((s = strpbrk(modes, "rwa+")) != NULL) {
mode = *s;
if (strpbrk(&s[1], "rwa+")) mode = 'e';
}
else mode = '\0';
if (mode != 'r' && mode != 'w') { errno = EINVAL; return NULL; }
fp = (hFILE_libcurl *) hfile_init(sizeof (hFILE_libcurl), modes, 0);
if (fp == NULL) return NULL;
fp->easy = curl_easy_init();
if (fp->easy == NULL) { errno = ENOMEM; goto error; }
fp->headers = NULL;
fp->file_size = -1;
fp->buffer.ptr.rd = NULL;
fp->buffer.len = 0;
fp->final_result = (CURLcode) -1;
fp->paused = fp->closing = fp->finished = 0;
// Make a route to the hFILE_libcurl* given just a CURL* easy handle
err = curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp);
if (mode == 'r') {
err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEFUNCTION, recv_callback);
err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp);
}
else {
err |= curl_easy_setopt(fp->easy, CURLOPT_READFUNCTION, send_callback);
err |= curl_easy_setopt(fp->easy, CURLOPT_READDATA, fp);
err |= curl_easy_setopt(fp->easy, CURLOPT_UPLOAD, 1L);
if (add_header(fp, "Transfer-Encoding: chunked") < 0) goto error;
}
if (tolower(url[0]) == 's' && url[1] == '3') {
// Construct the HTTP-Method/Content-MD5/Content-Type part of the
// message to be signed. This will be destroyed by add_s3_settings().
kstring_t message = { 0, 0, NULL };
kputs((mode == 'r')? "GET\n" : "PUT\n", &message);
kputc('\n', &message);
kputc('\n', &message);
if (add_s3_settings(fp, url, &message) < 0) goto error;
}
else
err |= curl_easy_setopt(fp->easy, CURLOPT_URL, url);
err |= curl_easy_setopt(fp->easy, CURLOPT_USERAGENT, curl.useragent.s);
if (fp->headers)
err |= curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, fp->headers);
err |= curl_easy_setopt(fp->easy, CURLOPT_FOLLOWLOCATION, 1L);
err |= curl_easy_setopt(fp->easy, CURLOPT_FAILONERROR, 1L);
if (hts_verbose >= 8)
err |= curl_easy_setopt(fp->easy, CURLOPT_VERBOSE, 1L);
if (err != 0) { errno = ENOSYS; goto error; }
errm = curl_multi_add_handle(curl.multi, fp->easy);
if (errm != CURLM_OK) { errno = multi_errno(errm); goto error; }
curl.nrunning++;
while (! fp->paused && ! fp->finished)
if (wait_perform() < 0) goto error_remove;
if (fp->finished && fp->final_result != CURLE_OK) {
errno = easy_errno(fp->easy, fp->final_result);
goto error_remove;
}
if (mode == 'r') {
double dval;
if (curl_easy_getinfo(fp->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
&dval) == CURLE_OK && dval >= 0.0)
fp->file_size = (off_t) (dval + 0.1);
}
fp->base.backend = &libcurl_backend;
return &fp->base;
error_remove:
save = errno;
(void) curl_multi_remove_handle(curl.multi, fp->easy);
curl.nrunning--;
errno = save;
error:
save = errno;
curl_easy_cleanup(fp->easy);
if (fp->headers) curl_slist_free_all(fp->headers);
hfile_destroy((hFILE *) fp);
errno = save;
return NULL;
}
