void vcf_file::open()
{
if (!compressed)
{
if (filename.substr(filename.size()-3) == ".gz")
{
warning("Filename ends in '.gz'. Shouldn't you be using --gzvcf?\n");
}
vcf_in.open(filename.c_str(), ios::in);
if (!vcf_in.is_open())
error("Could not open VCF file: " + filename, 0);
}
else
{
gzMAX_LINE_LEN = 1024*1024;
gz_readbuffer = new char[gzMAX_LINE_LEN];
gzvcf_in = gzopen(filename.c_str(), "rb");
if (gzvcf_in == NULL)
error("Could not open GZVCF file: " + filename, 0);
#ifdef ZLIB_VERNUM
string tmp(ZLIB_VERSION);
printLOG("Using zlib version: " + tmp + "\n");
#if (ZLIB_VERNUM >= 0x1240)
gzbuffer(gzvcf_in, gzMAX_LINE_LEN); // Included in zlib v1.2.4 and makes things MUCH faster
#else
printLOG("Versions of zlib >= 1.2.4 will be *much* faster when reading zipped VCF files.\n");
#endif
#endif
}
}
inline gzFile open_fastq(const char* filename) {
gzFile fd1;
fd1=gzopen(filename,"r");
if (fd1==NULL) {
fprintf(stderr,"\nError: Unable to open %s\n",filename);
exit(1);
}
gzbuffer(fd1,128000);
return(fd1);
}
static el_file_ptr gz_file_open(const char* file_name)
{
gzFile file;
el_file_ptr result;
Sint64 read, size;
file = gzopen(file_name, "rb");
if (!file)
{
LOG_ERROR("Can't open file '%s': %s", file_name, strerror(errno));
return NULL;
}
result = calloc(1, sizeof(el_file_t));
result->file_name = strdup(file_name);
size = 0;
#if (ZLIB_VERNUM >= 0x1235)
gzbuffer(file, 0x40000); // 256k
#endif
do
{
result->buffer = realloc(result->buffer, size + 0x40000);
read = gzread(file, result->buffer + size, 0x40000);
size += read;
}
while (gzeof(file) == 0);
result->buffer = realloc(result->buffer, size);
#ifdef FASTER_STARTUP
result->current = result->buffer;
result->end = result->buffer + size;
#else
result->size = size;
#endif
#ifndef FASTER_MAP_LOAD
result->crc32 = CrcCalc(result->buffer, result->size);
#endif
gzclose(file);
#ifdef FASTER_MAP_LOAD
LOG_DEBUG_VERBOSE("File '%s' [crc:0x%08X] opened.", file_name,
el_crc32(result));
#else
LOG_DEBUG_VERBOSE("File '%s' [crc:0x%08X] opened.", file_name,
result->crc32);
#endif
return result;
}
/*!
@see futil_open()
*/
gzFile futil_gzopen(const char *path, const char *mode)
{
ctx_assert(strcmp(path, "-") != 0 || strcmp(mode,"w") == 0);
gzFile gzout = strcmp(path, "-") == 0 ? gzdopen(fileno(stdout), mode)
: gzopen(path, mode);
if(gzout == NULL)
die("Cannot open gzfile: %s [%s]", futil_outpath_str(path), strerror(errno));
// Set buffer size
#if ZLIB_VERNUM >= 0x1240
gzbuffer(gzout, DEFAULT_IO_BUFSIZE);
#endif
return gzout;
}
bool
write_vmat_compressed(const String& fn, size_t xdim, size_t ydim, int map_cnt)
{
//Form1->Memo1->Lines->Add("saving compressed, compress bound: "+IntToStr(compressBound(64000)));
gzFile f = gzopen(fn.toUtf8().constData(), "w");
if (!f) {
Form1->Memo1->Lines->Add(" FATAL ERROR: Could not open compressed vmat file: "+fn);
return false;
}
#if ZLIB_VERNUM >= 0x1240
int ok = gzbuffer(f, gz_buf_size);
if (0 != ok) {
Form1->Memo1->Lines->Add(" FATAL ERROR: Could not set gz buffer for file: "+fn);
return false;
}
#endif
//ok = gzsetparams(f, lvl, strat);
gzwrite(f, magic, strlen(magic));
gzwrite(f, (const char*)&xdim, sizeof(xdim));
gzwrite(f, (const char*)&ydim, sizeof(ydim));
gzwrite(f, (const char*)&map_cnt, sizeof(map_cnt));
BFOC_size_t empty = 0;
for (size_t y=0; y<ydim; y++) {
for (size_t x=0; x<xdim; x++) {
if (!vmat[y][x]) {
gzwrite(f, (const char*)&empty, sizeof(empty));
} else {
// f << size << <list of index-value pairs>
const Biodiv_Features_Occur_Container& occ = vmat[y][x];
BFOC_size_t n = occ.size();
gzwrite(f, (const char*)&n, sizeof(n));
for (BFOC_size_t spp=occ.first(); spp!=occ.overflow(); spp=occ.next(spp)) {
//f << spp << vmat[y][x][spp];
gzwrite(f, (const char*)&spp, sizeof(spp));
gzwrite(f, (const char*)&(vmat[y][x][spp]), sizeof(typeof(vmat[y][x][spp])));
}
}
}
if (0 == (y+1)%REPORT_PERIOD)
Form1->Memo1->Lines->Add(" Written "+IntToStr(y+1)+" rows...");
}
gzclose(f);
return true;
}
ReadDataFile_FASTQ_gz::ReadDataFile_FASTQ_gz(const char *read_file_name,
const QualityEncoding qualities,
const uint32 max_reads,
const uint32 max_read_len,
const ReadEncoding flags)
: ReadDataFile_FASTQ_parser(read_file_name, qualities, max_reads, max_read_len, flags)
{
m_file = gzopen(read_file_name, "r");
if (!m_file) {
m_file_state = FILE_OPEN_FAILED;
} else {
m_file_state = FILE_OK;
}
gzbuffer(m_file, m_buffer_size);
}
// Open gzFile
// rb - read
// wb6 - write with compresion level 6
// wb9 - write with compresion level 9
// wbT - write without compression
gzFile open_gzfile(const char* name, const char* mode, uint64_t buf_size){
gzFile fh = NULL;
if( strcmp(name, "-") == 0 )
fh = gzdopen(fileno(stdin), mode);
else
fh = gzopen(name, mode);
if(fh == NULL)
return NULL;
if(gzbuffer(fh, buf_size) < 0)
return NULL;
return fh;
}
inline gzFile open_fixed_fastq(const char* filename) {
gzFile fd1=NULL;
if (fix_dot) {
// new filename
char new_filename[1024];
strncpy(&new_filename[0],filename,1024);
strcat(&new_filename[0],"_fix.fastq.gz");
fd1=gzopen(new_filename,"w");
if (fd1==NULL) {
fprintf(stderr,"\nError: Unable to open %s\n",filename);
exit(1);
}
gzbuffer(fd1,128000);
}
return(fd1);
}
SequenceDataFile_TXT_gz::SequenceDataFile_TXT_gz(
const char* read_file_name,
const QualityEncoding qualities,
const uint32 max_reads,
const uint32 max_read_len,
const SequenceEncoding flags,
const uint32 buffer_size)
: SequenceDataFile_TXT(read_file_name, qualities, max_reads, max_read_len, flags, buffer_size)
{
m_file = gzopen(read_file_name, "r");
if (!m_file) {
m_file_state = FILE_OPEN_FAILED;
} else {
m_file_state = FILE_OK;
}
gzbuffer(m_file, m_buffer_size);
}
gzstreambuf* open(const char* name, int open_mode) {
if (is_open()) {
return (gzstreambuf*) 0;
}
mode_ = open_mode;
// no append nor read/write mode
// if ((mode_ & std::ios::ate) || (mode_ & std::ios::app)
// || ((mode_ & std::ios::in) && (mode_ & std::ios::out))) {
// return (gzstreambuf*) 0;
// }
if ((mode_ & std::ios::app)
|| ((mode_ & std::ios::in) && (mode_ & std::ios::out))) {
return (gzstreambuf*) 0;
}
char fmode[10];
char* fmodeptr = fmode;
if (mode_ & std::ios::in) {
*fmodeptr++ = 'r';
} else if (mode_ & std::ios::out) {
*fmodeptr++ = 'w';
} else if(mode_ & std::ios::ate){
*fmodeptr++ = 'a';
}
*fmodeptr++ = 'b';
*fmodeptr = '\0';
file_ = gzopen(name, fmode);
#if ZLIB_VERNUM >= 0x1280
//if you couldn't set the buffer, throw
if (gzbuffer(file_, 128 * 1024)) {
throw std::runtime_error { std::string(__PRETTY_FUNCTION__)
+ ":couldn't set gz buffer" };
}
#endif
if (file_ == 0) {
return (gzstreambuf*) 0;
}
opened_ = 1;
return this;
}
static uproc_io_stream *
io_open(const char *path, const char *mode, enum uproc_io_type type)
{
uproc_io_stream *stream = malloc(sizeof *stream);
if (!stream) {
uproc_error(UPROC_ENOMEM);
return NULL;
}
stream->type = type;
stream->stdstream = false;
switch (stream->type) {
case UPROC_IO_GZIP:
#if HAVE_ZLIB_H
if (!(stream->s.gz = gzopen(path, mode))) {
/* gzopen sets errno to 0 if memory allocation failed */
if (!errno) {
uproc_error(UPROC_ENOMEM);
goto error;
}
goto error_errno;
}
(void) gzbuffer(stream->s.gz, GZIP_BUFSZ);
break;
#endif
case UPROC_IO_STDIO:
if (!(stream->s.fp = fopen(path, mode))) {
goto error_errno;
}
break;
default:
goto error;
}
return stream;
error_errno:
uproc_error_msg(UPROC_ERRNO, "can't open \"%s\" with mode \"%s\"",
path, mode);
error:
free(stream);
return NULL;
}
//----------------------------------------------------------------------------------
// Open the file
bool CFastqReader::OpenFiles()
{
if(in || in_gzip || in_bzip2)
return false;
// Uncompressed file
if(mode == m_plain)
{
if((in = fopen(input_file_name.c_str(), "rb")) == NULL)
return false;
}
// Gzip-compressed file
else if(mode == m_gzip)
{
if((in_gzip = gzopen(input_file_name.c_str(), "rb")) == NULL)
return false;
gzbuffer(in_gzip, gzip_buffer_size);
}
// Bzip2-compressed file
else if(mode == m_bzip2)
{
in = fopen(input_file_name.c_str(), "rb");
if(!in)
return false;
setvbuf(in, NULL, _IOFBF, bzip2_buffer_size);
if((in_bzip2 = BZ2_bzReadOpen(&bzerror, in, 0, 0, NULL, 0)) == NULL)
{
fclose(in);
return false;
}
}
// Reserve via PMM
pmm_fastq->reserve(part);
part_filled = 0;
return true;
}
void vcf_file::open_gz()
{
gzMAX_LINE_LEN = 1024*1024;
gz_readbuffer = new char[gzMAX_LINE_LEN];
if (stream)
gzfile_in = gzdopen(fileno(stdin), "r");
else
gzfile_in = gzopen(filename.c_str(), "rb");
if (gzfile_in == NULL)
LOG.error("Could not open GZVCF file: " + filename, 0);
#ifdef ZLIB_VERNUM
string tmp(ZLIB_VERSION);
LOG.printLOG("Using zlib version: " + tmp + "\n");
#if (ZLIB_VERNUM >= 0x1240)
gzbuffer(gzfile_in, gzMAX_LINE_LEN); // Included in zlib v1.2.4 and makes things MUCH faster
#else
LOG.printLOG("Versions of zlib >= 1.2.4 will be *much* faster when reading zipped VCF files.\n");
#endif
#endif
}
void bcf_file::open_gz()
{
int ret;
gzMAX_LINE_LEN = 1024*1024;
if (stream)
gzfile_in = gzdopen(fileno(stdin), "r");
else
gzfile_in = gzopen(filename.c_str(), "rb");
if (gzfile_in == NULL)
LOG.error("Could not open BGZF BCF file: " + filename, 0);
#ifdef ZLIB_VERNUM
string tmp(ZLIB_VERSION);
LOG.printLOG("Using zlib version: " + tmp + "\n");
#if (ZLIB_VERNUM >= 0x1240)
ret = gzbuffer(gzfile_in, gzMAX_LINE_LEN); // Included in zlib v1.2.4 and makes things MUCH faster
if (ret != 0)
LOG.warning("Unable to change zlib buffer size.");
#else
LOG.printLOG("Versions of zlib >= 1.2.4 will be *much* faster when reading compressed BCF files.\n");
#endif
#endif
}
bool
load_vmat_compressed(const String& fn, size_t xdim, size_t ydim, int map_cnt)
{
gzFile f = gzopen(fn.toUtf8().constData(), "r");
if (!f) {
Form1->Memo1->Lines->Add(" FATAL ERROR: Could not open compressed vmat file: "+fn);
return false;
}
#if ZLIB_VERNUM >= 0x1240
int ok = gzbuffer(f, gz_buf_size);
if (0 != ok) {
Form1->Memo1->Lines->Add(" FATAL ERROR: Could not set gz buffer for file: "+fn);
return false;
}
#endif
char* in_magic = new char[strlen(magic)];
gzread(f, (char*)in_magic, strlen(magic));
// TODO: do some check here
delete[] in_magic;
size_t in_xdim, in_ydim;
int in_map_cnt;
gzread(f, (char*)&in_xdim, sizeof(in_xdim));
gzread(f, (char*)&in_ydim, sizeof(in_ydim));
gzread(f, (char*)&in_map_cnt, sizeof(in_map_cnt));
if (in_xdim != xdim || in_ydim != ydim) {
Form1->Memo1->Lines->Add(" FATAL ERROR: dimensions do not match. In current setup, columns: "+
IntToStr(xdim)+", rows: "+IntToStr(ydim)+", whereas in vmat file, columns: "+
IntToStr(in_xdim)+", rows: "+IntToStr(in_ydim)+".");
return false;
} else {
Form1->Memo1->Lines->Add("Dimensions match, columns: "+IntToStr(xdim)+
", rows: "+IntToStr(ydim));
Form1->Memo1->Lines->Add("Features in vmat file: "+IntToStr(in_map_cnt)+
", features in current setup: "+IntToStr(map_cnt));
}
Form1->Memo1->Lines->Add("Size BFOC_size_t: "+IntToStr(sizeof(BFOC_size_t))+
", size key_t: "+IntToStr(sizeof(*Biodiv_Features_Occur_Container::priv_key))+
", size val_t: "+IntToStr(sizeof(*Biodiv_Features_Occur_Container::priv_val)));
BFOC_size_t len = 0;
for (size_t y=0; y<ydim; y++) {
for (size_t x=0; x<xdim; x++) {
gzread(f, (char*)&len, sizeof(len));
if (0 == len)
continue;
BFOC_size_t idx;
float val;
// resize the Biodiv_Features_Occur_Container& and load values
Biodiv_Features_Occur_Container& occ = vmat[y][x];
occ.reserve(len, len);
occ.priv_size = len;
for (BFOC_size_t table_idx=0; table_idx<len; table_idx++) {
gzread(f, (char*)&(occ.priv_key[table_idx]), sizeof(*occ.priv_key));
gzread(f, (char*)&(occ.priv_val[table_idx]), sizeof(*occ.priv_val));
}
}
if (0 == (y+1)%REPORT_PERIOD)
Form1->Memo1->Lines->Add(" Loaded "+IntToStr(y+1)+" rows...");
}
gzclose(f);
return true;
}
unsigned int post_file(const char *filetag) {
char jsonfile[BUFLEN] = NULLSTR;
char jsonfilegz[BUFLEN] = NULLSTR;
char jsonpfile[BUFLEN] = NULLSTR;
char jsonpfilegz[BUFLEN] = NULLSTR;
unsigned char filebuf[32768];
SNPRINTF(jsonfile, BUFLEN, "%s%s.json\0", JSON_DIR, filetag);
SNPRINTF(jsonfilegz, BUFLEN, "%s%s.json.gz\0", JSON_DIR, filetag);
SNPRINTF(jsonpfile, BUFLEN, "%s%s.jsonp\0", JSON_DIR, filetag);
SNPRINTF(jsonpfilegz, BUFLEN, "%s%s.jsonp.gz\0", JSON_DIR, filetag);
// printf("post process %s - ", filetag);
printf("post process - ");
if (ACCESS(jsonfile, 0) != 0) {
printf("err\n");
fprintf(stderr, "could not access file\n");
return 1;
}
FILE *j = fopen(jsonfile, "r");
if (j == NULL) {
printf("err\n");
fprintf(stderr, "error opening file\n");
return 1;
}
FILE *f = NULL;
if (JSONP_FLAG) {
f = fopen(jsonpfile, "w");
if (f == NULL) {
fclose(j);
printf("err\n");
fprintf(stderr, "error opening file\n");
return 1;
}
}
gzFile jz = NULL;
gzFile fz = NULL;
if (GZIP_FLAG) {
jz = gzopen(jsonfilegz, "wb9");
if (jz == NULL) {
printf("err\n");
fprintf(stderr, "error opening file\n");
return 1;
}
gzbuffer(jz, 65536);
}
if (GZIP_FLAG && JSONP_FLAG) {
fz = gzopen(jsonpfilegz, "wb9");
if (fz == NULL) {
printf("err\n");
fprintf(stderr, "error opening file\n");
return 1;
}
gzbuffer(fz, 65536);
}
if (JSONP_FLAG) fprintf(f, "EVEoj_");
if (GZIP_FLAG && JSONP_FLAG) gzprintf(fz, "EVEoj_");
if (JSONP_FLAG) fprintf(f, filetag);
if (GZIP_FLAG && JSONP_FLAG) gzprintf(fz, filetag);
if (JSONP_FLAG) fprintf(f, "_callback(");
if (GZIP_FLAG && JSONP_FLAG) gzprintf(fz, "_callback(");
size_t readsize = 32768;
while (readsize == 32768) {
readsize = fread(filebuf, sizeof(unsigned char), 32768, j);
if (JSONP_FLAG) fwrite(filebuf, sizeof(unsigned char), readsize, f);
if (GZIP_FLAG) gzwrite(jz, filebuf, readsize);
if (GZIP_FLAG && JSONP_FLAG) gzwrite(fz, filebuf, readsize);
}
if (GZIP_FLAG) gzclose_w(jz);
if (GZIP_FLAG && JSONP_FLAG) {
gzprintf(fz, ");");
gzclose_w(fz);
}
if (JSONP_FLAG) {
fprintf(f, ");");
fclose(f);
}
fclose(j);
printf("OK\n");
return 0;
}
bool CTarArcFile_GZip::open(const char *arcfile, const char *mode, int compress_level)
{
m_arcfile = arcfile;
gzFile f = NULL;
int fd = -1;
size64 rpmlen = 0;
char buf[16];
bool bReadMode=(NULL!=strchr(mode,'r'));
if(bReadMode){
rpmlen = rpm_getheadersize(arcfile);
if(rpmlen == -1){rpmlen = 0;}
fd = _open(arcfile, _O_BINARY|_O_RDONLY, 0);
if(fd == -1){return false;}
_lseeki64(fd, rpmlen, SEEK_CUR);
}else{
fd = _open(arcfile, _O_BINARY|_O_CREAT|_O_RDWR|_O_TRUNC, _S_IREAD | _S_IWRITE);
_snprintf(buf,COUNTOF(buf),"%s%d",mode,compress_level);
mode=buf; //モードの文字列をすり替え
}
f = gzdopen(fd, mode);
if(f==NULL){
_close(fd);
return false;
}
gzbuffer(f,1024*1024);
m_gzFile = f;
if(bReadMode){
/* retrieve GZIP header information(filename, time,...) */
//std::ifstream fs_r;
fast_fstream fs_r;
fs_r.open(arcfile, std::ios::in|std::ios::binary);
int c;
if(fs_r.fail()){return false;}
fs_r.seekg(rpmlen, std::ios_base::cur); /* skip rpm header */
if(fs_r.get()!=0x1f || fs_r.get()!=0x8b){return false;}
if((c=fs_r.get())==EOF){return false;}
m_gzip_compress_method = c;
if((c=fs_r.get())==EOF){return false;}
int flags = m_gzip_flags = c;
if((flags & GZIP_FLAG_ENCRYPTED)||(flags & GZIP_FLAG_CONTINUATION)||(flags & GZIP_FLAG_RESERVED)){return true;}
time_t stamp;
stamp = fs_r.get();
stamp |= (fs_r.get()<<8);
stamp |= (fs_r.get()<<16);
stamp |= (fs_r.get()<<24);
if(stamp<0){stamp=0;}
m_mtime = m_gzip_time_stamp = stamp;
m_gzip_ext_flag = fs_r.get();
m_gzip_os_type = fs_r.get();
if(flags & GZIP_FLAG_CONTINUATION){
m_gzip_part = fs_r.get();
}
if(flags & GZIP_FLAG_EXTRA_FIELD){
int len = fs_r.get();
while(len<10000 && (len--)>0){
fs_r.get();
}
}
if(flags & GZIP_FLAG_ORIG_NAME){
std::string fname;
while((c=fs_r.get())!=EOF && c!='\0'){
fname += c;
}
m_orig_filename = m_gzip_orig_name = fname;
}
if(flags & GZIP_FLAG_COMMENT){
std::string comment;
while((c=fs_r.get())!=EOF && c!='\0'){
comment += c;
}
m_gzip_comment = comment;
}
/*
fs_r.seekg(-4, std::ios_base::end);
size64 size;
size = ((unsigned __int64)fs_r.get());
size |= ((unsigned __int64)fs_r.get())<<8;
size |= ((unsigned __int64)fs_r.get())<<16;
size |= ((unsigned __int64)fs_r.get())<<24;
//ファイルサイズの下位32bitのみが格納されているため、4GB以上のファイルサイズを正しく復元できない
m_orig_filesize = size;*/
m_orig_filesize=-1;
}
return true;
//return (f != NULL);
}