/* not static because imported by hdf5file.c via forward decl. */ rc_t HDF5FileMake ( HDF5File **fp, hid_t dataset_handle, bool read_enabled, bool write_enabled ) { rc_t rc; HDF5File *f; if ( dataset_handle < 0 ) return RC ( rcFS, rcFile, rcConstructing, rcFileDesc, rcInvalid ); f = malloc ( sizeof *f ); if ( f == NULL ) return RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted ); rc = KFileInit ( & f -> dad, ( const KFile_vt* ) &vtHDF5File, "HDF5File", "no-name", read_enabled, write_enabled ); if ( rc == 0 ) { /* make the inner Arrayfile...*/ rc = HDF5ArrayFileMake ( & f -> array_file, & f -> dad, dataset_handle, read_enabled, write_enabled ); if ( rc == 0 ) { * fp = f; return 0; } KArrayFileRelease( & f -> array_file -> dad ); } free ( f ); return rc; }
static rc_t KBufWriteFileMake ( KBufWriteFile ** bp, const KFile *f, size_t bsize, const KFile_vt *vt, bool read_enabled, bool write_enabled ) { rc_t rc; KBufWriteFile *buf = malloc ( sizeof * buf - 1 + bsize ); if ( buf == NULL ) rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted ); else { memset ( buf, 0, sizeof * buf ); rc = KFileInit ( & buf -> dad, vt, "KBufWriteFile", "no-name", read_enabled, write_enabled ); if ( rc == 0 ) { rc = KFileAddRef ( f ); if ( rc == 0 ) { buf -> f = ( KFile* ) f; buf -> bsize = bsize; * bp = buf; return 0; } } free ( buf ); } return rc; }
rc_t SRAFastqFile_Open(const KFile** cself, const SRAListNode* sra, const FileOptions* opt) { rc_t rc = 0; SRAFastqFile* self; CALLOC( self, 1, sizeof( *self ) ); if( self == NULL ) { rc = RC( rcExe, rcFile, rcConstructing, rcMemory, rcExhausted ); } else { if ( ( rc = KFileInit( &self->dad, (const KFile_vt*)&SRAFastqFile_vtbl, "SRAFastqFile", "no-name", true, false ) ) == 0 ) { if ( ( rc = SRAListNode_TableOpen( sra, &self->stbl ) ) == 0 ) { if ( ( rc = SRATableGetKTableRead( self->stbl, &self->ktbl ) ) == 0 ) { if ( ( rc = KTableOpenIndexRead( self->ktbl, &self->kidx, opt->index ) ) == 0 ) { if ( ( rc = KLockMake( &self->lock ) ) == 0 ) { MALLOC( self->buf, opt->buffer_sz * ( opt->f.fastq.gzip ? 2 : 1 ) ); if ( self->buf == NULL ) { rc = RC( rcExe, rcFile, rcOpening, rcMemory, rcExhausted ); } else { self->file_sz = opt->file_sz; self->buffer_sz = opt->buffer_sz; if ( opt->f.fastq.gzip ) { self->gzipped = &self->buf[ opt->buffer_sz ]; } self->from = ~0; /* reset position beyond file end */ rc = FastqReaderMake( &self->reader, self->stbl, opt->f.fastq.accession, opt->f.fastq.colorSpace, opt->f.fastq.origFormat, false, opt->f.fastq.printLabel, opt->f.fastq.printReadId, !opt->f.fastq.clipQuality, false, opt->f.fastq.minReadLen, opt->f.fastq.qualityOffset, opt->f.fastq.colorSpaceKey, opt->f.fastq.minSpotId, opt->f.fastq.maxSpotId ); } } } } } if ( rc == 0 ) { *cself = &self->dad; } else { KFileRelease( &self->dad ); } } } return rc; }
static rc_t KSubFileMake (KSubFile ** self, KFile * original, uint64_t start, uint64_t size, bool read_enabled, bool write_enabled) { rc_t rc; KSubFile * pF; /* ----- */ assert (self != NULL); assert (original != NULL); /* assert (start >= size); */ /* ----- * the enables should be true or false */ assert ((read_enabled == true)||(read_enabled == false)); assert ((write_enabled == true)||(write_enabled == false)); /* ----- * get space for the object */ pF = malloc (sizeof (KSubFile)); if (pF == NULL) /* allocation failed */ { /* fail */ rc = RC (rcFS, rcFile, rcConstructing, rcMemory, rcExhausted); } else { rc = KFileInit (&pF->dad, /* initialize base class */ (const KFile_vt*)&vtKSubFile, /* VTable for KSubFile */ "KSubFile", "no-name", read_enabled, /* read allowed */ write_enabled); /* write disallowed */ if (rc == 0) { KFileAddRef (original); /* succeed */ pF->original = original; pF->start = start; pF->size = size; *self = pF; return 0; } /* fail */ free (pF); } return rc; }
static rc_t KNSManagerVMakeUndyingHttpFile(const KNSManager *self, const KFile **file, struct KStream *conn, ver_t vers, const char *url, va_list args) { char buffer[PATH_MAX] = ""; size_t num_writ = 0; rc_t rc = 0; KHttpUndyingFile *f = NULL; if (file == NULL) { return RC(rcNS, rcFile, rcConstructing, rcParam, rcNull); } *file = NULL; f = calloc(1, sizeof *f); if (f == NULL) { return RC(rcNS, rcFile, rcConstructing, rcMemory, rcExhausted); } rc = string_vprintf(buffer, sizeof buffer, &num_writ, url, args); if (rc == 0) { f->url = string_dup_measure(url, NULL); } if (f->url == NULL) { rc = RC(rcNS, rcFile, rcConstructing, rcMemory, rcExhausted); } f->vers = vers; if (rc == 0) { rc = KNSManagerAddRef(self); if (rc == 0) { f->mgr = self; } } if (rc == 0) { assert(conn == NULL); rc = Revive(f); } if (rc == 0) { KHttpUndyingFileSize(f, &f->size); DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_MGR), ( "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@4 %s KNSManagerMakeUndyingHttpFile: size = %ld" "\n", __FUNCTION__, f->size)); } if (rc == 0) { rc = KFileInit(&f->dad, (const KFile_vt*)&vtKHttpFile, "KHttpUndyingFile", url, true, false); } if (rc == 0) { *file = &f->dad; } else { KHttpUndyingFileDestroy(f); } return rc; }
rc_t KFileMakeChunkRead (const struct KFile ** pself, const struct KFile * original, uint64_t size, uint32_t num_chunks, struct KTocChunk * chunks) { rc_t rc; KSubChunkFile * self; /* ----- */ assert (pself != NULL); assert (original != NULL); *pself = NULL; rc = 0; /* ----- * get space for the object */ self = malloc (sizeof (KSubChunkFile) + ((num_chunks-1) * sizeof (KTocChunk))); if (self == NULL) /* allocation failed */ { /* fail */ rc = RC (rcFS, rcFile, rcConstructing, rcMemory, rcExhausted); } else { rc = KFileInit (&self->dad, /* initialize base class */ (const KFile_vt*)&vtKSubChunkFile, /* VTable for KSubChunkFile */ "KSubChunkFile", "no-name", true,false); /* read allowed,write disallowed */ if (rc == 0) { KFileAddRef (original); /* succeed */ self->size = size; self->original = original; self->num_chunks = num_chunks; memcpy ((struct KTocChunk*)self->chunks, chunks, num_chunks * sizeof (KTocChunk)); *pself = &self->dad; return 0; } /* fail */ free (self); } return rc; }
static rc_t CCFileMake (CCFile ** pself, KFile * original, rc_t * prc) { CCFile * self; rc_t rc; /* needs to be better */ assert (pself); assert (original); assert (prc); self = malloc (sizeof (CCFile)); if (self == NULL) /* allocation failed */ { /* fail */ rc = RC (rcFS, rcFile, rcConstructing, rcMemory, rcExhausted); } else { rc = KFileInit (&self->dad, /* initialize base class */ (const KFile_vt*)&vtCCFile,/* VTable for CCFile */ "CCFile", "no-name", original->read_enabled, original->write_enabled); if (rc == 0) { rc = KFileAddRef (original); if (rc == 0) { self->original = original; self->prc = prc; *pself = self; return *prc = 0; } } /* fail */ free (self); } *pself = NULL; *prc = rc; return rc; }
rc_t TextFile_Open( const KFile** cself, const FileOptions* opt ) { rc_t rc = 0; TextFile* self; CALLOC(self, 1, sizeof(*self)); if( self == NULL ) { rc = RC(rcExe, rcFile, rcConstructing, rcMemory, rcExhausted ); } else if ( ( rc = KFileInit( &self->dad, (const KFile_vt*)&TextFile_vtbl, "TextFile", "no-name", true, false ) ) != 0 || ( rc = StrDup( opt->f.txt64b, &self->data ) ) != 0 ) { KFileRelease( &self->dad ); } else { *cself = &self->dad; } return rc; }
/* read only version for decrypting of existing files */ KRYPTO_EXTERN rc_t CC KFileMakeWGAEncRead (const struct KFile ** pself, const struct KFile * encrypted, const char * key, size_t key_size) { rc_t rc; if (pself == NULL) { rc = RC (rcFS, rcFile, rcConstructing, rcSelf, rcNull); LOGERR (klogErr, rc, "key parameter for WGA encrypted file is empty"); return rc; } *pself = NULL; if ((encrypted == NULL)||(key == NULL)) { rc = RC (rcFS, rcFile, rcConstructing, rcParam, rcNull); LOGERR (klogErr, rc, "missing WGA encrypted file passed in to constructor"); } else if (key_size == 0) { rc = RC (rcFS, rcFile, rcConstructing, rcParam, rcInvalid); LOGERR (klogErr, rc, "missing WGA encrypted file passed in to constructor"); } else if (encrypted->read_enabled == 0) { rc = RC (rcFS, rcFile, rcConstructing, rcParam, rcIncorrect); LOGERR (klogErr, rc, "encrypted file not readable"); } else { KCipherManager * cipher_mgr; rc = KCipherManagerMake (&cipher_mgr); if (rc == 0) { KCipher * cipher; rc = KCipherManagerMakeCipher (cipher_mgr, &cipher, kcipher_AES); if (rc == 0) { size_t z; rc = KCipherBlockSize (cipher, &z); if (rc) { LOGERR (klogErr, rc, "unable to get block size for WGA " "encrypted file cipher passed in to constructor"); } else { if (z != ECB_BYTES) { rc = RC (rcFS, rcFile, rcConstructing, rcParam, rcInvalid); LOGERR (klogErr, rc, "wrong block size for WGA " "encrypted file cipher passed in to " "constructor"); } else { KWGAEncFile * self; self = calloc (sizeof (*self), sizeof (uint8_t)); if (self == NULL) { rc = RC (rcFS, rcFile, rcConstructing, rcMemory, rcExhausted); LOGERR (klogErr, rc, "out of memory while " "constructing decryptor"); } else { rc = KFileAddRef (encrypted); if (rc) LOGERR (klogErr, rc, "unable to add reference " "to encrypted file"); else { /* cast to strip const */ self->encrypted = encrypted; self->cipher = cipher; /* read the header of the encrypted file for * details about the decrypted file */ DEBUG_STS(("%s: calling KWGAEncFileHeaderRead\n", __func__)); rc = KWGAEncFileHeaderRead (self); if (rc == 0) { /* using the file header's stored encoding * key build a key from the parameter key */ DEBUG_STS(("%s: calling " "KWGAEncFileHeaderRead\n", __func__)); rc = KWGAEncFileKeyInit (self, key, key_size); } if (rc == 0) { rc = KFileInit (&self->dad, (const KFile_vt*) &vtKWGAEncFileRead, "KWGAEncFile", "no-name", true, false); if (rc) LOGERR (klogInt, rc, "Failed to initialize decrypting file"); else { *pself = &self->dad; self->buffer.offset = 0; self->buffer.valid = 0; KCipherManagerRelease (cipher_mgr); return 0; } } /* release of encrypted handled in destroy() */ } KWGAEncFileDestroyRead (self); } } } KCipherRelease (cipher); } } } return rc; }
static rc_t KCounterFileMake (KCounterFile ** pself, KFile * original, uint64_t * bytecounter, uint64_t * linecounter, bool force) { uint64_t fsize; rc_t rc; KCounterFile * self; /* ----- * we can not accept any of the three pointer parameters as NULL */ assert (pself != NULL); assert (original != NULL); assert (bytecounter != NULL); /* ----- * get space for the object */ self = malloc (sizeof (KCounterFile)); if (self == NULL) /* allocation failed */ { /* fail */ rc = RC (rcFS, rcFile, rcConstructing, rcMemory, rcExhausted); } else { rc = KFileInit (&self->dad, /* initialize base class */ (const KFile_vt*)&vtKCounterFile,/* VTable for KCounterFile */ original->read_enabled, original->write_enabled); if (rc == 0) { for (;;) { if (force) /* all sizes come from actual reads */ { fsize = 0; self->force = true; self->size_allowed = false; } else { rc = KFileSize(original,&fsize); if (GetRCState(rc) == rcUnsupported) { force = true; continue; } else if (rc) break; self->force = false; self->size_allowed = true; } self->max_position = fsize; self->original = original; self->bytecounter = bytecounter; self->linecounter = linecounter; if (linecounter) { self->line_state = ENOTHING; *linecounter = 0; } else { self->line_state = EBINARY; } *bytecounter = fsize; *pself = self; return 0; } } /* fail */ free (self); } *pself = NULL; return rc; }
LIB_EXPORT rc_t CC KLoaderFile_Make(const KLoaderFile **file, const KDirectory* dir, const char* filename, const uint8_t* md5_digest, bool read_ahead) { rc_t rc = 0; KLoaderFile* obj = NULL; if( file == NULL || dir == NULL || filename == NULL ) { rc = RC(rcApp, rcFile, rcConstructing, rcParam, rcNull); } else { *file = NULL; if( (obj = calloc(1, sizeof(*obj))) == NULL || (obj->filename = strdup(filename)) == NULL ) { rc = RC(rcApp, rcFile, rcConstructing, rcMemory, rcExhausted); } else { if( (rc = KDirectoryAddRef(dir)) == 0 ) { obj->dir = dir; if( md5_digest != NULL ) { obj->has_md5 = true; memcpy(obj->md5_digest, md5_digest, sizeof(obj->md5_digest)); } if( (rc = KLoaderFile_AllocateBuffer(obj)) == 0 ) { char resolved[4096]; char* ext = NULL; strcpy(resolved, filename); ext = strrchr(resolved, '.'); DBG(("%s adding %s\n", __func__, resolved)); rc = KLoadProgressbar_File(&obj->job, resolved, obj->dir); if( ext != NULL && strcmp(ext, ".gz") == 0 ) { obj->compress_type = compress_gzip; } else if( ext != NULL && strcmp(ext, ".bz2") == 0 ) { obj->compress_type = compress_bzip2; } if( rc != 0 && obj->compress_type != compress_none ) { *ext = '\0'; DBG(("%s retry adding as %s\n", __func__, resolved)); if( (rc = KLoadProgressbar_File(&obj->job, resolved, obj->dir)) == 0 ) { obj->has_md5 = false; obj->compress_type = compress_none; } } if( rc == 0 && (rc = KFileInit(&obj->dad, (const KFile_vt*)&KLoaderFile_vtbl, "KLoaderFile", filename, true, false)) == 0 ) { obj->file = &obj->dad; #if WINDOWS obj->ahead = false; #else obj->ahead = read_ahead; #endif obj->realname = strdup(resolved); if( obj->realname == NULL ) { rc = RC(rcApp, rcFile, rcConstructing, rcMemory, rcExhausted); } } } } } } if( rc == 0 ) { *file = obj; } else { *file = NULL; KLoaderFile_Release(obj, false); } return rc; }
/* MakeWrite * make a queue file for writing-behind on background thread * * when the file is created, a background thread is started that * waits for buffers to appear in the cross-thread queue. as the producer * thread writes, data are accumulated into buffers which are pushed * into the queue as they fill, and written in turn on the bg thread. * * the producer thread is throttled by queue capacity - determined by * "queue_bytes" and "block_size", such that if the queue is full, * the thread will sleep. the background thread is also throttled by * the queue in that it will sleep if the queue is empty with pending * data. * * the background thread will exit upon a permanent error, or if the * queue is sealed by the producer thread. * * when the file is collected in response to a release message, * the queue will be sealed against further inserts, pending buffers * will be written, the background thread will be joined, and * the source file will be released. * * the intended usage is serial writing of the file. random writes * will be accepted, but may reduce the queue efficiency. * * "qf" [ OUT ] - return parameter for queue file * * "dst" [ IN ] - destination file for write-behind on background thread. * must have write permissions. * * "queue_bytes" [ IN ] - the write-behind limit of the producer * thread, in bytes. this is the amount of data that will be queued * for the background thread before the producer thread sleeps. * * "block_size" [ IN, DEFAULT ZERO ] - optional parameter giving * desired block size when writing to "dst". this may be used * to tune writing for source data, e.g. 64K blocks for gzip. */ LIB_EXPORT rc_t CC KQueueFileMakeWrite ( KFile **qfp, KFile *dst, size_t queue_bytes, size_t block_size, uint32_t timeout_ms ) { rc_t rc; if ( qfp == NULL ) rc = RC ( rcApp, rcFile, rcConstructing, rcParam, rcNull ); else { if ( dst == NULL ) rc = RC ( rcApp, rcFile, rcConstructing, rcFile, rcNull ); else if ( ! dst -> write_enabled ) { if ( dst -> read_enabled ) rc = RC ( rcApp, rcFile, rcConstructing, rcFile, rcReadonly ); else rc = RC ( rcApp, rcFile, rcConstructing, rcFile, rcNoPerm ); } else { KQueueFile *qf = malloc ( sizeof * qf ); if ( qf == NULL ) rc = RC ( rcApp, rcFile, rcConstructing, rcMemory, rcExhausted ); else { rc = KFileInit ( & qf -> dad, ( const KFile_vt* ) & KQueueFileWrite_vt_v1, "KQueueFile", "no-name", false, true ); if ( rc == 0 ) { qf -> f = dst; rc = KFileAddRef ( dst ); if ( rc == 0 ) { uint32_t capacity; /* zero block size means default */ if ( block_size == 0 ) block_size = DEFAULT_BLOCK_SIZE; /* queue capacity is expressed in bytes originally translate to buffer count */ capacity = ( queue_bytes + block_size - 1 ) / block_size; if ( capacity == 0 ) capacity = 1; /* actual capacity will be a power of 2 */ rc = KQueueMake ( & qf -> q, capacity ); if ( rc == 0 ) { /* capture current size if supported */ qf -> rc_from_size_on_open = KFileSize ( dst, & qf -> apparent_size ); /* starting position not used */ qf -> start_pos = 0; /* requested buffer size */ qf -> b = NULL; qf -> bsize = block_size; /* timeout */ qf -> timeout_ms = timeout_ms == 0 ? DEFAULT_TIMEOUT_MS : timeout_ms; /* finally, start the background thread */ rc = KThreadMake ( & qf -> t, KQueueFileRunWrite, qf ); if ( rc == 0 ) { * qfp = & qf -> dad; return 0; } KQueueRelease ( qf -> q ); } KFileRelease ( qf -> f ); } } free ( qf ); } } * qfp = NULL; } return rc; }
LIB_EXPORT rc_t CC MakeLogFileV ( struct KDirectory * self, struct KFile const **log_file, struct KFile *to_wrap, bool append, bool timed, const char * path, va_list args ) { rc_t rc = 0; if ( log_file == NULL ) rc = RC ( rcFS, rcFile, rcAllocating, rcParam, rcNull ); else { *log_file = NULL; if ( self == NULL ) rc = RC ( rcFS, rcFile, rcAllocating, rcSelf, rcNull ); else if ( to_wrap == NULL || path == NULL ) rc = RC ( rcFS, rcFile, rcAllocating, rcParam, rcNull ); } if ( rc == 0 ) { rc = KFileAddRef ( to_wrap ); if ( rc == 0 ) { struct Recorder * rec; rc = MakeVRecorder ( self, &rec, 4096, append, path, args ); if ( rc == 0 ) { LogFile * lf = malloc ( sizeof * lf ); if ( lf == NULL ) rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted ); else { /* now we can enter everything into the rr - struct */ lf -> wrapped = to_wrap; lf -> rec = rec; lf -> timed = timed; if ( timed ) { rc = KFileInit ( &lf -> dad, ( const union KFile_vt * ) &vtLogFile_timed, "LogFile", "logfile", true, false ); } else { rc = KFileInit ( &lf -> dad, ( const union KFile_vt * ) &vtLogFile, "LogFile", "logfile", true, false ); } if ( rc != 0 ) free( ( void * ) lf ); else *log_file = ( const KFile * ) &lf -> dad; if ( rc != 0 ) ReleaseRecorder ( rec ); } } if ( rc != 0 ) KFileRelease ( to_wrap ); } } if ( rc != 0 ) { rc = KFileAddRef( to_wrap ); if ( rc == 0 ) *log_file = to_wrap; } return rc; }
LIB_EXPORT rc_t CC KFileMakeBzip2ForWrite (struct KFile **pnew_obj, struct KFile *compfile) { rc_t rc; if ( pnew_obj == NULL || compfile == NULL ) rc= RC ( rcFS, rcFile, rcConstructing, rcParam, rcNull ); else { KBZipFile *obj; *pnew_obj = NULL; obj = (KBZipFile*)calloc(1,sizeof(KBZipFile)); if (obj == NULL) { rc = RC (rcFS, rcFile, rcConstructing, rcMemory, rcExhausted); LOGERR (klogErr, rc, "memory exhausted building bzip2 " "file object"); } else { rc = KFileInit(&obj->dad, (const KFile_vt*)&KBZipFile_vt_v1, "KBZipFile", "no-name", false, true); if (rc == 0) { bz_stream* strm; int zret; strm = &obj->strm; zret = BZ2_bzCompressInit(strm, 9, /* blockSize100k */ 1, /* verbosity */ 30); /* workFactor */ switch (zret) { case BZ_OK: obj->completed = true; rc = KFileAddRef (compfile); if (rc == 0) { obj->file = compfile; *pnew_obj = &obj->dad; return 0; } break; case BZ_CONFIG_ERROR: rc = RC (rcFS, rcFile, rcConstructing, rcLibrary, rcCorrupt); LOGERR (klogFatal, rc, "bzip2 library miscompiled"); break; case BZ_PARAM_ERROR: rc = RC (rcFS, rcFile, rcConstructing, rcParam, rcInvalid); LOGERR (klogInt, rc, "coding error bzip2 file object"); break; case BZ_MEM_ERROR: rc = RC (rcFS, rcFile, rcConstructing, rcMemory, rcExhausted); LOGERR (klogErr, rc, "memory exhausted building bzip2 " "file object"); break; default: rc = RC (rcFS, rcFile, rcConstructing, rcLibrary, rcUnexpected); LOGERR (klogFatal, rc, "bzip2 library return unexpected " "error"); break; } } } KBZipFileDestroy (obj); } return rc; }