static
rc_t GetNewPassword(const struct KFile* pwd_in, struct KFile* pwd_out, char* buf)
{
rc_t rc = KFileWrite ( pwd_out, 0, KR_PWD_PROMPT_1, string_measure(KR_PWD_PROMPT_1, NULL), NULL);
if (rc == 0)
{
char buf1[MaxPwdSize];
size_t last_pos = 0;
rc = ReadPassword(pwd_in, & last_pos, buf1, MaxPwdSize);
if (rc == 0)
{
rc = KFileWrite ( pwd_out,
string_measure(KR_PWD_PROMPT_1, NULL),
KR_PWD_PROMPT_2, string_measure(KR_PWD_PROMPT_2, NULL), NULL );
if (rc == 0)
{
char buf2[MaxPwdSize];
rc = ReadPassword(pwd_in, & last_pos, buf2, sizeof(buf2));
if (rc == 0)
{
size_t pwd_size = string_measure(buf1, NULL);
if (string_cmp(buf1, pwd_size, buf2, string_measure(buf2, NULL), MaxPwdSize) != 0)
rc = RC(rcApp, rcEncryptionKey, rcCreating, rcParam, rcInconsistent);
else
string_copy(buf, MaxPwdSize, buf1, pwd_size + 1);
}
}
}
}
return rc;
}
static bool CC write_to_file_cb( stat_row * row, void * f_data )
{
write_ctx * wctx = ( write_ctx * ) f_data;
char buffer[ 256 ];
size_t num_writ;
rc_t rc = string_printf ( buffer, sizeof buffer, &num_writ,
"%s\t%u\t%u\t%s\t%u\t%u\t%u\t%u\t%u\t%u\n",
row->spotgroup,
row->base_pos,
row->n_read,
row->dimer,
row->gc_content,
row->hp_run,
row->max_qual_value,
row->quality,
row->count,
row->mismatch_count );
if ( rc == 0 )
{
size_t f_writ;
rc = KFileWrite ( wctx->out, wctx->pos, buffer, num_writ, &f_writ );
if ( rc == 0 )
{
uint32_t percent = progressbar_percent( wctx->entries, ++wctx->lines, wctx->fract_digits );
update_progressbar( wctx->progress, wctx->fract_digits, percent );
wctx->pos += f_writ;
}
}
return ( rc == 0 );
}
static rc_t compress_loop( const KFile *src, KFile *dst )
{
rc_t rc = 0;
uint64_t pos = 0;
size_t bsize = 4096;
size_t num_read = 1;
char * buffer = malloc( bsize );
if ( buffer == NULL )
return RC( rcExe, rcFile, rcPacking, rcMemory, rcExhausted );
while ( rc == 0 && num_read > 0 )
{
rc = KFileRead ( src, pos, buffer, bsize, &num_read );
DISP_RC( rc, "KFileRead() failed" );
if ( rc == 0 && num_read > 0 )
{
size_t num_writ;
rc = KFileWrite ( dst, pos, buffer, num_read, &num_writ );
DISP_RC( rc, "KFilewrite() failed" );
pos += num_read;
}
}
OUTMSG (( "%lu bytes copied\n", pos ));
free( buffer );
return rc;
}
static
rc_t KBufWriteFileFlush ( KBufWriteFile *self, uint64_t dpos )
{
rc_t rc;
size_t total_writ, partial;
/* flush buffer */
for ( rc = 0, total_writ = 0; total_writ < self -> num_valid; total_writ += partial )
{
rc = KFileWrite ( self -> f, dpos + total_writ,
& self -> buff [ total_writ ],
self -> num_valid - total_writ, & partial );
if ( rc != 0 )
break;
if ( partial == 0 )
{
rc = RC ( rcFS, rcFile, rcWriting, rcTransfer, rcIncomplete );
break;
}
}
if ( rc == 0 )
self -> num_valid = 0;
return rc;
}
static
rc_t CC KBufWriteFileSetSize ( KBufWriteFile *self, uint64_t size )
{
if ( self -> pos + self -> num_valid > size )
{
if ( self -> pos < size )
{
size_t total, num_writ, to_write = ( size_t ) ( size - self -> pos );
for ( total = 0; total < to_write; total += num_writ )
{
rc_t rc = KFileWrite ( self -> f, self -> pos + total,
& self -> buff [ total ], to_write - total, & num_writ );
if ( rc != 0 )
return rc;
if ( num_writ == 0 )
return RC ( rcFS, rcFile, rcReading, rcTransfer, rcIncomplete );
}
}
self -> pos = 0;
self -> num_valid = 0;
}
return KFileSetSize ( self -> f, size );
}
static rc_t LogFileWrite_timed ( LogFile *self, uint64_t pos, const void *buffer, size_t size, size_t *num_writ )
{
KTimeMs_t ms = KTimeMsStamp ();
rc_t rc = KFileWrite ( self -> wrapped, pos, buffer, size, num_writ );
ms = KTimeMsStamp () - ms;
WriteToRecorder ( self -> rec, "W\t%lu\t%lu\t%lu\t%lu\n", pos, size, *num_writ, ms );
return rc;
}
static
rc_t CC KU64Index_WriteFunc( void *param, const void *buffer, size_t size, size_t *num_writ )
{
KU64Index_PersistData* pd = param;
rc_t rc = KFileWrite(pd->file, pd->pos, buffer, size, num_writ);
pd->pos += *num_writ;
return rc;
}
static
rc_t copy_file (const KFile * fin, KFile *fout)
{
rc_t rc;
uint8_t buff [64 * 1024];
size_t num_read;
uint64_t inpos;
uint64_t outpos;
assert (fin != NULL);
assert (fout != NULL);
inpos = 0;
outpos = 0;
do
{
rc = KFileRead (fin, inpos, buff, sizeof (buff), &num_read);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to read from directory structure in creating archive at $(P)",
PLOG_U64(P), inpos));
break;
}
else if (num_read > 0)
{
size_t to_write;
inpos += (uint64_t)num_read;
STSMSG (2, ("Read %zu bytes to %lu", num_read, inpos));
/* PLOGMSG (klogDebug10, "Read $(B) Bytes for $(T)", PLOG_2(PLOG_U64(B),PLOG_U64(T)), num_read, inpos); */
to_write = num_read;
while (to_write > 0)
{
size_t num_writ;
rc = KFileWrite (fout, outpos, buff, num_read, &num_writ);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to write to archive in creating archive at $(P)",
PLOG_U64(P), outpos));
break;
}
outpos += num_writ;
/* PLOGMSG (klogDebug10, "Wrote $(B) Bytes for $(T)", PLOG_2(PLOG_U64(B),PLOG_U64(T)), num_writ, outpos); */
to_write -= num_writ;
}
}
/* else */
/* PLOGMSG (klogDebug10, "Read $(B) Bytes for $(T)", PLOG_2(PLOG_U64(B),PLOG_U64(T)), num_read, inpos); */
if (rc != 0)
break;
} while (num_read != 0);
return rc;
}
static
rc_t copy_file (const KFile * fin, KFile *fout)
{
rc_t rc;
uint8_t buff [64 * 1024];
size_t num_read;
uint64_t inpos;
uint64_t outpos;
uint64_t fsize;
assert (fin != NULL);
assert (fout != NULL);
inpos = 0;
outpos = 0;
rc = KFileSize (fin, &fsize);
if (rc != 0)
return rc;
do
{
rc = KFileRead (fin, inpos, buff, sizeof (buff), &num_read);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to read from directory structure in creating archive at $(P)",
PLOG_U64(P), inpos));
break;
}
else if (num_read > 0)
{
size_t to_write;
inpos += (uint64_t)num_read;
to_write = num_read;
while (to_write > 0)
{
size_t num_writ;
rc = KFileWrite (fout, outpos, buff, num_read, &num_writ);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to write to archive in creating archive at $(P)",
PLOG_U64(P), outpos));
break;
}
outpos += num_writ;
to_write -= num_writ;
}
}
if (rc != 0)
break;
} while (num_read != 0);
return rc;
}
rc_t CopierDoOne (Copier * self)
{
rc_t rc = 0;
const Buffer * b;
LOGMSG (klogDebug10, "CopierDoOne");
rc = Quitting();
if (rc == 0)
{
LOGMSG (klogDebug10, "call BufferQPopBuffer");
rc = BufferQPopBuffer (self->q, &b, NULL);
if (rc == 0)
{
size_t w;
size_t z;
LOGMSG (klogDebug10, "call BufferContentGetSize");
z = BufferContentGetSize (b);
rc = KFileWrite (self->f, self->o, b, z, &w);
self->o += w;
if (w != z)
rc = RC (rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete);
else
rc = BufferRelease (b);
}
/* ow this is ugly! */
/* is the rc a "exhausted" on a timeout? */
else if ((GetRCObject(rc) == rcTimeout) && (GetRCState(rc) == rcExhausted))
{
rc = 0;
LOGMSG (klogDebug10, "CopierDoOne timeout");
/* if so is the queue also sealed? */
if (BufferQSealed (self->q) == true)
{
LOGMSG (klogDebug10, "CopierDoOne sealed");
/* if both then we are done and so signal */
rc = KFileRelease (self->f);
PLOGMSG (klogDebug10, "CopierDoOne back from KFileRelease $(rc)",PLOG_U32(rc),rc);
if (rc == 0)
{
self->f = NULL;
rc = BufferQRelease (self->q);
if (rc == 0)
{
self->q = NULL;
rc = RC (rcExe, rcNoTarg, rcCopying, rcNoTarg, rcDone );
}
}
}
}
else
LOGMSG (klogDebug10, "CopierDoOne pop failure");
}
else
LOGMSG (klogDebug10, "CopierDoOne: quitting");
return rc;
}
/* ----------------------------------------------------------------------
* Write
* write file at known position
*
* "pos" [ IN ] - starting position within file
*
* "buffer" [ IN ] and "size" [ IN ] - data to be written
*
* "num_writ" [ OUT, NULL OKAY ] - optional return parameter
* giving number of bytes actually written
*
*/
static
rc_t CC CCFileWrite (CCFile *self, uint64_t pos,
const void *buffer, size_t bsize,
size_t *num_writ)
{
rc_t rc;
rc = KFileWrite (self->original, pos, buffer, bsize, num_writ);
if (*self->prc == 0)
*self->prc = rc;
return rc;
}
/* ======================================================================
* subroutine functions for KBZipFile methods
*/
static
rc_t KBZipFileWriteInt (KBZipFile *self,
int action,
size_t *pnumwrit)
{
bz_stream *strm;
unsigned avail_in;
int ret;
rc_t rc = 0;
assert (self);
assert (pnumwrit);
*pnumwrit = 0;
strm = &self->strm;
avail_in = strm->avail_in;
ret = 0;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been read in */
do
{
uint32_t num_comp;
size_t written;
int zret;
/* compress one internal buffers worth */
strm->next_out = self->buff;
strm->avail_out = sizeof (self->buff);
zret = BZ2_bzCompress (strm, action); /* no bad return value */
/* state not clobbered */
assert(zret == BZ_OK || zret == BZ_RUN_OK
|| zret == BZ_FINISH_OK || zret == BZ_STREAM_END);
/* compression used the sizeof of the outbuffer - the amount
* it says it didn't use */
num_comp = sizeof(self->buff) - strm->avail_out;
rc = KFileWrite (self->file, self->filePosition, self->buff, num_comp, &written);
self->filePosition += written;
*pnumwrit = avail_in - strm->avail_in;
} while (strm->avail_out == 0);
assert (strm->avail_in == 0); /* all input will be used */
return rc;
}
rc_t write_output_file( KDirectory *dir, statistic * data,
const char * path, uint64_t * written )
{
write_ctx wctx;
rc_t rc;
if ( written != NULL )
{
*written = 0;
}
wctx.out = NULL;
wctx.pos = 0;
wctx.lines = 0;
rc = KDirectoryCreateFile ( dir, &wctx.out, false, 0664, kcmInit, path );
if ( rc != 0 )
LogErr( klogInt, rc, "KDirectoryCreateFile() failed\n" );
else
{
char buffer[ 256 ];
size_t num_writ;
rc = string_printf ( buffer, sizeof buffer, &num_writ,
"SPOTGROUP\tCYCLE\tNRead\tDIMER\tGC_CONTENT\tHP_RUN\tMaxQ\tQuality\tTOTAL\tMISMATCH\n" );
if ( rc == 0 )
{
size_t f_writ;
rc = KFileWrite ( wctx.out, wctx.pos, buffer, num_writ, &f_writ );
if ( rc == 0 )
{
if ( written != NULL ) *written = f_writ;
wctx.pos += f_writ;
make_progressbar( &wctx.progress );
wctx.entries = data->entries;
wctx.fract_digits = progressbar_calc_fract_digits( wctx.entries );
foreach_statistic( data, write_to_file_cb, &wctx );
destroy_progressbar( wctx.progress );
OUTMSG(( "\n" ));
KFileRelease ( wctx.out );
if ( written != NULL )
{
*written = wctx.lines;
}
}
}
}
return rc;
}
static
rc_t copy_file_skey_md5_kludge (const KFile * fin, KFile *fout)
{
/* size of HEX digest plus spzce plus * */
#define READ_SIZE 34
static const uint8_t skey[] = "skey\n";
uint8_t buff [256];
uint64_t tot_read, tot_writ;
size_t num_read, num_writ;
rc_t rc;
assert (fin);
assert (fout);
for (tot_read = 0 ; tot_read < READ_SIZE; tot_read += num_read)
{
rc = KFileRead (fin, tot_read, buff, READ_SIZE - tot_read, &num_read);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to read from directory structure in creating archive at $(P)",
PLOG_U64(P), tot_read));
break;
}
if (num_read == 0)
break;
}
if (rc == 0)
{
if (tot_read == READ_SIZE)
{
memcpy (buff + READ_SIZE, skey, sizeof (skey));
tot_read += sizeof (skey) - 1;
}
for (tot_writ = 0; tot_writ < tot_read; tot_writ += num_writ)
{
rc = KFileWrite (fout, tot_writ, buff + tot_writ,
(uint32_t)(tot_read - tot_writ), &num_writ);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to write to archive in creating archive at $(P)",
PLOG_U64(P), num_writ));
break;
}
}
}
return rc;
}
static
rc_t GetPassword(const struct KFile* pwd_in, struct KFile* pwd_out, char* buf)
{
rc_t rc = KFileWrite ( pwd_out, 0, KR_PWD_PROMPT_1, string_measure(KR_PWD_PROMPT_1, NULL), NULL);
if (rc == 0)
{
char buf1[MaxPwdSize];
size_t last_pos = 0;
rc = ReadPassword(pwd_in, & last_pos, buf1, MaxPwdSize);
if (rc == 0)
string_copy(buf, MaxPwdSize, buf1, string_measure(buf1, NULL) + 1);
}
return rc;
}
static rc_t write_header_line( lane * l, const char * line, size_t len )
{
size_t num_writ_file;
rc_t rc = KFileWrite ( l->reads, l->write_pos, line, len, &num_writ_file );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot write output for header" );
}
else
{
l->write_pos += num_writ_file;
}
return rc;
}
static rc_t CC stdout_redir_callback ( void* self, const char* buffer, size_t bufsize, size_t* num_writ )
{
rc_t rc = 0;
stdout_redir * writer = ( stdout_redir * ) self;
do {
rc = KFileWrite( writer->kfile, writer->pos, buffer, bufsize, num_writ );
if ( rc == 0 )
{
buffer += *num_writ;
bufsize -= *num_writ;
writer->pos += *num_writ;
}
} while ( rc == 0 && bufsize > 0 );
return rc;
}
/* Write
* writes block to idx2 at end of file
*
* "pos" [ OUT ] - location at which block was written
*
* "buffer" [ IN ] and "bytes" [ IN ] - describes data buffer
*
* return values:
*/
rc_t KColumnIdx2Write ( KColumnIdx2 *self,
uint64_t *pos, const void *buffer, size_t bytes )
{
rc_t rc;
size_t num_writ;
* pos = self -> eof;
rc = KFileWrite ( self -> f,
self -> eof, buffer, bytes, & num_writ );
if ( rc == 0 )
{
if ( num_writ != bytes )
rc = RC ( rcDB, rcIndex, rcWriting, rcTransfer, rcIncomplete );
}
return rc;
}
static rc_t CC fileWriter
(void* data, const char* buffer, size_t bytes, size_t* num_writ)
{
rc_t rc = 0;
SFile* self = (SFile*)data;
size_t dummy;
if (num_writ == NULL)
{ num_writ = &dummy; }
*num_writ = 0;
assert(self);
if (self->magic != MAGIC) {
return rc;
}
rc = KFileWrite(self->f, self->pos, buffer, bytes, num_writ);
self->pos += *num_writ;
return rc;
}
/* ----------------------------------------------------------------------
* Write
* write file at known position
*
* "pos" [ IN ] - starting position within file
*
* "buffer" [ IN ] and "size" [ IN ] - data to be written
*
* "num_writ" [ OUT, NULL OKAY ] - optional return parameter
* giving number of bytes actually written
*
* Unsupported as we now treat archives as READ ONLY
*/
static
rc_t CC KSubFileWrite (KSubFile *self, uint64_t pos,
const void *buffer, size_t bsize,
size_t *num_writ)
{
size_t to_write;
assert (self != NULL);
assert (buffer != NULL);
assert (num_writ != NULL);
*num_writ = 0;
if (pos >= ( uint64_t ) self->size)
return 0;
to_write = bsize;
if ((pos + bsize) > self->size)
to_write = ( uint64_t ) self -> size - pos;
return KFileWrite (self->original, self->start + pos, buffer, to_write, num_writ);
}
/* ----------------------------------------------------------------------
* Write
* write file at known position
*
* "pos" [ IN ] - starting position within file
*
* "buffer" [ IN ] and "size" [ IN ] - data to be written
*
* "num_writ" [ OUT, NULL OKAY ] - optional return parameter
* giving number of bytes actually written
*
* Unsupported as we now treat archives as READ ONLY
*/
static
rc_t CC KCounterFileWrite (KCounterFile *self, uint64_t pos,
const void *buffer, size_t bsize,
size_t *num_writ)
{
uint64_t max_position;
uint64_t temp_max_position;
rc_t rc;
/* -----
* self and buffer were validated as not NULL before calling here
*
* So get the KTTOCNode type: chunked files and contiguous files
* are read differently.
*/
assert (self != NULL);
assert (self->original != NULL);
assert (buffer != NULL);
assert (num_writ != NULL);
assert (bsize != 0);
max_position = self->max_position;
if ((self->dad.read_enabled) && (pos > max_position) && (! self->size_allowed))
{
rc = KCounterFileSeek (self, pos);
if (rc != 0)
return rc;
}
rc = KFileWrite (self->original, pos, buffer, bsize, num_writ);
temp_max_position = pos + *num_writ;
if (temp_max_position > max_position)
{
*self->bytecounter = self->max_position = temp_max_position;
}
return rc;
}
LIB_EXPORT void CC XMLLogger_Release(const XMLLogger* cself)
{
if( cself != NULL ) {
XMLLogger* self = (XMLLogger*)cself;
KLogFmtHandlerSet(self->log.fmt.formatter, 0, self->log.fmt.data);
KLogLibFmtHandlerSet(self->loglib.fmt.formatter, 0, self->loglib.fmt.data);
KLogHandlerSet(self->log.wrt.writer, self->log.wrt.data);
KLogLibHandlerSet(self->loglib.wrt.writer, self->loglib.wrt.data);
KStsFmtHandlerSet(self->sts.fmt.formatter, 0, self->sts.fmt.data);
KStsLibFmtHandlerSet(self->stslib.fmt.formatter, 0, self->stslib.fmt.data);
KStsHandlerSet(self->sts.wrt.writer, self->sts.wrt.data);
KStsLibHandlerSet(self->stslib.wrt.writer, self->stslib.wrt.data);
/* make log valid XML */
if( self->file.file != NULL ) {
if( self->file.pos > 0 ) {
KFileWrite(self->file.file, self->file.pos, "</Log>\n", 7, NULL);
}
KFileRelease(self->file.file);
}
free(self);
}
}
rc_t KU64IndexPersist_v3(KU64Index_v3* self, bool proj, KDirectory *dir, const char *path, bool use_md5)
{
KU64Index_PersistData pd;
char tmpname[256];
char tmpmd5name[256];
char md5path[256];
self->rc = 0;
memset(&pd, 0, sizeof(KU64Index_PersistData));
self->rc = KDirectoryResolvePath(dir, false, tmpname, sizeof(tmpname), "%s.tmp", path);
if( self->rc == 0 ) {
self->rc = KDirectoryCreateFile(dir, &pd.file, true, 0664, kcmInit, "%s", tmpname);
if(use_md5 && self->rc == 0 ) {
KMD5SumFmt *km = NULL;
size_t tmplen = snprintf(tmpmd5name, sizeof(tmpmd5name), "%s.md5", tmpname);
KFile* kf = NULL;
if( tmplen >= sizeof(tmpmd5name) ) {
self->rc = RC(rcDB, rcIndex, rcPersisting, rcName, rcExcessive);
} else {
tmplen = snprintf(md5path, sizeof(md5path), "%s.md5", path);
if( tmplen >= sizeof(md5path) ) {
self->rc = RC(rcDB, rcIndex, rcPersisting, rcName, rcExcessive);
} else {
self->rc = KDirectoryCreateFile(dir, &kf, true, 0664, kcmInit, "%s", tmpmd5name);
if( self->rc == 0 ) {
self->rc = KMD5SumFmtMakeUpdate(&km, kf);
if( self->rc == 0 ) {
KMD5File * k5f;
kf = NULL;
self->rc = KMD5FileMakeWrite(&k5f, pd.file, km, path);
if( self->rc == 0 ) {
pd.file_md5 = k5f;
pd.file = KMD5FileToKFile(k5f);
}
/* release pass or fail */
KMD5SumFmtRelease(km);
} else {
KFileRelease ( kf );
}
} else {
KFileRelease ( kf );
}
}
}
if( self->rc != 0 ) {
KFileRelease(pd.file);
}
}
if( self->rc == 0 ) {
struct KIndexFileHeader_v3 head;
size_t writ = 0;
KDBHdrInit(&head.h, 3);
head.index_type = kitU64;
self->rc = KFileWrite(pd.file, pd.pos, &head, sizeof(struct KIndexFileHeader_v3), &writ);
if( self->rc == 0 ) {
pd.pos += writ;
if( use_md5 ) {
KMD5FileBeginTransaction(pd.file_md5);
}
self->rc = BSTreePersist(&self->tree, NULL, KU64Index_WriteFunc, &pd, KU64Index_AuxFunc, &pd);
}
KFileRelease(pd.file);
pd.file = NULL;
}
}
if( self->rc == 0 ) {
self->rc = KDirectoryRename(dir, false, tmpname, path);
if( self->rc == 0 ) {
if ( use_md5 ) {
self->rc = KDirectoryRename(dir, false, tmpmd5name, md5path);
}
if( self->rc == 0 ) {
/* done */
return 0;
}
}
}
/* whack temporary file */
KDirectoryRemove(dir, false, "%s", tmpname);
if( use_md5 ) {
KDirectoryRemove(dir, false, "%s", tmpmd5name);
}
return self->rc;
}
static
rc_t run (void)
{
VFSManager * manager;
rc_t rc = 0, orc = 0;
STSMSG (1, ("Make VFSManager"));
rc = VFSManagerMake (&manager);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0)
{
#if 1
static const char name[] = "test-kfs-manager-data-file";
#else
static const char name[] =
"ncbi-kfs:test-kfs-manager-data-file?enc&pwfile=password";
#endif
VPath * path;
STSMSG (1, ("Make test VPath file '%s'",name));
rc = VPathMake (&path, name);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0)
{
KFile * file;
STSMSG (1, ("Open File for write using manager and path"));
rc = VFSManagerCreateFile (manager, &file, false, 0666, kcmCreate,
path);
STSMSG (2, ("rc %R", rc, rc));
if (rc == 0)
{
char buff[4096];
size_t ix;
size_t num_writ;
uint64_t tot_writ = 0;
for (ix = 0; ix < sizeof buff; ++ix)
buff[ix] = 'A' + (ix%26);
STSMSG (1, ("writing to file"));
for (ix = 0; ix < 32; ++ix)
{
rc = KFileWrite (file, tot_writ, buff, sizeof buff, &num_writ);
if (rc == 0)
tot_writ += num_writ;
};
STSMSG (1, ("Release file - it should whack"));
orc = KFileRelease (file);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
#if 1
STSMSG (1, ("Remove file"));
orc = VFSManagerRemove (manager, true, path);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
#endif
}
STSMSG (1, ("Release VPath - it should Whack"));
orc = VPathRelease (path);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
}
STSMSG (1, ("Release VFSManager - it should Whack"));
orc = VFSManagerRelease (manager);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
}
return rc;
}
static
rc_t CC KBufWriteFileWrite ( KBufWriteFile *self, uint64_t pos,
const void *buffer, size_t size, size_t *num_writ )
{
rc_t rc;
size_t total, partial, trailing;
const uint8_t *bbuff = buffer;
uint64_t send = pos + size;
/* case 1: empty write */
if ( size == 0 )
{
* num_writ = 0;
return 0;
}
/* perform write */
for ( rc = 0, total = trailing = 0; total < size; total += partial )
{
size_t boff = total - trailing;
uint64_t dpos = self -> pos;
uint64_t dend = self -> pos + self -> num_valid;
uint64_t dlim = self -> pos + self -> bsize;
uint64_t spos = pos + boff;
/* case 2: left within buffer */
if ( dpos <= spos && spos <= dend && spos < dlim )
{
partial = ( size_t ) ( ( dlim < send ? dlim : send ) - spos );
memcpy ( & self -> buff [ spos - dpos ], & bbuff [ boff ], partial );
if ( spos + partial > dend )
{
dend = spos + partial;
self -> num_valid = ( size_t ) ( dend - dpos );
}
}
/* case 3: right within buffer */
else if ( spos < dpos && dpos < send && send <= dlim )
{
partial = ( size_t ) ( send - dpos );
memcpy ( & self -> buff [ spos - dpos ], & bbuff [ boff ], partial );
if ( send > dend )
{
dend = send;
self -> num_valid = ( size_t ) ( dend - dpos );
}
/* adjust other components for trailing copy */
assert ( trailing == 0 );
trailing = partial;
send = dpos;
}
/* case 4: no intersection */
else if ( send <= dpos || dend <= spos )
{
/* flush buffer as-is */
rc = KBufWriteFileFlush ( self, dpos );
if ( rc != 0 )
break;
/* empty buffer centered on this write */
self -> pos = spos;
/* if write is >= buffer size, just write it directy */
partial = size - total;
if ( partial < self -> bsize )
{
memcpy ( self -> buff, & bbuff [ boff ], partial );
self -> num_valid = partial;
}
else
{
rc = KFileWrite ( self -> f, spos, & bbuff [ boff ], partial, & partial );
if ( rc != 0 )
break;
if ( partial == 0 )
{
rc = RC ( rcFS, rcFile, rcWriting, rcTransfer, rcIncomplete );
break;
}
}
}
/* case 5: completely engulfs */
else
{
assert ( spos < dpos && dlim < send );
rc = KFileWrite ( self -> f, spos, & bbuff [ boff ], ( size_t ) ( send - spos ), & partial );
if ( rc != 0 )
break;
if ( partial == 0 )
{
rc = RC ( rcFS, rcFile, rcWriting, rcTransfer, rcIncomplete );
break;
}
/* anything in the buffer is invalid now */
self -> num_valid = 0;
}
}
* num_writ = total;
return rc;
}
static rc_t CC KHttpUndyingFileWrite ( KHttpUndyingFile *self, uint64_t pos,
const void *buffer, size_t size, size_t *num_writ )
{
return KFileWrite(GetUnderlyingFile(self), pos, buffer, size, num_writ);
}
LIB_EXPORT rc_t CC XMLLogger_Make2(const XMLLogger** self, KDirectory* dir, const char* logpath, const int fd)
{
rc_t rc = 0;
XMLLogger* obj;
KDirectory* my_dir = NULL;
const uint32_t lopt = klogFmtTimestamp | klogFmtSeverity |
klogFmtMessage | klogFmtAppName | klogFmtAppVersion | klogFmtReasonShort;
const uint32_t sopt = kstsFmtTimestamp | kstsFmtMessage | kstsFmtAppName | kstsFmtAppVersion;
KLogFmtFlagsSet(lopt);
KLogLibFmtFlagsSet(lopt);
KStsFmtFlagsSet(sopt);
KStsLibFmtFlagsSet(sopt);
obj = calloc(1, sizeof(*obj));
if( obj == NULL ) {
rc = RC(rcApp, rcLog, rcAllocating, rcMemory, rcExhausted);
} else if( fd < 0 && dir == NULL && (rc = KDirectoryNativeDir(&my_dir)) != 0 ) {
} else if( fd >= 0 && (rc = KFileMakeFDFileWrite(&obj->file.file, false, fd)) != 0 ) {
} else if( logpath != NULL && fd < 0 && (rc = KDirectoryCreateFile(dir ? dir : my_dir, &obj->file.file, false, 0644, kcmInit, "%s", logpath)) != 0 ) {
} else {
obj->file.pos = 0;
obj->log.file = &obj->file;
obj->log.fmt.formatter = KLogFmtWriterGet();
obj->log.fmt.data = KLogFmtDataGet();
obj->log.wrt.writer = KLogWriterGet();
obj->log.wrt.data = KLogDataGet();
obj->loglib.file = &obj->file;
obj->loglib.fmt.formatter = KLogLibFmtWriterGet();
obj->loglib.fmt.data = KLogLibFmtDataGet();
obj->loglib.wrt.writer = KLogLibWriterGet();
obj->loglib.wrt.data = KLogLibDataGet();
obj->sts.file = &obj->file;
obj->sts.fmt.formatter = KStsFmtWriterGet();
obj->sts.fmt.data = KStsFmtDataGet();
obj->sts.wrt.writer = KStsWriterGet();
obj->sts.wrt.data = KStsDataGet();
obj->stslib.file = &obj->file;
obj->stslib.fmt.formatter = KStsLibFmtWriterGet();
obj->stslib.fmt.data = KStsLibFmtDataGet();
obj->stslib.wrt.writer = KStsLibWriterGet();
obj->stslib.wrt.data = KStsLibDataGet();
if( (rc = KLogFmtHandlerSet(LoaderXMLFormatter, lopt, &obj->log)) == 0 &&
(rc = KLogLibFmtHandlerSet(LoaderXMLFormatter, lopt, &obj->loglib)) == 0 &&
(rc = KStsFmtHandlerSet(LoaderXMLFormatter, sopt, &obj->sts)) == 0 &&
(rc = KStsLibFmtHandlerSet(LoaderXMLFormatter, sopt, &obj->stslib)) == 0 ) {
/* make log valid XML */
if( obj->file.file != NULL ) {
size_t num_writ = 0;
rc = KFileWrite(obj->file.file, obj->file.pos, "<Log>\n", 6, &num_writ);
obj->file.pos += num_writ;
}
}
}
KDirectoryRelease(my_dir);
if( rc == 0 ) {
*self = obj;
if( fd >= 0 ) {
DBG(("XML Log file set to handle %d\n", fd));
} else if( logpath != NULL) {
DBG(("XML Log file set to %s\n", logpath));
}
} else {
XMLLogger_Release(obj);
*self = NULL;
}
return rc;
}
static rc_t cg_dump_write_spot( cg_dump_opts * opts, cg_dump_ctx * cg_ctx, uint64_t row_id, lane * l )
{
uint32_t elem_bits, boff, read_len;
const char * read;
rc_t rc = VCursorCellDataDirect( cg_ctx->seq_cur, row_id, cg_ctx->seq_read_idx, &elem_bits, (const void**)&read, &boff, &read_len );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "cannot read READ in row #$(row_id)", "row_id=%lu", row_id ) );
}
else
{
uint32_t qual_len;
const char * qual;
rc = VCursorCellDataDirect( cg_ctx->seq_cur, row_id, cg_ctx->seq_qual_idx, &elem_bits, (const void**)&qual, &boff, &qual_len );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "cannot read QUALITY in row #$(row_id)", "row_id=%lu", row_id ) );
}
else
{
if ( ( read_len != 70 ) && ( qual_len != 70 ) )
{
rc = RC( rcExe, rcDatabase, rcReading, rcRange, rcInvalid );
(void)LOGERR( klogErr, rc, "len of read/quality columns do not match cg-length of 2 x 35" );
}
else
{
char buffer[ 1024 ];
size_t num_writ_buf;
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf,
"%lu\t0\t%.35s\t%.35s\n%lu\t1\t%.35s\t%.35s\n",
row_id, read, qual, row_id, &(read[35]), &(qual[35]) );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "cannot generate output in row #$(row_id)", "row_id=%lu", row_id ) );
}
else
{
if ( opts->comp != oc_null )
{
if ( l->spot_count >= opts->cutoff )
{
KFileRelease( l->reads );
l->chunk++;
l->spot_count = 0;
l->write_pos = 0;
rc = make_read_file( opts, cg_ctx->lookup, cg_ctx->out_dir, l );
}
if ( rc == 0 )
{
size_t num_writ_file;
rc = KFileWrite ( l->reads, l->write_pos, buffer, num_writ_buf, &num_writ_file );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "cannot write output in row #$(row_id)", "row_id=%lu", row_id ) );
}
else
{
l->write_pos += num_writ_file;
l->spot_count ++;
}
}
}
}
}
}
}
return rc;
}
static rc_t CreateConfig(char* argv0) {
const KFile* std_in = NULL;
KDirectory* native = NULL;
KDirectory* dir = NULL;
rc_t rc = 0;
char* location = NULL;
char* mod = NULL;
char* wmod = NULL;
char* refseq = NULL;
if (rc == 0) {
rc = KDirectoryNativeDir(&native);
}
if (rc == 0) {
const char* def = NULL;
char cwd[PATH_MAX + 1] = "";
const char* home = getenv("HOME");
if (home)
{ def = home; }
else {
rc = VPathGetCWD(cwd, sizeof cwd);
if (rc == 0 && cwd[0])
{ def = cwd; }
else
{ def = "."; }
}
while (rc == 0) {
char buffer[PATH_MAX + 1];
rc = In("Specify configuration files directory", def, &location);
if (rc == 0) {
rc = KDirectoryOpenDirUpdate(native, &dir, false, location);
if (rc == 0) {
rc = KDirectoryVVisit
(dir, false, scan_config_dir, buffer, ".", NULL);
if (rc != 0) {
if (rc ==
RC(rcExe, rcDirectory, rcListing, rcFile, rcExists)
&& buffer[0])
{
PLOGERR(klogErr, (klogErr, rc,
"Configuration file found: $(dir)/$(name)",
"dir=%s,name=%s", location, buffer));
rc = 0;
buffer[0] = '\0';
continue;
}
else {
PLOGERR(klogErr, (klogErr, rc, "$(dir)/$(name)",
"dir=%s,name=%s", location, buffer));
}
}
break;
}
else if (GetRCObject(rc) == rcPath &&
(GetRCState(rc) == rcIncorrect || GetRCState(rc) == rcNotFound))
{
PLOGERR(klogErr,
(klogErr, rc, "$(path)", "path=%s", location));
rc = 0;
}
else { DISP_RC(rc, location); }
}
}
}
while (rc == 0) {
const KDirectory* dir = NULL;
rc = In("Specify refseq installation directory", NULL, &refseq);
if (rc != 0)
{ break; }
rc = KDirectoryOpenDirRead(native, &dir, false, refseq);
if (rc == 0) {
RELEASE(KDirectory, dir);
break;
}
else if (GetRCObject(rc) == rcPath
&& GetRCState(rc) == rcIncorrect)
{
PLOGERR(klogErr,
(klogErr, rc, "$(path)", "path=%s", refseq));
rc = 0;
}
DISP_RC(rc, refseq);
}
if (rc == 0) {
char buffer[512];
const char path[] = "vdb-config.kfg";
uint64_t pos = 0;
KFile* f = NULL;
rc = KDirectoryCreateFile(dir, &f, false, 0664, kcmCreate, path);
DISP_RC(rc, path);
if (rc == 0) {
int n = snprintf(buffer, sizeof buffer,
"refseq/servers = \"%s\"\n", refseq);
if (n >= sizeof buffer) {
rc = RC(rcExe, rcFile, rcWriting, rcBuffer, rcInsufficient);
}
else {
size_t num_writ = 0;
rc = KFileWrite(f, pos, buffer, strlen(buffer), &num_writ);
pos += num_writ;
}
}
if (rc == 0) {
const char buffer[] = "refseq/volumes = \".\"\n";
size_t num_writ = 0;
rc = KFileWrite(f, pos, buffer, strlen(buffer), &num_writ);
pos += num_writ;
}
if (rc == 0 && mod && mod[0]) {
int n = snprintf(buffer, sizeof buffer,
"vdb/module/paths = \"%s\"\n", mod);
if (n >= sizeof buffer) {
rc = RC(rcExe, rcFile, rcWriting, rcBuffer, rcInsufficient);
}
else {
size_t num_writ = 0;
rc = KFileWrite(f, pos, buffer, strlen(buffer), &num_writ);
pos += num_writ;
}
}
if (rc == 0 && wmod && wmod[0]) {
int n = snprintf(buffer, sizeof buffer,
"vdb/wmodule/paths = \"%s\"\n", wmod);
if (n >= sizeof buffer) {
rc = RC(rcExe, rcFile, rcWriting, rcBuffer, rcInsufficient);
}
else {
size_t num_writ = 0;
rc = KFileWrite(f, pos, buffer, strlen(buffer), &num_writ);
pos += num_writ;
}
}
RELEASE(KFile, f);
}
free(mod);
free(wmod);
free(refseq);
free(location);
RELEASE(KDirectory, dir);
RELEASE(KDirectory, native);
RELEASE(KFile, std_in);
DestroyStdin();
return rc;
}
static
rc_t CC LoaderXMLFormatter( void* data, KWrtHandler* writer,
size_t argc, const wrt_nvp_t args[],
size_t envc, const wrt_nvp_t envs[])
{
rc_t rc = 0;
size_t i, remaining, num_writ = 0;
XMLFormatterData* self = (XMLFormatterData*)data;
char buffer[4096];
const char* severity, *msg_val = NULL;
bool severity_std = false;
char* pbuffer;
const char* const tag_nvp_name = "severity";
const wrt_nvp_t* msg_nvp = NULL;
static const char* tags[] = {
"fatal",
"system",
"internal",
"error",
"warning",
"info"
};
msg_nvp = wrt_nvp_find(envc, envs, "message");
if( msg_nvp != NULL ) {
msg_val = msg_nvp->value;
}
severity = wrt_nvp_find_value(argc, args, tag_nvp_name);
if( severity == NULL ) {
severity = wrt_nvp_find_value(envc, envs, tag_nvp_name);
if( severity != NULL ) {
severity_std = true;
/* translate std severity name to full name */
for(i = 0; i < sizeof(tags)/sizeof(tags[0]); i++) {
if( strncmp(severity, tags[i], strlen(severity)) == 0 ) {
severity = tags[i];
break;
}
}
}
}
if( severity == NULL ) {
severity = "status";
}
#define FIX_UP() if(rc != 0){break;} remaining -= num_writ; pbuffer += num_writ
pbuffer = buffer;
remaining = sizeof(buffer);
do {
size_t k, pq = envc;
const wrt_nvp_t* p = envs;
const char* subst = NULL;
rc = LogInsertSpace("<", pbuffer, remaining, &num_writ);
FIX_UP();
rc = LogInsertSpace(severity, pbuffer, remaining, &num_writ);
FIX_UP();
/* print env first and than args! */
for(k = 0; rc == 0 && k < 2; k++) {
for(i = 0; i < pq; i++ ) {
if( strcmp(p[i].name, tag_nvp_name) == 0 ) {
continue;
}
if( p == args ) {
if( i == 0 && msg_nvp != NULL ) {
/* grab args attr buffer start */
subst = pbuffer;
}
if( severity_std ) {
/* allow only specific attributes from message into xml log
for LOG calls with standard severity */
int x, yes = 0;
static const char* allowed_attr[] = {
"file", "line", "offset",
"spot", "spot_name", "spotname"
};
for(x = 0; x < sizeof(allowed_attr)/sizeof(allowed_attr[0]); x++) {
if( strcasecmp(p[i].name, allowed_attr[x]) == 0 ) {
yes = 1;
break;
}
}
if( !yes ) {
continue;
}
}
}
rc = LogInsertSpace(" ", pbuffer, remaining, &num_writ);
FIX_UP();
rc = XMLLogger_Encode(p[i].name, pbuffer, remaining, &num_writ);
FIX_UP();
rc = LogInsertSpace("=\"", pbuffer, remaining, &num_writ);
FIX_UP();
rc = XMLLogger_Encode(p[i].value, pbuffer, remaining, &num_writ);
FIX_UP();
rc = LogInsertSpace("\"", pbuffer, remaining, &num_writ);
FIX_UP();
}
p = args;
pq = argc;
}
if( subst != NULL && subst[0] != '\0' ) {
/* hack 'message' to print curr argv to std log as text attr="value" */
((wrt_nvp_t*)msg_nvp)->value = subst + 1; /* \0 terminated pre LogInsertSpace behavior */
if( (rc = self->fmt.formatter(self->fmt.data, &self->wrt, 0, NULL, envc, envs)) != 0 ) {
break;
}
((wrt_nvp_t*)msg_nvp)->value = msg_val;
}
rc = LogInsertSpace("/>\n", pbuffer, remaining, &num_writ);
FIX_UP();
} while(false);
if( self->file->file != NULL ) {
if( rc != 0 ) {
pbuffer = buffer;
remaining = sizeof(buffer);
rc = string_printf(pbuffer, remaining, & num_writ, "<error severity=\"err\" message=\"XML log failed: %R\"/>\n", rc);
pbuffer += num_writ <= remaining ? num_writ : 0;
}
rc = KFileWrite(self->file->file, self->file->pos, buffer, pbuffer - buffer, &num_writ);
self->file->pos += num_writ;
}
rc = self->fmt.formatter(self->fmt.data, &self->wrt, argc, args, envc, envs);
return rc;
}