/*
* determine the encryption type for KFile f with pathname name
*/
rc_t EncryptionTypeCheck (const KFile * f, const char * name, EncScheme * scheme)
{
size_t num_read;
rc_t rc;
char head [128];
assert (f != NULL);
assert (name != NULL);
rc = KFileReadAll (f, 0, head, sizeof head, &num_read);
if (rc)
{
PLOGERR (klogErr, (klogErr, rc, "Unable to read head of "
"'$(F)'", "F=%s", name));
*scheme = encError;
return rc;
}
rc = KFileIsEnc (head, num_read);
if (rc == 0)
{
*scheme = encEncFile;
}
else
{
rc = KFileIsWGAEnc (head, num_read);
if (rc == 0)
*scheme = encWGAEncFile;
else
*scheme = encNone;
}
return 0;
}
/*
* determine the archive type for KFile f with pathname name
*
* This could be extended to handle tar files with a larger head size and
* and some less simple checks for a tar header block at the start of
* the file.
*/
ArcScheme ArchiveTypeCheck (const KFile * f)
{
size_t num_read;
rc_t rc;
char head [128];
IsArchive = false;
rc = KFileReadAll (f, 0, head, sizeof head, &num_read);
if (rc)
{
LOGERR (klogErr, rc, "Unable to read head of decrypted file");
return arcError;
}
rc = KFileIsSRA (head, num_read);
if (rc == 0)
{
/* OUTMSG (("+++++ ARCHIVE\n")); */
/* a hack... */
IsArchive = true;
return arcSRAFile;
}
/* OUTMSG (("----- not an archive\n")); */
return arcNone;
}
rc_t CC
TCheckFileContent ( struct KFile * file, const uint8_t* filler, size_t filler_size )
{
rc_t rc = 0;
uint64_t content_size;
rc = KFileSize ( file, &content_size );
if (rc != 0)
{
return rc;
}
if (content_size == 0)
{
// do nothing
return rc;
}
const size_t buffer_size = 32*1024;
uint8_t buffer[buffer_size], reference_buffer[buffer_size];
size_t filler_sizes_in_buffer = buffer_size / filler_size;
assert(buffer_size % filler_size == 0);
assert(buffer_size / filler_size > 0);
for (size_t i = 0; i < filler_sizes_in_buffer; ++i)
{
memmove(reference_buffer + i*filler_size, filler, filler_size);
}
uint64_t num_read_total = 0;
size_t num_read;
while (num_read_total < content_size)
{
size_t to_read = content_size - num_read_total;
if (to_read > buffer_size)
{
to_read = buffer_size;
}
rc = KFileReadAll ( file, num_read_total, buffer, to_read, &num_read );
if (rc != 0)
{
break;
}
assert(num_read == to_read);
if (memcmp(reference_buffer, buffer, num_read) != 0)
{
return RC( rcKrypto, rcFile, rcValidating, rcEncryption, rcCorrupt );
}
num_read_total += num_read;
}
return rc;
}
static rc_t perform_file_load( const p_context ctx, KDirectory *dir, const TableWriterRefSeq* wr )
{
rc_t rc = 0;
uint32_t pt = KDirectoryPathType ( dir, "%s", ctx->src );
if ( pt != kptFile )
{
rc = RC( rcExe, rcFile, rcConstructing, rcParam, rcInvalid );
LOGERR( klogErr, rc, "input-file not found" );
}
else
{
const KFile * f;
rc = KDirectoryOpenFileRead ( dir, &f, "%s", ctx->src );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "cannot open input-file for read" );
}
else
{
data_ctx dctx;
rc = init_data_context( ctx, wr, &dctx );
if ( rc == 0 )
{
uint64_t pos = 0;
bool done = false;
while ( !done )
{
size_t num_read;
char buffer[ 8192 ];
rc = KFileReadAll ( f, pos, buffer, sizeof buffer, &num_read );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc, "$(path) at $(pos)", "path=%s,pos=%lu", ctx->src, pos ) );
done = true;
}
else
{
on_curl_data( buffer, 1, num_read, &dctx );
if ( num_read < sizeof( buffer ) )
done = true;
else
pos += num_read;
}
}
finish_data_context( ctx, &dctx );
if ( dctx.rc == 0 )
{
dctx.rc = rc;
}
}
KFileRelease( f );
}
}
return rc;
}
/*
* Copy a file from a const KFile * to a KFile * with the paths for the two
* for logging purposes
*
* return rc_t = 0 for success
* return rc_t != 0 for failure
*/
rc_t CopyFile (const KFile * src, KFile * dst, const char * source, const char * dest)
{
rc_t rc;
uint8_t buff [256 * 1024];
size_t num_read;
size_t num_writ;
uint64_t pos;
for (pos = 0; ; pos += num_read)
{
rc = Quitting ();
if (rc)
{
LOGMSG (klogFatal, "Received quit");
break;
}
rc = KFileReadAll (src, pos, buff, sizeof (buff), &num_read);
if (rc)
{
PLOGERR (klogErr,
(klogErr, rc,
"Failed to read from file $(F) at $(P)",
"F=%s,P=%lu", source, pos));
break;
}
if (num_read == 0)
break;
rc = KFileWriteAll (dst, pos, buff, num_read, &num_writ);
if (rc)
{
PLOGERR (klogErr,
(klogErr, rc,
"Failed to write to file $(F) at $(P)",
"F=%s,P=%lu", dest, pos));
break;
}
if (num_writ != num_read)
{
rc = RC (rcExe, rcFile, rcWriting, rcFile, rcInsufficient);
PLOGERR (klogErr,
(klogErr, rc,
"Failed to write all to file $(F) at $(P)",
"F=%s,P=%lu", dest, pos));
break;
}
}
return rc;
}
rc_t CC
TCopyFile ( struct KFile * file_dst, struct KFile * file_src )
{
rc_t rc = 0;
uint64_t size_src;
rc = KFileSize ( file_src, &size_src );
if (rc != 0)
{
return rc;
}
if (size_src == 0)
{
// do nothing
return rc;
}
const size_t buffer_size = 32*1024;
uint8_t buffer[buffer_size];
uint64_t num_copied_total = 0;
size_t num_read, num_written;
while (num_copied_total < size_src)
{
size_t to_copy = size_src - num_copied_total;
if (to_copy > buffer_size)
{
to_copy = buffer_size;
}
rc = KFileReadAll ( file_src, num_copied_total, buffer, to_copy, &num_read );
if (rc != 0)
{
break;
}
assert(num_read == to_copy);
rc = KFileWriteAll ( file_dst, num_copied_total, buffer, to_copy, &num_written );
if (rc != 0)
{
break;
}
assert(num_written == to_copy);
num_copied_total += num_written;
}
return rc;
}
/* Init
*/
#if USE_BSTREE_IN_COLUMN_IDX1
static
rc_t KColumnIdx1Inflate ( KColumnIdx1 *self,
const KColBlockLoc *buffer, uint32_t count )
{
uint32_t i;
for ( i = 0; i < count; ++ i )
{
KColumnIdx1Node *exist, *n = malloc ( sizeof * n );
if ( n == NULL )
return RC ( rcDB, rcColumn, rcConstructing, rcMemory, rcExhausted );
n -> loc = buffer [ i ];
if ( BSTreeInsertUnique ( & self -> bst,
& n -> n, ( BSTNode** ) & exist, KColumnIdx1NodeSort ) )
{
free ( n );
return RC ( rcDB, rcColumn, rcConstructing, rcIndex, rcCorrupt );
}
++ self -> count;
}
return 0;
}
#endif /* USE_BSTREE_IN_COLUMN_IDX1 */
static
void KColumnIdx1Swap ( KColBlockLoc *buffer, uint32_t count )
{
uint32_t i;
for ( i = 0; i < count; ++ i )
{
buffer [ i ] . pg = bswap_64 ( buffer [ i ] . pg );
buffer [ i ] . u . gen = bswap_32 ( buffer [ i ] . u . gen );
buffer [ i ] . id_range = bswap_32 ( buffer [ i ] . id_range );
buffer [ i ] . start_id = bswap_64 ( buffer [ i ] . start_id );
}
}
#if ! USE_BSTREE_IN_COLUMN_IDX1
static
rc_t KColumnIdx1Init ( KColumnIdx1 *self, uint32_t off, uint32_t count )
{
rc_t rc=0;
self -> data = malloc ( count * sizeof * self -> data );
if ( self -> data == NULL )
rc = RC ( rcDB, rcColumn, rcConstructing, rcMemory, rcExhausted );
else
{
size_t num_read;
uint32_t i, cnt;
for ( rc = 0, i = 0; i < count; off += ( uint32_t ) num_read, i += cnt )
{
cnt = count - i;
rc = KFileReadAll ( self -> f, off,
& self -> data [ i ], cnt * sizeof * self->data, & num_read );
if ( rc != 0 )
break;
if ( num_read == 0 )
break;
if ( ( num_read % sizeof * self->data ) != 0 )
{
rc = RC ( rcDB, rcColumn, rcConstructing, rcIndex, rcCorrupt );
break;
}
if ( self -> bswap )
KColumnIdx1Swap ( & self->data [ i ], cnt );
}
if ( rc == 0 ) {
self -> count = count;
self -> loaded = true;
} else {
free ( self->data );
}
}
return rc;
}
#else /* USE_BSTREE_IN_COLUMN_IDX1 */
static
rc_t KColumnIdx1Init ( KColumnIdx1 *self, uint32_t off, uint32_t count )
{
rc_t rc;
KColBlockLoc *buffer = malloc ( 2048 * sizeof * buffer );
if ( buffer == NULL )
rc = RC ( rcDB, rcColumn, rcConstructing, rcMemory, rcExhausted );
else
{
size_t num_read;
uint32_t i, cnt;
for ( rc = 0, i = 0; i < count; off += ( uint32_t ) num_read, i += cnt )
{
cnt = count - i;
if ( cnt > 2048 )
cnt = 2048;
rc = KFileReadAll ( self -> f, off,
buffer, cnt * sizeof * buffer, & num_read );
if ( rc != 0 )
break;
if ( num_read == 0 )
break;
if ( ( num_read % sizeof * buffer ) != 0 )
{
rc = RC ( rcDB, rcColumn, rcConstructing, rcIndex, rcCorrupt );
break;
}
if ( self -> bswap )
KColumnIdx1Swap ( buffer, cnt );
rc = KColumnIdx1Inflate ( self, buffer, cnt );
if ( rc != 0 )
break;
}
if ( rc == 0 )
self -> loaded = true;
free ( buffer );
}
return rc;
}
static
rc_t KColumnIdx1Init ( KColumnIdx1 *self, uint32_t off, uint32_t count )
{
rc_t rc=0;
self -> data = malloc ( count * sizeof * self -> data );
if ( self -> data == NULL )
rc = RC ( rcDB, rcColumn, rcConstructing, rcMemory, rcExhausted );
else
{
size_t num_read;
uint32_t i, cnt;
for ( rc = 0, i = 0; i < count; off += ( uint32_t ) num_read, i += cnt )
{
cnt = count - i;
rc = KFileReadAll ( self -> f, off,
& self -> data [ i ], cnt * sizeof * self->data, & num_read );
if ( rc != 0 )
break;
if ( num_read == 0 )
break;
if ( ( num_read % sizeof * self->data ) != 0 )
{
rc = RC ( rcDB, rcColumn, rcConstructing, rcIndex, rcCorrupt );
break;
}
if ( self -> bswap )
KColumnIdx1Swap ( & self->data [ i ], cnt );
}
if ( rc == 0 ) {
self -> count = count;
self -> loaded = true;
} else {
free ( self->data );
}
}
return rc;
}
/*
* determine the encryption type for KFile f with pathname name
*/
rc_t EncryptionTypeCheck (const KFile * f, const char * name, EncScheme * scheme)
{
size_t num_read;
rc_t rc;
char head [128];
assert (f != NULL);
assert (name != NULL);
rc = KFileReadAll (f, 0, head, sizeof head, &num_read);
if (rc)
{
PLOGERR (klogErr, (klogErr, rc, "Unable to read head of "
"'$(F)'", "F=%s", name));
*scheme = encError;
return rc;
}
/* looks for files with NCBInenc or NCBIsenc signatures */
rc = KFileIsEnc (head, num_read);
if (rc == 0)
{
/* looks for files with just NCBIsenc signatures */
rc = KFileIsSraEnc (head, num_read);
*scheme = (rc == 0) ? encSraEncFile : encEncFile;
}
else
{
rc = KFileIsWGAEnc (head, num_read);
if (rc == 0)
*scheme = encWGAEncFile;
else
*scheme = encNone;
}
return 0;
}
/* Open
*/
rc_t KColumnIdx1OpenRead ( KColumnIdx1 *self, const KDirectory *dir,
uint64_t *data_eof, uint32_t *idx0_count, uint64_t *idx2_eof,
size_t *pgsize, int32_t *checksum )
{
rc_t rc;
#if USE_BSTREE_IN_COLUMN_IDX1
BSTreeInit ( & self -> bst );
#else
self -> last_found = 0;
self -> data = NULL;
#endif
self -> fidx = NULL;
self -> count = 0;
self -> vers = 0;
#if LAZY_LOAD_COLUMN_IDX1
self -> load_off = 0;
self -> load_rc = 0;
#endif
self -> bswap = false;
self -> loaded = false;
rc = KDirectoryVOpenFileRead ( dir, & self -> f, "idx1", NULL );
if ( rc == 0 )
{
KColumnHdr hdr;
size_t num_bytes;
rc = KFileReadAll ( self -> f, 0, & hdr, sizeof hdr, & num_bytes );
if ( rc == 0 )
{
if ( num_bytes == 0 )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcNotFound );
else if ( num_bytes < KColumnHdrMinSize ( hdr ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
if ( GetRCState ( rc ) == rcIncorrect && GetRCObject ( rc ) == rcByteOrder )
{
self -> bswap = true;
hdr . dad . endian = bswap_32 ( hdr . dad . endian );
hdr . dad . version = bswap_32 ( hdr . dad . version );
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
}
if ( rc == 0 )
{
uint32_t off, count;
switch ( hdr . dad . version )
{
case 1:
if ( num_bytes < KColumnHdrOffset ( hdr, v1 ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
if ( self -> bswap )
{
hdr . u . v1 . data_eof = bswap_64 ( hdr . u . v1 . data_eof );
hdr . u . v1 . idx2_eof = bswap_64 ( hdr . u . v1 . idx2_eof );
hdr . u . v1 . num_blocks = bswap_32 ( hdr . u . v1 . num_blocks );
hdr . u . v1 . page_size = bswap_32 ( hdr . u . v1 . page_size );
}
* data_eof = hdr . u . v1 . data_eof;
* idx2_eof = hdr . u . v1 . idx2_eof;
* pgsize = hdr . u . v1 . page_size;
* checksum = hdr . u . v1 . checksum;
count = hdr . u . v1 . num_blocks;
off = KColumnHdrOffset ( hdr, v1 );
/* not supported in v1 */
* idx0_count = 0;
}
break;
default:
rc = KDirectoryVOpenFileRead ( dir, ( const KFile** ) & self -> fidx, "idx", NULL );
if ( rc == 0 )
{
off = sizeof hdr . dad;
rc = KFileReadAll ( self -> fidx, 0, & hdr, sizeof hdr, & num_bytes );
if ( rc == 0 )
{
if ( num_bytes < KColumnHdrOffset ( hdr, v2 ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
rc = KDBHdrValidate ( & hdr . dad,
num_bytes, 2, KCOL_CURRENT_VERSION );
if ( GetRCState ( rc ) == rcIncorrect && GetRCObject ( rc ) == rcByteOrder )
{
if ( ! self->bswap ) /* catch mis-matched endianess */
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
hdr . dad . endian = bswap_32 ( hdr . dad . endian );
hdr . dad . version = bswap_32 ( hdr . dad . version );
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
}
}
else if ( self -> bswap ) /* catch mis-matched endianess */
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
if ( rc == 0 ) switch ( hdr . dad . version )
{
case 2:
if ( self -> bswap )
{
hdr. u . v2 . data_eof = bswap_64 ( hdr. u . v2 . data_eof );
hdr. u . v2 . idx2_eof = bswap_64 ( hdr. u . v2 . idx2_eof );
hdr. u . v2 . num_blocks = bswap_32 ( hdr. u . v2 . num_blocks );
hdr. u . v2 . page_size = bswap_32 ( hdr. u . v2 . page_size );
}
* data_eof = hdr . u . v2 . data_eof;
* idx2_eof = hdr . u . v2 . idx2_eof;
* pgsize = hdr . u . v2 . page_size;
* checksum = hdr . u . v2 . checksum;
count = hdr . u . v2 . num_blocks;
* idx0_count = 0;
break;
case 3:
if ( self -> bswap )
{
hdr. u . v3 . data_eof = bswap_64 ( hdr. u . v3 . data_eof );
hdr. u . v3 . idx2_eof = bswap_64 ( hdr. u . v3 . idx2_eof );
hdr. u . v3 . idx0_count = bswap_32 ( hdr. u . v3 . idx0_count );
hdr. u . v3 . num_blocks = bswap_32 ( hdr. u . v3 . num_blocks );
hdr. u . v3 . page_size = bswap_32 ( hdr. u . v3 . page_size );
}
* data_eof = hdr . u . v3 . data_eof;
* idx2_eof = hdr . u . v3 . idx2_eof;
* idx0_count = hdr . u . v3 . idx0_count;
* pgsize = hdr . u . v3 . page_size;
* checksum = hdr . u . v3 . checksum;
count = hdr . u . v3 . num_blocks;
break;
default:
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcBadVersion );
}
}
}
}
}
if ( rc == 0 )
{
self -> vers = hdr . dad . version;
#if LAZY_LOAD_COLUMN_IDX1
self -> load_off = off;
self -> count = count;
#else
rc = KColumnIdx1Init ( self, off, count );
if ( rc == 0 )
#endif
return rc;
}
KFileRelease ( self -> fidx );
self -> fidx = NULL;
}
}
}
KFileRelease ( self -> f );
self -> f = NULL;
}
return rc;
}
static rc_t VDBDependenciesReportDepend1(const VDBDependencies *self,
const ReportFuncs *f,
uint32_t count, uint32_t indent, bool toreport, uint32_t *missing)
{
KDirectory *dir = NULL;
rc_t rc = 0;
uint32_t i = ~0;
assert(missing);
*missing = 0;
for (i = 0; i < count; ++i) {
bool isMissing = false;
bool local = false;
const char* seq_id = "";
const char* path = NULL;
rc = VDBDependenciesSeqId(self, &seq_id, i);
if (rc != 0 && toreport)
reportErrorStrInt(indent, rc, "VDBDependenciesSeqId",
"origin", "VDatabaseListDependencies", "idx", i);
if (rc == 0) {
rc = VDBDependenciesLocal(self, &local, i);
if (rc != 0 && toreport) {
reportErrorStrInt(indent, rc,
"VDBDependenciesLocal",
"origin", "VDatabaseListDependencies", "idx", i);
}
}
if (rc == 0) {
rc = VDBDependenciesPath(self, &path, i);
if (rc != 0 && toreport) {
reportErrorStrInt(indent, rc, "VDBDependenciesPath",
"origin", "VDatabaseListDependencies", "idx", i);
}
}
if (rc == 0) {
if (!local && (path == NULL || path[0] == '\0')) {
isMissing = true;
++*missing;
}
if (toreport) {
bool reported = false;
if (!isMissing && !local) {
rc_t rc = 0;
bool readable = false;
uint64_t size = ~0;
KTime_t date = 0;
bool ready = false;
if (dir == NULL)
{ rc = KDirectoryNativeDir(&dir); }
assert(!local && path && path[0]);
if (dir != NULL) {
rc = KDirectoryFileSize(dir, &size, "%s", path);
if (rc == 0)
{ rc = KDirectoryDate(dir, &date, "%s", path); }
if (rc == 0) {
const KFile* f = NULL;
rc = KDirectoryOpenFileRead(dir, &f, "%s", path);
if (rc == 0) {
char buffer[1024];
size_t num_read = 0;
size_t bsize =
size > sizeof buffer ? sizeof buffer : size;
rc = KFileReadAll(f,
0, buffer, bsize, &num_read);
if (rc == 0 && num_read != bsize) {
rc = RC(rcVDB, rcFile, rcReading,
rcBuffer, rcExcessive);
}
if (rc == 0)
{ readable = true; }
}
KFileRelease(f);
}
if (rc == 0)
{ ready = true; }
}
if (ready) {
KTime kt;
memset(&kt, 0, sizeof kt);
KTimeLocal(&kt, date);
report(indent, "Dependency", 7,
"index", 'd', i,
"seq_id", 's', seq_id,
"local", 's', local ? "true" : "false",
"path", 's', path,
"size", 'u', size,
"date", 'T', &kt,
"readable", 's', readable ? "true" : "false");
reported = true;
}
else {
report(indent, "Dependency", 5,
"index", 'd', i,
"seq_id", 's', seq_id,
"local", 's', local ? "true" : "false",
"path", 's', path,
"error", 'R', rc);
reported = true;
}
}
if (!reported) {
report(indent, "Dependency", 4,
"index", 'd', i,
"seq_id", 's', seq_id,
"local", 's', local ? "true" : "false",
"path", 's', path == NULL ? "" : path);
}
}
}
if (rc != 0 && !toreport)
{ break; }
}
RELEASE(KDirectory, dir);
return rc;
}
rc_t KColumnIdx2LocateBlob ( const KColumnIdx2 *self,
KColBlobLoc *loc, const KColBlockLoc *bloc,
int64_t first, int64_t upper, bool bswap )
{
rc_t rc;
uint32_t count;
size_t block_size, orig;
uint32_t slot=0;
void *block;
/* check within page cache */
if(self -> cstorage.elem_count > 0){
KColumnIdx2BlockCache * cache=(KColumnIdx2BlockCache *)self -> cstorage.base;
assert(self->last < self->cstorage.elem_count);
if(bloc -> start_id == cache[self->last].start_id){
rc = KColIdxBlockLocateBlob ( & cache[self->last].iblk, loc, bloc, ( uint32_t ) cache[self->last].count, first, upper );
if ( rc == 0) return 0;
} else {
uint32_t lower,upper,pivot;
if(bloc -> start_id > cache[self->last].start_id){
lower=self->last+1;
upper=self->cstorage.elem_count;
} else {
lower=0;
upper = self->last;
}
while(lower < upper){
pivot = (lower + upper) / 2;
if(bloc -> start_id == cache[pivot].start_id){
KColumnIdx2 * nc_self=(KColumnIdx2 *)self;
nc_self->last = pivot;
rc = KColIdxBlockLocateBlob ( & cache[self->last].iblk, loc, bloc, ( uint32_t ) cache[self->last].count, first, upper );
if ( rc == 0) return 0;
goto BSEARCH_DONE;
} else if(bloc -> start_id < cache[pivot].start_id){
upper=pivot;
} else {
lower =pivot+1;
}
}
assert(lower == upper );
slot = upper;
}
}
BSEARCH_DONE:
/* file may be empty or non-existent */
if ( self -> eof == 0 )
return RC ( rcDB, rcColumn, rcSelecting, rcBlob, rcNotFound );
/* TBD - compression not supported */
if ( bloc -> u . blk . compressed )
return RC ( rcDB, rcColumn, rcSelecting, rcData, rcUnsupported );
/* determine the number of entries in block */
orig = bloc -> u . blk . size;
count = KColBlockLocEntryCount ( bloc, & orig );
/* determine the size to allocate */
block_size = KColBlockLocAllocSize ( bloc, orig, count );
/* allocate a block */
block = malloc ( block_size );
if ( block == NULL )
rc = RC ( rcDB, rcColumn, rcSelecting, rcMemory, rcExhausted );
else
{
size_t num_read;
rc = KFileReadAll ( self -> f, bloc -> pg, block, orig, & num_read );
if ( rc == 0 )
{
if ( num_read != orig )
rc = RC ( rcDB, rcIndex, rcReading, rcTransfer, rcIncomplete );
else
{
KColIdxBlock iblk;
rc = KColIdxBlockInit ( & iblk, bloc, orig, block, block_size, bswap );
if ( rc == 0 )
{
rc = KColIdxBlockLocateBlob ( & iblk,
loc, bloc, count, first, upper );
if ( rc == 0 )
{
KColumnIdx2BlockCache * cache;
KDataBufferResize(&((KColumnIdx2 *)self)->cstorage,self->cstorage.elem_count+1);
cache=(KColumnIdx2BlockCache *)self -> cstorage.base;
if(slot < self->cstorage.elem_count -1){ /** not adding to the end **/
memmove(cache+slot+1,cache+slot,sizeof(*cache)*(self->cstorage.elem_count - slot - 1));
}
cache += slot;
cache -> block = block;
cache -> start_id = bloc -> start_id;
cache -> count = count;
cache -> iblk = iblk;
return 0;
}
}
}
}
free ( block );
}
return rc;
}
/* not KDB specific - just uses vfs/krypto/kfs objects */
static
rc_t KDBOpenFileAsDirectory (const KDirectory * dir,
const char * path,
const KDirectory ** pdir,
uint32_t rcobj)
{
const KFile * file;
const KFile * f;
const KDirectory * ldir;
bool encrypted = false;
rc_t rc;
*pdir = NULL;
rc = KDirectoryOpenFileRead (dir, &file, path);
if (rc == 0)
{
rc = KFileRandomAccess(file);
if (rc)
rc = RC (rcDB, rcMgr, rcOpening, rcobj, rcUnsupported);
else
{
size_t tz;
char tbuff [4096];
char pbuff [4096 + 1];
rc = KFileReadAll (file, 0, tbuff, sizeof tbuff, &tz);
if (rc == 0)
{
if (KFileIsEnc (tbuff, tz) == 0)
{
encrypted = true;
rc = KDBOpenFileGetPassword (pbuff, sizeof (pbuff) - 1);
if (rc == 0)
{
KKey key;
rc = KKeyInitRead (&key, kkeyAES128, pbuff, string_size (pbuff));
if (rc == 0)
{
rc = KEncFileMakeRead (&f, file, &key);
if (rc == 0)
{
/* KEncFileMakeRead adds a reference */
KFileRelease (file);
file = f;
rc = KFileReadAll (file, 0, tbuff, sizeof tbuff, &tz);
}
}
}
}
else if (KFileIsWGAEnc (tbuff, tz) == 0)
{
encrypted = true;
rc = KDBOpenFileGetPassword (pbuff, sizeof (pbuff) - 1);
if (rc == 0)
{
rc = KFileMakeWGAEncRead (&f, file, pbuff, string_size (pbuff));
if (rc == 0)
{
/* KFileMakeWGAEncRead adds a reference */
KFileRelease (file);
file = f;
rc = KFileReadAll (file, 0, tbuff, sizeof tbuff, &tz);
}
}
}
/* else not a handled encryption or unencrypted: we can't distinguish too much */
if (rc == 0)
{
if (KFileIsSRA (tbuff, tz) == 0)
{
rc = KDirectoryOpenSraArchiveReadUnbounded_silent_preopened (dir,
&ldir,
false,
file,
path);
}
else
{
rc = KDirectoryOpenTarArchiveRead_silent_preopened (dir, &ldir, false,
file, path);
}
/* not an archive type we handle or a bad archive */
if (rc)
{
if (encrypted)
rc = RC ( rcDB, rcMgr, rcOpening, rcEncryptionKey, rcIncorrect );
else
rc = RC ( rcDB, rcMgr, rcOpening, rcPath, rcIncorrect );
}
else
{
/*
* release our ownership of the KFile that but archive will
* keep theirs
*/
KFileRelease (file);
*pdir = ldir;
return 0;
}
}
}
}
KFileRelease (file);
}
return rc;
}
/* return configured password as ASCIZ
* opertates on vfs/kfs/kfg objects, not kdb objects */
static
rc_t KDBOpenFileGetPassword (char * pw, size_t pwz)
{
VFSManager * mgr;
rc_t rc;
assert (pw);
assert (pwz);
pw[0] = '\0';
rc = VFSManagerMake (&mgr);
if (rc)
; /* failure to make VFS manager: pass along rc */
else
{
size_t pwfz;
char pwf [4096 + 1];
rc = VFSManagerGetConfigPWFile (mgr, pwf, sizeof (pwf) - 1, &pwfz);
if (rc)
/* failure to get password file path: tweak rc */
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcNotFound);
else
{
VPath * pwp;
pwf [pwfz] = '\0'; /* force to ASCIZ */
#if 0
rc = VPathMakeSysPath (&pwp, pwf);
#else
rc = VFSManagerMakePath (mgr, &pwp, pwf);
#endif
if (rc)
; /* failure to construct a path from the string */
else
{
const KFile * pwf;
rc = VFSManagerOpenFileRead (mgr, &pwf, pwp);
if (rc)
/* failure to open password file */
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcNotOpen);
else
{
size_t z;
char pwb [4098]; /* arbitrarily using 4096 as maximum
allowed length */
/* at this point we are only getting the password from a
* file but in the future if we can get it from a pipe of
* some sort we can't count on the ReadAll to really know
* if we hit end of file and not just a pause in the
* streaming. VFS/KFS 2 will have to fix this somehow
*/
rc = KFileReadAll (pwf, 0, pwb, sizeof pwb, &z);
if (rc)
; /* failure to read password file: pass along rc */
else
{
/* trim off EOL if present */
char * pc;
pwb[z] = '\0'; /* force ASCIZ */
pc = string_chr (pwb, z, '\r');
if (pc)
{
*pc = '\0';
z = 1 + pc - pwb;
}
pc = string_chr (pwb, z, '\n');
if (pc)
{
*pc = '\0';
z = 1 + pc - pwb;
}
if (z == 0)
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcTooShort);
else if (pwz < z) /* pwz came in as 4096 */
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcTooLong);
else
{
memmove (pw, pwb, z+1);
}
}
KFileRelease (pwf);
}
VPathRelease (pwp);
}
}
VFSManagerRelease (mgr);
}
return rc;
}
/* RunRead
* runs read loop on background thread
*/
static
rc_t CC KQueueFileRunRead ( const KThread *t, void *data )
{
KQueueFile *self = data;
rc_t rc;
bool loop;
uint64_t pos;
size_t num_read;
QFMSG ( "BG: %s: running read-ahead loop\n", __func__ );
for ( rc = 0, loop = true, pos = self -> start_pos; loop; pos += num_read )
{
timeout_t tm;
KQueueFileBuffer *b = malloc ( sizeof * b - sizeof b -> data + self -> bsize );
if ( b == NULL )
{
rc = RC ( rcApp, rcFile, rcReading, rcMemory, rcExhausted );
QFMSG ( "BG: %s: failed to allocate %zu byte buffer\n", __func__, self -> bsize );
b = malloc ( sizeof * b );
if ( b == NULL )
break;
num_read = 0;
b -> rc = rc;
QFMSG ( "BG: %s: created empty buffer with rc = %R.\n", __func__, b -> rc );
loop = false;
}
else
{
QFMSG ( "BG: %s: reading %zu byte buffer\n", __func__, self -> bsize );
b -> rc = KFileReadAll ( self -> f, pos, b -> data, self -> bsize, & num_read );
QFMSG ( "BG: %s: read %zu bytes, rc = %R\n", __func__, num_read, b -> rc );
if ( b -> rc != 0 || num_read == 0 )
loop = false;
}
/* update buffer data */
b -> pos = pos;
b -> bytes = num_read;
#if _DEBUGGING
b -> align = 0;
#endif
PushAgain:
/* timeout is in mS */
QFMSG ( "BG: %s: initializing timeout for %,umS\n", __func__, self->timeout_ms );
TimeoutInit ( & tm, self->timeout_ms );
/* push buffer */
QFMSG ( "BG: %s: pushing buffer...\n", __func__ );
rc = KQueuePush ( self -> q, b, & tm );
QFMSG ( "BG: %s: ...done: rc = %R\n", __func__, rc );
if ( rc != 0 )
{
/* if queue has been sealed by fg */
if ( GetRCState ( rc ) == rcReadonly )
{
rc = 0;
QFMSG ( "BG: %s: clearing rc\n", __func__ );
} else if( GetRCObject(rc) == ( enum RCObject )rcTimeout && GetRCState(rc) == rcExhausted ) {
goto PushAgain;
}
QFMSG ( "BG: %s: dousing buffer\n", __func__ );
free ( b );
break;
}
}
/* going to exit thread */
QFMSG ( "BG: %s: sealing thread\n", __func__ );
KQueueSeal ( self -> q );
QFMSG ( "BG: %s: exit with rc = %R\n", __func__, rc );
return rc;
}
/* EncryptionKey
* return any associated encryption key
*
* attempts to copy NUL-terminated key into provided buffer
*
* "buffer" [ OUT ] and "bsize" [ IN ] - encryption key output parameter
*
* "key_size" [ OUT, NULL OKAY ] - returns the key size in
* bytes, excluding any NUL termination.
*/
LIB_EXPORT rc_t CC KRepositoryEncryptionKey ( const KRepository *self,
char *buffer, size_t bsize, size_t *key_size )
{
rc_t rc;
if ( self == NULL )
rc = RC ( rcKFG, rcNode, rcAccessing, rcSelf, rcNull );
else
{
const KConfigNode *node;
if ( key_size != NULL )
* key_size = 0;
rc = KConfigNodeOpenNodeRead ( self -> node, & node, "encryption-key" );
if ( rc == 0 )
{
size_t num_read, remaining;
rc = KConfigNodeRead ( node, 0, buffer, bsize, & num_read, & remaining );
KConfigNodeRelease ( node );
if ( rc == 0 )
{
if ( key_size != NULL )
* key_size = num_read + remaining;
if ( remaining != 0 )
rc = RC ( rcKFG, rcNode, rcAccessing, rcBuffer, rcInsufficient );
else if ( num_read < bsize )
memset ( & buffer [ num_read ], 0, bsize - num_read );
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
char path [ 4096 ];
rc_t rc2 = KRepositoryEncryptionKeyFile ( self, path, sizeof path, NULL );
if ( rc2 == 0 )
{
KDirectory *wd;
rc2 = KDirectoryNativeDir ( & wd );
if ( rc2 == 0 )
{
const KFile *keyFile;
rc2 = KDirectoryOpenFileRead ( wd, & keyFile, path );
KDirectoryRelease ( wd );
if ( rc2 == 0 )
{
size_t num_read;
rc = KFileReadAll ( keyFile, 0, buffer, bsize, & num_read );
if ( rc == 0 )
{
if ( num_read == bsize )
{
uint64_t eof;
rc = KFileSize ( keyFile, & eof );
if ( rc == 0 )
num_read = ( size_t ) eof;
else
num_read = 0;
rc = RC ( rcKFG, rcFile, rcReading, rcBuffer, rcInsufficient );
memset ( buffer, 0, bsize );
}
else if ( num_read == 0 )
{
rc = RC ( rcKFG, rcFile, rcReading, rcFile, rcEmpty );
memset ( buffer, 0, bsize );
}
else
{
char *eoln = string_chr ( buffer, num_read, '\n' );
if ( eoln != NULL )
{
if ( eoln == buffer )
num_read = 0;
else if ( eoln [ -1 ] == '\r' )
num_read = eoln - buffer - 1;
else
num_read = eoln - buffer;
}
if ( key_size != NULL )
* key_size = num_read;
memset ( & buffer [ num_read ], 0, bsize - num_read );
}
}
KFileRelease ( keyFile );
}
}
}
}
}
return rc;
}
static
rc_t CC
_ReadAll ( const char * Path, char ** Buf, size_t * BufSize )
{
rc_t RCt;
const struct KFile * File;
struct KDirectory * Dir;
char * RetBuf;
size_t Size, NumRead;
RCt = 0;
File = NULL;
Dir = NULL;
RetBuf = NULL;
Size = 0;
NumRead = 0;
RCt = KDirectoryNativeDir ( & Dir );
if ( RCt == 0 ) {
RCt = KDirectoryFileSize ( Dir, & Size, Path );
if ( RCt == 0 ) {
if ( Size == 0 ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcInvalid );
}
else {
RetBuf = calloc ( Size, sizeof ( char ) );
if ( RetBuf == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcExhausted );
}
else {
RCt = KDirectoryOpenFileRead ( Dir, & File, Path );
if ( RCt == 0 ) {
RCt = KFileReadAll (
File,
0,
RetBuf,
Size,
& NumRead
);
if ( RCt == 0 ) {
* Buf = RetBuf;
* BufSize = Size;
}
}
}
}
}
KDirectoryRelease ( Dir );
}
if ( RCt != 0 ) {
Buf = NULL;
BufSize = 0;
if ( RetBuf != NULL ) {
free ( RetBuf );
}
}
return RCt;
} /* _ReadAll () */
static
rc_t CC
_TaskRead ( const struct XTask * self )
{
rc_t RCt;
const char * Name;
uint64_t Offset;
size_t Size, NumRead;
struct KDirectory * Directory;
const struct KFile * File;
uint64_t Time;
RCt = 0;
Name = NULL;
Offset = 0;
Size = NumRead = 0;
Directory = NULL;
File = NULL;
Time = 0;
if ( self == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcNull );
}
if ( self -> tasker == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcInvalid );
}
RCt = XTaskerNewJob ( self -> tasker, & Name, & Offset, & Size );
if ( RCt == 0 ) {
if ( Name != NULL ) {
RCt = _TaskRealloc ( self, Size );
if ( RCt == 0 ) {
RCt = KDirectoryNativeDir ( & Directory );
if ( RCt == 0 ) {
RCt = KDirectoryOpenFileRead (
Directory,
& File,
Name
);
if ( RCt == 0 ) {
Time = XTmNow ();
RCt = KFileReadAll (
File,
Offset,
self -> buffer,
Size,
& NumRead
);
if ( RCt == 0 ) {
XStatsGood (
self -> tasker -> stat,
NumRead,
( XTmNow () - Time )
);
}
else {
XStatsBad ( self -> tasker -> stat );
}
KFileRelease ( File );
}
KDirectoryRelease ( Directory );
}
}
}
}
return RCt;
} /* _TaskRead () */
static
rc_t aws_find_nodes ( KConfigNode *aws_node, const char *aws_path )
{
KDirectory *wd;
rc_t rc = KDirectoryNativeDir ( &wd );
if ( rc == 0 )
{
char *buffer;
size_t num_read;
uint64_t buf_size;
const KFile *credentials, *config;
rc = KDirectoryOpenFileRead ( wd, &credentials, "%s%s", aws_path, "/credentials" );
if ( rc == 0 )
{
rc = KFileSize ( credentials, &buf_size );
if ( rc == 0 )
{
buffer = malloc ( buf_size );
if ( buffer != NULL )
{
rc = KFileReadAll ( credentials, 0, buffer, ( size_t ) buf_size, &num_read );
if ( rc == 0 )
aws_parse_file ( credentials, aws_node, buffer, num_read, true );
free ( buffer );
}
}
KFileRelease ( credentials );
}
rc = KDirectoryOpenFileRead ( wd, &config, "%s%s", aws_path, "/config" );
if ( rc == 0 )
{
rc = KFileSize ( config, &buf_size );
if ( rc == 0 )
{
buffer = malloc ( buf_size );
if ( buffer != NULL )
{
rc = KFileReadAll ( config, 0, buffer, ( size_t ) buf_size, &num_read );
if ( rc == 0 )
aws_parse_file ( config, aws_node, buffer, num_read, false );
free ( buffer );
}
}
KFileRelease ( config );
}
KDirectoryRelease ( wd );
}
return rc;
}
/* LocateBlob
* locate an existing blob
*/
rc_t KColumnIdx2LocateBlob ( const KColumnIdx2 *self,
KColBlobLoc *loc, const KColBlockLoc *bloc,
int64_t first, int64_t upper, bool bswap )
{
rc_t rc;
/* compression not supported */
if ( bloc -> u . blk . compressed )
rc = RC ( rcDB, rcIndex, rcSelecting, rcNoObj, rcUnsupported );
else
{
uint64_t buffer [ 1024 / 8 ]; /* make sure is uint64_t aligned */
void *block = buffer;
/* determine the number of entries in block */
size_t orig = bloc -> u . blk . size;
uint32_t count = KColBlockLocEntryCount ( bloc, & orig );
/* determine the size to allocate */
size_t block_size = KColBlockLocAllocSize ( bloc, orig, count );
/* allocate a block */
if ( block_size > sizeof buffer )
block = malloc ( block_size );
if ( block == NULL )
rc = RC ( rcDB, rcIndex, rcSelecting, rcMemory, rcExhausted );
else
{
size_t num_read;
rc = KFileReadAll ( self -> f, bloc -> pg, block, orig, & num_read );
if ( rc == 0 )
{
if ( num_read != orig )
rc = RC ( rcDB, rcIndex, rcSelecting, rcTransfer, rcIncomplete );
else
{
KColIdxBlock iblk;
rc = KColIdxBlockInit ( & iblk, bloc, orig, block, block_size, bswap );
if ( rc == 0 )
{
uint32_t span;
int64_t start_id;
int slot = KColIdxBlockFind ( & iblk,
bloc, count, first, & start_id, & span );
if ( slot < 0 )
rc = RC ( rcDB, rcIndex, rcSelecting, rcRange, rcNotFound );
else if ( upper > ( start_id + span ) )
rc = RC ( rcDB, rcIndex, rcSelecting, rcRange, rcInvalid );
else
{
loc -> start_id = start_id;
loc -> id_range = span;
KColIdxBlockGet ( & iblk,
bloc, count, slot, & loc -> pg, & span );
loc -> u . blob . size = span;
}
}
}
}
if ( block != buffer )
free ( block );
}
}
return rc;
}
/* local fill the buffer with block or more */
static
rc_t KWGAEncFileReadInt (KWGAEncFile * self, uint64_t pos, size_t bsize)
{
uint64_t adjpos;
size_t tot_read;
size_t num_read;
rc_t rc = 0;
assert (self);
assert (bsize);
assert (bsize <= 32 * 1024);
assert (128%16 == 0);
memset (self->buffer.data, 0, sizeof self->buffer.data);
tot_read = num_read = 0;
adjpos = pos + sizeof (KWGAEncFileHeader);
#if 0
do
{
rc = KFileRead (self->encrypted, adjpos + tot_read,
self->buffer.data + tot_read, bsize - tot_read,
&num_read);
if (rc)
return rc;
tot_read += num_read;
} while ((tot_read < bsize) && (num_read > 0));
#else
rc = KFileReadAll (self->encrypted, adjpos, self->buffer.data, bsize,
&tot_read);
#endif
self->buffer.offset = pos;
self->buffer.valid = (uint32_t)tot_read;
if (tot_read & 15)
rc = RC (rcKrypto, rcFile, rcReading, rcSize, rcInsufficient);
else if (tot_read > 0)
{
#if 1
#if RETAINED_COMPATIBILTY_WITH_ERROR
KCipherDecryptECB (self->cipher, self->buffer.data, self->buffer.data,
(uint32_t)(tot_read / ECB_BYTES));
#else
/* Well this is wrong for even being wrong now */
KCipherDecryptECB (self->cipher, self->buffer.data, self->buffer.data,
tot_read);
#endif
#else
uint32_t count;
uint32_t ix;
/* this loop can be replaced by the KCipherDecryptECB
* with care taken to match the error in the original
* C++
*/
#if RETAINED_COMPATIBILTY_WITH_ERROR
count = tot_read / ECB_BYTES;
#else
/* do all full 16 byte blocks plus a last partial block */
count = (tot_read + (ECB_BYTES-1)) / ECB_BYTES;
#endif
for (ix = 0; ix < count; ++ix)
{
uint8_t * pb = self->buffer.data + (ix * ECB_BYTES);
rc = KCipherDecrypt (self->cipher, pb, pb);
if (rc)
break;
}
#endif
}
return rc;
}
static
rc_t copy_file (const char * src, const char * dst, const KFile * fin, KFile *fout)
{
rc_t rc;
uint8_t buff [64 * 1024];
size_t num_read;
uint64_t inpos;
uint64_t outpos;
assert (src);
assert (dst);
assert (fin);
assert (fout);
inpos = 0;
outpos = 0;
#if 1
for (inpos = 0; ; inpos += num_read)
{
rc = Quitting ();
if (rc)
{
LOGMSG (klogFatal, "Received quit");
break;
}
else
{
rc = KFileReadAll (fin, inpos, buff, sizeof (buff), &num_read);
if (rc)
{
PLOGERR (klogErr,
(klogErr, rc,
"Failed to read from file $(F) at $(P)",
"F=%s,P=%lu", src, inpos));
break;
}
else if (num_read)
{
size_t num_writ;
rc = KFileWriteAll (fout, inpos, buff, num_read, &num_writ);
if (rc)
{
PLOGERR (klogErr,
(klogErr, rc,
"Failed to write to file $(F) at $(P)",
"F=%s,P=%lu", dst, outpos));
break;
}
else if (num_writ != num_read)
{
rc = RC (rcExe, rcFile, rcWriting, rcFile, rcInsufficient);
PLOGERR (klogErr,
(klogErr, rc,
"Failed to write all to file $(F) at $(P)",
"F=%s,P=%lu", dst, outpos));
break;
}
}
else
break;
}
}
#else
do
{
rc = Quitting ();
if (rc)
{
LOGMSG (klogFatal, "Received quit");
break;
}
rc = KFileRead (fin, inpos, buff, sizeof (buff), &num_read);
if (rc)
{
PLOGERR (klogErr,
(klogErr, rc,
"Failed to read from file $(F) at $(P)",
"F=%s,P=%lu", src, inpos));
break;
}
else if (num_read > 0)
{
size_t to_write;
inpos += (uint64_t)num_read;
STSMSG (2, ("Read %zu bytes to %lu", num_read, inpos));
to_write = num_read;
while (to_write > 0)
{
size_t num_writ;
rc = KFileWrite (fout, outpos, buff, num_read, &num_writ);
if (rc)
{
PLOGERR (klogErr,
(klogErr, rc,
"Failed to write to file $(F) at $(P)",
"F=%s,P=%lu", dst, outpos));
break;
}
outpos += num_writ;
to_write -= num_writ;
}
}
if (rc)
break;
} while (num_read != 0);
#endif
return rc;
}
