/* Open
*/
rc_t KColumnDataOpenRead ( KColumnData *self,
const KDirectory *dir, uint64_t eof, size_t pgsize )
{
rc_t rc = KDirectoryVOpenFileRead ( dir,
& self -> f, "data", NULL );
#if DATA_READ_FILE_BUFFER
if ( rc == 0 )
{
const KFile * orig = self -> f;
rc = KBufFileMakeRead ( & self -> f, self -> f, DATA_READ_FILE_BUFFER );
if ( rc == 0 )
{
KFileRelease ( orig );
}
else
{
self -> f = orig;
rc = 0;
}
}
#endif
if ( rc == 0 )
rc = KColumnDataInit ( self, eof, pgsize );
return rc;
}
/* OpenFileRead
* opens an existing file with read-only access
*
* "f" [ OUT ] - return parameter for newly opened file
*
* "path" [ IN ] - NUL terminated string in directory-native
* character set denoting target file
*/
inline rc_t OpenFileRead ( struct KFile const **f,
const char *path, ... ) const throw ()
{
va_list args;
va_start ( args, path );
rc_t rc = KDirectoryVOpenFileRead ( this, f, path, args );
va_end ( args );
return rc;
}
rc_t make_lookup_reader( const KDirectory *dir, const struct index_reader * index,
struct lookup_reader ** reader, size_t buf_size, const char * fmt, ... )
{
rc_t rc;
const struct KFile * f = NULL;
va_list args;
va_start ( args, fmt );
rc = KDirectoryVOpenFileRead( dir, &f, fmt, args );
if ( rc != 0 )
{
char tmp[ 4096 ];
size_t num_writ;
rc_t rc1 = string_vprintf( tmp, sizeof tmp, &num_writ, fmt, args );
if ( rc1 != 0 )
ErrMsg( "make_lookup_reader.KDirectoryVOpenFileRead( '?' ) -> %R", rc );
else
ErrMsg( "make_lookup_reader.KDirectoryVOpenFileRead( '%s' ) -> %R", tmp, rc );
}
else
{
const struct KFile * temp_file = NULL;
rc = KBufFileMakeRead( &temp_file, f, buf_size );
KFileRelease( f );
if ( rc != 0 )
{
ErrMsg( "make_lookup_reader.KBufFileMakeRead() -> %R", rc );
}
else
{
lookup_reader * r = calloc( 1, sizeof * r );
if ( r == NULL )
{
KFileRelease( temp_file );
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
ErrMsg( "make_lookup_reader.calloc( %d ) -> %R", ( sizeof * r ), rc );
}
else
{
r->f = temp_file;
r->index = index;
rc = make_SBuffer( &r->buf, 4096 );
if ( rc == 0 )
*reader = r;
else
release_lookup_reader( r );
}
}
}
va_end ( args );
return rc;
}
/* Open
*/
rc_t KColumnIdx2OpenRead ( KColumnIdx2 *self,
const KDirectory *dir, uint64_t eof )
{
rc_t rc;
rc = KDataBufferMake ( & self -> cstorage, sizeof ( KColumnIdx2BlockCache ) * 8, 0 );
if ( rc != 0 )
{
memset ( self, 0, sizeof * self );
return rc;
}
self -> last = 0;
if ( eof == 0 )
{
#if 0
KDataBufferResize(&self->cstorage,0);
self -> last = 0;
#endif
self -> eof = 0;
self -> f = NULL;
return 0;
}
rc = KDirectoryVOpenFileRead ( dir,
& self -> f, "idx2", NULL );
#if IDX2_READ_FILE_BUFFER
if ( rc == 0 )
{
const KFile * orig = self -> f;
rc = KBufFileMakeRead ( & self -> f, self -> f, IDX2_READ_FILE_BUFFER );
if ( rc == 0 )
{
KFileRelease ( orig );
}
else
{
self -> f = orig;
rc = 0;
}
}
#endif
if ( rc == 0 )
rc = KColumnIdx2Init ( self, eof );
return rc;
}
static
rc_t CC extract_action (const KDirectory * dir, const char * path, void * _adata)
{
rc_t rc;
extract_adata * adata;
KPathType type;
char link [2 * 4096]; /* we'll truncate? */
uint32_t access;
rc = 0;
adata = _adata;
STSMSG (1, ("extract_action: %s\n", path));
type = KDirectoryPathType (dir, path);
if (type & kptAlias)
{
rc = KDirectoryVResolveAlias (dir, false, link, sizeof (link),
path, NULL);
if (rc == 0)
{
rc = KDirectoryVAccess (dir, &access, path, NULL);
if (rc == 0)
{
rc = KDirectoryCreateAlias (adata->dir, access, kcmCreate|kcmParents,
link, path);
}
}
}
else
{
switch (type & ~kptAlias)
{
case kptNotFound:
rc = RC (rcExe, rcDirectory, rcAccessing, rcPath, rcNotFound);
KOutMsg ("%s: %s type kptNotFouns %R\n", __func__, path, rc);
break;
case kptBadPath:
rc = RC (rcExe, rcDirectory, rcAccessing, rcPath, rcInvalid);
break;
case kptFile:
rc = KDirectoryVAccess (dir, &access, path, NULL);
if (rc == 0)
{
const KFile * fin;
KFile * fout;
rc = KDirectoryVCreateFile (adata->dir, &fout, false, access,
kcmCreate|kcmParents,
path, NULL);
if (rc == 0)
{
rc = KDirectoryVOpenFileRead (dir, &fin, path, NULL);
if (rc == 0)
{
#if USE_SKEY_MD5_FIX
/* KLUDGE!!!! */
size_t pathz, skey_md5z;
static const char skey_md5[] = "skey.md5";
pathz = string_size (path);
skey_md5z = string_size(skey_md5);
if ( pathz >= skey_md5z && strcmp ( & path [ pathz - skey_md5z ], skey_md5 ) == 0 )
rc = copy_file_skey_md5_kludge (fin, fout);
else
#endif
rc = copy_file (fin, fout);
KFileRelease (fin);
}
KFileRelease (fout);
}
}
break;
case kptDir:
rc = KDirectoryVAccess (dir, &access, path, NULL);
if (rc == 0)
{
rc = KDirectoryCreateDir (adata->dir, 0700,
kcmCreate|kcmParents,
path, NULL);
if (rc == 0)
{
rc = step_through_dir (dir, path, adata->filter, adata->fdata,
extract_action, adata);
if (rc == 0)
rc = KDirectoryVSetAccess (adata->dir, false, access, 0777, path, NULL);
}
}
break;
case kptCharDev:
case kptBlockDev:
case kptFIFO:
/* shouldn't get here */
return 0;
}
}
return rc;
}
/* Open
*/
rc_t KColumnIdx1OpenRead ( KColumnIdx1 *self, const KDirectory *dir,
uint64_t *data_eof, uint32_t *idx0_count, uint64_t *idx2_eof,
size_t *pgsize, int32_t *checksum )
{
rc_t rc;
#if USE_BSTREE_IN_COLUMN_IDX1
BSTreeInit ( & self -> bst );
#else
self -> last_found = 0;
self -> data = NULL;
#endif
self -> fidx = NULL;
self -> count = 0;
self -> vers = 0;
#if LAZY_LOAD_COLUMN_IDX1
self -> load_off = 0;
self -> load_rc = 0;
#endif
self -> bswap = false;
self -> loaded = false;
rc = KDirectoryVOpenFileRead ( dir, & self -> f, "idx1", NULL );
if ( rc == 0 )
{
KColumnHdr hdr;
size_t num_bytes;
rc = KFileReadAll ( self -> f, 0, & hdr, sizeof hdr, & num_bytes );
if ( rc == 0 )
{
if ( num_bytes == 0 )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcNotFound );
else if ( num_bytes < KColumnHdrMinSize ( hdr ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
if ( GetRCState ( rc ) == rcIncorrect && GetRCObject ( rc ) == rcByteOrder )
{
self -> bswap = true;
hdr . dad . endian = bswap_32 ( hdr . dad . endian );
hdr . dad . version = bswap_32 ( hdr . dad . version );
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
}
if ( rc == 0 )
{
uint32_t off, count;
switch ( hdr . dad . version )
{
case 1:
if ( num_bytes < KColumnHdrOffset ( hdr, v1 ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
if ( self -> bswap )
{
hdr . u . v1 . data_eof = bswap_64 ( hdr . u . v1 . data_eof );
hdr . u . v1 . idx2_eof = bswap_64 ( hdr . u . v1 . idx2_eof );
hdr . u . v1 . num_blocks = bswap_32 ( hdr . u . v1 . num_blocks );
hdr . u . v1 . page_size = bswap_32 ( hdr . u . v1 . page_size );
}
* data_eof = hdr . u . v1 . data_eof;
* idx2_eof = hdr . u . v1 . idx2_eof;
* pgsize = hdr . u . v1 . page_size;
* checksum = hdr . u . v1 . checksum;
count = hdr . u . v1 . num_blocks;
off = KColumnHdrOffset ( hdr, v1 );
/* not supported in v1 */
* idx0_count = 0;
}
break;
default:
rc = KDirectoryVOpenFileRead ( dir, ( const KFile** ) & self -> fidx, "idx", NULL );
if ( rc == 0 )
{
off = sizeof hdr . dad;
rc = KFileReadAll ( self -> fidx, 0, & hdr, sizeof hdr, & num_bytes );
if ( rc == 0 )
{
if ( num_bytes < KColumnHdrOffset ( hdr, v2 ) )
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
rc = KDBHdrValidate ( & hdr . dad,
num_bytes, 2, KCOL_CURRENT_VERSION );
if ( GetRCState ( rc ) == rcIncorrect && GetRCObject ( rc ) == rcByteOrder )
{
if ( ! self->bswap ) /* catch mis-matched endianess */
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
else
{
hdr . dad . endian = bswap_32 ( hdr . dad . endian );
hdr . dad . version = bswap_32 ( hdr . dad . version );
rc = KDBHdrValidate ( & hdr . dad, num_bytes, 1, KCOL_CURRENT_VERSION );
}
}
else if ( self -> bswap ) /* catch mis-matched endianess */
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcCorrupt );
if ( rc == 0 ) switch ( hdr . dad . version )
{
case 2:
if ( self -> bswap )
{
hdr. u . v2 . data_eof = bswap_64 ( hdr. u . v2 . data_eof );
hdr. u . v2 . idx2_eof = bswap_64 ( hdr. u . v2 . idx2_eof );
hdr. u . v2 . num_blocks = bswap_32 ( hdr. u . v2 . num_blocks );
hdr. u . v2 . page_size = bswap_32 ( hdr. u . v2 . page_size );
}
* data_eof = hdr . u . v2 . data_eof;
* idx2_eof = hdr . u . v2 . idx2_eof;
* pgsize = hdr . u . v2 . page_size;
* checksum = hdr . u . v2 . checksum;
count = hdr . u . v2 . num_blocks;
* idx0_count = 0;
break;
case 3:
if ( self -> bswap )
{
hdr. u . v3 . data_eof = bswap_64 ( hdr. u . v3 . data_eof );
hdr. u . v3 . idx2_eof = bswap_64 ( hdr. u . v3 . idx2_eof );
hdr. u . v3 . idx0_count = bswap_32 ( hdr. u . v3 . idx0_count );
hdr. u . v3 . num_blocks = bswap_32 ( hdr. u . v3 . num_blocks );
hdr. u . v3 . page_size = bswap_32 ( hdr. u . v3 . page_size );
}
* data_eof = hdr . u . v3 . data_eof;
* idx2_eof = hdr . u . v3 . idx2_eof;
* idx0_count = hdr . u . v3 . idx0_count;
* pgsize = hdr . u . v3 . page_size;
* checksum = hdr . u . v3 . checksum;
count = hdr . u . v3 . num_blocks;
break;
default:
rc = RC ( rcDB, rcColumn, rcOpening, rcIndex, rcBadVersion );
}
}
}
}
}
if ( rc == 0 )
{
self -> vers = hdr . dad . version;
#if LAZY_LOAD_COLUMN_IDX1
self -> load_off = off;
self -> count = count;
#else
rc = KColumnIdx1Init ( self, off, count );
if ( rc == 0 )
#endif
return rc;
}
KFileRelease ( self -> fidx );
self -> fidx = NULL;
}
}
}
KFileRelease ( self -> f );
self -> f = NULL;
}
return rc;
}
static
rc_t CCCopyDoFile (CCCopy * self)
{
const KFile * original;
rc_t rc = 0;
enum KCreateMode mode;
PLOGMSG (klogDebug9, "CCCopyDoFile $(f)", PLOG_S(f), self->path);
if (! self->force)
{
/* if not forced replace mode we fail on existing file */
mode = kcmCreate;
}
else
{
uint32_t tt;
tt = KDirectoryPathType (self->out, self->path);
switch (tt)
{
default:
PLOGMSG (klogWarn, "File exists and will be replaced in output directory $(f)",
PLOG_S(f), self->path);
break;
/* if the path to the file or the file do not exist no warning */
case kptNotFound:
case kptBadPath:
break;
}
tt = KDirectoryPathType (self->xml, self->path);
switch (tt)
{
default:
PLOGMSG (klogWarn, "File exists and might be replaced in xml directory $(f)",
PLOG_S(f), self->path);
break;
/* if the path to the file or the file do not exist no warning */
case kptNotFound:
case kptBadPath:
break;
}
/* forced mode we create with init instead of create forcing a delete/create effect */
mode = kcmInit;
}
/* open original source for read */
rc = KDirectoryVOpenFileRead (self->in, &original, self->path, NULL);
if (rc == 0)
{
KFile * copy;
/* create copy output for write */
rc = KDirectoryVCreateFile (self->out, ©, false, 0644, mode|kcmParents,
self->path, NULL);
if (rc == 0)
{
KFile * fm;
/* create parallel <path>.md5 */
rc = KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5",
self->path);
if (rc == 0)
{
KMD5SumFmt * md5f;
/* make the .md5 an MD5 sum format file */
rc = KMD5SumFmtMakeUpdate (&md5f, fm);
if (rc == 0)
{
union u
{
KFile * kf;
KMD5File * mf;
} outf;
/* combine the copy and MD5 file into our special KFile */
rc = KMD5FileMakeWrite (&outf.mf, copy, md5f, self->path);
if (rc == 0)
{
const KFile * inf;
/* release this outside reference to the MD5SumFMT leaving
* only the one internal to the KMD5File */
KMD5SumFmtRelease (md5f);
/* create the KTeeFile that copies reads from the
* original as writes to the copy. Reads will be
* made by the cataloging process */
rc = KFileMakeTeeRead (&inf, original, outf.kf);
if (rc == 0)
{
CCCat * po;
KTime_t mtime;
/* try to get a modification time for this pathname */
rc = KDirectoryVDate (self->in, &mtime, self->path, NULL);
if (rc != 0)
mtime = 0; /* default to ? 0? */
/* create the cataloger giving it the infile which
* is the KTeeFile, Indirectory, the XML directory,
* and the original path for the file */
rc = CCCatMake (&po, self->in, self->xml, inf, self->md5,
self->ff, mtime, self->tree, false, 0, self->path);
if (rc == 0)
{
/* do the catalog (and thus copy) */
rc = CCCatDo(po);
/* release the cataloger object */
CCCatRelease (po);
}
else
pLOGERR (klogDebug6, rc, "failure in CCCatMake $(P)",
PLOG_S(P), self->path);
/* release the infile which will complete a copy
* regardless of the state of the cataloger */
KFileRelease (inf);
/* return rc; */
}
else
{
KFileRelease (outf.kf);
KFileRelease (original);
pLOGERR (klogDebug4, rc, "failure with kfilemaketeeread $(P)",
PLOG_S(P), self->path);
} /* rc = KFileMakeTeeRead (&inf, original, outf.kf);*/
}
else
{
KFileRelease (copy);
KMD5SumFmtRelease (md5f);
pLOGERR (klogDebug4, rc, "failure with KMD5FileMakeWrite $(P)",
PLOG_S(P), self->path);
} /* KMD5FileMakeWrite (&outf.mf, copy, md5f, self->path); */
} /* KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5", */
else
pLOGERR (klogDebug4, rc, "failure with KMD5SumFmtMakeUpdate $(P)",
PLOG_S(P), self->path);
KFileRelease (fm);
} /* KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5", */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryCreateFile $(P).md5",
PLOG_S(P), self->path);
KFileRelease (copy);
} /* rc = KDirectoryVCreateFile (self->out, ©, false, 0644, mode|kcmParents, */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryVCreateFile $(P)",
PLOG_S(P), self->path);
KFileRelease (original);
} /* rc = KDirectoryVOpenFileRead (self->in, &original, self->path, NULL); */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryVOpenFileRead $(pP)",
PLOG_S(P), self->path);
return rc;
}
inline rc_t OpenFileRead ( struct KFile const **f, const char *path,
va_list args ) const throw ()
{ return KDirectoryVOpenFileRead ( this, f, path, args ); }
