static
rc_t CopierMakeF (Copier * cp, KDirectory * dir, const char * path)
{
rc_t rc, orc;
KFile * fm;
rc = KDirectoryCreateFile (dir, &fm, true, 0666, kcmCreate|kcmParents,
"%s.md5", path);
if (rc == 0)
{
KMD5SumFmt * md5f;
rc = KMD5SumFmtMakeUpdate (&md5f, fm);
if (rc == 0)
{
KFile * f;
rc = KDirectoryCreateFile (dir, &f, false, 0666, kcmCreate,
"%s", path);
if (rc == 0)
{
KMD5File * fmd5;
rc = KMD5FileMakeWrite (&fmd5, f, md5f, path);
if (rc == 0)
{
cp->f = KMD5FileToKFile (fmd5);
orc = KMD5SumFmtRelease (md5f);
if (orc)
LOGERR (klogInt, orc, "Failure releasing MD5 format");
cp->o = 0; /* start of file */
return rc;
}
orc = KFileRelease (f);
if (orc)
LOGERR (klogInt, orc, "Failure releasing Copier file");
}
orc = KMD5SumFmtRelease (md5f);
if (orc)
LOGERR (klogInt, orc, "Failure releasing MD5 format");
}
orc = KFileRelease (fm);
if (orc)
LOGERR (klogInt, orc, "Failure releasing MD5SUM file");
}
return rc;
}
/* Whack
*/
static
rc_t KDatabaseWhack ( KDatabase *self )
{
rc_t rc = 0;
KDBManager *mgr = self -> mgr;
KSymbol * symb;
assert ( mgr != NULL );
KRefcountWhack ( & self -> refcount, "KDatabase" );
/* release dad */
if ( self -> dad != NULL )
{
rc = KDatabaseSever ( self -> dad );
if ( rc != 0 )
return rc;
self -> dad = NULL;
}
/* shut down md5 sum file if it is open */
KMD5SumFmtRelease ( self -> md5 ), self -> md5 = NULL;
/* remove from mgr */
symb = KDBManagerOpenObjectFind (mgr, self->path);
if (symb != NULL)
{
rc = KDBManagerOpenObjectDelete (mgr, symb);
if (rc == 0)
{
/* release manager
should never fail */
rc = KDBManagerSever ( mgr );
if ( rc != 0 )
rc = KDBManagerOpenObjectAdd (mgr, symb);
else
{
/* complete */
KDirectoryRelease ( self -> dir );
free ( self );
return 0;
}
}
}
KRefcountInit ( & self -> refcount, 1, "KDatabase", "whack", "kdb" );
return rc;
}
/* Whack
*/
static
rc_t KDatabaseWhack ( KDatabase *self )
{
rc_t rc = 0;
/* unused: const KDBManager *mgr; */
KRefcountWhack ( & self -> refcount, "KDatabase" );
/* shut down md5 fmt file */
if ( self -> md5 != NULL )
{
rc = KMD5SumFmtRelease ( self -> md5 );
if ( rc == 0 )
self -> md5 = NULL;
}
if ( rc == 0 )
{
/* release dad */
if ( self -> dad != NULL )
{
rc = KDatabaseSever ( self -> dad );
if ( rc == 0 )
self -> dad = NULL;
}
/* remove from mgr */
if ( rc == 0 )
{
/* rc = KDBManagerSever ( self -> mgr, & self -> sym ); */
rc = KDBManagerSever ( self -> mgr );
/* complete */
if ( rc == 0 )
{
KDirectoryRelease ( self -> dir );
free ( self );
return 0;
}
}
}
KRefcountInit ( & self -> refcount, 1, "KDatabase", "whack", "kdb" );
return rc;
}
rc_t ProcessOneRelease (const ProcessOne * cself)
{
ProcessOne * self = (ProcessOne *)cself;
rc_t rc = 0;
if (self != NULL)
{
PLOGMSG (klogDebug10, "ProcessOneRelease $(f)", PLOG_S(f), self->path);
if (atomic32_dec_and_test (&self->refcount))
{
KDirectoryRelease (self->dir);
KDirectoryRelease (self->xml);
KMD5SumFmtRelease (self->md5);
CCFileFormatRelease (self->ff);
KFileRelease (self->file);
free (self);
}
}
return rc;
}
rc_t CCCopyRelease (const CCCopy * cself)
{
CCCopy * self = (CCCopy *)cself;
rc_t rc = 0;
LOGMSG (klogDebug9, "Enter: CCCopyRelease");
if (self != NULL)
{
if (atomic32_dec_and_test (&self->refcount))
{
KDirectoryRelease (self->in);
KDirectoryRelease (self->out);
KDirectoryRelease (self->xml);
KMD5SumFmtRelease (self->md5);
CCFileFormatRelease (self->ff);
free (self);
}
}
return rc;
}
rc_t KU64IndexPersist_v3(KU64Index_v3* self, bool proj, KDirectory *dir, const char *path, bool use_md5)
{
KU64Index_PersistData pd;
char tmpname[256];
char tmpmd5name[256];
char md5path[256];
self->rc = 0;
memset(&pd, 0, sizeof(KU64Index_PersistData));
self->rc = KDirectoryResolvePath(dir, false, tmpname, sizeof(tmpname), "%s.tmp", path);
if( self->rc == 0 ) {
self->rc = KDirectoryCreateFile(dir, &pd.file, true, 0664, kcmInit, "%s", tmpname);
if(use_md5 && self->rc == 0 ) {
KMD5SumFmt *km = NULL;
size_t tmplen = snprintf(tmpmd5name, sizeof(tmpmd5name), "%s.md5", tmpname);
KFile* kf = NULL;
if( tmplen >= sizeof(tmpmd5name) ) {
self->rc = RC(rcDB, rcIndex, rcPersisting, rcName, rcExcessive);
} else {
tmplen = snprintf(md5path, sizeof(md5path), "%s.md5", path);
if( tmplen >= sizeof(md5path) ) {
self->rc = RC(rcDB, rcIndex, rcPersisting, rcName, rcExcessive);
} else {
self->rc = KDirectoryCreateFile(dir, &kf, true, 0664, kcmInit, "%s", tmpmd5name);
if( self->rc == 0 ) {
self->rc = KMD5SumFmtMakeUpdate(&km, kf);
if( self->rc == 0 ) {
KMD5File * k5f;
kf = NULL;
self->rc = KMD5FileMakeWrite(&k5f, pd.file, km, path);
if( self->rc == 0 ) {
pd.file_md5 = k5f;
pd.file = KMD5FileToKFile(k5f);
}
/* release pass or fail */
KMD5SumFmtRelease(km);
} else {
KFileRelease ( kf );
}
} else {
KFileRelease ( kf );
}
}
}
if( self->rc != 0 ) {
KFileRelease(pd.file);
}
}
if( self->rc == 0 ) {
struct KIndexFileHeader_v3 head;
size_t writ = 0;
KDBHdrInit(&head.h, 3);
head.index_type = kitU64;
self->rc = KFileWrite(pd.file, pd.pos, &head, sizeof(struct KIndexFileHeader_v3), &writ);
if( self->rc == 0 ) {
pd.pos += writ;
if( use_md5 ) {
KMD5FileBeginTransaction(pd.file_md5);
}
self->rc = BSTreePersist(&self->tree, NULL, KU64Index_WriteFunc, &pd, KU64Index_AuxFunc, &pd);
}
KFileRelease(pd.file);
pd.file = NULL;
}
}
if( self->rc == 0 ) {
self->rc = KDirectoryRename(dir, false, tmpname, path);
if( self->rc == 0 ) {
if ( use_md5 ) {
self->rc = KDirectoryRename(dir, false, tmpmd5name, md5path);
}
if( self->rc == 0 ) {
/* done */
return 0;
}
}
}
/* whack temporary file */
KDirectoryRemove(dir, false, "%s", tmpname);
if( use_md5 ) {
KDirectoryRemove(dir, false, "%s", tmpmd5name);
}
return self->rc;
}
/* OpenDBUpdate
* VOpenDBUpdate
* open a database for read/write
*
* "db" [ OUT ] - return parameter for newly opened database
*
* "path" [ IN ] - NUL terminated string in
* wd-native character set giving path to database
*/
static
rc_t KDBManagerVOpenDBUpdateInt ( KDBManager *self,
KDatabase **db, KDirectory *wd, const char *path, va_list args )
{
char dbpath [ 4096 ];
rc_t rc = KDirectoryVResolvePath ( wd, true,
dbpath, sizeof dbpath, path, args );
if ( rc == 0 )
{
KSymbol * sym;
KFile *f;
KMD5SumFmt * md5 = NULL;
/* if already open, refuse */
sym = KDBManagerOpenObjectFind (self, dbpath);
if (sym != NULL)
{
rc_t obj;
switch (sym->type)
{
default:
obj = rcPath;
break;
case kptDatabase:
obj = rcDatabase;
break;
case kptTable:
obj = rcTable;
break;
case kptColumn:
obj = rcColumn;
break;
case kptIndex:
obj = rcIndex;
break;
case kptMetadata:
obj = rcMetadata;
break;
}
return RC ( rcDB, rcMgr, rcOpening, obj, rcBusy );
}
/* only open existing dbs */
switch (KDBPathType ( /*self,*/ wd, NULL, dbpath ) )
{
case kptNotFound:
return RC ( rcDB, rcMgr, rcOpening, rcDatabase, rcNotFound );
case kptBadPath:
return RC ( rcDB, rcMgr, rcOpening, rcPath, rcInvalid );
case kptFile:
case kptFile | kptAlias:
/* if we find a file, vary the failure if it is an archive that is a database
* or a non related file */
if ( KDBOpenPathTypeRead ( self, wd, dbpath, NULL, kptDatabase, NULL, false ) == 0 )
return RC ( rcDB, rcMgr, rcOpening, rcDirectory, rcUnauthorized );
/* fall through */
default:
return RC ( rcDB, rcMgr, rcOpening, rcPath, rcIncorrect );
case kptDatabase:
case kptDatabase | kptAlias:
break;
}
/* test now for locked directory */
rc = KDBWritable (wd, dbpath);
switch (GetRCState (rc))
{
default:
#if 0
return RC ( rcDB, rcMgr, rcOpening, rcDatabase, rcNoPerm ); /* TBD: better state? */
#endif
case rcLocked:
return RC ( rcDB, rcMgr, rcOpening, rcDatabase, rcLocked );
case rcReadonly:
return RC ( rcDB, rcMgr, rcOpening, rcDatabase, rcReadonly );
case 0:
rc = 0;
break;
}
rc = KDirectoryOpenFileWrite ( wd, &f, true, "%s/md5", dbpath );
if ( rc == 0 )
{
rc = KMD5SumFmtMakeUpdate ( & md5, f );
if ( rc != 0 )
KFileRelease ( f );
}
else if ( GetRCState ( rc ) == rcNotFound )
rc = 0;
if ( rc == 0 )
rc = KDBManagerMakeDBUpdate ( self, db, wd, dbpath, md5 );
KMD5SumFmtRelease ( md5 );
}
return rc;
}
/* CreateDB
* VCreateDB
* create a new or open an existing database
*
* "db" [ OUT ] - return parameter for newly opened database
*
* "cmode" [ IN ] - creation mode
*
* "path" [ IN ] - NUL terminated string in
* wd-native character set giving path to database
*/
static
rc_t KDBManagerVCreateDBInt ( KDBManager *self,
KDatabase **db, KDirectory *wd, KCreateMode cmode,
const char *path, va_list args )
{
char dbpath [ 4096 ];
rc_t rc = KDirectoryVResolvePath ( wd, true,
dbpath, sizeof dbpath, path, args );
if ( rc == 0 )
{
/* we won't try accession resolution here */
int type = KDBPathType ( /*NULL,*/ wd, NULL, dbpath );
switch ( type )
{
case kptNotFound:
/* first good path */
break;
case kptBadPath:
return RC ( rcDB, rcMgr, rcCreating, rcPath, rcInvalid );
case kptDatabase:
case kptDatabase | kptAlias:
/* found so is not good if we want to create new and not
* clear/init or open old
*/
if ((cmode & kcmValueMask) == kcmCreate)
return RC ( rcDB, rcMgr, rcCreating, rcDatabase, rcExists );
if (KDBManagerOpenObjectBusy (self, dbpath))
return RC ( rcDB, rcMgr, rcCreating, rcDatabase, rcBusy );
/* test now for locked directory */
rc = KDBWritable ( wd, dbpath );
if (rc)
{
switch (GetRCState(rc))
{
default:
return rc;
case rcLocked:
return RC ( rcDB, rcMgr, rcCreating, rcDatabase, rcLocked );
case rcReadonly:
return RC ( rcDB, rcMgr, rcCreating, rcDatabase, rcReadonly );
case rcNotFound:
/* not found is good but probably unreachable */
break;
case 0:
rc = 0;
break;
}
}
/* second good path */
break;
case kptTable:
case kptTable | kptAlias:
return RC (rcDB, rcMgr, rcCreating, rcTable, rcExists);
case kptColumn:
case kptColumn | kptAlias:
return RC (rcDB, rcMgr, rcCreating, rcColumn, rcExists);
case kptIndex:
case kptIndex | kptAlias:
return RC (rcDB, rcMgr, rcCreating, rcIndex, rcExists);
case kptMetadata:
case kptMetadata | kptAlias:
return RC (rcDB, rcMgr, rcCreating, rcMetadata, rcExists);
case kptFile:
case kptFile | kptAlias:
/* if we find a file, vary the failure if it is an archive that is a database
* or a non related file */
if ( KDBOpenPathTypeRead ( self, wd, dbpath, NULL, kptDatabase, NULL, false ) == 0 )
return RC ( rcDB, rcMgr, rcCreating, rcDirectory, rcUnauthorized );
/* fall through */
default:
return RC ( rcDB, rcMgr, rcCreating, rcPath, rcIncorrect );
}
/* [re]create directory */
rc = KDirectoryCreateDir ( wd, 0775, cmode, "%s", dbpath );
if (rc == 0)
/* create tbl subdirectory as required for db */
rc = KDirectoryCreateDir ( wd, 0775, kcmOpen, "%s/tbl", dbpath );
if ( rc == 0 )
{
KMD5SumFmt *md5 = NULL;
if ( ( cmode & kcmMD5 ) != 0 )
{
KFile * f;
/* if needed create the md5 digest file */
rc = KDirectoryCreateFile ( wd, &f, true, 0664, kcmOpen, "%s/md5", dbpath );
if ( rc == 0 )
{
/* create a formatter around file
formatter will own "f" afterward */
rc = KMD5SumFmtMakeUpdate ( & md5, f );
/* if failed to create formatter, release "f" */
if ( rc != 0 )
KFileRelease ( f );
}
}
if ( rc == 0 )
rc = KDBManagerMakeDBUpdate ( self, db, wd, dbpath, md5 );
KMD5SumFmtRelease ( md5 );
}
}
return rc;
}
rc_t CopyFileToFile( const KDirectory *top, const char *inname, KDirectory *targettop, const char *outname )
{
const KFile *in = NULL;
KFile *out = NULL;
KFile *md5file = NULL;
KMD5File *md5out = NULL;
KMD5SumFmt *md5sumfmt = NULL;
char md5filename[1024];
rc_t rc = 0;
uint32_t mode = 0;
uint32_t pathtype = 0;
uint32_t failed = 0;
if (PathIsMD5File(top, inname)) {
/* Skip it */
return 0;
}
rc = KDirectoryOpenFileRead( top, &in, "%s", inname );
if (rc != 0) {
failed = rc;
goto FAIL;
}
mode = DEFAULTMODE;
rc = KDirectoryAccess( top, &mode, inname);
if (rc != 0) {
failed = rc;
goto FAIL;
}
/*
* Not sure here -- does kcmInit re-initialize the file mode as we specify?
* Or does it preserve the existing mode (and do we want it to)?
*/
if (clobber_protections) {
pathtype = KDirectoryPathType( targettop, "%s", outname );
if ((pathtype & ~kptAlias) == kptFile) {
rc = KDirectorySetAccess( targettop, false, mode, 0777, "%s", outname);
if (rc != 0) {
failed = rc;
goto FAIL;
}
}
}
rc = KDirectoryCreateFile( targettop, &out, false, mode, (force? kcmInit: kcmCreate), "%s", outname );
if (rc != 0) {
failed = rc;
goto FAIL;
}
sprintf(md5filename, "%s.md5", outname);
rc = KDirectoryCreateFile( targettop, &md5file, false, DEFAULTMODE, (force? kcmInit: kcmCreate), "%s", md5filename);
if (rc != 0) {
failed = rc;
goto FAIL;
}
rc = KMD5SumFmtMakeUpdate( &md5sumfmt, md5file);
if (rc != 0) {
failed = rc;
goto FAIL;
}
rc = KMD5FileMakeWrite( &md5out, out, md5sumfmt, outname );
if (rc != 0) {
failed = rc;
goto FAIL;
}
{
uint64_t rpos = 0;
uint64_t wpos = 0;
size_t numread;
while (true) {
rc = KFileRead( in, rpos, buffer, BUFSIZE, &numread );
/* fprintf(stderr, "Read %d bytes.\n", numread); */
if (rc == 0 && numread == 0)
break;
if (rc != 0) {
failed = rc;
goto FAIL;
}
rpos += numread;
{
size_t numwritten = 0;
int written = 0;
while (written < numread) {
rc = KFileWrite( (KFile *)md5out, wpos, buffer+written, numread-written, &numwritten );
if (rc != 0) {
failed = rc;
break;
}
if (numwritten == 0) {
fprintf(stderr, "Didn't write anything.\n");
failed = -1;
goto FAIL;
}
wpos += numwritten;
written += numwritten;
}
}
}
}
/* Success also, check the value of failed to see if failed */
FAIL:
if (NULL != md5out) {
KFileRelease((KFile *)md5out);
md5out = NULL;
}
/*KFileRelease(out); */
if (NULL != md5sumfmt) {
KMD5SumFmtRelease(md5sumfmt);
md5sumfmt = NULL;
}
/* KFileRelease(md5file); */
if (NULL != in) {
KFileRelease(in);
in = NULL;
}
/* KDirectoryRelease(top); */
if (failed) {
KDirectoryRemove( targettop, false, "%s", md5filename );
KDirectoryRemove( targettop, false, "%s", outname);
}
return failed;
}
static
rc_t CCCopyDoFile (CCCopy * self)
{
const KFile * original;
rc_t rc = 0;
enum KCreateMode mode;
PLOGMSG (klogDebug9, "CCCopyDoFile $(f)", PLOG_S(f), self->path);
if (! self->force)
{
/* if not forced replace mode we fail on existing file */
mode = kcmCreate;
}
else
{
uint32_t tt;
tt = KDirectoryPathType (self->out, self->path);
switch (tt)
{
default:
PLOGMSG (klogWarn, "File exists and will be replaced in output directory $(f)",
PLOG_S(f), self->path);
break;
/* if the path to the file or the file do not exist no warning */
case kptNotFound:
case kptBadPath:
break;
}
tt = KDirectoryPathType (self->xml, self->path);
switch (tt)
{
default:
PLOGMSG (klogWarn, "File exists and might be replaced in xml directory $(f)",
PLOG_S(f), self->path);
break;
/* if the path to the file or the file do not exist no warning */
case kptNotFound:
case kptBadPath:
break;
}
/* forced mode we create with init instead of create forcing a delete/create effect */
mode = kcmInit;
}
/* open original source for read */
rc = KDirectoryVOpenFileRead (self->in, &original, self->path, NULL);
if (rc == 0)
{
KFile * copy;
/* create copy output for write */
rc = KDirectoryVCreateFile (self->out, ©, false, 0644, mode|kcmParents,
self->path, NULL);
if (rc == 0)
{
KFile * fm;
/* create parallel <path>.md5 */
rc = KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5",
self->path);
if (rc == 0)
{
KMD5SumFmt * md5f;
/* make the .md5 an MD5 sum format file */
rc = KMD5SumFmtMakeUpdate (&md5f, fm);
if (rc == 0)
{
union u
{
KFile * kf;
KMD5File * mf;
} outf;
/* combine the copy and MD5 file into our special KFile */
rc = KMD5FileMakeWrite (&outf.mf, copy, md5f, self->path);
if (rc == 0)
{
const KFile * inf;
/* release this outside reference to the MD5SumFMT leaving
* only the one internal to the KMD5File */
KMD5SumFmtRelease (md5f);
/* create the KTeeFile that copies reads from the
* original as writes to the copy. Reads will be
* made by the cataloging process */
rc = KFileMakeTeeRead (&inf, original, outf.kf);
if (rc == 0)
{
CCCat * po;
KTime_t mtime;
/* try to get a modification time for this pathname */
rc = KDirectoryVDate (self->in, &mtime, self->path, NULL);
if (rc != 0)
mtime = 0; /* default to ? 0? */
/* create the cataloger giving it the infile which
* is the KTeeFile, Indirectory, the XML directory,
* and the original path for the file */
rc = CCCatMake (&po, self->in, self->xml, inf, self->md5,
self->ff, mtime, self->tree, false, 0, self->path);
if (rc == 0)
{
/* do the catalog (and thus copy) */
rc = CCCatDo(po);
/* release the cataloger object */
CCCatRelease (po);
}
else
pLOGERR (klogDebug6, rc, "failure in CCCatMake $(P)",
PLOG_S(P), self->path);
/* release the infile which will complete a copy
* regardless of the state of the cataloger */
KFileRelease (inf);
/* return rc; */
}
else
{
KFileRelease (outf.kf);
KFileRelease (original);
pLOGERR (klogDebug4, rc, "failure with kfilemaketeeread $(P)",
PLOG_S(P), self->path);
} /* rc = KFileMakeTeeRead (&inf, original, outf.kf);*/
}
else
{
KFileRelease (copy);
KMD5SumFmtRelease (md5f);
pLOGERR (klogDebug4, rc, "failure with KMD5FileMakeWrite $(P)",
PLOG_S(P), self->path);
} /* KMD5FileMakeWrite (&outf.mf, copy, md5f, self->path); */
} /* KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5", */
else
pLOGERR (klogDebug4, rc, "failure with KMD5SumFmtMakeUpdate $(P)",
PLOG_S(P), self->path);
KFileRelease (fm);
} /* KDirectoryCreateFile (self->out, &fm, true, 0644, mode, "%s.md5", */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryCreateFile $(P).md5",
PLOG_S(P), self->path);
KFileRelease (copy);
} /* rc = KDirectoryVCreateFile (self->out, ©, false, 0644, mode|kcmParents, */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryVCreateFile $(P)",
PLOG_S(P), self->path);
KFileRelease (original);
} /* rc = KDirectoryVOpenFileRead (self->in, &original, self->path, NULL); */
else
pLOGERR (klogDebug4, rc, "failure with KDirectoryVOpenFileRead $(pP)",
PLOG_S(P), self->path);
return rc;
}
