static rc_t on_clear_path( visit_ctx * obj )
{
rc_t rc = 0;
clear_data * data = obj->data;
/* the caller is written to first hit the files/subdirs, then the directory that contains them */
if ( obj->path_type == kptDir )
{
if ( obj->options->remove_dirs )
{
rc = KDirectoryRemove ( obj->dir, false, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryRemove( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
data->removed_directories++;
}
}
else if ( obj->path_type == kptFile )
{
uint64_t file_size;
rc = KDirectoryFileSize ( obj->dir, &file_size, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryFileSize( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
rc = KDirectoryRemove ( obj->dir, false, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryRemove( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
if ( obj->options->detailed )
rc = KOutMsg( "FILE: '%s' removed (%,u bytes)\n", obj->path, file_size );
data->removed_files++;
data->removed_size += file_size;
if ( obj->options->max_remove > 0 && data->removed_size >= obj->options->max_remove )
{
obj->terminate = true;
KOutMsg( "the maximum of %,lu bytes to be removed is reached now\n", obj->options->max_remove );
}
}
}
}
return rc;
}
rc_t CC KMain (int argc, char * argv[])
{
rc_t rc = 0;
KDirectory* wd;
KLogLevelSet(klogInfo);
LogMsg ( klogInfo, "KeyringServer: starting");
rc = KDirectoryNativeDir (&wd);
if (rc == 0)
{
KFile* lockedFile;
const char* dataDir;
char lockFileName[MAX_PATH];
if (argc < 2 || argv[1] == NULL)
dataDir = KeyRingDefaultDataDir;
else
dataDir = argv[1];
rc = string_printf(lockFileName, sizeof(lockFileName)-1, NULL, "%s/keyring_lock", dataDir);
if (rc == 0)
{
rc = KDirectoryCreateExclusiveAccessFile(wd, &lockedFile, true, 0600, kcmOpen, "%s", lockFileName);
if (rc == 0)
{
KNSManager* mgr;
rc = KNSManagerMake(&mgr);
if (rc == 0)
{
rc = string_printf(keyRingFileName, sizeof(keyRingFileName)-1, NULL, "%s/keyring", dataDir);
if (rc == 0)
rc = Server(mgr);
KNSManagerRelease(mgr);
}
else
LogErr(klogErr, rc, "KeyringServer: KNSManagerMake failed");
KFileRelease(lockedFile);
LogMsg ( klogInfo, "KeyringServer: removing lock file.");
KDirectoryRemove(wd, true, "%s", lockFileName);
}
else
{ /*TODO: check for stale lock file*/
LogMsg ( klogInfo, "KeyringServer: another instance appears to be running.");
rc = 0;
}
}
else
LogErr ( klogErr, rc, "KeyringServer: failed to build the lock file name" );
KDirectoryRelease(wd);
}
else
LogErr(klogErr, rc, "KeyringServer: KDirectoryNativeDir failed");
LogMsg ( klogInfo, "KeyringServer: finishing");
return rc;
}
/* CreateLockFile
* attempts to create a KLockFile
*
* "lock" [ OUT ] - return parameter for newly created lock file
*
* "path" [ IN ] - NUL terminated string in directory-native
* character set denoting lock file
*/
LIB_EXPORT rc_t CC KDirectoryVCreateLockFile ( KDirectory *self,
KLockFile **lock, const char *path, va_list args )
{
rc_t rc;
if ( lock == NULL )
rc = RC ( rcFS, rcFile, rcLocking, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcFS, rcFile, rcLocking, rcSelf, rcNull );
else if ( path == NULL )
rc = RC ( rcFS, rcFile, rcLocking, rcPath, rcNull );
else if ( path [ 0 ] == 0 )
rc = RC ( rcFS, rcFile, rcLocking, rcPath, rcEmpty );
else
{
char full [ 4096 ];
rc = KDirectoryVResolvePath ( self, false, full, sizeof full, path, args );
if ( rc == 0 )
{
KFile *lock_file;
rc = KDirectoryCreateFile ( self, & lock_file, false, 0600, kcmCreate | kcmParents, "%s", full );
if ( rc == 0 )
{
rc_t rc2;
/* no longer need the file - not going to write to it anyway */
KFileRelease ( lock_file );
/* we have the lock */
rc = KLockFileMake ( lock, self, full );
if ( rc == 0 )
return 0;
/* must unlink lockfile */
rc2 = KDirectoryRemove ( self, true, "%s", full );
if ( rc2 != 0 )
/* issue a report */;
}
else if ( GetRCState ( rc ) == rcExists )
{
/* map the rc to kproc type values */
rc = RC ( rcFS, rcFile, rcLocking, rcLocking, rcBusy );
}
else
{
rc = ResetRCContext ( rc, rcFS, rcFile, rcLocking );
}
}
}
* lock = NULL;
}
return rc;
}
rc_t CreateDatabase(KKeyRing* self)
{ /* database is presumed write-locked */
rc_t rc;
assert(self);
KDirectoryRemove(self->wd, true, "%s", tmp_path); /* in case exists */
rc = KeyRingDatabaseSave(self->data, self->wd, tmp_path);
if (rc == 0)
{
rc_t rc2;
rc = ArchiveAndEncrypt(self->wd, tmp_path, self->path, self->passwd);
rc2 = KDirectoryRemove(self->wd, true, "%s", tmp_path);
if (rc == 0)
rc = rc2;
}
return rc;
}
static rc_t delete_tmp_files( const sorter_params * params, uint32_t count )
{
rc_t rc = 0;
char buffer[ 4096 ];
uint32_t i;
for ( i = 0; rc == 0 && i < count; ++ i )
{
make_pool_src_filename( params, i + 1, buffer, sizeof buffer );
if ( rc == 0 )
rc = KDirectoryRemove( params->dir, true, "%s", buffer );
if ( rc != 0 )
ErrMsg( "KDirectoryRemove( 'tmp_%d.dat' ) -> %R", rc );
}
return rc;
}
static rc_t remove_path( KDirectory * dir, const char * path, bool quiet )
{
rc_t rc;
if ( !quiet )
{
PLOGMSG( klogInfo, ( klogInfo, "removing '$(path)'", "path=%s", path ));
}
rc = KDirectoryRemove ( dir, true, path );
if ( rc != 0 )
{
LOGERR( klogErr, rc, "remove_path:KDirectoryRemove() failed" );
}
return rc;
}
static rc_t on_unlock_path( visit_ctx * obj )
{
rc_t rc = 0;
if ( obj->path_type == kptFile && string_ends_in( obj->path, ".lock" ) )
{
unlock_data * data = obj->data;
rc = KDirectoryRemove ( obj->dir, false, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryRemove( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
data->lock_count++;
rc = KOutMsg( "%s deleted\n", obj->path );
}
}
return rc;
}
static rc_t cg_dump_remove_file( cg_dump_opts * opts, cg_dump_ctx * cg_ctx )
{
rc_t rc;
if ( opts->overwrite )
{
rc = KDirectoryRemove ( cg_ctx->dir, true, "%s", cg_ctx->dst );
if ( rc != 0 )
{
(void)PLOGERR( klogErr, ( klogErr, rc, "cannot remove file '$(dt)'",
"dt=%s", cg_ctx->dst ) );
}
}
else
{
rc = RC( rcExe, rcNoTarg, rcReading, rcParam, rcInvalid );
(void)PLOGERR( klogErr, ( klogErr, rc, "output-directory-name exists as file '$(dt)', use force-switch",
"dt=%s", cg_ctx->dst ) );
KOutMsg( "output-directory-name exists as file '%s', use force-switch", cg_ctx->dst );
}
return rc;
}
rc_t KU64IndexPersist_v3(KU64Index_v3* self, bool proj, KDirectory *dir, const char *path, bool use_md5)
{
KU64Index_PersistData pd;
char tmpname[256];
char tmpmd5name[256];
char md5path[256];
self->rc = 0;
memset(&pd, 0, sizeof(KU64Index_PersistData));
self->rc = KDirectoryResolvePath(dir, false, tmpname, sizeof(tmpname), "%s.tmp", path);
if( self->rc == 0 ) {
self->rc = KDirectoryCreateFile(dir, &pd.file, true, 0664, kcmInit, "%s", tmpname);
if(use_md5 && self->rc == 0 ) {
KMD5SumFmt *km = NULL;
size_t tmplen = snprintf(tmpmd5name, sizeof(tmpmd5name), "%s.md5", tmpname);
KFile* kf = NULL;
if( tmplen >= sizeof(tmpmd5name) ) {
self->rc = RC(rcDB, rcIndex, rcPersisting, rcName, rcExcessive);
} else {
tmplen = snprintf(md5path, sizeof(md5path), "%s.md5", path);
if( tmplen >= sizeof(md5path) ) {
self->rc = RC(rcDB, rcIndex, rcPersisting, rcName, rcExcessive);
} else {
self->rc = KDirectoryCreateFile(dir, &kf, true, 0664, kcmInit, "%s", tmpmd5name);
if( self->rc == 0 ) {
self->rc = KMD5SumFmtMakeUpdate(&km, kf);
if( self->rc == 0 ) {
KMD5File * k5f;
kf = NULL;
self->rc = KMD5FileMakeWrite(&k5f, pd.file, km, path);
if( self->rc == 0 ) {
pd.file_md5 = k5f;
pd.file = KMD5FileToKFile(k5f);
}
/* release pass or fail */
KMD5SumFmtRelease(km);
} else {
KFileRelease ( kf );
}
} else {
KFileRelease ( kf );
}
}
}
if( self->rc != 0 ) {
KFileRelease(pd.file);
}
}
if( self->rc == 0 ) {
struct KIndexFileHeader_v3 head;
size_t writ = 0;
KDBHdrInit(&head.h, 3);
head.index_type = kitU64;
self->rc = KFileWrite(pd.file, pd.pos, &head, sizeof(struct KIndexFileHeader_v3), &writ);
if( self->rc == 0 ) {
pd.pos += writ;
if( use_md5 ) {
KMD5FileBeginTransaction(pd.file_md5);
}
self->rc = BSTreePersist(&self->tree, NULL, KU64Index_WriteFunc, &pd, KU64Index_AuxFunc, &pd);
}
KFileRelease(pd.file);
pd.file = NULL;
}
}
if( self->rc == 0 ) {
self->rc = KDirectoryRename(dir, false, tmpname, path);
if( self->rc == 0 ) {
if ( use_md5 ) {
self->rc = KDirectoryRename(dir, false, tmpmd5name, md5path);
}
if( self->rc == 0 ) {
/* done */
return 0;
}
}
}
/* whack temporary file */
KDirectoryRemove(dir, false, "%s", tmpname);
if( use_md5 ) {
KDirectoryRemove(dir, false, "%s", tmpmd5name);
}
return self->rc;
}
static
rc_t FileInPlace (KDirectory * cwd, const char * leaf, bool try_rename)
{
rc_t rc;
bool is_tmp;
STSMSG (1, ("%scrypting file in place %s",De,leaf));
rc = 0;
is_tmp = IsTmpFile (leaf);
if (is_tmp)
{
STSMSG (1, ("%s is a vdb-decrypt/vdb-encrypt temporary file and will "
"be ignored", leaf));
TmpFoundFlag = true;
if (ForceFlag)
; /* LOG OVERWRITE */
else
; /* LOG TMP */
}
if (!is_tmp || ForceFlag)
{
char temp [MY_MAX_PATH];
rc = KDirectoryResolvePath (cwd, false, temp, sizeof temp, ".%s%s",
leaf, TmpExt);
if (rc)
PLOGERR (klogErr, (klogErr, rc, "unable to resolve '.$(S)$(E)'",
"S=%s,E=%s",leaf,TmpExt));
else
{
KPathType kpt;
uint32_t kcm;
kcm = kcmCreate|kcmParents;
kpt = KDirectoryPathType (cwd, temp);
if (kpt != kptNotFound)
{
/* log busy */
if (ForceFlag)
{
kcm = kcmInit|kcmParents;
/* log force */
kpt = kptNotFound;
}
}
if (kpt == kptNotFound)
{
const KFile * infile;
rc = KDirectoryOpenFileRead (cwd, &infile, "%s", leaf);
if (rc)
PLOGERR (klogErr, (klogErr, rc, "Unable to resolve '$(F)'",
"F=%s",leaf));
else
{
EncScheme scheme;
rc = EncryptionTypeCheck (infile, leaf, &scheme);
if (rc == 0)
{
ArcScheme ascheme;
bool changed;
bool do_this_file;
char new_name [MY_MAX_PATH + sizeof EncExt];
do_this_file = DoThisFile (infile, scheme, &ascheme);
strcpy (new_name, leaf);
if (try_rename)
changed = NameFixUp (new_name);
else
changed = false;
/* KOutMsg ("### %d \n", changed); */
if (!do_this_file)
{
if (changed)
{
STSMSG (1, ("renaming %s to %s", leaf, new_name));
rc = KDirectoryRename (cwd, false, leaf, new_name);
}
else
STSMSG (1, ("skipping %s",leaf));
}
else
{
KFile * outfile;
rc = KDirectoryCreateExclusiveAccessFile (cwd, &outfile,
false, 0600, kcm,
temp);
if (rc)
;
else
{
const KFile * Infile;
KFile * Outfile;
rc = CryptFile (infile, &Infile, outfile, &Outfile, scheme);
if (rc == 0)
{
STSMSG (1, ("copying %s to %s", leaf, temp));
rc = CopyFile (Infile, Outfile, leaf, temp);
if (rc == 0)
{
uint32_t access;
KTime_t date;
rc = KDirectoryAccess (cwd, &access, "%s", leaf);
if (rc == 0)
rc = KDirectoryDate (cwd, &date, "%s", leaf);
KFileRelease (infile);
KFileRelease (outfile);
KFileRelease (Infile);
KFileRelease (Outfile);
if (rc == 0)
{
STSMSG (1, ("renaming %s to %s", temp, new_name));
rc = KDirectoryRename (cwd, true, temp, new_name);
if (rc)
LOGERR (klogErr, rc, "error renaming");
else
{
if (changed)
KDirectoryRemove (cwd, false, "%s", leaf);
/*rc =*/
KDirectorySetAccess (cwd, false, access,
0777, "%s", new_name);
KDirectorySetDate (cwd, false, date, "%s", new_name);
/* gonna ignore an error here I think */
return rc;
}
}
}
}
KFileRelease (outfile);
}
}
}
KFileRelease (infile);
}
}
}
}
return rc;
}
static
rc_t CC KRemoveLockFileTaskExecute ( KRemoveLockFileTask *self )
{
return KDirectoryRemove ( self -> dir, true, "%s", self -> path );
}
static
void remove_out_file (const char * path)
{
KDirectoryRemove (kdir, true, path);
}
rc_t matcher_execute( matcher* self, const p_matcher_input in )
{
VSchema * dflt_schema;
const VTable * src_table;
rc_t rc;
if ( self == NULL )
return RC( rcExe, rcNoTarg, rcResolving, rcSelf, rcNull );
if ( in->manager == NULL || in->add_schemas == NULL ||
in->cfg == NULL || in->columns == NULL ||
in->src_path == NULL || in->dst_path == NULL ||
in->dst_tabname == NULL )
return RC( rcExe, rcNoTarg, rcResolving, rcParam, rcNull );
rc = matcher_build_column_vector( self, in->columns );
if ( rc != 0 ) return rc;
rc = matcher_exclude_columns( self, in->excluded_columns );
if ( rc != 0 ) return rc;
rc = helper_parse_schema( in->manager, &dflt_schema, in->add_schemas );
if ( rc != 0 ) return rc;
rc = VDBManagerOpenTableRead( in->manager, &src_table, dflt_schema, "%s", in->src_path );
if ( rc == 0 )
{
const VSchema * src_schema;
rc = VTableOpenSchema ( src_table, &src_schema );
if ( rc == 0 )
{
rc = matcher_read_src_types( self, src_table, src_schema );
if ( rc == 0 )
{
if ( in->legacy_schema != NULL )
rc = VSchemaParseFile ( dflt_schema, "%s", in->legacy_schema );
if ( rc == 0 )
{
VTable * dst_table;
KCreateMode cmode = kcmParents;
const VSchema * dst_schema = src_schema;
if ( in->legacy_schema != NULL )
dst_schema = dflt_schema;
if ( in->force_unlock )
VDBManagerUnlock ( in->manager, "%s", in->dst_path );
if ( in->force_kcmInit )
cmode |= kcmInit;
else
cmode |= kcmCreate;
rc = VDBManagerCreateTable( in->manager, &dst_table,
dst_schema, in->dst_tabname, cmode, "%s", in->dst_path );
if ( rc == 0 )
{
rc = matcher_read_dst_types( self, dst_table, dst_schema );
if ( rc == 0 )
{
rc = matcher_make_type_matrix( self );
if ( rc == 0 )
rc = matcher_match_matrix( self, src_schema, in->cfg );
}
VTableRelease( dst_table );
if ( !(in->force_kcmInit) )
KDirectoryRemove ( in->dir, true, "%s", in->dst_path );
}
}
}
VSchemaRelease( src_schema );
}
VTableRelease( src_table );
}
VSchemaRelease( dflt_schema );
return rc;
}
rc_t CopyFileToFile( const KDirectory *top, const char *inname, KDirectory *targettop, const char *outname )
{
const KFile *in = NULL;
KFile *out = NULL;
KFile *md5file = NULL;
KMD5File *md5out = NULL;
KMD5SumFmt *md5sumfmt = NULL;
char md5filename[1024];
rc_t rc = 0;
uint32_t mode = 0;
uint32_t pathtype = 0;
uint32_t failed = 0;
if (PathIsMD5File(top, inname)) {
/* Skip it */
return 0;
}
rc = KDirectoryOpenFileRead( top, &in, "%s", inname );
if (rc != 0) {
failed = rc;
goto FAIL;
}
mode = DEFAULTMODE;
rc = KDirectoryAccess( top, &mode, inname);
if (rc != 0) {
failed = rc;
goto FAIL;
}
/*
* Not sure here -- does kcmInit re-initialize the file mode as we specify?
* Or does it preserve the existing mode (and do we want it to)?
*/
if (clobber_protections) {
pathtype = KDirectoryPathType( targettop, "%s", outname );
if ((pathtype & ~kptAlias) == kptFile) {
rc = KDirectorySetAccess( targettop, false, mode, 0777, "%s", outname);
if (rc != 0) {
failed = rc;
goto FAIL;
}
}
}
rc = KDirectoryCreateFile( targettop, &out, false, mode, (force? kcmInit: kcmCreate), "%s", outname );
if (rc != 0) {
failed = rc;
goto FAIL;
}
sprintf(md5filename, "%s.md5", outname);
rc = KDirectoryCreateFile( targettop, &md5file, false, DEFAULTMODE, (force? kcmInit: kcmCreate), "%s", md5filename);
if (rc != 0) {
failed = rc;
goto FAIL;
}
rc = KMD5SumFmtMakeUpdate( &md5sumfmt, md5file);
if (rc != 0) {
failed = rc;
goto FAIL;
}
rc = KMD5FileMakeWrite( &md5out, out, md5sumfmt, outname );
if (rc != 0) {
failed = rc;
goto FAIL;
}
{
uint64_t rpos = 0;
uint64_t wpos = 0;
size_t numread;
while (true) {
rc = KFileRead( in, rpos, buffer, BUFSIZE, &numread );
/* fprintf(stderr, "Read %d bytes.\n", numread); */
if (rc == 0 && numread == 0)
break;
if (rc != 0) {
failed = rc;
goto FAIL;
}
rpos += numread;
{
size_t numwritten = 0;
int written = 0;
while (written < numread) {
rc = KFileWrite( (KFile *)md5out, wpos, buffer+written, numread-written, &numwritten );
if (rc != 0) {
failed = rc;
break;
}
if (numwritten == 0) {
fprintf(stderr, "Didn't write anything.\n");
failed = -1;
goto FAIL;
}
wpos += numwritten;
written += numwritten;
}
}
}
}
/* Success also, check the value of failed to see if failed */
FAIL:
if (NULL != md5out) {
KFileRelease((KFile *)md5out);
md5out = NULL;
}
/*KFileRelease(out); */
if (NULL != md5sumfmt) {
KMD5SumFmtRelease(md5sumfmt);
md5sumfmt = NULL;
}
/* KFileRelease(md5file); */
if (NULL != in) {
KFileRelease(in);
in = NULL;
}
/* KDirectoryRelease(top); */
if (failed) {
KDirectoryRemove( targettop, false, "%s", md5filename );
KDirectoryRemove( targettop, false, "%s", outname);
}
return failed;
}
static rc_t produce_final_db_output( tool_ctx_t * tool_ctx )
{
struct temp_registry * registry = NULL;
join_stats stats;
rc_t rc = make_temp_registry( ®istry, tool_ctx -> cleanup_task ); /* temp_registry.c */
clear_join_stats( &stats ); /* helper.c */
/* join SEQUENCE-table with lookup-table === this is the actual purpos of the tool === */
/* --------------------------------------------------------------------------------------------
produce special-output ( SPOT_ID,READ,SPOT_GROUP ) by iterating over the SEQUENCE - table:
produce fastq-output by iterating over the SEQUENCE - table:
--------------------------------------------------------------------------------------------
each thread iterates over a slice of the SEQUENCE-table
for each SPOT it may look up an entry in the lookup-table to get the READ
if it is not stored in the SEQ-tbl
-------------------------------------------------------------------------------------------- */
if ( rc == 0 )
rc = execute_db_join( tool_ctx -> dir,
tool_ctx -> accession_path,
tool_ctx -> accession_short,
&stats,
&tool_ctx -> lookup_filename[ 0 ],
&tool_ctx -> index_filename[ 0 ],
tool_ctx -> temp_dir,
registry,
tool_ctx -> cursor_cache,
tool_ctx -> buf_size,
tool_ctx -> num_threads,
tool_ctx -> show_progress,
tool_ctx -> fmt,
& tool_ctx -> join_options ); /* join.c */
/* from now on we do not need the lookup-file and it's index any more... */
if ( tool_ctx -> lookup_filename[ 0 ] != 0 )
KDirectoryRemove( tool_ctx -> dir, true, "%s", &tool_ctx -> lookup_filename[ 0 ] );
if ( tool_ctx -> index_filename[ 0 ] != 0 )
KDirectoryRemove( tool_ctx -> dir, true, "%s", &tool_ctx -> index_filename[ 0 ] );
/* STEP 4 : concatenate output-chunks */
if ( rc == 0 )
rc = temp_registry_merge( registry,
tool_ctx -> dir,
tool_ctx -> output_filename,
tool_ctx -> buf_size,
tool_ctx -> show_progress,
tool_ctx -> force,
tool_ctx -> compress ); /* temp_registry.c */
/* in case some of the partial results have not been deleted be the concatenator */
if ( registry != NULL )
destroy_temp_registry( registry ); /* temp_registry.c */
if ( rc == 0 )
print_stats( &stats ); /* helper.c */
return rc;
}