static rc_t split_argument_into_path_and_readgroup( const char *argument, char ** path, char ** attribute ) { rc_t rc; char * colon_ptr = string_chr ( argument, string_size ( argument ), ':' ); if ( colon_ptr == NULL ) { /* we do not have a colon in the argument, that means: there is no uri-syntax involved ---> we can split the "old fashioned way" at the equal-sign */ rc = split_argument( argument, path, attribute, '=' ); } else { VFSManager * mgr; rc_t rc = VFSManagerMake ( & mgr ); *path = NULL; *attribute = NULL; if ( rc == 0 ) { VPath * vpath; rc = VFSManagerMakePath ( mgr, &vpath, argument ); if ( rc == 0 ) { rc = test_split_vpath_into_path_and_readgroup( vpath, argument, path, attribute ); VPathRelease( vpath ); } VFSManagerRelease ( mgr ); } } return rc; }
/* Make */ static rc_t SRAMgrInitPath ( SRAMgr *mgr, const KDirectory *wd ) { #if OLD_SRAPATH_MGR /* try to make the path manager */ rc_t rc = SRAPathMake ( & mgr -> _pmgr, wd ); if ( GetRCState ( rc ) == rcNotFound && GetRCTarget ( rc ) == rcDylib ) { /* we are operating outside of the archive */ assert ( mgr -> _pmgr == NULL ); rc = 0; } return rc; #else KConfig *kfg; rc_t rc = KConfigMake ( & kfg, NULL ); if ( rc == 0 ) { VFSManager *vfs; rc = VFSManagerMake ( & vfs ); if ( rc == 0 ) { rc = VFSManagerMakeResolver ( vfs, ( VResolver** ) & mgr -> _pmgr, kfg ); VFSManagerRelease ( vfs ); } KConfigRelease ( kfg ); } if ( rc != 0 ) mgr -> _pmgr = NULL; return 0; #endif }
rc_t report_on_reference( Args * args, bool extended ) { uint32_t count; rc_t rc = ArgsParamCount( args, &count ); if ( rc != 0 ) { LOGERR( klogInt, rc, "ArgsParamCount() failed" ); } else { KDirectory *dir; rc = KDirectoryNativeDir( &dir ); if ( rc != 0 ) { LOGERR( klogInt, rc, "KDirectoryNativeDir() failed" ); } else { const VDBManager *vdb_mgr; rc = VDBManagerMakeRead ( &vdb_mgr, dir ); if ( rc != 0 ) { LOGERR( klogInt, rc, "VDBManagerMakeRead() failed" ); } else { VFSManager * vfs_mgr; rc = VFSManagerMake ( &vfs_mgr ); if ( rc != 0 ) { (void)LOGERR( klogErr, rc, "cannot make vfs-manager" ); } else { uint32_t idx; for ( idx = 0; idx < count && rc == 0; ++idx ) { const char *param = NULL; rc = ArgsParamValue( args, idx, ¶m ); if ( rc != 0 ) { LOGERR( klogInt, rc, "ArgsParamvalue() failed" ); } else { /* rc value not used, because it can be something that has no references */ report_references( vdb_mgr, vfs_mgr, param, extended ); } } VFSManagerRelease ( vfs_mgr ); } VDBManagerRelease( vdb_mgr ); } KDirectoryRelease( dir ); } } return rc; }
rc_t report_deletes( Args * args, uint32_t min_len ) { uint32_t count; rc_t rc = ArgsParamCount( args, &count ); if ( rc != 0 ) { LOGERR( klogInt, rc, "ArgsParamCount() failed" ); } else { KDirectory *dir; rc = KDirectoryNativeDir( &dir ); if ( rc != 0 ) { LOGERR( klogInt, rc, "KDirectoryNativeDir() failed" ); } else { const VDBManager *vdb_mgr; rc = VDBManagerMakeRead ( &vdb_mgr, dir ); if ( rc != 0 ) { LOGERR( klogInt, rc, "VDBManagerMakeRead() failed" ); } else { VFSManager * vfs_mgr; rc = VFSManagerMake ( &vfs_mgr ); if ( rc != 0 ) { (void)LOGERR( klogErr, rc, "cannot make vfs-manager" ); } else { uint32_t idx; for ( idx = 0; idx < count && rc == 0; ++idx ) { const char *param = NULL; rc = ArgsParamValue( args, idx, (const void **)¶m ); if ( rc != 0 ) { LOGERR( klogInt, rc, "ArgsParamvalue() failed" ); } else { rc = report_deletes_spec( vdb_mgr, vfs_mgr, param, min_len ); } } VFSManagerRelease ( vfs_mgr ); } VDBManagerRelease( vdb_mgr ); } KDirectoryRelease( dir ); } } return rc; }
static void ref_walker_release( struct ref_walker * self ) { KDirectoryRelease( self->dir ); VDBManagerRelease( self->vmgr ); VSchemaRelease( self->vschema ); AlignMgrRelease ( self->amgr ); VFSManagerRelease ( self->vfs_mgr ); VNamelistRelease ( self->sources ); free_ref_regions( &self->regions ); free( ( void * )self->spot_group ); }
LIB_EXPORT rc_t LegacyVPathMakeSysPath ( VPath ** new_path, const char * sys_path ) { VFSManager * vfs; rc_t rc = VFSManagerMake ( & vfs ); if ( rc == 0 ) { rc = VFSManagerMakeSysPath ( vfs, new_path, sys_path ); VFSManagerRelease ( vfs ); } return rc; }
LIB_EXPORT rc_t CC XFS_VfsManagerDispose () { rc_t RCt; struct VFSManager * Manager; RCt = 0; Manager = _sVfsManager; if ( Manager != NULL ) { _sVfsManager = NULL; RCt = VFSManagerRelease ( Manager ); } return RCt; } /* XFS_VfsManagerDispose () */
static rc_t StartFileSystem (const char * src, const char * dst) { VFSManager * vmanager; rc_t rc; rc = VFSManagerMake (&vmanager); if (rc) LOGERR (klogErr, rc, "Failed to open file system"); else { rc = VFSManagerGetKryptoPassword (vmanager, Password, sizeof Password, &PasswordSize); if (rc != 0) LOGERR (klogErr, rc, "unable to obtain a password"); else { rc = KKeyInitRead (&Key, kkeyAES128, Password, PasswordSize); if (rc) LOGERR (klogErr, rc, "Unable to make encryption/decryption key"); else { KDirectory * cwd; rc = VFSManagerGetCWD (vmanager, &cwd); if (rc) LOGERR (klogInt, rc, "unable to access current directory"); else { rc = Start (cwd, src, dst); KDirectoryRelease (cwd); } } } VFSManagerRelease (vmanager); } return rc; }
/* KMain */ rc_t CC KMain ( int argc, char *argv [] ) { Args * args; rc_t rc; rc = ArgsMakeAndHandle (&args, argc, argv, 0); if (rc) LOGERR (klogInt, rc, "failed to parse arguments"); else do { uint32_t acount; rc = ArgsParamCount (args, &acount); if (rc) { LOGERR (klogInt, rc, "failed to count parameters"); break; } if (acount == 0) { rc = MiniUsage (args); break; } else { VFSManager* mgr; rc = VFSManagerMake(&mgr); if (rc) LOGERR ( klogErr, rc, "failed to create VFSManager object" ); else { VResolver * resolver; rc = VFSManagerGetResolver (mgr, &resolver); if (rc == 0) { uint32_t ix; for ( ix = 0; ix < acount; ++ ix ) { const char * pc; rc = ArgsParamValue (args, ix, &pc ); if (rc) LOGERR (klogInt, rc, "failed to retrieve parameter value"); else { const VPath * upath = NULL; rc = VFSManagerMakePath ( mgr, (VPath**)&upath, "%s", pc); if (rc == 0) { const VPath * local; const VPath * remote; rc = VResolverQuery (resolver, eProtocolHttp, upath, &local, &remote, NULL); if (rc == 0) { const String * s; if (local != NULL) rc = VPathMakeString (local, &s); else rc = VPathMakeString (remote, &s); if (rc == 0) { OUTMSG (("%S\n", s)); free ((void*)s); } VPathRelease (local); VPathRelease (remote); } else { KDirectory * cwd; rc_t orc = VFSManagerGetCWD (mgr, &cwd); if (orc == 0) { KPathType kpt = KDirectoryPathType(cwd, "%s", pc); switch (kpt &= ~kptAlias) { case kptNotFound: STSMSG(1, ("'%s': not found while " "searching the file system", pc)); break; case kptBadPath: STSMSG(1, ("'%s': bad path while " "searching the file system", pc)); break; default: STSMSG(1, ("'%s': " "found in the file system", pc)); rc = 0; break; } } if (orc == 0 && rc == 0) { if (rc != 0) { PLOGMSG(klogErr, (klogErr, "'$(name)': not found", "name=%s", pc)); } else { char resolved[PATH_MAX] = ""; rc = KDirectoryResolvePath(cwd, true, resolved, sizeof resolved, "%s", pc); if (rc == 0) { STSMSG(1, ("'%s': found in " "the current directory at '%s'", pc, resolved)); OUTMSG (("%s\n", resolved)); } else { STSMSG(1, ("'%s': cannot resolve " "in the current directory", pc)); OUTMSG (("./%s\n", pc)); } } } KDirectoryRelease(cwd); } } RELEASE(VPath, upath); } } VResolverRelease (resolver); } VFSManagerRelease(mgr); } } ArgsWhack (args); } while (0); return rc; }
static rc_t run (void) { VFSManager * manager; rc_t rc = 0, orc = 0; STSMSG (1, ("Make VFSManager")); rc = VFSManagerMake (&manager); STSMSG (2, ("rc %R", orc, orc)); if (rc == 0) { #if 1 static const char name[] = "test-kfs-manager-data-file"; #else static const char name[] = "ncbi-kfs:test-kfs-manager-data-file?enc&pwfile=password"; #endif VPath * path; STSMSG (1, ("Make test VPath file '%s'",name)); rc = VPathMake (&path, name); STSMSG (2, ("rc %R", orc, orc)); if (rc == 0) { KFile * file; STSMSG (1, ("Open File for write using manager and path")); rc = VFSManagerCreateFile (manager, &file, false, 0666, kcmCreate, path); STSMSG (2, ("rc %R", rc, rc)); if (rc == 0) { char buff[4096]; size_t ix; size_t num_writ; uint64_t tot_writ = 0; for (ix = 0; ix < sizeof buff; ++ix) buff[ix] = 'A' + (ix%26); STSMSG (1, ("writing to file")); for (ix = 0; ix < 32; ++ix) { rc = KFileWrite (file, tot_writ, buff, sizeof buff, &num_writ); if (rc == 0) tot_writ += num_writ; }; STSMSG (1, ("Release file - it should whack")); orc = KFileRelease (file); STSMSG (2, ("rc %R", orc, orc)); if (rc == 0) rc = orc; #if 1 STSMSG (1, ("Remove file")); orc = VFSManagerRemove (manager, true, path); STSMSG (2, ("rc %R", orc, orc)); if (rc == 0) rc = orc; #endif } STSMSG (1, ("Release VPath - it should Whack")); orc = VPathRelease (path); STSMSG (2, ("rc %R", orc, orc)); if (rc == 0) rc = orc; } STSMSG (1, ("Release VFSManager - it should Whack")); orc = VFSManagerRelease (manager); STSMSG (2, ("rc %R", orc, orc)); if (rc == 0) rc = orc; } return rc; }
/* * Try to locate in RefSeq Archives: * check for pattern '(\w{4}\d{2})[\.\d]+'; the archive is $1 * use the scheme "x-ncbi-legrefseq" for vfs to recognize special case */ static rc_t SRAPathFullREFSEQArchive(NCBISRAPath const *self, char const rep[], char const vol[], char const accession[], char path[], size_t path_max ) { size_t const rep_sz = strlen(rep); size_t const vol_sz = strlen(vol); char const *const rep_sep = (rep_sz > 0 && rep[rep_sz - 1] != '/') ? "/" : ""; char const *const vol_sep = (vol_sz > 0 && vol[vol_sz - 1] != '/') ? "/" : ""; size_t sz; unsigned i; VFSManager *vfs; rc_t rc = VFSManagerMake(&vfs); VPath *vpath; KDirectory const *dir; KPathType type; if (rc) return rc; for (i = 0; i < 4; ++i) { int const ch = accession[i]; if (ch == 0 || !isalpha(ch)) return RC(rcSRA, rcMgr, rcAccessing, rcPath, rcIncorrect); } for ( ; ; ++i) { int const ch = accession[i]; if (ch == 0) break; if (ch != '.' && !isdigit(ch)) return RC(rcSRA, rcMgr, rcAccessing, rcPath, rcIncorrect); } if (i < 8) return RC(rcSRA, rcMgr, rcAccessing, rcPath, rcIncorrect); rc = string_printf(path, path_max, &sz, "x-ncbi-legrefseq:%s%s%s%s%.6s", rep, rep_sep, vol, vol_sep, accession); if (rc) return rc; i = sz; rc = VPathMake(&vpath, path + 17); if (rc) return rc; rc = VFSManagerOpenDirectoryRead(vfs, &dir, vpath); VPathRelease(vpath); VFSManagerRelease(vfs); if (rc) return rc; type = KDirectoryPathType(dir, "tbl/%s", accession); KDirectoryRelease(dir); if (type != kptDir) return RC(rcSRA, rcMgr, rcAccessing, rcPath, rcIncorrect); rc = string_printf(path + i, path_max - i, &sz, "#tbl/%s", accession); return rc; }
/* return configured password as ASCIZ * opertates on vfs/kfs/kfg objects, not kdb objects */ static rc_t KDBOpenFileGetPassword (char * pw, size_t pwz) { VFSManager * mgr; rc_t rc; assert (pw); assert (pwz); pw[0] = '\0'; rc = VFSManagerMake (&mgr); if (rc) ; /* failure to make VFS manager: pass along rc */ else { size_t pwfz; char pwf [4096 + 1]; rc = VFSManagerGetConfigPWFile (mgr, pwf, sizeof (pwf) - 1, &pwfz); if (rc) /* failure to get password file path: tweak rc */ rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcNotFound); else { VPath * pwp; pwf [pwfz] = '\0'; /* force to ASCIZ */ #if 0 rc = VPathMakeSysPath (&pwp, pwf); #else rc = VFSManagerMakePath (mgr, &pwp, pwf); #endif if (rc) ; /* failure to construct a path from the string */ else { const KFile * pwf; rc = VFSManagerOpenFileRead (mgr, &pwf, pwp); if (rc) /* failure to open password file */ rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcNotOpen); else { size_t z; char pwb [4098]; /* arbitrarily using 4096 as maximum allowed length */ /* at this point we are only getting the password from a * file but in the future if we can get it from a pipe of * some sort we can't count on the ReadAll to really know * if we hit end of file and not just a pause in the * streaming. VFS/KFS 2 will have to fix this somehow */ rc = KFileReadAll (pwf, 0, pwb, sizeof pwb, &z); if (rc) ; /* failure to read password file: pass along rc */ else { /* trim off EOL if present */ char * pc; pwb[z] = '\0'; /* force ASCIZ */ pc = string_chr (pwb, z, '\r'); if (pc) { *pc = '\0'; z = 1 + pc - pwb; } pc = string_chr (pwb, z, '\n'); if (pc) { *pc = '\0'; z = 1 + pc - pwb; } if (z == 0) rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcTooShort); else if (pwz < z) /* pwz came in as 4096 */ rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcTooLong); else { memmove (pw, pwb, z+1); } } KFileRelease (pwf); } VPathRelease (pwp); } } VFSManagerRelease (mgr); } return rc; }
static rc_t tokenize_file_and_progname_into_argv( const char * filename, const char * progname, int * argc, char *** argv ) { rc_t rc2, rc = 0; VFSManager *vfs_mgr; ( *argv ) = NULL; ( *argc ) = 0; rc = VFSManagerMake ( &vfs_mgr ); if ( rc != 0 ) LOGERR( klogInt, rc, "VFSManagerMake() failed" ); else { VPath * vfs_path; rc = VFSManagerMakePath ( vfs_mgr, &vfs_path, "%s", filename ); if ( rc != 0 ) LOGERR( klogInt, rc, "VPathMake() failed" ); else { struct KFile const *my_file; rc = VFSManagerOpenFileRead ( vfs_mgr, &my_file, vfs_path ); if ( rc != 0 ) LOGERR( klogInt, rc, "VFSManagerOpenFileRead() failed" ); else { tokenzr *t; uint64_t pos = 0; char buffer[ 4096 + 1 ]; size_t num_read; rc = make_tokenzr( &t, argc, argv ); if ( rc != 0 ) LOGERR( klogInt, rc, "make_tokenzr() failed" ); else { if ( progname != NULL ) rc = add_string_to_argv( t, progname, string_size( progname ) ); if ( rc == 0 ) { do { rc = KFileRead ( my_file, pos, buffer, ( sizeof buffer ) - 1, &num_read ); if ( rc != 0 ) LOGERR( klogInt, rc, "KFileRead() failed" ); else if ( num_read > 0 ) { buffer[ num_read ] = 0; rc = tokenize_buffer( t, buffer, num_read ); if ( rc != 0 ) LOGERR( klogInt, rc, "tokenize_buffer() failed" ); pos += num_read; } } while ( rc == 0 && num_read > 0 ); } if ( rc == 0 && t->used > 0 ) { rc = add_token_to_argv( t ); if ( rc != 0 ) LOGERR( klogInt, rc, "add_token_to_argv() failed" ); } free_tokenzr( t ); } rc2 = KFileRelease ( my_file ); if ( rc2 != 0 ) LOGERR( klogInt, rc2, "KFileRelease() failed" ); } rc2 = VPathRelease ( vfs_path ); if ( rc2 != 0 ) LOGERR( klogInt, rc2, "VPathRelease() failed" ); } rc2 = VFSManagerRelease ( vfs_mgr ); if ( rc2 != 0 ) LOGERR( klogInt, rc2, "VFSManagerRelease() failed" ); } return rc; }
/* ResolveTablePath * takes either an accession or path * substitutes any arguments * resolves via SRAPath mgr if present */ rc_t ResolveTablePath ( const SRAMgr *mgr, char *path, size_t psize, const char *spec, va_list args ) { #if OLD_SRAPATH_MGR int len; char tblpath [ 4096 ]; const SRAPath *pmgr = mgr -> _pmgr; /* if no path manager or if the spec string has embedded path separators, then this can't be an accession - just print it out */ if ( mgr -> _pmgr == NULL || strchr( spec, '/' ) != NULL ) { len = vsnprintf ( path, psize, spec, args ); if ( len < 0 || ( size_t ) len >= psize ) return RC ( rcSRA, rcTable, rcOpening, rcPath, rcExcessive ); return 0; } /* create a copy - not likely to be too large */ len = vsnprintf ( tblpath, sizeof tblpath, spec, args ); if ( len < 0 || ( size_t ) len >= sizeof tblpath ) return RC ( rcSRA, rcTable, rcOpening, rcPath, rcExcessive ); /* test if the path exists in current directory, i.e. with assumed dot */ if ( ! SRAPathTest ( pmgr, tblpath ) ) { rc_t rc = SRAPathFind ( pmgr, tblpath, path, psize ); if ( rc == 0 ) return 0; } /* use the path given */ if ( ( size_t ) len >= psize ) return RC ( rcSRA, rcTable, rcOpening, rcBuffer, rcInsufficient ); strcpy ( path, tblpath ); return 0; #else VFSManager *vfs; rc_t rc = VFSManagerMake ( & vfs ); if ( rc == 0 ) { VPath *accession; const VPath *tblpath = NULL; rc = VFSManagerVMakePath ( vfs, & accession, spec, args ); if ( rc == 0 ) { rc = VResolverLocal ( ( const VResolver* ) mgr -> _pmgr, accession, & tblpath ); if ( rc == 0 ) { size_t size; rc = VPathReadPath ( tblpath, path, psize, & size ); VPathRelease ( tblpath ); } VPathRelease ( accession ); } VFSManagerRelease ( vfs ); } return rc; #endif }
int LowLevelTest() { cout << "Running LowLevelTest()." << endl; const AlignAccessMgr* mgr = 0; CALL(AlignAccessMgrMake(&mgr)); VFSManager* vfs_mgr; CALL(VFSManagerMake(&vfs_mgr)); VPath* bam_path = 0; #ifdef _MSC_VER # define BAM_FILE "//traces04/1kg_pilot_data/ftp/pilot_data/data/NA10851/alignment/NA10851.SLX.maq.SRP000031.2009_08.bam" #else # define BAM_FILE "/netmnt/traces04/1kg_pilot_data/ftp/pilot_data/data/NA10851/alignment/NA10851.SLX.maq.SRP000031.2009_08.bam" #endif cout << "Testing BAM file: "<<BAM_FILE<<endl; CALL(VFSManagerMakeSysPath(vfs_mgr, &bam_path, BAM_FILE)); VPath* bai_path = 0; CALL(VFSManagerMakeSysPath(vfs_mgr, &bai_path, BAM_FILE ".bai")); const AlignAccessDB* bam = 0; CALL(AlignAccessMgrMakeIndexBAMDB(mgr, &bam, bam_path, bai_path)); #ifdef _MSC_VER InitializeCriticalSection(&sdk_mutex); #endif const size_t kNumCursors = 8; const size_t kNumThreads = 3; SThreadInfo tinfo[kNumCursors]; const char* ids[kNumCursors] = { "NT_113960", "NT_113945", "NT_113880", "NT_113960", "NT_113960", "NT_113960", "NT_113945", "NT_113880" }; for ( size_t i = 0; i < kNumThreads; ++i ) { tinfo[i].init(i, bam, ids[i]); } for ( size_t i = 0; i < kNumThreads; ++i ) { cout << "Starting thread " << i << " for " << ids[i] << endl; #ifdef _MSC_VER tinfo[i].thread_id = CreateThread(NULL, 0, read_thread_func, &tinfo[i], 0, NULL); #else pthread_create(&tinfo[i].thread_id, 0, read_thread_func, &tinfo[i]); #endif } for ( size_t i = 0; i < kNumThreads; ++i ) { cout << "Waiting for thread " << i << endl; void* ret = 0; #ifdef _MSC_VER WaitForSingleObject(tinfo[i].thread_id, INFINITE); CloseHandle(tinfo[i].thread_id); #else pthread_join(tinfo[i].thread_id, &ret); #endif cout << "Align count: " << tinfo[i].count << endl; } CALL(AlignAccessDBRelease(bam)); CALL(VPathRelease(bam_path)); CALL(VPathRelease(bai_path)); CALL(AlignAccessMgrRelease(mgr)); CALL(VFSManagerRelease(vfs_mgr)); cout << "Success." << endl; return 0; }
static rc_t nenctool (const char * srcstr, const char * dststr, bool force) { VFSManager * mgr; rc_t rc; rc = VFSManagerMake (&mgr); if (rc) LOGERR (klogInt, rc, "failed to create file system manager"); else { VPath * srcpath; rc = VFSManagerMakePath (mgr, &srcpath, "%s", srcstr); if (rc) PLOGERR (klogErr, (klogErr, rc, "Failed to parse source path '$(path)'", "path=%s", srcstr)); else { VPath * dstpath; rc = VFSManagerMakePath (mgr, &dstpath, "%s", dststr); if (rc) PLOGERR (klogErr, (klogErr, rc, "Failed to parse destination path '$(path)'", "path=%s", dststr)); else { const KFile * srcfile; rc = VFSManagerOpenFileRead (mgr, &srcfile, srcpath); if (rc) PLOGERR (klogErr, (klogErr, rc, "Failed to open source path '$(path)'", "path=%s", srcstr)); else { KFile * dstfile; rc = VFSManagerCreateFile (mgr, &dstfile, false, 0666, kcmParents | (force ? kcmInit : kcmCreate), dstpath); if (rc) PLOGERR (klogErr, (klogErr, rc, "failed to open destination path '$(path)'", "path=%s", dststr)); else { rc = copy_file (srcstr, dststr, srcfile, dstfile); if (rc) { PLOGERR (klogErr, (klogErr, rc, "failed to copy '$(S)' to '$(D)'", "S=%s,D=%s", srcstr, dststr)); VFSManagerRemove (mgr, true, dstpath); } KFileRelease (dstfile); } KFileRelease (srcfile); } VPathRelease (dstpath); } VPathRelease (srcpath); } VFSManagerRelease (mgr); } return rc; }