static rc_t vds_read_line( const KFile * f, uint64_t * pos, size_t * len, char * buffer, size_t buflen )
{
size_t num_read;
rc_t rc = KFileRead( f, *pos, buffer, buflen, &num_read );
if ( rc == 0 && num_read > 0 )
{
char * nl = string_chr ( buffer, num_read, '\n' );
char * cr = string_chr ( buffer, num_read, '\r' );
if ( nl != NULL )
{
if ( cr != NULL )
{
if ( nl < cr )
*len = ( nl - buffer );
else
*len = ( cr - buffer );
}
else
*len = ( nl - buffer );
}
else if ( cr != NULL )
{
*len = ( cr - buffer );
}
else
*len = 0;
*pos += ( *len + 1 );
}
return rc;
}
static rc_t split_argument_into_path_and_readgroup( const char *argument, char ** path, char ** attribute )
{
rc_t rc;
char * colon_ptr = string_chr ( argument, string_size ( argument ), ':' );
if ( colon_ptr == NULL )
{
/* we do not have a colon in the argument, that means: there is no uri-syntax involved
---> we can split the "old fashioned way" at the equal-sign */
rc = split_argument( argument, path, attribute, '=' );
}
else
{
VFSManager * mgr;
rc_t rc = VFSManagerMake ( & mgr );
*path = NULL;
*attribute = NULL;
if ( rc == 0 )
{
VPath * vpath;
rc = VFSManagerMakePath ( mgr, &vpath, argument );
if ( rc == 0 )
{
rc = test_split_vpath_into_path_and_readgroup( vpath, argument, path, attribute );
VPathRelease( vpath );
}
VFSManagerRelease ( mgr );
}
}
return rc;
}
/****************************************************************************************
splits an argument
example: "/path/file=grp1" into path = "/path/file" and attribute = "grp1"
or
example: "/path/file" into path = "/path/file" and attribute = NULL
****************************************************************************************/
static rc_t split_argument( const char *argument, char ** path, char ** attribute, char delim )
{
if ( argument == NULL || path == NULL || attribute == NULL )
return RC( rcApp, rcNoTarg, rcConstructing, rcParam, rcNull );
else
{
char * delim_ptr = string_chr ( argument, string_size ( argument ), delim );
if ( delim_ptr == NULL )
{
*path = string_dup_measure( argument, NULL );
*attribute = NULL;
}
else
{
size_t len = string_size( argument );
size_t len1 = ( delim_ptr - argument );
*path = string_dup ( argument, len1 );
if ( delim_ptr < argument + len - 1 )
*attribute = string_dup ( delim_ptr + 1, len - ( len1 + 1 ) );
else
*attribute = NULL;
}
}
return 0;
}
static const char * col_name_without_type( const char * colname )
{
const char * res = colname;
const char * s = string_chr( colname, string_size( colname ), ')' );
if ( s != NULL )
res = ++s;
return res;
}
bool KDBIsPathUri (const char * path)
{
const char * pc;
size_t z;
z = string_size (path);
if (NULL != (pc = string_chr (path, z, ':')))
return true;
if (NULL != (pc = string_chr (path, z, '?')))
return true;
if (NULL != (pc = string_chr (path, z, '#')))
return true;
return false;
}
/* remove the first line, including it's line-feed */
static void remove_first_line( char * s, size_t * len )
{
char * p = string_chr ( s, *len, '\n' );
if ( p )
{
*len -= ( p - s ) + 1;
memmove( s, p + 1, *len );
}
}
static
VRemoteProtocols parseProtocol ( const char * protocol, rc_t * prc )
{
VRemoteProtocols protocols = 0;
int n = 0;
int j = 0;
uint32_t sz = 0;
assert ( protocol && prc );
sz = string_measure ( protocol, NULL );
while ( sz > 0 ) {
bool found = false;
int l = sz;
char * c = string_chr ( protocol, sz, ',' );
if ( c != NULL )
l = c - protocol;
for ( j = 0;
j < sizeof PROTOCOLS / sizeof PROTOCOLS [ 0 ]; ++ j )
{
const TProtocol * p = & PROTOCOLS [ j ];
uint32_t max_chars = p -> s > l ? p -> s : l;
if (string_cmp ( protocol, l, p -> n, p -> s, max_chars ) == 0) {
VRemoteProtocols next = p -> p << ( 3 * n );
protocols = next + protocols;
++n;
assert ( l >= p -> s );
protocol += p -> s;
sz -= p -> s;
found = true;
break;
}
}
if ( ! found ) {
* prc = RC ( rcExe, rcArgv, rcParsing, rcParam, rcInvalid );
PLOGERR ( klogErr, ( klogErr, * prc, "Bad protocol '$(p)'",
"p=%.*s", l, protocol ) );
break;
}
if ( c != NULL ) {
++ protocol;
-- sz;
}
}
return protocols;
}
Object obj_get(Object self, Object k) {
Object v;
switch (TM_TYPE(self)) {
case TYPE_STR: {
DictNode* node;
if (TM_TYPE(k) == TYPE_NUM) {
double d = GET_NUM(k);
int n = d;
if (n < 0) {
n += GET_STR_LEN(self);
}
if (n >= GET_STR_LEN(self) || n < 0)
tm_raise("String_get: index overflow ,len=%d,index=%d, str=%o",
GET_STR_LEN(self), n, self);
return string_chr(0xff & GET_STR(self)[n]);
} else if ((node = dict_get_node(GET_DICT(tm->str_proto), k)) != NULL) {
return method_new(node->val, self);
}
break;
}
case TYPE_LIST:{
DictNode* node;
if (TM_TYPE(k) == TYPE_NUM) {
return list_get(GET_LIST(self), GET_NUM(k));
} else if ((node = dict_get_node(GET_DICT(tm->list_proto), k))!=NULL) {
return method_new(node->val, self);
}
break;
}
case TYPE_DICT:{
DictNode* node;
node = dict_get_node(GET_DICT(self), k);
if (node != NULL) {
return node->val;
} else if ((node = dict_get_node(GET_DICT(tm->dict_proto), k))!=NULL) {
return method_new(node->val, self);
}
break;
}
case TYPE_FUNCTION:
return get_func_attr(GET_FUNCTION(self), k);
case TYPE_DATA:
return GET_DATA(self)->get(GET_DATA(self), k);
}
tm_raise("keyError %o", k);
return NONE_OBJECT;
}
int string_last_chr(string_ptr str, char ch)
{
int pos = 0;
int result = -1;
for (;;)
{
pos = string_chr(str, pos, ch);
if (pos >= 0)
{
result = pos;
pos++;
}
else
break;
}
return result;
}
/*
* Check a file path name for ending in the extension used by this program
*
* return true if it ends with the extension and false if it does not
*/
static
bool IsTmpFile (const char * path)
{
const char * pc;
pc = strrchr (path, '.');
if (pc == NULL)
return false;
if (strcmp (pc, TmpExt) == 0)
return true;
pc = string_chr (path, pc - path, '.');
if (pc == NULL)
return false;
return (strcmp (pc, TmpLockExt) == 0);
}
static
rc_t validate_obj ( const Sprintf *obj, bool fmt_only )
{
uint32_t i;
/* validate the format blocks */
for ( i = 0; obj -> fmt [ i ] . type != sptTerm; ++ i )
{
/* at this point, only looking for bad formats */
if ( obj -> fmt [ i ] . type == sptLiteral )
{
if ( obj -> fmt [ i ] . u . l . text == NULL )
return RC ( rcXF, rcFunction, rcConstructing, rcString, rcNull );
if ( obj -> fmt [ i ] . u . l . size == 0 )
return RC ( rcXF, rcFunction, rcConstructing, rcString, rcEmpty );
if ( string_chr ( obj -> fmt [ i ] . u . l . text, obj -> fmt [ i ] . u . l . size, 0 ) != NULL )
return RC ( rcXF, rcFunction, rcConstructing, rcString, rcInvalid );
}
}
return 0;
}
static
rc_t aws_extract_key_value_pair ( const String *source, String *key, String *val )
{
String k, v;
const char *start = source -> addr;
const char *end = start + source -> size;
char *eql = string_chr ( start, source -> size, '=' );
if ( eql == NULL )
return RC ( rcKFG, rcChar, rcSearching, rcFormat, rcInvalid );
/* key */
StringInit ( &k, start, eql - start, string_len ( start, eql - start ) );
StringTrim ( key, &k );
start = eql + 1;
/* value */
StringInit ( &v, start, end - start, string_len ( start, end - start ) );
StringTrim ( val, &v );
return 0;
}
rc_t get_common_options( Args * args, common_options *opts )
{
rc_t rc = get_str_option( args, OPTION_OUTF, &opts->output_file );
if ( rc == 0 )
rc = get_str_option( args, OPTION_INF, &opts->input_file );
if ( rc == 0 )
rc = get_bool_option( args, OPTION_GZIP, &opts->gzip_output, false );
if ( rc == 0 )
rc = get_bool_option( args, OPTION_BZIP, &opts->bzip_output, false );
if ( rc == 0 )
rc = get_bool_option( args, OPTION_NO_MT, &opts->no_mt, false );
if ( rc == 0 )
rc = get_str_option( args, OPTION_SCHEMA, &opts->schema_file );
if ( rc == 0 )
{
const char * table2use = NULL;
rc = get_str_option( args, OPTION_TABLE, &table2use );
opts->tab_select = primary_ats;
if ( rc == 0 && table2use != NULL )
{
size_t l = string_size ( table2use );
opts->tab_select = 0;
if ( ( string_chr ( table2use, l, 'p' ) != NULL )||
( string_chr ( table2use, l, 'P' ) != NULL ) )
{ opts->tab_select |= primary_ats; };
if ( ( string_chr ( table2use, l, 's' ) != NULL )||
( string_chr ( table2use, l, 'S' ) != NULL ) )
{ opts->tab_select |= secondary_ats; };
if ( ( string_chr ( table2use, l, 'e' ) != NULL )||
( string_chr ( table2use, l, 'E' ) != NULL ) )
{ opts->tab_select |= evidence_ats; };
}
}
return rc;
}
static bool does_table_have_column( VTable const * tbl, char const column[] )
{
KNamelist * column_names;
bool res = false;
rc_t rc = VTableListReadableColumns ( tbl, &column_names );
if ( rc == 0 )
{
uint32_t count;
rc = KNamelistCount ( column_names, &count );
if ( rc == 0 && count > 0 )
{
uint32_t idx;
size_t col_name_size;
const char * col_name = string_chr ( column, string_size( column ), ')' );
if ( col_name == NULL )
col_name = column;
else
col_name++;
col_name_size = string_size( col_name );
for ( idx = 0; idx < count && rc == 0 && !res; ++idx )
{
const char * name;
rc = KNamelistGet ( column_names, idx, &name );
if ( rc == 0 && name != NULL )
{
int cmp = string_cmp( col_name, col_name_size,
name, string_size( name ), 0xFFFF );
if ( cmp == 0 )
res = true;
}
}
}
KNamelistRelease ( column_names );
}
return res;
}
static
rc_t SRAPathAddPath ( DLList *list, const char *path, uint8_t alg )
{
rc_t rc;
if ( path == NULL )
rc = RC ( rcSRA, rcMgr, rcUpdating, rcString, rcNull );
else if ( path [ 0 ] == 0 )
rc = RC ( rcSRA, rcMgr, rcUpdating, rcString, rcEmpty );
else
{
/* treat path as a Unix-style multi-path */
size_t size = string_size ( path );
while ( 1 )
{
/* find separator */
const char *sep = string_chr ( path, size, ':' );
if ( sep == NULL )
break;
/* add sub-path */
rc = SRAPathAddSubPath ( list, path, sep - path, alg );
if ( rc != 0 )
return rc;
/* consume ':' */
++ sep;
/* pop from string */
size -= sep - path;
path = sep;
}
return SRAPathAddSubPath ( list, path, size, alg );
}
return rc;
}
/* FastFind
* Uses heuristics to select the repository most likely to contain the accession, then tries to locate the acecssion in the repository.
*/
static
rc_t FindFast( const NCBISRAPath *cself, const char *accession, char *path, size_t path_max, size_t *rep_len )
{
/*TODO: look up cache first */
/* recognize known naming schemes */
size_t size = string_size(accession);
if ( string_cmp(accession, size, "SRR", 3, 3) == 0 )
return SRAPathFindInRepoByType(cself, accession, path, path_max, rep_len, alg_ncbi, alg_ncbi);
if ( string_cmp(accession, size, "ERR", 3, 3) == 0 )
return SRAPathFindInRepoByType(cself, accession, path, path_max, rep_len, alg_ncbi, alg_ebi);
if ( string_cmp(accession, size, "DRR", 3, 3) == 0 )
return SRAPathFindInRepoByType(cself, accession, path, path_max, rep_len, alg_ncbi, alg_ddbj);
if ( string_chr(accession, size, '.') != NULL )
return SRAPathFindInRepoByType(cself, accession, path, path_max, rep_len, alg_refseq, alg_none);
if ( size > 2 && isdigit(accession[size-1]) && isdigit(accession[size-2]) && ! isdigit(accession[size-3]) )
return SRAPathFindInRepoByType(cself, accession, path, path_max, rep_len, alg_wgs, alg_none);
return RC ( rcSRA, rcMgr, rcSelecting, rcPath, rcNotFound );
}
static
rc_t RefSeqMgr_KfgReadRepositories(const KConfig* kfg, char* paths, size_t paths_sz)
{
/* servers are children of refseq/repository, e.g.: /refseq/repository/main="..." */
/* volumes are in refseq/repository/<serverName>/volumes, e.g.: /refseq/repository/main/volumes="..." */
/* all server/volume combinations are returned in paths separated by ':' */
rc_t rc = 0;
const KConfigNode *node;
#define KFG_PATH "/refseq/repository/"
paths[0] = 0;
rc = KConfigOpenNodeRead ( kfg, & node, KFG_PATH );
if ( rc == 0 )
{
KNamelist* children;
rc = KConfigNodeListChild ( node, &children );
if ( rc == 0 )
{
uint32_t count;
rc = KNamelistCount ( children, &count );
if ( rc == 0 )
{
uint32_t i;
for (i = 0; i < count; ++i) /* for all servers */
{
const char* name;
rc = KNamelistGet ( children, i, &name );
if ( rc == 0 )
{
#define BufSize 4096
char server[ BufSize ];
char buf[ BufSize ];
size_t num_writ;
rc = string_printf(buf, BufSize, &num_writ, KFG_PATH "%s", name);
if (rc == 0)
{
rc = RefSeqMgr_ConfigValue ( kfg, buf, server, sizeof(server) );
if (rc == 0)
{
rc = string_printf(buf, BufSize, &num_writ, KFG_PATH "%s/volumes", name);
if (rc == 0)
{
char volumes[ BufSize ];
rc = RefSeqMgr_ConfigValue ( kfg, buf, volumes, sizeof(volumes) );
if (rc == 0)
{ /* create a server/volume pair for every combination, append to paths, ':' - separate */
char *vol_rem = volumes;
char *vol_sep;
do {
char const *volume = vol_rem;
vol_sep = string_chr(volume, string_size(volume), ':');
if(vol_sep) {
vol_rem = vol_sep + 1;
*vol_sep = 0;
}
string_copy(paths + string_size(paths), paths_sz - string_size(paths), server, string_size(server));
if (paths[string_size(paths)-1] != '/')
{
string_copy(paths + string_size(paths), paths_sz - string_size(paths), "/", 1);
}
string_copy(paths + string_size(paths), paths_sz - string_size(paths), volume, string_size(volume));
string_copy(paths + string_size(paths), paths_sz - string_size(paths), ":", 1);
} while(vol_sep);
}
}
}
}
#undef BufSize
}
if ( rc != 0 )
{
break;
}
}
}
KNamelistRelease ( children );
}
KConfigNodeRelease ( node );
}
if (GetRCState(rc) == rcNotFound)
{
paths[0] = '\0';
return 0;
}
return 0;
}
static
void aws_parse_file ( const KFile *self, KConfigNode *aws_node,
char *buffer, size_t buf_size, bool isCredentialsFile )
{
char *sep;
const char *start = buffer;
const char *end = start + buf_size;
for ( ; start < end; start = sep + 1 )
{
rc_t rc;
String string, trim;
String key, value;
sep = string_chr ( start, end - start, '\n' );
if ( sep == NULL )
sep = ( char * ) end;
StringInit ( &string, start, sep - start, string_len ( start, sep - start ) );
StringTrim ( &trim, &string );
/* check for comment line and skip */
if ( StringLength ( & trim ) != 0 && trim . addr [ 0 ] == '#' )
continue;
/* check for key/value pairs and skip if none found */
rc = aws_extract_key_value_pair ( &trim, &key, &value );
if ( rc != 0 )
continue;
/* now check keys we are looking for and populate the node*/
if ( isCredentialsFile )
{
String access_key_id, secret_access_key;
CONST_STRING ( &access_key_id, "aws_access_key_id" );
CONST_STRING ( &secret_access_key, "aws_secret_access_key" );
if ( StringCaseEqual ( &key, &access_key_id ) )
{
rc = aws_KConfigNodeUpdateChild ( aws_node, &key, &value );
if ( rc != 0 )
return;
}
if ( StringCaseEqual ( &key, &secret_access_key ) )
{
rc = aws_KConfigNodeUpdateChild ( aws_node, &key, &value );
if ( rc != 0 )
return;
}
}
else
{
String region, output;
CONST_STRING ( ®ion, "region" );
CONST_STRING ( &output, "output" );
if ( StringCaseEqual ( &key, ®ion ) )
{
rc = aws_KConfigNodeUpdateChild ( aws_node, &key, &value );
if ( rc != 0 )
return;
}
if ( StringCaseEqual ( &key, &output ) )
{
rc = aws_KConfigNodeUpdateChild ( aws_node, &key, &value );
if ( rc != 0 )
return;
}
}
}
}
LIB_EXPORT rc_t CC aspera_get(
const char *ascp_bin, const char *private_file, const char *aSrc,
const char *dest, AscpOptions *opt)
{
char path[PATH_MAX] = "";
AscpOptions dummy;
bool status = false;
int64_t prev = -1;
int attempt = 0;
KDirectory *dir = NULL;
TQuitting *quitting = NULL;
const char *src = aSrc;
rc_t rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
if (ascp_bin == NULL || private_file == NULL ||
src == NULL || dest == NULL)
{
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
if (opt == NULL) {
memset(&dummy, 0, sizeof dummy);
opt = &dummy;
}
if (opt->ascp_options == NULL && opt->target_rate[0] == '\0') {
KConfig *cfg = NULL;
rc_t rc = KConfigMake(&cfg, NULL);
DISP_RC(rc, "cannot KConfigMake");
if (rc == 0) {
rc = _KConfigGetAscpRate(cfg,
opt->target_rate, sizeof opt->target_rate);
DISP_RC(rc, "cannot get aspera max rate");
}
RELEASE(KConfig, cfg);
}
sStatus = status = opt->status;
quitting = opt->quitting;
{
/* remove trailing #... or ?... from src path:
it could come from Revolver */
size_t s = string_size(aSrc);
const char *n = string_chr(aSrc, s, '#');
const char *q = string_chr(aSrc, s, '?');
if (q != NULL && (n == NULL || q < n)) {
n = q;
}
if (n != NULL) {
if (n - aSrc + 1 > sizeof path) {
return RC(rcNS, rcFile, rcCopying, rcBuffer, rcInsufficient);
}
else {
#if _DEBUGGING
size_t s =
#endif
string_copy(path, sizeof path, aSrc, n - aSrc);
assert(s <= sizeof path);
src = path;
}
}
}
while (true) {
rc = run_ascp(ascp_bin, private_file, src, dest, opt);
if (rc == 0) {
if (status) {
STSMSG(STS_DBG, ("ascp finished with success"));
}
break;
}
else if (rc == SILENT_RC(rcExe,
rcProcess, rcExecuting, rcMemory, rcExhausted))
{
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
break;
}
else {
rc_t rc2 = 0;
uint64_t size = 0;
if (quitting != NULL) {
rc2 = quitting();
if (rc2 != 0) {
break;
}
}
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
rc2 = KDirectoryFileSize(dir, &size, "%s", dest);
if (rc2 ==
SILENT_RC(rcFS, rcDirectory, rcAccessing, rcPath, rcNotFound))
{
if (prev < 0) {
if (status) {
STSMSG(0, ("fasp download failed. "
"File not found. Retrying..."));
}
prev = 0;
}
else {
if (status) {
STSMSG(0, ("fasp download failed. File not found."));
}
break;
}
}
else if (rc2 != 0 || (int64_t)size < prev) {
if (status) {
STSMSG(0, ("fasp download failed. KDirectoryFileSize "
"after ascp run: rc = %ld, size = %ld", rc, size));
}
break;
}
else if ((int64_t)size > prev) {
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying...", size));
}
attempt = 0;
prev = size;
}
else {
if (attempt++ > 3) {
break;
}
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying %d...", size, attempt));
}
}
}
}
RELEASE(KDirectory, dir);
return rc;
}
rc_t ascpParse(const char *buf, size_t len, const char *filename,
EAscpState *state, String *line)
{
bool failure = false;
const char *p = buf;
int64_t l = len;
assert(buf && len && filename && state && line);
StringInit(line, NULL, 0, 0);
while (true) {
const char *n = string_chr(p, l, '\n');
const char *r = string_chr(p, l, '\r');
if (n == NULL) {
if (r != NULL) {
n = r;
}
}
else {
if (r != NULL) {
if (r < n) {
n = r;
}
}
}
if (n != NULL) {
StringInit(line, p, n - p, (uint32_t)(n - p));
l -= n - p + 1;
}
else {
StringInit(line, p, l, (uint32_t)l);
}
if (line->addr && line->len && line->addr[line->len - 1] == '\r') {
line->size = line->len - 1;
line->len = line->len - 1;
}
if (line->addr && line->len && line->addr[0] == '\r') {
++line->addr;
line->size = line->len - 1;
line->len = line->len - 1;
}
if (line->len != 0) {
SAscpState full;
rc_t rc = parseAscpLine(line, &full, filename);
if (rc != 0) {
return rc;
}
switch (full.s) {
case eChild:
break;
case eUnknown:
switch (*state) {
case eKeyStart:
case eKeyMayBeIn:
case eKeyIn:
*state = eKeyMayBeIn;
break;
case eCompleted:
case eFailed:
case eWriteFailed:
*state = eEnd;
/* report to user */
break;
case eProgress:
if (sStatus) {
OUTMSG(("\n"));
}
/* no break; */
default:
*state = eUnknown;
/* report to user */
break;
}
break;
case eFailed:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
failure = true;
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* no break; */
break;
case eWriteFailed:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
failure = true;
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* no break; */
break;
case eCompleted:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
failure = false;
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* no break; */
break;
case eProgress:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\r"));
}
}
*state = full.s;
if (sStatus) {
OUTMSG(("%s", full.msg));
}
break;
case eEnd:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* report to user */
break;
default:
*state = full.s;
break;
}
SAscpStateFini(&full);
}
if (n == NULL || l <= 0) {
break;
}
if (*state == eKeyEnd) {
String end;
if (_StringHas(line, "Store key in cache? (y/n) ", &end)) {
if (n > end.addr + end.len) {
l += n - end.addr + end.len;
n = end.addr + end.len - 1;
}
}
}
p = n + 1;
if (p >= buf + len) {
break;
}
}
if (sStatus) {
STSMSG(STS_FIN, ("%.*s", len, buf));
}
if (failure) {
/* ignore it */
}
return 0;
}
static rc_t parseAscpLine(const String *s, SAscpState *state, const char *name)
{
bool debug = true;
if (!sStatus) {
debug = false;
}
assert(s && state);
memset(state, 0, sizeof *state);
if (_StringHas(s, "CHILD: ", NULL)) {
if (sStatus) {
OUTMSG(("%.*s\n", s->len, s->addr));
}
state->s = eChild;
}
else if (_StringStartsWith(s, "Cannot open log file: ")) {
if (debug) OUTMSG(("matched: LOG: '%.*s'\n", s->len, s->addr));
state->s = eLog;
}
else if (_StringStartsWith(s, "The server's host key is not")) {
if (debug) OUTMSG(("matched: KeySTART: '%.*s'\n", s->len, s->addr));
state->s = eKeyStart;
}
else if (_StringHas(s, "no guarantee that the server is th", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "think it is.", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "The server's rsa2 key fingerprint ", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "ssh-rsa 1024 ", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "If you trust this host, enter ", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "PuTTY's cache and carry on connect", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "If you want to carry on connecting", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "adding the key to the cache, enter", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, " you do not trust this host, press", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "connection.", NULL)) {
if (debug) OUTMSG(("matched: KeyIN: '%.*s'\n", s->len, s->addr));
state->s = eKeyIn;
}
else if (_StringHas(s, "Store key in cache? (y/n) ", NULL)) {
if (debug) OUTMSG(("matched: KeyEND: '%.*s'\n", s->len, s->addr));
state->s = eKeyEnd;
}
else if (string_chr(s->addr, s->len, '%') != NULL) {
if (debug) OUTMSG(("matched: PROGRESS: '%.*s'\n", s->len, s->addr));
state->s = eProgress;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringStartsWith(s, "Completed: ")) {
if (debug) OUTMSG(("matched: COMPLETED: '%.*s'\n", s->len, s->addr));
state->s = eCompleted;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringStartsWith(s, "Partial Completion: ")) {
if (debug) OUTMSG(("matched: END: '%.*s'\n", s->len, s->addr));
state->s = eEnd;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringStartsWith(s, "Connection abandoned.")) {
/* printed in caller printf("matched: END: '%.*s'\n", s->len, s->addr); */
state->s = eFailed;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringHas(s, "failed to open connection to remot", NULL)) {
/* printed in caller printf("matched: END: '%.*s'\n", s->len, s->addr); */
state->s = eFailed;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringHas(s, "exiting", NULL)) {
state->s = eFailed;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringStartsWith(s, "Session Stop (Error: Disk write ")) {
if (debug) OUTMSG(("matched: Disk write failed: '%.*s'\n", s->len, s->addr));
state->s = eWriteFailed;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringStartsWith(s, "Session Stop ")) {
if (debug) OUTMSG(("matched: COMPLETED: '%.*s'\n", s->len, s->addr));
state->s = eFailed;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringHas(s, " bits/sec), in 1 file", NULL)) {
if (debug) OUTMSG(("matched: END: '%.*s'\n", s->len, s->addr));
state->s = eEnd;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else if (_StringStartsWith(s, name)) {
/* in the beginning:
line starting by dest file name, then some white characters */
if (debug) OUTMSG(("matched: PROGRESS: '%.*s'\n", s->len, s->addr));
state->s = eProgress;
if ((state->msg = string_dup(s->addr, s->len)) == NULL) {
return RC(rcNS, rcStorage, rcAllocating, rcMemory, rcExhausted);
}
}
else {
OUTMSG(("LINE = (%d) '%.*s'\n", s->len, s->len, s->addr));
/* assert(0); */
}
return 0;
}
/* EncryptionKey
* return any associated encryption key
*
* attempts to copy NUL-terminated key into provided buffer
*
* "buffer" [ OUT ] and "bsize" [ IN ] - encryption key output parameter
*
* "key_size" [ OUT, NULL OKAY ] - returns the key size in
* bytes, excluding any NUL termination.
*/
LIB_EXPORT rc_t CC KRepositoryEncryptionKey ( const KRepository *self,
char *buffer, size_t bsize, size_t *key_size )
{
rc_t rc;
if ( self == NULL )
rc = RC ( rcKFG, rcNode, rcAccessing, rcSelf, rcNull );
else
{
const KConfigNode *node;
if ( key_size != NULL )
* key_size = 0;
rc = KConfigNodeOpenNodeRead ( self -> node, & node, "encryption-key" );
if ( rc == 0 )
{
size_t num_read, remaining;
rc = KConfigNodeRead ( node, 0, buffer, bsize, & num_read, & remaining );
KConfigNodeRelease ( node );
if ( rc == 0 )
{
if ( key_size != NULL )
* key_size = num_read + remaining;
if ( remaining != 0 )
rc = RC ( rcKFG, rcNode, rcAccessing, rcBuffer, rcInsufficient );
else if ( num_read < bsize )
memset ( & buffer [ num_read ], 0, bsize - num_read );
}
}
else if ( GetRCState ( rc ) == rcNotFound )
{
char path [ 4096 ];
rc_t rc2 = KRepositoryEncryptionKeyFile ( self, path, sizeof path, NULL );
if ( rc2 == 0 )
{
KDirectory *wd;
rc2 = KDirectoryNativeDir ( & wd );
if ( rc2 == 0 )
{
const KFile *keyFile;
rc2 = KDirectoryOpenFileRead ( wd, & keyFile, path );
KDirectoryRelease ( wd );
if ( rc2 == 0 )
{
size_t num_read;
rc = KFileReadAll ( keyFile, 0, buffer, bsize, & num_read );
if ( rc == 0 )
{
if ( num_read == bsize )
{
uint64_t eof;
rc = KFileSize ( keyFile, & eof );
if ( rc == 0 )
num_read = ( size_t ) eof;
else
num_read = 0;
rc = RC ( rcKFG, rcFile, rcReading, rcBuffer, rcInsufficient );
memset ( buffer, 0, bsize );
}
else if ( num_read == 0 )
{
rc = RC ( rcKFG, rcFile, rcReading, rcFile, rcEmpty );
memset ( buffer, 0, bsize );
}
else
{
char *eoln = string_chr ( buffer, num_read, '\n' );
if ( eoln != NULL )
{
if ( eoln == buffer )
num_read = 0;
else if ( eoln [ -1 ] == '\r' )
num_read = eoln - buffer - 1;
else
num_read = eoln - buffer;
}
if ( key_size != NULL )
* key_size = num_read;
memset ( & buffer [ num_read ], 0, bsize - num_read );
}
}
KFileRelease ( keyFile );
}
}
}
}
}
return rc;
}
/* return configured password as ASCIZ
* opertates on vfs/kfs/kfg objects, not kdb objects */
static
rc_t KDBOpenFileGetPassword (char * pw, size_t pwz)
{
VFSManager * mgr;
rc_t rc;
assert (pw);
assert (pwz);
pw[0] = '\0';
rc = VFSManagerMake (&mgr);
if (rc)
; /* failure to make VFS manager: pass along rc */
else
{
size_t pwfz;
char pwf [4096 + 1];
rc = VFSManagerGetConfigPWFile (mgr, pwf, sizeof (pwf) - 1, &pwfz);
if (rc)
/* failure to get password file path: tweak rc */
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcNotFound);
else
{
VPath * pwp;
pwf [pwfz] = '\0'; /* force to ASCIZ */
#if 0
rc = VPathMakeSysPath (&pwp, pwf);
#else
rc = VFSManagerMakePath (mgr, &pwp, pwf);
#endif
if (rc)
; /* failure to construct a path from the string */
else
{
const KFile * pwf;
rc = VFSManagerOpenFileRead (mgr, &pwf, pwp);
if (rc)
/* failure to open password file */
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcNotOpen);
else
{
size_t z;
char pwb [4098]; /* arbitrarily using 4096 as maximum
allowed length */
/* at this point we are only getting the password from a
* file but in the future if we can get it from a pipe of
* some sort we can't count on the ReadAll to really know
* if we hit end of file and not just a pause in the
* streaming. VFS/KFS 2 will have to fix this somehow
*/
rc = KFileReadAll (pwf, 0, pwb, sizeof pwb, &z);
if (rc)
; /* failure to read password file: pass along rc */
else
{
/* trim off EOL if present */
char * pc;
pwb[z] = '\0'; /* force ASCIZ */
pc = string_chr (pwb, z, '\r');
if (pc)
{
*pc = '\0';
z = 1 + pc - pwb;
}
pc = string_chr (pwb, z, '\n');
if (pc)
{
*pc = '\0';
z = 1 + pc - pwb;
}
if (z == 0)
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcTooShort);
else if (pwz < z) /* pwz came in as 4096 */
rc = RC (rcDB, rcMgr, rcOpening, rcEncryptionKey, rcTooLong);
else
{
memmove (pw, pwb, z+1);
}
}
KFileRelease (pwf);
}
VPathRelease (pwp);
}
}
VFSManagerRelease (mgr);
}
return rc;
}
static void _doc_process_tag(doc_ptr doc, doc_tag_ptr tag)
{
if ( tag->type == DOC_TAG_CLOSE_COLOR
|| tag->type == DOC_TAG_CLOSE_STYLE
|| tag->type == DOC_TAG_CLOSE_INDENT )
{
doc_pop_style(doc);
}
else if (tag->type == DOC_TAG_COLOR)
{
assert(tag->arg);
if (tag->arg_size == 1)
{
if (tag->arg[0] == '*')
doc_pop_style(doc);
else
{
doc_style_t style = *doc_current_style(doc); /* copy */
switch (tag->arg[0])
{
case 'd': style.color = TERM_DARK; break;
case 'w': style.color = TERM_WHITE; break;
case 's': style.color = TERM_SLATE; break;
case 'o': style.color = TERM_ORANGE; break;
case 'r': style.color = TERM_RED; break;
case 'g': style.color = TERM_GREEN; break;
case 'b': style.color = TERM_BLUE; break;
case 'u': style.color = TERM_UMBER; break;
case 'D': style.color = TERM_L_DARK; break;
case 'W': style.color = TERM_L_WHITE; break;
case 'v': style.color = TERM_VIOLET; break;
case 'y': style.color = TERM_YELLOW; break;
case 'R': style.color = TERM_L_RED; break;
case 'G': style.color = TERM_L_GREEN; break;
case 'B': style.color = TERM_L_BLUE; break;
case 'U': style.color = TERM_L_UMBER; break;
}
doc_push_style(doc, &style);
}
}
else
{
string_ptr arg = string_copy_sn(tag->arg, tag->arg_size);
doc_style_t style = *doc_current_style(doc); /* copy */
doc_style_f f = _get_doc_style_f(doc, string_buffer(arg));
if (f)
f(&style);
{/* We don't copy the named style, just its color. */
/* Also, we damn well better push a style or we'll be upset
when the good little user pops! */
doc_style_t copy = *doc_current_style(doc);
copy.color = style.color;
doc_push_style(doc, ©);
}
string_free(arg);
}
}
else if (tag->type == DOC_TAG_INDENT)
{
doc_style_t style = *doc_current_style(doc);
style.left = doc->cursor.x;
doc_push_style(doc, &style);
}
else
{
string_ptr arg = string_copy_sn(tag->arg, tag->arg_size);
switch (tag->type)
{
case DOC_TAG_STYLE:
if (tag->arg_size == 1 && tag->arg[0] == '*')
doc_pop_style(doc);
else
{/* Better silently add one if name doesn't exist ... */
doc_style_t copy = *doc_current_style(doc);
doc_style_f f = _get_doc_style_f(doc, string_buffer(arg));
if (f)
f(©);
doc_push_style(doc, ©);
}
break;
case DOC_TAG_VAR:
_doc_process_var(doc, string_buffer(arg));
break;
case DOC_TAG_TAB:
{
int pos = atoi(string_buffer(arg)) + doc_current_style(doc)->left;
if (pos > doc->cursor.x)
doc_insert_space(doc, pos - doc->cursor.x);
else
doc_rollback(doc, doc_pos_create(pos, doc->cursor.y));
break;
}
case DOC_TAG_TOPIC:
{
doc_bookmark_ptr mark = malloc(sizeof(doc_bookmark_t));
mark->name = arg; /* steal ownership */
arg = NULL;
mark->pos = doc->cursor;
vec_add(doc->bookmarks, mark);
break;
}
case DOC_TAG_LINK:
{
doc_link_ptr link = malloc(sizeof(doc_link_t));
int split = string_chr(arg, 0, '#');
int ch = 'a' + int_map_count(doc->links);
if (split >= 0)
{
substring_t left = string_left(arg, split);
substring_t right = string_right(arg, string_length(arg) - split - 1);
link->file = substring_copy(&left);
link->topic = substring_copy(&right);
}
else
{
link->file = arg; /* steal ownership */
arg = NULL;
link->topic = NULL;
}
link->location.start = doc->cursor;
int_map_add(doc->links, ch, link);
{
/* TODO: This is flawed. Here's a real world example:
"(see below <link:birth.txt#PrimaryStats>)."
Can you see the problem? We might line break after "[a]" right
before ").". Instead, "[a])." should be treated as the current
word. To fix this, we'll need a parser with a token queue
that we can push onto, but this raises storage issues.
*/
string_ptr s = string_alloc_format("<style:link>[%c]</style>", ch);
doc_insert(doc, string_buffer(s));
string_free(s);
link->location.stop = doc->cursor;
}
break;
}
}
string_free(arg);
}
}
rc_t KNSProxiesVSetHTTPProxyPath ( KNSProxies * self,
const char * fmt, va_list args, bool clear )
{
rc_t rc = 0;
if ( clear )
rc = KNSProxiesHttpProxyClear ( self );
if ( rc == 0 && fmt != NULL && fmt [ 0 ] != 0 ) {
size_t psize;
char path [ 4096 * 2 ];
const char * p = path;
rc = string_vprintf ( path, sizeof path, & psize, fmt, args );
if ( rc == 0 ) {
while ( psize != 0 ) {
size_t s = psize;
uint16_t proxy_port = 0;
const char * colon = NULL;
const char * comma = string_chr ( p, psize, ',' );
if ( comma != NULL ) {
#ifdef MULTIPLE_PROXIES
s = comma - p;
#else
rc = RC ( rcNS, rcMgr, rcUpdating, rcPath, rcInvalid );
break;
#endif
}
colon = string_chr ( p, s, ':' );
if ( colon != NULL ) {
char * end = NULL;
const char * port_spec = NULL;
long port_num = 0;
int have = colon - p;
int remains = s - have;
if ( remains > 2 ) {
assert ( colon [ 0 ] == ':' );
if ( colon [ 1 ] == '/' && colon [ 2 ] == '/' ) {
/* strip off the scheme from proxy specification: it is ignored now */
psize -= have + 3;
p = colon + 3;
if ( psize == 0 )
return RC ( rcNS, rcMgr, rcUpdating,
rcPath, rcInvalid );
continue;
}
}
port_spec = colon + 1;
/* it is true that some day we might read symbolic port names... */
port_num = strtol ( port_spec, & end, 10 );
if ( port_num <= 0 || port_num >= 0x10000) {
PLOGERR ( klogErr, ( klogErr, rc,
"Proxy '$(proxy)' was ignored",
"proxy=%.*s", ( int ) s, p ) );
rc = RC ( rcNS, rcMgr, rcUpdating, rcPath, rcInvalid );
}
else if ( end [ 0 ] != 0 && comma == NULL ) {
if ( * end != '/' && * end != '?' ) {
/* skip everyting after '/' or '?' */
PLOGERR ( klogErr, ( klogErr, rc,
"Proxy '$(proxy)' was ignored",
"proxy=%.*s", ( int ) s, p ) );
rc = RC ( rcNS, rcMgr, rcUpdating,
rcPath, rcInvalid );
}
}
if ( rc == 0 ) {
proxy_port = ( uint16_t ) port_num;
s = colon - p;
}
}
if ( rc == 0 )
rc = KNSProxiesAddHttpProxyPath ( self, p, s, proxy_port );
if ( comma == NULL)
psize = 0;
else {
s = comma - p + 1;
if ( s > psize )
psize = 0;
else {
psize -= s;
p += s;
}
}
}
}
}
return rc;
}
/* ----------------------------------------------------------------------
*/
static
void CC build_vpath_one (void * item, void * data)
{
const char * path;
rc_data * prc;
path = item;
prc = data;
if (prc->rc == 0)
{
static const char ccid [] = "copycat-id:";
const size_t cz = sizeof (ccid) - 1;
size_t pz;
const char * hier;
const char * ppath;
VPath * vpath;
rc_t rc;
char vbuff [8193];
rc = 0;
ppath = path;
pz = string_size (path);
hier = string_chr (path, pz, ':');
if ((hier != NULL) &&
(0 == string_cmp (path, (hier+1) - path, ccid, cz /*sizeof (ccid) - 1*/, cz)))
{
static const char nfile[] = "ncbi-file:";
char * qmark;
size_t s, r/*, q */;
char ibuff [8192];
char rbuff [8192];
++hier;
s = hier - path;
r = string_copy (ibuff, sizeof (ibuff), hier, pz - s);
qmark = string_chr (ibuff, r, '?');
if (qmark == NULL) /* this is more future with parts */
qmark = string_chr (ibuff, r, '#');
if (qmark != NULL)
*qmark = '\0';
rc = KDirectoryResolveAlias (options.root, true, rbuff, sizeof (rbuff), "%s", ibuff);
if (rc)
PLOGERR (klogErr, (klogErr, rc, "error resolving file id '$(I)", "I=%s", ibuff));
else
{
char * slash;
size_t z;
slash = string_chr (rbuff+1, sizeof (rbuff), '/');
if (slash == NULL)
/* we won't extract the root */
return;
++slash;
z = string_size (slash);
if (z == 0)
return;
if (qmark)
{
s = string_copy (vbuff, sizeof (vbuff), nfile, sizeof (nfile));
r = string_copy (vbuff + s, sizeof (vbuff) - s, slash, z);
/*q = */string_copy (vbuff + s + r, (sizeof (vbuff) - s) - r, qmark, pz - (qmark - path));
}
else
{
s = string_copy (vbuff, sizeof (vbuff), slash, z);
}
ppath = vbuff;
}
}
if (rc == 0)
{
rc = VFSManagerMakePath (options.vfsmgr, &vpath, "%s", ppath);
if (rc)
;
else
{
rc = VectorAppend (&options.pathvpath, NULL, vpath);
if (rc)
{
VPathRelease (vpath);
}
}
}
prc->rc = rc;
}
}
rc_t get_a_sequence (param_block * pb)
{
char * eol;
rc_t rc = 0;
if (pb->seq == NULL) /* first call */
{
const void * annoying;
uint64_t file_pos;
size_t map_size;
uint64_t file_size;
rc = KFileSize (pb->file, &file_size);
if (rc)
return rc;
rc = KMMapAddrRead (pb->mmap, &annoying);
if (rc)
return rc;
pb->seq = annoying;
if (pb->seq == NULL)
return 0;
rc = KMMapPosition (pb->mmap, &file_pos);
if (rc)
return rc;
if (file_pos != 0)
{
rc = RC (rcExe, rcMemMap, rcAccessing, rcOffset, rcInvalid);
return rc;
}
rc = KMMapSize (pb->mmap, &map_size);
if (rc)
return rc;
if (map_size != file_size)
{
rc = RC (rcExe, rcMemMap, rcAccessing, rcFile, rcInvalid);
return rc;
}
pb->eof = pb->seq + map_size;
}
else
{
pb->seq += pb->seq_size + 1;
if (pb->seq >= pb->eof)
{
pb->seq = NULL;
return 0;
}
}
eol = string_chr (pb->seq, pb->eof - pb->seq, '\n');
if (eol == NULL)
pb->seq_size = pb->eof - pb->seq;
else
pb->seq_size = eol - pb->seq;
return rc;
}
