static rc_t read_key_and_len( struct lookup_reader * reader, uint64_t pos, uint64_t *key, size_t *len )
{
size_t num_read;
char buffer[ 10 ];
rc_t rc = KFileRead( reader->f, pos, buffer, sizeof buffer, &num_read );
if ( rc != 0 )
{
ErrMsg( "read_key_and_len.KFileRead( at %ld, to_read %u ) -> %R", pos, sizeof buffer, rc );
}
else if ( num_read != sizeof buffer )
{
if ( num_read == 0 )
rc = SILENT_RC( rcVDB, rcNoTarg, rcReading, rcId, rcNotFound );
else
rc = SILENT_RC( rcVDB, rcNoTarg, rcReading, rcFormat, rcInvalid );
}
else
{
uint16_t dna_len;
size_t packed_len;
memmove( key, buffer, sizeof *key );
dna_len = buffer[ 8 ];
dna_len <<= 8;
dna_len |= buffer[ 9 ];
packed_len = ( dna_len & 1 ) ? ( dna_len + 1 ) >> 1 : dna_len >> 1;
*len = ( ( sizeof *key ) + ( sizeof dna_len ) + packed_len );
}
return rc;
}
// interpret exception processed by Utils::HandleException:
// filter out some errors like invalid db - users don't want to
// see such errors as actual errors
int InterpretException ( int64_t rcCodeUtil, bool bSilent, char const* szErrDesc )
{
if ( rcCodeUtil == Utils::rcUnknown ||
rcCodeUtil == Utils::rcErrorStdExc ||
rcCodeUtil == Utils::rcInvalid)
{
if ( ! bSilent )
LOGMSG ( klogErr, szErrDesc );
return 3;
}
else if (rcCodeUtil == SILENT_RC( rcDB,rcMgr,rcOpening,rcDatabase,rcIncorrect ) ||
rcCodeUtil == SILENT_RC( rcVFS,rcTree,rcResolving,rcPath,rcNotFound ))
{
if ( ! bSilent )
LOGMSG ( klogWarn, szErrDesc );
return 0;
}
else
{
if ( ! bSilent )
LOGMSG ( klogErr, szErrDesc );
return 3;
}
}
LIB_EXPORT rc_t CC AlignmentIteratorNext ( AlignmentIterator *self )
{
rc_t rc = 0;
if ( self == NULL )
rc = RC( rcAlign, rcIterator, rcPositioning, rcSelf, rcNull );
else
{
self->flags = 0;
self->rel_ref_pos++;
if ( self->rel_ref_pos >= self->ref_len )
rc = SILENT_RC( rcAlign, rcIterator, rcPositioning, rcItem, rcDone );
else
{
if ( self->skip > 0 )
{
self->flags |= align_iter_skip;
self->skip--;
}
else if ( self->inserts > 0 )
{
self->seq_pos += ( self->inserts + 1 );
self->inserts = 0;
}
else
{
self->seq_pos++;
}
al_iter_adjust_next( self );
}
}
return rc;
}
LIB_EXPORT rc_t CC ReferenceIteratorNextPos ( ReferenceIterator *self, bool skip_empty )
{
rc_t rc = 0;
if ( self == NULL )
rc = RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
else
{
self->current_rec = NULL;
if ( self->need_init )
{
rc = first_ref_iter_nxt_pos( self, skip_empty );
}
else
{
/* increment the current position */
self->current_pos++;
if ( self->current_pos <= self->last_pos )
{
/* jump over gaps, if requested ... */
if ( self->depth == 0 && skip_empty )
{
self->current_pos = self->nxt_avail_pos;
}
/* increment the internal alignment-iterator of every placement-record */
inc_alignment_iterators( &self->spot_groups, self->current_pos );
/* loop through the list to look if we have to remove records,
that do end before this new position */
self->depth = remove_invalid_records( self->refobj, &self->spot_groups, self->current_pos );
rc = fill_recordlist( self, self->current_pos );
if ( rc == 0 )
{
self->current_spot_group = NULL;
/* set our sights to the next position... */
rc = PlacementSetIteratorNextAvailPos ( self->pl_set_iter, &self->nxt_avail_pos, NULL );
if ( GetRCState( rc ) == rcDone )
{
if ( self->depth > 0 )
{
rc = 0;
}
else if ( !skip_empty )
{
if ( self->current_pos <= self->last_pos ) rc = 0;
}
}
}
}
else
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
clear_spot_group_list( &self->spot_groups );
}
}
}
return rc;
}
static rc_t print_fastq_1_read( join_stats * stats,
struct join_results * results,
const fastq_rec * rec,
const join_options * jo,
uint32_t dst_id,
uint32_t read_id )
{
rc_t rc = 0;
if ( rec -> read . len != rec -> quality . len )
{
ErrMsg( "row #%ld : READ.len(%u) != QUALITY.len(%u)\n", rec -> row_id, rec -> read . len, rec -> quality . len );
stats -> reads_invalid++;
if ( jo -> terminate_on_invalid )
return SILENT_RC( rcApp, rcNoTarg, rcReading, rcItem, rcInvalid );
}
if ( filter1( stats, rec, jo ) )
{
if ( join_results_match( results, &( rec -> read ) ) )
{
rc = join_results_print_fastq_v1( results,
rec -> row_id,
dst_id,
read_id,
jo -> rowid_as_name ? NULL : &( rec -> name ),
&( rec -> read ),
&( rec -> quality ) );
if ( rc == 0 )
stats -> reads_written++;
}
}
return rc;
}
/*--------------------------------------------------------------------------
* xc to rc
*/
static
rc_t extract_rc_xobj ( const XCObj * xobj )
{
while ( xobj -> dad != NULL )
xobj = xobj -> dad;
return SILENT_RC ( 0, 0, 0, xobj -> rc_obj, 0 );
}
static
rc_t extract_rc_xstate ( const XCState * xstate )
{
while ( xstate -> dad != NULL )
xstate = xstate -> dad;
return SILENT_RC ( 0, 0, 0, 0, xstate -> rc_state );
}
/* with a printf-style error message
* records filename and lineno
* uses string_vprintf to build error_msg from fmt + args
* initializes rc with an unknown error code
*/
Exception :: Exception ( const char * _filename, uint32_t _lineno, const char *fmt, ... )
throw ()
: filename ( _filename )
, lineno ( _lineno )
, rc ( SILENT_RC ( rcExe, rcNoTarg, rcExecuting, rcNoObj, rcUnknown ) )
{
}
static rc_t loop_until_key_found( struct lookup_reader * reader, uint64_t key_to_find,
uint64_t *key_found , uint64_t *offset )
{
rc_t rc = 0;
bool done = false;
uint64_t curr = *offset;
while ( !done && rc == 0 )
{
size_t found_len;
rc = read_key_and_len( reader, curr, key_found, &found_len );
if ( keys_equal( key_to_find, *key_found ) )
{
done = true;
*offset = curr;
}
else if ( key_to_find > *key_found )
curr += found_len;
else
{
done = true;
rc = SILENT_RC( rcVDB, rcNoTarg, rcReading, rcId, rcNotFound );
}
}
return rc;
}
LIB_EXPORT rc_t CC PlacementSetIteratorNextWindow ( PlacementSetIterator *self,
INSDC_coord_zero *first_pos, INSDC_coord_len *len )
{
rc_t rc = 0;
if ( first_pos != NULL ) {
*first_pos = 0;
}
if ( len != NULL ) {
*len = 0;
}
if ( self == NULL )
return RC( rcAlign, rcIterator, rcReleasing, rcSelf, rcNull );
self->current_entry = NULL; /* what is the current pi-entry, we are handling ? */
if ( self->current_ref == NULL )
{
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
pl_set_iter_clear_curr_ref_window( self );
/* !!! here we are taking the next window from the top of the list */
self->current_window = ( pi_window * )DLListPopHead ( &(self->current_ref->pi_windows) );
/* check if we have reached the last window on this reference... */
if ( self->current_window == NULL )
{
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
/* point to the first entry in this window... */
self->current_entry = ( pi_entry * )DLListHead( &(self->current_window->pi_entries) );
/* if the caller wants to know first_pos / len */
if ( first_pos != NULL )
{
*first_pos = self->current_window->w.first;
}
if ( len != NULL )
{
*len = self->current_window->w.len;
}
return rc;
}
LIB_EXPORT rc_t CC PlacementSetIteratorNextRecordAt ( PlacementSetIterator *self,
INSDC_coord_zero pos, const PlacementRecord **rec )
{
rc_t rc = 0;
pi_window * pw;
bool done;
if ( rec == NULL )
return RC( rcAlign, rcIterator, rcAccessing, rcParam, rcNull );
*rec = NULL;
if ( self == NULL )
return RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
if ( self->current_ref == NULL )
{
/* no more reference to iterator over! the iterator is done! */
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
if ( self->current_window == NULL )
{
/* no more windows to iterator over! the iterator is done! */
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
pw = self->current_window;
done = false;
do
{
if ( self->current_entry == NULL )
{
self->current_entry = ( pi_entry * )DLListHead( &(pw->pi_entries) );
}
done = ( self->current_entry == NULL );
rc = ( done ? SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone ) : 0 );
if ( rc == 0 )
{
rc = PlacementIteratorNextRecordAt ( self->current_entry->pi, pos, rec );
done = ( GetRCState( rc ) != rcDone );
if ( !done )
{
self->current_entry = ( pi_entry * )DLNodeNext( ( DLNode * )self->current_entry );
done = ( self->current_entry == NULL );
rc = ( done ? SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone ) : 0 );
}
}
} while ( !done );
return rc;
}
DepFixture(const char *path)
: mgr(NULL)
, vmgr(NULL)
, resolver(NULL)
, siteless(false)
{
rc_t rc = 0;
KDirectory *wd = NULL;
if (KDirectoryNativeDir(&wd)) {
FAIL("failed to KDirectoryNativeDir");
}
const KDirectory *dir = NULL;
KConfig *cfg = NULL;
if (KDirectoryOpenDirRead(wd, &dir, false, path)) {
FAIL("failed to KDirectoryOpenDirRead()");
}
if (KConfigMake(&cfg, dir)) {
FAIL("failed to KConfigMake()");
}
RELEASE(KDirectory, dir);
if (VFSManagerMakeFromKfg(&vmgr, cfg)) {
FAIL("failed to VFSManagerMakeFromKfg()");
}
if (VFSManagerGetResolver(vmgr, &resolver)) {
FAIL("failed to VFSManagerGetResolver");
}
String *result = NULL;
rc = KConfigReadString(cfg, "repository/site/main/tracearc/root",
&result);
if (rc != 0) {
if (rc == SILENT_RC(rcKFG, rcNode, rcOpening, rcPath, rcNotFound)) {
rc = 0;
siteless = true;
}
else {
FAIL(
"failed to KConfigReadString(repository/site/main/tracearc/root)");
}
}
else {
assert(result);
KPathType t = KDirectoryPathType(wd, result->addr);
if (t != kptDir) {
siteless = true;
}
}
RELEASE(String, result);
RELEASE(KConfig, cfg);
if (VDBManagerMakeReadWithVFSManager(&mgr, NULL, vmgr)) {
FAIL("failed to VDBManagerMakeReadWithVFSManager()");
}
RELEASE(KDirectory, wd);
}
LIB_EXPORT rc_t CC PlacementSetIteratorNextAvailPos ( const PlacementSetIterator *cself,
INSDC_coord_zero *pos, INSDC_coord_len *len )
{
rc_t rc = 0;
if ( cself == NULL )
rc = RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
else
{
if ( pos == NULL )
rc = RC( rcAlign, rcIterator, rcAccessing, rcParam, rcNull );
else
{
PlacementSetIterator *self = ( PlacementSetIterator * )cself;
if ( self->current_ref == NULL || self->current_window == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
/* loop through all the pi_entry int the current_pi_ref */
pi_ref_nxt_avail_pos_ctx ctx;
ctx.count = 0;
ctx.rc = 0;
ctx.min_pos = 0;
ctx.min_len = 0;
ctx.min_pos_initialized = false;
DLListForEach ( &(self->current_window->pi_entries),
false, nxt_avail_pos_cb, &ctx );
rc = ctx.rc;
if ( ctx.count == 0 )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
*pos = ctx.min_pos;
if ( len != NULL )
{
*len = ctx.min_len;
}
}
}
}
}
return rc;
}
KRYPTO_EXTERN rc_t CC KFileIsWGAEnc (const void * buffer, size_t buffer_size)
{
const uint8_t * ph;
const uint8_t * pt;
const uint8_t * pb;
size_t ix;
size_t lim;
if ((buffer == NULL) || (buffer_size == 0))
return RC (rcFS, rcFile, rcIdentifying, rcParam, rcNull);
/* bare minimum size to identify we decide is the first 8
* obsfucated ASCII bytes */
if (buffer_size < (sizeof (header_const.magic) - 1))
return RC (rcFS, rcFile, rcIdentifying, rcBuffer, rcInsufficient);
/* a match is ph[X] ^ pt[X] == pb[X] at specially identified points
* we could have put the ^ into a constant for a tad faster operation
* but meh...
*/
ph = (const uint8_t*)&header_const;
pt = (const uint8_t*)&header_table;
pb = (const uint8_t*)buffer;
ix = offsetof (struct KWGAEncFileHeader, magic);
lim = ix + sizeof (header_const.magic);
if (lim > buffer_size)
lim = buffer_size;
for (; (ix < lim) && (ix < buffer_size); ++ix)
if ((ph[ix] ^ pt[ix]) != pb[ix])
return SILENT_RC (rcFS, rcFile, rcIdentifying, rcFile, rcWrongType);
if (buffer_size < offsetof (struct KWGAEncFileHeader, fer_enc))
return 0;
if ((ph[offsetof (struct KWGAEncFileHeader, fer_enc)] ^ pt[offsetof (struct KWGAEncFileHeader, fer_enc)])
!= pb[offsetof (struct KWGAEncFileHeader, fer_enc)])
return RC (rcFS, rcFile, rcIdentifying, rcFile, rcWrongType);
ix = offsetof (struct KWGAEncFileHeader, reserved);
if (buffer_size < ix)
return 0;
lim = ix + sizeof (header_const.reserved);
if (lim > buffer_size)
lim = buffer_size;
for (; ix < lim; ++ix)
if ((ph[ix] ^ pt[ix]) != pb[ix])
return RC (rcFS, rcFile, rcIdentifying, rcFile, rcWrongType);
return 0;
}
rc_t get_packed_and_key_from_lookup_reader( struct lookup_reader * reader,
uint64_t * key, SBuffer * packed_bases )
{
rc_t rc;
if ( reader == NULL || key == NULL || packed_bases == NULL )
{
rc = RC( rcVDB, rcNoTarg, rcReading, rcParam, rcInvalid );
ErrMsg( "get_packed_and_key_from_lookup_reader() -> %R", rc );
}
else
{
size_t num_read;
char buffer1[ 10 ];
rc = KFileRead( reader->f, reader->pos, buffer1, sizeof buffer1, &num_read );
if ( rc != 0 )
ErrMsg( "KFileRead( at %ld, to_read %u ) -> %R", reader->pos, sizeof buffer1, rc );
else if ( num_read != sizeof buffer1 )
rc = SILENT_RC( rcVDB, rcNoTarg, rcReading, rcFormat, rcInvalid );
else
{
uint16_t dna_len;
size_t to_read;
char * dst = ( char * )packed_bases->S.addr;
memmove( key, buffer1, sizeof *key );
dna_len = buffer1[ 8 ];
dna_len <<= 8;
dna_len |= buffer1[ 9 ];
dst[ 0 ] = buffer1[ 8 ];
dst[ 1 ] = buffer1[ 9 ];
dst += 2;
to_read = ( dna_len & 1 ) ? ( dna_len + 1 ) >> 1 : dna_len >> 1;
if ( to_read > ( packed_bases->buffer_size - 2 ) )
to_read = ( packed_bases->buffer_size - 2 );
if ( rc == 0 )
{
rc = KFileRead( reader->f, reader->pos + 10, dst, to_read, &num_read );
if ( rc != 0 )
ErrMsg( "KFileRead( at %ld, to_read %u ) -> %R", reader->pos + 10, to_read, rc );
else if ( num_read != to_read )
{
rc = RC( rcVDB, rcNoTarg, rcReading, rcFormat, rcInvalid );
ErrMsg( "KFileRead( %ld ) %d vs %d -> %R", reader->pos + 10, num_read, to_read, rc );
}
else
{
packed_bases->S.len = packed_bases->S.size = num_read + 2;
reader->pos += ( num_read + 10 );
}
}
}
}
return rc;
}
LIB_EXPORT rc_t CC ReferenceIteratorGetPlacement ( ReferenceIterator *self,
const PlacementRecord **rec )
{
rc_t rc = 0;
if ( self == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
}
else if ( rec == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcParam, rcNull );
}
else
{
if ( self->current_spot_group == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
if ( self->current_rec != NULL )
{
/* remove the 'previous' current-rec! */
DLListPopHead ( &self->current_spot_group->records );
PlacementRecordWhack ( self->current_rec );
self->depth--;
self->current_rec = NULL;
}
self->current_rec = ( PlacementRecord * )DLListHead( &self->current_spot_group->records );
if ( self->current_rec == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
*rec = self->current_rec;
}
}
}
return rc;
}
LIB_EXPORT rc_t CC ReferenceIteratorNextPlacement ( ReferenceIterator *self,
const PlacementRecord **rec )
{
rc_t rc = 0;
if ( self == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
}
else if ( rec == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcParam, rcNull );
}
else
{
if ( self->current_spot_group == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
if ( self->current_rec == NULL )
{
self->current_rec = ( PlacementRecord * )DLListHead( &self->current_spot_group->records );
}
else
{
self->current_rec = ( PlacementRecord * )DLNodeNext( ( DLNode * )self->current_rec );
}
if ( self->current_rec == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
*rec = self->current_rec;
}
}
}
return rc;
}
LIB_EXPORT rc_t CC ReferenceIteratorNextSpotGroup ( ReferenceIterator *self,
const char ** name, size_t * len )
{
rc_t rc = 0;
if ( self->current_spot_group == NULL )
{
self->current_spot_group = ( spot_group * )DLListHead( &self->spot_groups );
if ( self->current_spot_group == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
}
else
{
spot_group *nxt = ( spot_group * )DLNodeNext( ( DLNode * ) self->current_spot_group );
if ( nxt == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
self->current_spot_group = nxt;
}
}
self->current_rec = NULL;
if ( rc == 0 && self->current_spot_group != NULL )
{
if ( name != NULL )
{
*name = self->current_spot_group->name;
}
if ( len != NULL )
{
*len = self->current_spot_group->len;
}
}
return rc;
}
LIB_EXPORT rc_t CC PlacementSetIteratorNextReference ( PlacementSetIterator *self,
INSDC_coord_zero *first_pos, INSDC_coord_len *len, struct ReferenceObj const ** refobj )
{
rc_t rc = 0;
if ( refobj != NULL ) {
*refobj = NULL;
}
if ( self == NULL )
return RC( rcAlign, rcIterator, rcReleasing, rcSelf, rcNull );
pl_set_iter_clear_curr_ref_window( self );
pl_set_iter_clear_curr_ref( self );
self->current_entry = NULL; /* what is the current pi-entry, we are handling ? */
/* !!! here we are taking the next reference from the top of the list */
self->current_ref = ( pi_ref * )DLListPopHead ( &self->pi_refs );
if ( self->current_ref == NULL )
{
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
if ( first_pos != NULL ) *first_pos = self->current_ref->outer.first;
if ( len != NULL) *len = self->current_ref->outer.len;
/* if the caller wants to know the ref-obj... */
if ( refobj != NULL )
{
pi_window *pw = ( pi_window * )DLListHead( &(self->current_ref->pi_windows) );
if ( pw != NULL )
{
pi_entry * pie = ( pi_entry * )DLListHead( &(pw->pi_entries) );
if ( pie != NULL )
{
rc = PlacementIteratorRefObj( pie->pi, refobj );
}
}
}
return rc;
}
static String* _KConfigAscpString(const KConfig *self,
const char *path, const char *name)
{
String *ascp = NULL;
rc_t rc = KConfigReadString(self, path, &ascp);
if (rc == 0) {
assert(ascp);
/* STSMSG(STS_INFO, ("Using %s from configuration: '%s'",
name, ascp->addr)); */
return ascp;
}
else {
if (rc != SILENT_RC(rcKFG, rcNode, rcOpening, rcPath, rcNotFound)) {
DISP_RC(rc, path);
}
else {
/* STSMSG(STS_DBG, ("'%s': not found in configuration", path)); */
}
free(ascp);
return NULL;
}
}
static bool _KConfigAscpDisabled(const KConfig *self, bool status) {
bool disabled = false;
const char path[] = "tools/ascp/disabled";
rc_t rc = KConfigReadBool(self, path, &disabled);
if (rc != 0) {
if (rc != SILENT_RC(rcKFG, rcNode, rcOpening, rcPath, rcNotFound)) {
DISP_RC(rc, path);
}
else {
if (status) {
STSMSG(STS_DBG, ("'%s': not found in configuration", path));
}
}
disabled = false;
}
else {
if (status) {
STSMSG(STS_DBG, ("'%s' = '%s'", path, disabled ? "true" : "false"));
}
}
return disabled;
}
static bool reportToUserSffFromNot454Run(rc_t rc, char* argv0, bool silent) {
assert( argv0 );
if ( rc == SILENT_RC( rcSRA, rcFormatter, rcConstructing,
rcData, rcUnsupported ) )
{
const char* name = strpbrk( argv0, "/\\" );
const char* last_name = name;
if ( last_name )
{
++last_name;
}
while ( name )
{
name = strpbrk( last_name, "/\\" );
if ( name )
{
last_name = name;
if ( last_name )
{
++last_name;
}
}
}
name = last_name ? last_name : argv0;
if ( strcmp( "sff-dump", name ) == 0 )
{
if (!silent) {
OUTMSG((
"This run cannot be transformed into SFF format.\n"
"Conversion cannot be completed because the source lacks\n"
"one or more of the data series required by the SFF format.\n"
"You should be able to dump it as FASTQ by running fastq-dump.\n"
"\n"));
}
return true;
}
}
return false;
}
/* ------------------------------------------------------------------------------------------ */
static rc_t print_fastq_n_reads_split( join_stats * stats,
struct join_results * results,
const fastq_rec * rec,
const join_options * jo )
{
rc_t rc = 0;
String R, Q;
uint32_t read_id_0 = 0;
uint32_t offset = 0;
uint32_t read_len_sum = 0;
if ( rec -> read . len != rec -> quality . len )
{
ErrMsg( "row #%ld : READ.len(%u) != QUALITY.len(%u)\n", rec -> row_id, rec -> read . len, rec -> quality . len );
stats -> reads_invalid++;
if ( jo -> terminate_on_invalid )
return SILENT_RC( rcApp, rcNoTarg, rcReading, rcItem, rcInvalid );
}
while ( read_id_0 < rec -> num_read_len )
read_len_sum += rec -> read_len[ read_id_0++ ];
if ( rec -> read . len != read_len_sum )
{
ErrMsg( "row #%ld : READ.len(%u) != sum(READ_LEN)(%u)\n", rec -> row_id, rec -> read . len, read_len_sum );
stats -> reads_invalid++;
if ( jo -> terminate_on_invalid )
return SILENT_RC( rcApp, rcNoTarg, rcReading, rcItem, rcInvalid );
}
read_id_0 = 0;
while ( rc == 0 && read_id_0 < rec -> num_read_len )
{
if ( rec -> read_len[ read_id_0 ] > 0 )
{
if ( filter( stats, rec, jo, read_id_0 ) )
{
R . addr = &rec -> read . addr[ offset ];
R . size = rec -> read_len[ read_id_0 ];
R . len = ( uint32_t )R . size;
if ( join_results_match( results, &R ) )
{
Q . addr = &rec -> quality . addr[ offset ];
Q . size = rec -> read_len[ read_id_0 ];
Q . len = ( uint32_t )Q . size;
if ( join_results_match( results, &( rec -> read ) ) )
rc = join_results_print_fastq_v1( results,
rec -> row_id,
0,
read_id_0 + 1,
jo -> rowid_as_name ? NULL : &( rec -> name ),
&R,
&Q );
if ( rc == 0 )
stats -> reads_written++;
}
}
offset += rec -> read_len[ read_id_0 ];
}
else
stats -> reads_zero_length++;
read_id_0++;
}
return rc;
}
LIB_EXPORT rc_t CC aspera_get(
const char *ascp_bin, const char *private_file, const char *aSrc,
const char *dest, AscpOptions *opt)
{
char path[PATH_MAX] = "";
AscpOptions dummy;
bool status = false;
int64_t prev = -1;
int attempt = 0;
KDirectory *dir = NULL;
TQuitting *quitting = NULL;
const char *src = aSrc;
rc_t rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
if (ascp_bin == NULL || private_file == NULL ||
src == NULL || dest == NULL)
{
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
if (opt == NULL) {
memset(&dummy, 0, sizeof dummy);
opt = &dummy;
}
if (opt->ascp_options == NULL && opt->target_rate[0] == '\0') {
KConfig *cfg = NULL;
rc_t rc = KConfigMake(&cfg, NULL);
DISP_RC(rc, "cannot KConfigMake");
if (rc == 0) {
rc = _KConfigGetAscpRate(cfg,
opt->target_rate, sizeof opt->target_rate);
DISP_RC(rc, "cannot get aspera max rate");
}
RELEASE(KConfig, cfg);
}
sStatus = status = opt->status;
quitting = opt->quitting;
{
/* remove trailing #... or ?... from src path:
it could come from Revolver */
size_t s = string_size(aSrc);
const char *n = string_chr(aSrc, s, '#');
const char *q = string_chr(aSrc, s, '?');
if (q != NULL && (n == NULL || q < n)) {
n = q;
}
if (n != NULL) {
if (n - aSrc + 1 > sizeof path) {
return RC(rcNS, rcFile, rcCopying, rcBuffer, rcInsufficient);
}
else {
#if _DEBUGGING
size_t s =
#endif
string_copy(path, sizeof path, aSrc, n - aSrc);
assert(s <= sizeof path);
src = path;
}
}
}
while (true) {
rc = run_ascp(ascp_bin, private_file, src, dest, opt);
if (rc == 0) {
if (status) {
STSMSG(STS_DBG, ("ascp finished with success"));
}
break;
}
else if (rc == SILENT_RC(rcExe,
rcProcess, rcExecuting, rcMemory, rcExhausted))
{
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
break;
}
else {
rc_t rc2 = 0;
uint64_t size = 0;
if (quitting != NULL) {
rc2 = quitting();
if (rc2 != 0) {
break;
}
}
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
rc2 = KDirectoryFileSize(dir, &size, "%s", dest);
if (rc2 ==
SILENT_RC(rcFS, rcDirectory, rcAccessing, rcPath, rcNotFound))
{
if (prev < 0) {
if (status) {
STSMSG(0, ("fasp download failed. "
"File not found. Retrying..."));
}
prev = 0;
}
else {
if (status) {
STSMSG(0, ("fasp download failed. File not found."));
}
break;
}
}
else if (rc2 != 0 || (int64_t)size < prev) {
if (status) {
STSMSG(0, ("fasp download failed. KDirectoryFileSize "
"after ascp run: rc = %ld, size = %ld", rc, size));
}
break;
}
else if ((int64_t)size > prev) {
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying...", size));
}
attempt = 0;
prev = size;
}
else {
if (attempt++ > 3) {
break;
}
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying %d...", size, attempt));
}
}
}
}
RELEASE(KDirectory, dir);
return rc;
}
static rc_t print_fastq_n_reads_split_3( join_stats * stats,
struct join_results * results,
const fastq_rec * rec,
const join_options * jo )
{
rc_t rc = 0;
String R, Q;
uint32_t read_id_0 = 0;
uint32_t write_id_1 = 1;
uint32_t valid_reads = 0;
uint32_t valid_bio_reads = 0;
uint32_t offset = 0;
uint32_t read_len_sum = 0;
if ( rec -> read . len != rec -> quality . len )
{
ErrMsg( "row #%ld : READ.len(%u) != QUALITY.len(%u)\n", rec -> row_id, rec -> read . len, rec -> quality . len );
stats -> reads_invalid++;
if ( jo -> terminate_on_invalid )
return SILENT_RC( rcApp, rcNoTarg, rcReading, rcItem, rcInvalid );
}
while ( read_id_0 < rec -> num_read_len )
{
read_len_sum += rec -> read_len[ read_id_0 ];
if ( rec -> read_len[ read_id_0 ] > 0 )
{
valid_reads++;
if ( ( rec -> read_type[ read_id_0 ] & READ_TYPE_BIOLOGICAL ) == READ_TYPE_BIOLOGICAL )
{
if ( jo -> min_read_len > 0 )
{
if ( rec -> read_len[ read_id_0 ] >= jo -> min_read_len )
valid_bio_reads++;
}
else
valid_bio_reads++;
}
}
read_id_0++;
}
if ( rec -> read . len != read_len_sum )
{
ErrMsg( "row #%ld : READ.len(%u) != sum(READ_LEN)(%u)\n", rec -> row_id, rec -> read . len, read_len_sum );
stats -> reads_invalid++;
if ( jo -> terminate_on_invalid )
return SILENT_RC( rcApp, rcNoTarg, rcReading, rcItem, rcInvalid );
}
if ( valid_reads == 0 )
return rc;
read_id_0 = 0;
while ( rc == 0 && read_id_0 < rec -> num_read_len )
{
if ( rec -> read_len[ read_id_0 ] > 0 )
{
bool process = filter( stats, rec, jo, read_id_0 );
if ( process )
{
R . addr = &rec -> read . addr[ offset ];
R . size = rec -> read_len[ read_id_0 ];
R . len = ( uint32_t )R . size;
if ( join_results_match( results, &R ) )
{
Q . addr = &rec -> quality . addr[ offset ];
Q . size = rec -> read_len[ read_id_0 ];
Q . len = ( uint32_t )Q . size;
if ( valid_bio_reads < 2 )
write_id_1 = 0;
rc = join_results_print_fastq_v1( results,
rec -> row_id,
write_id_1,
read_id_0 + 1,
jo -> rowid_as_name ? NULL : &( rec -> name ),
&R,
&Q );
if ( rc == 0 )
stats -> reads_written++;
}
if ( write_id_1 > 0 )
write_id_1++;
}
offset += rec -> read_len[ read_id_0 ];
}
else
stats -> reads_zero_length++;
read_id_0++;
}
return rc;
}
static rc_t align_info(const Params* prm) {
rc_t rc = 0;
const VDatabase* db = NULL;
const VDBManager* mgr = NULL;
const KDBManager *kmgr = NULL;
VSchema* schema = NULL;
bool is_db = false;
if (prm == NULL)
{ return RC(rcExe, rcQuery, rcExecuting, rcParam, rcNull); }
if (rc == 0) {
rc = VDBManagerMakeRead(&mgr, NULL);
DISP_RC(rc, "while calling VDBManagerMakeRead");
}
if (rc == 0) {
rc = VDBManagerOpenKDBManagerRead(mgr, &kmgr);
DISP_RC(rc, "while calling VDBManagerOpenKDBManagerRead");
}
if (rc == 0) {
rc = VDBManagerMakeSRASchema(mgr, &schema);
DISP_RC(rc, "while calling VDBManagerMakeSRASchema");
}
if (rc == 0) {
/* const char path[] = "/panfs/pan1/trace_work/scratch/XXX000013"; */
rc = VDBManagerOpenDBRead(mgr, &db, schema, prm->dbPath);
if (rc == 0) {
is_db = true;
}
else if (rc == SILENT_RC(rcDB, rcMgr, rcOpening, rcDatabase, rcIncorrect)) {
PLOGMSG(klogWarn, (klogWarn,
"'$(path)' is not a database", "path=%s", prm->dbPath));
rc = 0;
}
else {
PLOGERR(klogErr,
(klogErr, rc, "$(path)", "path=%s", prm->dbPath));
}
}
if (is_db) {
if (rc == 0) {
if (prm->paramRef) {
const VDBDependencies* dep = NULL;
uint32_t count = 0;
int i = 0;
rc = VDatabaseListDependencies(db, &dep, false);
DISP_RC2(rc, prm->dbPath,
"while calling VDatabaseListDependencies");
if (rc == 0) {
rc = VDBDependenciesCount(dep, &count);
DISP_RC2(rc, prm->dbPath,
"while calling VDBDependenciesCount");
}
for (i = 0; i < count && rc == 0; ++i) {
bool circular = false;
const char* name = NULL;
const char* path = NULL;
const char* remote = NULL;
bool local = false;
const char* seqId = NULL;
rc = VDBDependenciesCircular(dep, &circular, i);
if (rc != 0) {
DISP_RC2(rc, prm->dbPath,
"while calling VDBDependenciesCircular");
break;
}
rc = VDBDependenciesName(dep, &name, i);
if (rc != 0) {
DISP_RC2(rc, prm->dbPath,
"while calling VDBDependenciesName");
break;
}
rc = VDBDependenciesPath(dep, &path, i);
if (rc != 0) {
DISP_RC2(rc, prm->dbPath,
"while calling VDBDependenciesPath");
break;
}
rc = VDBDependenciesLocal(dep, &local, i);
if (rc != 0) {
DISP_RC2(rc, prm->dbPath,
"while calling VDBDependenciesLocal");
break;
}
rc = VDBDependenciesSeqId(dep, &seqId, i);
if (rc != 0) {
DISP_RC2(rc, prm->dbPath,
"while calling VDBDependenciesSeqId");
break;
}
rc = VDBDependenciesPathRemote(dep, &remote, i);
if (rc != 0) {
DISP_RC2(rc, prm->dbPath,
"while calling VDBDependenciesRemote");
break;
}
OUTMSG(("%s,%s,%s,%s", seqId, name,
(circular ? "true" : "false"),
(local ? "local" : "remote")));
if (path && path[0]) {
OUTMSG((":%s", path));
}
else if (remote && remote[0]) {
OUTMSG(("::%s", remote));
}
OUTMSG(("\n"));
}
DESTRUCT(VDBDependencies, dep);
}
if (prm->paramBamHeader) {
rc_t rc3 = bam_header(db);
if (rc3 != 0 && rc == 0)
{ rc = rc3; }
}
if (prm->paramQuality) {
rc_t rc3 = qual_stats(prm, db);
if (rc3 != 0 && rc == 0)
{ rc = rc3; }
}
}
}
DESTRUCT(VSchema, schema);
DESTRUCT(KDBManager, kmgr);
DESTRUCT(VDBManager, mgr);
DESTRUCT(VDatabase, db);
return rc;
}
static
rc_t SetPwd(const char *password, bool no_prompt)
{
rc_t rc = 0;
VFSManager *mgr = NULL;
assert(password);
if (rc == 0) {
rc = VFSManagerMake(&mgr);
DISP_RC(rc, "while calling VFSManagerMake");
}
if (rc == 0) {
char pwd_dir[PATH_MAX] = "";
rc = VFSManagerUpdateKryptoPassword
(mgr, password, strlen(password), pwd_dir, sizeof pwd_dir);
if (rc) {
if (no_prompt) {
LOGERR(klogInt, rc, "while updating the password");
}
else if (rc == SILENT_RC(rcVFS, rcEncryptionKey, rcUpdating,
rcDirectory, rcExcessive))
{
OUTMSG((
"\nSecurity warning:\n"
"Directory \"%s\" has excessive access permissions.\n"
"Run \"chmod 700 %s\" to repair\n"
"or ask your system administrator for assistance.\n",
pwd_dir, pwd_dir));
rc = 0;
}
else if (rc == SILENT_RC(rcVFS, rcEncryptionKey, rcUpdating,
rcSize, rcExcessive))
{
OUTMSG((
"\nError:\n"
"Password is too long.\n"
"Maximum password size is %ld.\n",
VFS_KRYPTO_PASSWORD_MAX_SIZE));
}
else if (rc == SILENT_RC(rcVFS, rcEncryptionKey, rcUpdating,
rcEncryptionKey, rcInvalid))
{
OUTMSG((
"\nError:\n"
"Invalid character in password\n"
"(CR/LF are not allowed)\n"));
}
else if (rc == SILENT_RC(rcVFS, rcEncryptionKey, rcUpdating,
rcPath, rcExcessive))
{
OUTMSG((
"\nError:\n"
"Cannot set the password.\n"
"Path to password file is too long.\n"
"Your configuration should be updated.\n"));
}
else if (rc == SILENT_RC(rcVFS, rcEncryptionKey, rcUpdating,
rcPath, rcIncorrect))
{
OUTMSG((
"\nError:\n"
"Cannot set the password.\n"
"Existing path to password is not a file.\n"
"Your configuration should be updated.\n"));
}
else if (rc == SILENT_RC(rcVFS, rcEncryptionKey, rcUpdating,
rcPath, rcCorrupt))
{
OUTMSG((
"\nError:\n"
"Cannot set the password.\n"
"Unknown file type for configured path/file name.\n"
"Your configuration should be updated.\n"));
}
else if (rc == SILENT_RC(rcVFS, rcEncryptionKey, rcWriting,
rcFile, rcInsufficient))
{
OUTMSG((
"\nError:\n"
"Incomplete writes to temporary password file.\n"
"Cannot set the password.\n"));
}
else {
OUTMSG(("\nCannot set the password. Please consult\n"
"configuration page at\n"
"http://www.ncbi.nlm.nih.gov/Traces/sra/sra.cgi?view=toolkit_doc&f=std or\n"
"https://github.com/ncbi/sra-tools/wiki/Toolkit-Configuration\n"));
}
}
}
RELEASE(VFSManager, mgr);
return rc;
}
rc_t run_ascp(const char *path, const char *key,
const char *src, const char *dest, const AscpOptions *opt)
{
const char *host = NULL;
const char *user = NULL;
const char *maxRate = NULL;
bool cache_key = false;
uint64_t heartbeat = 0;
const char *acc = NULL;
uint64_t srcSz = 0;
uint64_t id = 0;
TProgress *callback = NULL;
TQuitting *quitting = NULL;
rc_t rc = 0;
pid_t nPid = 0;
int pipeto[2]; /* pipe to feed the exec'ed program input */
int pipefrom[2]; /* pipe to get the exec'ed program output */
#define ARGV_SZ 64
char *argv[ARGV_SZ];
char extraOptions[4096] = "";
int i = 0;
int ret = 0;
i = 0;
if (opt != NULL) {
acc = opt->name;
cache_key = opt->cache_key;
callback = opt->callback;
heartbeat = opt->heartbeat;
host = opt->host;
id = opt->id;
quitting = opt->quitting;
srcSz = opt->src_size;
user = opt->user;
if (opt->ascp_options != NULL) {
size_t s = string_size(opt->ascp_options);
if (s >= sizeof extraOptions) {
LOGERR(klogErr,
RC(rcExe, rcProcess, rcCreating, rcBuffer, rcInsufficient),
"extra ascp options are ignored");
maxRate = opt->target_rate;
}
else {
string_copy
(extraOptions, sizeof extraOptions, opt->ascp_options, s);
}
}
else {
maxRate = opt->target_rate;
}
}
if (acc == NULL) {
acc = dest;
}
if (heartbeat > 0) {
heartbeat /= 1000;
if (heartbeat == 0) {
heartbeat = 1;
}
}
if (pipe(pipeto) != 0) {
perror("pipe() to");
rc = RC(rcExe, rcFileDesc, rcCreating, rcFileDesc, rcFailed);
LOGERR(klogErr, rc, "while pipe");
return rc;
}
if (pipe(pipefrom) != 0) {
perror("pipe() from");
rc = RC(rcExe, rcFileDesc, rcCreating, rcFileDesc, rcFailed);
LOGERR(klogErr, rc, "while pipe");
return rc;
}
argv[i++] = (char*)path;
argv[i++] = "-i";
argv[i++] = (char*)key;
argv[i++] = "-pQTk1";
if (maxRate != NULL && maxRate[0] != '\0') {
argv[i++] = "-l";
argv[i++] = (char*)maxRate;
}
if (user != NULL) {
argv[i++] = "--user";
argv[i++] = (char*)user;
}
if (host != NULL) {
argv[i++] = "--host";
argv[i++] = (char*)user;
}
if (extraOptions[0] != '\0') {
bool done = false;
char *c = extraOptions;
while (!done) {
while (true) {
if (*c == '\0') {
break;
}
else if (isspace(*c)) {
++c;
}
else {
break;
}
}
if (*c == '\0') {
break;
}
else {
argv[i++] = c;
}
while (true) {
if (*c == '\0') {
done = true;
break;
}
else if (isspace(*c)) {
*(c++) = '\0';
break;
}
++c;
}
if (i > ARGV_SZ - 4) {
LOGERR(klogErr,
RC(rcExe, rcProcess, rcCreating, rcBuffer, rcInsufficient),
"too mary extra ascp options - some of them are ignored");
break;
}
}
}
argv[i++] = (char*)src;
argv[i++] = (char*)dest;
argv[i++] = NULL;
logevp(path, argv);
if (quitting) {
rc = quitting();
}
if (rc != 0) {
return rc;
}
nPid = fork();
if (nPid < 0 ) {
perror("fork() 1");
rc = RC(rcExe, rcProcess, rcCreating, rcProcess, rcFailed);
LOGERR(klogErr, rc, "after fork");
return rc;
}
else if (nPid == 0) {
/* dup pipe read/write to stdin/stdout */
dup2(pipeto [0], STDIN_FILENO);
dup2(pipefrom[1], STDOUT_FILENO);
dup2(pipefrom[1], STDERR_FILENO);
close(pipeto[1]);
ret = execvp(path, argv);
STSMSG(STS_DBG, ("CHILD: Done %s %s %s = %d", path, src, dest, ret));
exit(EXIT_FAILURE);
}
else {
bool progressing = false;
bool writeFailed = false;
EAscpState state = eStart;
const char y[] = "y\n";
const char n[] = "n\n";
int status = 0;
int w = 0;
int fd = pipefrom[0];
const char *answer = n;
String line;
StringInit(&line, NULL, 0, 0);
if (cache_key) {
answer = y;
}
{
int flags = fcntl(fd, F_GETFL, 0);
fcntl(fd, F_SETFL, flags | O_NONBLOCK);
}
close(pipeto[0]);
close(pipefrom[1]);
assert(sizeof y == sizeof n);
{
int hang = 0;
uint64_t prev = 0;
KTime_t tPrev = 0;
char sfSrc = 'B';
uint64_t hSrc = humanize(srcSz, &sfSrc, NULL);
int sig = 0;
uint64_t i = 0;
KDirectory *dir = NULL;
rc_t rc = KDirectoryNativeDir(&dir);
DISP_RC(rc, "KDirectoryNativeDir");
if (rc != 0) {
return rc;
}
while (w == 0) {
bool quit = false;
w = waitpid(nPid, &status, WNOHANG);
if (w == 0) {
bool got = false;
rc_t rc = 0;
if (quitting) {
rc = quitting();
}
if (rc != 0 || quit) {
if (sig == 0) {
sig = SIGINT;
}
else if (sig >= SIGKILL) {
break;
}
else {
++sig;
}
if (progressing) {
OUTMSG(("\n"));
}
PLOGMSG(klogInfo, (klogInfo, "^C pressed: "
"Senging $(sgn) to ascp", "sgn=%s", sig));
kill(nPid, sig);
}
while (true) {
char buf[4096];
int s = read(fd, buf, sizeof buf);
if (s == 0) {
break;
}
else if (s < 0) {
if (errno != EAGAIN) {
if (progressing) {
OUTMSG(("\n"));
}
perror("read(child)");
}
break;
}
ascpParse(buf, s, dest, &state, &line);
switch (state) {
case eKeyEnd:
write(pipeto[1], answer, sizeof y - 1);
break;
case eWriteFailed:
writeFailed = true;
break;
default:
break;
}
got = true;
}
if (!got) {
sleep(1);
++i;
if ((heartbeat > 0 && i >= heartbeat) || (i > 99)) {
uint64_t size = 0;
rc_t rc = KDirectoryFileSize(dir, &size, "%s", dest);
if (rc != 0) {
size = 0;
}
else {
if (size != prev) {
prev = size;
tPrev = 0;
hang = 0;
}
else {
KTime_t date = 0;
rc_t rc = KDirectoryDate(dir, &date, "%s", dest);
if (rc == 0) {
tPrev = date;
if ((KTimeStamp() - date) > 60 * 99) {
/* no file update during 99' */
if (hang == 0) {
write(pipeto[1],
answer, sizeof y - 1);
++hang;
}
else if (hang < 9) {
++hang;
sig = 0;
}
else {
if (sig == 0) {
sig = SIGINT;
}
else {
++sig;
}
if (progressing) {
OUTMSG(("\n"));
}
if (sig > SIGKILL) {
rc = RC(rcExe,
rcProcess, rcExecuting,
rcProcess,rcDetached);
return rc;
}
PLOGMSG(klogInfo, (klogInfo,
"Senging $(sgn) to ascp",
"sgn=%s", sig));
kill(nPid, sig);
}
}
}
}
}
if (heartbeat > 0) {
if (callback) {
quit = !callback(id,
eAscpStateRunning, size, 0);
}
else {
progress(acc, size, srcSz, hSrc, sfSrc,
tPrev);
}
progressing = true;
}
i = 0;
}
}
}
}
RELEASE(KDirectory, dir);
}
if (progressing) {
OUTMSG(("\n"));
}
while (1) {
char buf[4096];
int s = read(fd, buf, sizeof buf);
if (s == 0) {
break;
}
else if (s < 0) {
if (errno != EAGAIN) {
perror("read(child)");
break;
}
continue;
}
ascpParse(buf, s, dest, &state, &line);
if (state == eWriteFailed) {
writeFailed = true;
}
}
STSMSG( STS_DBG, ("ascp exited with pid=%d status=%d", w, status));
if (WIFEXITED(status)) {
STSMSG(STS_DBG, ("ascp exited with exit status %d",
WEXITSTATUS(status)));
}
else {
STSMSG(STS_DBG, ("ascp has not terminated correctly"));
}
if (w == -1) {
perror("waitpid");
rc = RC(rcExe, rcProcess, rcWaiting, rcProcess, rcFailed);
LOGERR(klogErr, rc, "after waitpid");
exit(EXIT_FAILURE);
}
if (WIFEXITED(status)) {
if (WEXITSTATUS(status) == 0) {
STSMSG(STS_DBG, ("ascp succeed"));
if (callback) {
callback(id, eAscpStateExitSuccess, 0, 0);
}
} else if (writeFailed) {
rc = RC(rcExe, rcProcess, rcExecuting, rcMemory, rcExhausted);
if (callback) {
callback(id, eAscpStateExitWriteFailure, 0, 0);
}
}
else {
if (rc == 0) {
rc = RC(rcExe, rcProcess, rcWaiting, rcProcess, rcFailed);
}
PLOGERR(klogErr, (klogErr, rc,
"ascp failed with $(ret)", "ret=%d", WEXITSTATUS(status)));
if (callback) {
callback(id, eAscpStateExitFailure, 0, 0);
}
}
} else if (WIFSIGNALED(status)) {
if (rc == 0) {
if (quitting) {
rc = quitting();
if (rc == 0) {
rc = RC(rcExe,
rcProcess, rcWaiting, rcProcess, rcFailed);
}
}
}
if (rc != SILENT_RC(rcExe, rcProcess, rcExecuting,
rcProcess, rcCanceled))
{
PLOGERR(klogErr, (klogErr, rc, "ascp killed by signal $(sig)",
"sig=%d", WTERMSIG(status)));
if (callback) {
callback(id, eAscpStateExitFailure, 0, 0);
}
}
}
}
return rc;
}
LIB_EXPORT rc_t CC aspera_get(
const char *ascp_bin, const char *private_file, const char *src,
const char *dest, AscpOptions *opt)
{
AscpOptions dummy;
bool status = false;
uint64_t prev = 0;
int attempt = 0;
KDirectory *dir = NULL;
TQuitting *quitting = NULL;
rc_t rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
if (ascp_bin == NULL || private_file == NULL ||
src == NULL || dest == NULL)
{
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
if (opt == NULL) {
memset(&dummy, 0, sizeof dummy);
opt = &dummy;
}
if (opt->target_rate == NULL) {
KConfig *cfg = NULL;
rc_t rc = KConfigMake(&cfg, NULL);
DISP_RC(rc, "cannot KConfigMake");
if (rc == 0) {
rc = _KConfigGetAscpRate(cfg, &opt->target_rate);
DISP_RC(rc, "cannot get aspera max rate");
}
RELEASE(KConfig, cfg);
}
sStatus = status = opt->status;
quitting = opt->quitting;
while (true) {
rc = run_ascp(ascp_bin, private_file, src, dest, opt);
if (rc == 0) {
if (status) {
STSMSG(STS_DBG, ("ascp finished with success"));
}
break;
}
else if (rc == SILENT_RC(rcExe,
rcProcess, rcExecuting, rcMemory, rcExhausted))
{
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
break;
}
else {
rc_t rc = 0;
uint64_t size = 0;
if (quitting != NULL) {
rc = quitting();
if (rc != 0) {
break;
}
}
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
rc = KDirectoryFileSize(dir, &size, dest);
if (rc != 0 || size < prev) {
if (status) {
STSMSG(0, ("KDirectoryFileSize after ascp run: "
"rc = %ld, size = %ld", rc, size));
}
break;
}
else if (size > prev) {
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying...", size));
}
attempt = 0;
prev = size;
} else {
if (attempt++ > 3) {
break;
}
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying %d...", size, attempt));
}
}
}
}
RELEASE(KDirectory, dir);
return rc;
}
static rc_t report_repo( visit_ctx * octx, KRepCategory category )
{
rc_t rc, rc1;
KRepositoryVector repos;
const char * hint;
VectorInit ( &repos, 0, 5 );
switch ( category )
{
case krepUserCategory : hint = MAIN_CAT_USER;
rc = KRepositoryMgrUserRepositories( octx->repo_mgr, &repos );
break;
case krepSiteCategory : hint = MAIN_CAT_SITE;
rc = KRepositoryMgrSiteRepositories( octx->repo_mgr, &repos );
break;
case krepRemoteCategory : hint = MAIN_CAT_REMOTE;
rc = KRepositoryMgrRemoteRepositories( octx->repo_mgr, &repos );
break;
}
if ( rc != 0 )
{
if ( rc == SILENT_RC( rcKFG, rcNode, rcOpening, rcPath, rcNotFound ) )
{
KOutMsg("\n%s:\n", hint);
KOutMsg("\tnot found in configuration\n");
rc = 0;
}
else
{
PLOGERR( klogErr, ( klogErr, rc,
"KRepositoryMgr<$(hint)>repositories() failed in $(func)",
"hint=%s,func=%s", hint, __func__ ) );
}
}
else
{
uint32_t idx;
bool disabled = KRepositoryMgrCategoryDisabled ( octx->repo_mgr, category );
rc = KOutMsg( "\n%s:\n", hint );
if ( rc == 0 && disabled )
rc = KOutMsg( "\tglobally disabled\n" );
for ( idx = 0; idx < VectorLength( &repos ) && rc == 0; ++idx )
{
const KRepository *repo = VectorGet( &repos, idx );
if ( repo != NULL )
{
char repo_name[ 1024 ];
rc = KRepositoryDisplayName ( repo, repo_name, sizeof repo_name, NULL );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KRepositoryName() for $(hint) failed in $(func)", "hint=%s,func=%s", hint, __func__ ) );
}
else
{
KRepSubCategory sub_cat = KRepositorySubCategory ( repo );
bool disabled = KRepositoryDisabled ( repo );
bool cache_enabled = KRepositoryCacheEnabled ( repo );
const char * sub_cat_ptr = SUB_CAT_UNKNOWN;
switch( sub_cat )
{
case krepMainSubCategory : sub_cat_ptr = SUB_CAT_MAIN; break;
case krepAuxSubCategory : sub_cat_ptr = SUB_CAT_AUX; break;
case krepProtectedSubCategory : sub_cat_ptr = SUB_CAT_PROT; break;
default : sub_cat_ptr = SUB_CAT_UNKNOWN; break;
}
rc = KOutMsg( "\t%s.%s: %s, cache-%s",
sub_cat_ptr, repo_name,
( disabled ? "disabled" : "enabled" ),
( cache_enabled ? "enabled" : "disabled" ) );
if ( rc == 0 )
{
if ( octx->options->detailed )
{
/* it is OK if we cannot find the root of a repository... */
char where[ 4096 ];
rc1 = KRepositoryRoot ( repo, where, sizeof where, NULL );
if ( rc1 == 0 )
rc = KOutMsg( ", at %s", where );
else
{
rc1 = KRepositoryResolver ( repo, where, sizeof where, NULL );
if ( rc1 == 0 )
rc = KOutMsg( ", at %s", where );
}
}
}
if ( rc == 0 )
rc = KOutMsg( "\n" );
}
}
}
}
{
rc1 = KRepositoryVectorWhack ( &repos );
if ( rc1 != 0 )
{
PLOGERR( klogErr, ( klogErr, rc1,
"KRepositoryVectorWhack() for $(hint) failed in $(func)", "hint=%s,func=%s", hint, __func__ ) );
}
}
return rc;
}