static uint32_t remove_invalid_records( const struct ReferenceObj * const refobj,
DLList * list, INSDC_coord_zero pos )
{
uint32_t res = 0;
spot_group * sg = ( spot_group * )DLListHead( list );
while ( sg != NULL )
{
spot_group *nxt = ( spot_group * )DLNodeNext( ( DLNode * )sg );
PlacementRecord *rec = ( PlacementRecord * )DLListHead( &sg->records );
while ( rec != NULL )
{
PlacementRecord *nxt_rec = ( PlacementRecord * )DLNodeNext( ( DLNode * )rec );
INSDC_coord_zero end_pos = ( rec->pos + rec->len );
bool remove = ( end_pos <= pos );
if ( !remove )
{
AlignmentIterator * al_iter = PlacementRecordCast ( rec, placementRecordExtension0 );
int32_t state = AlignmentIteratorState ( al_iter, NULL );
remove = ( ( state & align_iter_invalid ) == align_iter_invalid );
}
if ( remove )
{
DLListUnlink ( &sg->records, ( DLNode * )rec );
PlacementRecordWhack ( rec );
}
else
{
res++;
}
rec = nxt_rec;
}
sg = nxt;
}
return res;
}
/* FindInRepo
* find accession in a repository
*/
static
rc_t SRAPathFindInRepo ( const NCBISRAPath *self, NCBIRepository *repo, const char *accession,
char *path, size_t path_max, size_t *rep_len, int vol_type )
{
SRAPathString *srv;
PATH_DEBUG (("SRAPathFindInRepo(%s)\n", AlgToStr(repo->type)));
/* look for accession on a rep-server */
for ( srv = ( SRAPathString* ) DLListHead ( & repo -> repsrv ); srv != NULL; srv = ( SRAPathString* ) DLNodeNext ( & srv -> n ) )
{
/* try with this server */
rc_t rc = SRAPathFindOnServer ( self, repo, srv, accession, path, path_max, vol_type );
if ( rc == 0 )
{
/* make sure server is at head of list */
if ( DLNodePrev ( & srv -> n ) != NULL )
{
DLListUnlink ( & repo -> repsrv, & srv -> n );
DLListPushHead ( & repo -> repsrv, & srv -> n );
}
if ( rep_len != NULL )
* rep_len = strlen ( srv -> path );
return 0;
}
if ( GetRCState ( rc ) != rcNotFound )
return rc;
}
return RC ( rcSRA, rcMgr, rcSelecting, rcPath, rcNotFound );
}
static
rc_t CC NCBISRAPathFull ( const NCBISRAPath *self, const char *rep, const char *vol,
const char *accession, char *path, size_t path_max )
{
NCBIRepository *repo;
bool found;
rc_t rc;
/* loop through repositories */
for ( repo = ( NCBIRepository* ) DLListHead ( & self -> repos );
repo != NULL; repo = ( NCBIRepository* ) DLNodeNext ( & repo -> n ) )
{
rc = ApplyAlg( self, rep, vol, accession, path, path_max, repo, &found);
if (found)
{
return rc;
}
}
/* try the default repository */
rc = ApplyAlg( self, rep, vol, accession, path, path_max, self -> dflt_repo, &found);
if (found)
{
return rc;
}
/* internal version */
return SRAPathFullInt ( self, rep, vol, accession, path, path_max, 1024 );
}
static void inc_alignment_iterators( DLList * list, INSDC_coord_zero pos )
{
spot_group * sg = ( spot_group * )DLListHead( list );
while ( sg != NULL )
{
spot_group *nxt = ( spot_group * )DLNodeNext( ( DLNode * )sg );
PlacementRecord *rec = ( PlacementRecord * )DLListHead( &sg->records );
while ( rec != NULL )
{
PlacementRecord *nxt_rec = ( PlacementRecord * )DLNodeNext( ( DLNode * )rec );
AlignmentIterator * al_iter = PlacementRecordCast ( rec, placementRecordExtension0 );
if ( rec->pos <= pos && al_iter != NULL )
{
AlignmentIteratorNext ( al_iter );
}
rec = nxt_rec;
}
sg = nxt;
}
}
LIB_EXPORT rc_t CC PlacementSetIteratorNextReference ( PlacementSetIterator *self,
INSDC_coord_zero *first_pos, INSDC_coord_len *len, struct ReferenceObj const ** refobj )
{
rc_t rc = 0;
if ( refobj != NULL ) {
*refobj = NULL;
}
if ( self == NULL )
return RC( rcAlign, rcIterator, rcReleasing, rcSelf, rcNull );
pl_set_iter_clear_curr_ref_window( self );
pl_set_iter_clear_curr_ref( self );
self->current_entry = NULL; /* what is the current pi-entry, we are handling ? */
/* !!! here we are taking the next reference from the top of the list */
self->current_ref = ( pi_ref * )DLListPopHead ( &self->pi_refs );
if ( self->current_ref == NULL )
{
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
if ( first_pos != NULL ) *first_pos = self->current_ref->outer.first;
if ( len != NULL) *len = self->current_ref->outer.len;
/* if the caller wants to know the ref-obj... */
if ( refobj != NULL )
{
pi_window *pw = ( pi_window * )DLListHead( &(self->current_ref->pi_windows) );
if ( pw != NULL )
{
pi_entry * pie = ( pi_entry * )DLListHead( &(pw->pi_entries) );
if ( pie != NULL )
{
rc = PlacementIteratorRefObj( pie->pi, refobj );
}
}
}
return rc;
}
/* FindInRepoByType
* find accession in a repository of a given type
*/
static
rc_t SRAPathFindInRepoByType ( const NCBISRAPath *self, const char *accession, char *path, size_t path_max, size_t *rep_len, int repo_type, int vol_type )
{
/* loop through all repositories */
NCBIRepository *repo;
for ( repo = ( NCBIRepository* ) DLListHead ( & self -> repos ); repo != NULL; repo = ( NCBIRepository* ) DLNodeNext ( & repo -> n ) )
{
if ( repo->type == repo_type && SRAPathFindInRepo(self, repo, accession, path, path_max, rep_len, vol_type) == 0 )
return 0;
}
return RC ( rcSRA, rcMgr, rcSelecting, rcPath, rcNotFound );
}
LIB_EXPORT rc_t CC PlacementSetIteratorNextRecordAt ( PlacementSetIterator *self,
INSDC_coord_zero pos, const PlacementRecord **rec )
{
rc_t rc = 0;
pi_window * pw;
bool done;
if ( rec == NULL )
return RC( rcAlign, rcIterator, rcAccessing, rcParam, rcNull );
*rec = NULL;
if ( self == NULL )
return RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
if ( self->current_ref == NULL )
{
/* no more reference to iterator over! the iterator is done! */
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
if ( self->current_window == NULL )
{
/* no more windows to iterator over! the iterator is done! */
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
pw = self->current_window;
done = false;
do
{
if ( self->current_entry == NULL )
{
self->current_entry = ( pi_entry * )DLListHead( &(pw->pi_entries) );
}
done = ( self->current_entry == NULL );
rc = ( done ? SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone ) : 0 );
if ( rc == 0 )
{
rc = PlacementIteratorNextRecordAt ( self->current_entry->pi, pos, rec );
done = ( GetRCState( rc ) != rcDone );
if ( !done )
{
self->current_entry = ( pi_entry * )DLNodeNext( ( DLNode * )self->current_entry );
done = ( self->current_entry == NULL );
rc = ( done ? SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone ) : 0 );
}
}
} while ( !done );
return rc;
}
LIB_EXPORT rc_t CC PlacementSetIteratorNextWindow ( PlacementSetIterator *self,
INSDC_coord_zero *first_pos, INSDC_coord_len *len )
{
rc_t rc = 0;
if ( first_pos != NULL ) {
*first_pos = 0;
}
if ( len != NULL ) {
*len = 0;
}
if ( self == NULL )
return RC( rcAlign, rcIterator, rcReleasing, rcSelf, rcNull );
self->current_entry = NULL; /* what is the current pi-entry, we are handling ? */
if ( self->current_ref == NULL )
{
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
pl_set_iter_clear_curr_ref_window( self );
/* !!! here we are taking the next window from the top of the list */
self->current_window = ( pi_window * )DLListPopHead ( &(self->current_ref->pi_windows) );
/* check if we have reached the last window on this reference... */
if ( self->current_window == NULL )
{
return SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
/* point to the first entry in this window... */
self->current_entry = ( pi_entry * )DLListHead( &(self->current_window->pi_entries) );
/* if the caller wants to know first_pos / len */
if ( first_pos != NULL )
{
*first_pos = self->current_window->w.first;
}
if ( len != NULL )
{
*len = self->current_window->w.len;
}
return rc;
}
static rc_t add_to_pi_ref( pi_ref * pr, window * w, PlacementIterator *pi )
{
rc_t rc = 0;
pi_window * pw = find_pi_window( &pr->pi_windows, w );
if ( pw == NULL )
rc = make_pi_window( &pw, &pr->pi_windows, w );
if ( rc == 0 )
rc = add_to_pi_window( pw, pi );
if ( rc == 0 )
{
/* keep track of the outer window... */
if ( DLListHead( &pr->pi_windows ) == NULL )
{
/* first window ?*/
pr->outer.first = w->first;
pr->outer.len = w->len;
}
else
{
if ( w->first < pr->outer.first )
pr->outer.first = w->first;
if ( w->first + w->len > pr->outer.first + pr->outer.len )
pr->outer.len = ( ( w->first + w->len ) - pr->outer.first ) + 1;
}
}
else if ( ( pw != NULL )&&( GetRCState( rc ) == rcDone ) )
{
/* add_to_pi_window() was not successful because iterator has no
alignments int the requested window, that means we have to delete
the window if it is empty */
if ( pw->count == 0 )
{
/* first we have to take the pw out of the pr->pi_windows - list...
it was pushed at the tail of it, so we pop it from there */
DLListPopTail( &pr->pi_windows );
/* because it is empty ( count == 0 ) we can just free it now */
free( pw );
}
}
return rc;
}
LIB_EXPORT rc_t CC ReferenceIteratorGetPlacement ( ReferenceIterator *self,
const PlacementRecord **rec )
{
rc_t rc = 0;
if ( self == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
}
else if ( rec == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcParam, rcNull );
}
else
{
if ( self->current_spot_group == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
if ( self->current_rec != NULL )
{
/* remove the 'previous' current-rec! */
DLListPopHead ( &self->current_spot_group->records );
PlacementRecordWhack ( self->current_rec );
self->depth--;
self->current_rec = NULL;
}
self->current_rec = ( PlacementRecord * )DLListHead( &self->current_spot_group->records );
if ( self->current_rec == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
*rec = self->current_rec;
}
}
}
return rc;
}
LIB_EXPORT rc_t CC ReferenceIteratorNextPlacement ( ReferenceIterator *self,
const PlacementRecord **rec )
{
rc_t rc = 0;
if ( self == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcSelf, rcNull );
}
else if ( rec == NULL )
{
rc = RC( rcAlign, rcIterator, rcAccessing, rcParam, rcNull );
}
else
{
if ( self->current_spot_group == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
if ( self->current_rec == NULL )
{
self->current_rec = ( PlacementRecord * )DLListHead( &self->current_spot_group->records );
}
else
{
self->current_rec = ( PlacementRecord * )DLNodeNext( ( DLNode * )self->current_rec );
}
if ( self->current_rec == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
*rec = self->current_rec;
}
}
}
return rc;
}
LIB_EXPORT rc_t CC ReferenceIteratorNextSpotGroup ( ReferenceIterator *self,
const char ** name, size_t * len )
{
rc_t rc = 0;
if ( self->current_spot_group == NULL )
{
self->current_spot_group = ( spot_group * )DLListHead( &self->spot_groups );
if ( self->current_spot_group == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
}
else
{
spot_group *nxt = ( spot_group * )DLNodeNext( ( DLNode * ) self->current_spot_group );
if ( nxt == NULL )
{
rc = SILENT_RC( rcAlign, rcIterator, rcAccessing, rcOffset, rcDone );
}
else
{
self->current_spot_group = nxt;
}
}
self->current_rec = NULL;
if ( rc == 0 && self->current_spot_group != NULL )
{
if ( name != NULL )
{
*name = self->current_spot_group->name;
}
if ( len != NULL )
{
*len = self->current_spot_group->len;
}
}
return rc;
}
/* Find
* finds location of run within rep-server/volume matrix
*
* "accession" [ IN ] - NUL terminated run accession,
* e.g. "SRR000001"
*
* "path" [ OUT ] and "path_max" [ IN ] - return buffer for
* NUL-terminated full path to accession.
*
* returns 0 if path exists, rc state rcNotFound if
* path cannot be found, and rcInsufficient if buffer is
* too small.
*/
static
rc_t CC NCBISRAPathFindWithRepLen ( const NCBISRAPath *cself, const char *accession, char *path, size_t path_max, size_t *rep_len )
{
rc_t rc;
NCBIRepository *repo;
PATH_DEBUG(("NCBISRAPathFindWithRepLen(%s)\n", accession));
rc = FindFast( cself, accession, path, path_max, rep_len );
if ( rc == 0 )
return 0;
/* loop through all repositories */
for ( repo = ( NCBIRepository* ) DLListHead ( & cself -> repos ); repo != NULL; repo = ( NCBIRepository* ) DLNodeNext ( & repo -> n ) )
{
rc = SRAPathFindInRepo(cself, repo, accession, path, path_max, rep_len, alg_none);
if ( rc == 0 )
return 0;
}
/* default repository */
return SRAPathFindInRepo(cself, cself -> dflt_repo, accession, path, path_max, rep_len, alg_none);
}
/* FindOnServer
* find accession on rep-server
*/
static
rc_t SRAPathFindOnServer ( const NCBISRAPath *self, const NCBIRepository *repo, const SRAPathString *srv,
const char *accession, char *path, size_t path_max, int vol_type )
{
const SRAPathString *vol;
PATH_DEBUG (("SRAPathFindOnServer(%s)\n", srv->path));
for ( vol = ( const SRAPathString* ) DLListHead ( & repo -> vols );
vol != NULL; vol = ( const SRAPathString* ) DLNodeNext ( & vol -> n ) )
{
if ( vol_type == alg_none || vol_type == vol->alg )
{
rc_t rc;
PATH_DEBUG (("SRAPathFindOnServer trying volume %s\n", vol->path));
switch ( vol -> alg )
{
case alg_ebi:
rc = SRAPathFullEBI ( self, srv -> path, vol -> path, accession, path, path_max );
break;
case alg_refseq:
rc = SRAPathFullREFSEQ ( self, srv -> path, vol -> path, accession, path, path_max );
if ( rc == 0) /* check for existence of accession at the root of the volume, and if not found try to apply the default path-building scheme */
{
switch ( KDirectoryPathType ( self -> dir, path ) )
{
case kptNotFound:
case kptBadPath:
rc = SRAPathFullREFSEQArchive( self, srv -> path, vol -> path, accession, path, path_max );
if (rc == 0) {
PATH_DEBUG (("SRAPathFindOnServer: found(%s)\n", path));
return 0;
}
break;
default:
return 0;
}
}
break;
case alg_wgs:
rc = SRAPathFullWGS ( self, srv -> path, vol -> path, accession, path, path_max );
break;
default:
rc = SRAPathFullInt ( self, srv -> path, vol -> path, accession, path, path_max, 1024 );
break;
}
if ( rc == 0 )
{
switch ( KDirectoryPathType ( self -> dir, path ) )
{
case kptNotFound:
case kptBadPath:
break;
default:
PATH_DEBUG (("SRAPathFindOnServer: found(%s)\n", path));
return 0;
}
}
else
{
if ( GetRCState( rc ) == rcInsufficient )
return rc;
}
}
}
return RC ( rcSRA, rcMgr, rcSelecting, rcPath, rcNotFound );
}
static void remove_unneeded_alignments (PileupIteratorState* pileup_state, uint64_t ref_pos, char* error_buf, size_t error_buf_size)
{
/*int64_t max_removed_id = 0;*/
#if CACHE_IMPL_AS_LIST == 1
Alignment_CacheItem* item = ( Alignment_CacheItem* ) DLListHead ( & pileup_state->cache_alignment.list_alignments );
for (; item != NULL; )
{
uint64_t local_ref_pos = ref_pos - item->seq_start;
if ( item->start + item->len <= local_ref_pos ) /* not "<" here because local_ref_pos here is the old position that will be incremented right after we exit from this function */
{
Alignment_CacheItem* item_cur = item;
item = (Alignment_CacheItem*) DLNodeNext( & item->node );
#if USE_BLOB_CACHE_FOR_ALIGNMENTS == 1
if ( item->row_id > max_removed_id )
max_removed_id = item->row_id;
#endif
DLListUnlink ( & pileup_state->cache_alignment.list_alignments, & item_cur->node);
Alignment_CacheItemWhack ( & item_cur->node, NULL );
-- pileup_state->cache_alignment.size;
}
else
item = (Alignment_CacheItem*) DLNodeNext( & item->node );
}
#else
#if 0
/* Improved (?) inplace version */
/* i - current item in current state cache (cache size can decrease during this algorithm, so as i)
all cache[j]: j < i are already processed in the cache - i.e. they present in new cache state
size - updated size of cache
gap_start - beginning index of first consecutive elements found to be removed from the current cache
(so it's an semi-open interval [start, i) that needs to be removed
from the cache)
*/
size_t i = 0;
Alignment_Cache* cache = & pileup_state->cache_alignment;
size_t size = cache->size;
size_t gap_start = 0;
for (; i < size;)
{
Alignment_CacheItem const* item = & cache->data [i];
uint64_t local_ref_pos = ref_pos - item->seq_start;
if ( item->start + item->len > local_ref_pos ) /* not ">=" here because local_ref_pos here is the old position that will be incremented right after we exit from this function */
{
/* check if we have a gap before i and if yes - move memory up */
if ( gap_start != i )
{
memmove ( & cache->data[gap_start], & cache->data[i], (size-i)*sizeof(Alignment_CacheItem) );
size -= i - gap_start;
i = gap_start;
}
++i;
gap_start = i;
}
else
{
++i;
}
}
if ( gap_start != i )
size = gap_start;
cache->size = size;
#else
size_t i_src, i_dst;
Alignment_Cache* cache = & pileup_state->cache_alignment;
size_t size = cache->size;
for (i_src = 0; i_src < size; ++ i_src)
{
Alignment_CacheItem const* item = & cache->data [i_src];
uint64_t local_ref_pos = ref_pos - item->seq_start;
if ( item->start + item->len > local_ref_pos ) /* not ">=" here because local_ref_pos here is the old position that will be incremented right after we exit from this function */
break;
}
for (i_dst = 0; i_src < size; ++ i_src)
{
Alignment_CacheItem const* item = & cache->data [i_src];
uint64_t local_ref_pos = ref_pos - item->seq_start;
if ( item->start + item->len > local_ref_pos ) /* not ">=" here because local_ref_pos here is the old position that will be incremented right after we exit from this function */
{
if (i_dst != i_src)
cache->data [i_dst] = cache->data [i_src];
++ i_dst;
}
}
cache->size = i_dst;
#endif
#endif
#if USE_BLOB_CACHE_FOR_ALIGNMENTS == 2
if ( max_removed_id > 0 )
{
size_t i_src, i_dst;
BlobItem* blobs = pileup_state->blobs_alignments.blobs;
size_t size = pileup_state->blobs_alignments.size;
for (i_src = 0; i_src < size; ++ i_src)
{
BlobItem const* item = & blobs [i_src];
int64_t start_id;
uint64_t count;
rc_t rc = VBlobIdRange ( item->blob, & start_id, & count );
if ( rc != 0 )
{
rc_t res = string_printf ( error_buf, error_buf_size, NULL,
"ERROR: VBlobIdRange failed with error: 0x%08x (%u) [%R]", rc, rc, rc);
if (res == rcBuffer || res == rcInsufficient)
error_buf [ error_buf_size - 1 ] = '\0';
return rc;
}
if ( start_id + (int64_t)count > max_removed_id )
{
/* Current blob contains ids greater than last removed one
we don't want to look further because most probably
all the following blobs will also contain ids that we will
need to keep in the cache, so we exit immediately
*/
break;
}
else
{
VBlobRelease ( item->blob );
}
}
for (i_dst = 0; i_src < size; ++ i_src)
{
Alignment_CacheItem const* item = & cache->data [i_src];
uint64_t local_ref_pos = ref_pos - item->seq_start;
if ( item->start + item->len > local_ref_pos ) /* not ">=" here because local_ref_pos here is the old position that will be incremented right after we exit from this function */
{
if (i_dst != i_src)
cache->data [i_dst] = cache->data [i_src];
++ i_dst;
}
}
cache->size = i_dst;
}
#endif
}