error_t
pager_read_page (struct user_pager_info *pager,
vm_offset_t page,
vm_address_t *buf,
int *write_lock)
{
int pfn = page / vm_page_size;
size_t nread;
/* We never request write locks. */
*write_lock = 0;
expand_map (pager, page);
if (!pager->map[pfn])
vm_allocate (mach_task_self (), buf, vm_page_size, 1);
else
{
store_read (backing_store, pager->map[pfn], vm_page_size,
(void **)buf, &nread);
if (nread != vm_page_size)
{
munmap ((caddr_t) *buf, nread);
return EIO;
}
}
return 0;
}
static void btree_ui_out(struct btree_info *ui, const char *filename)
{
FILE *out = stdout;
if (filename) {
printf("output %s\n", filename);
out = fopen(filename, "w+");
if (out == NULL)
out = stdout;
}
#if 1
btree_print(ui->tree, (void (*)(void*, void*))userinfo_print, out);
#else
{
int i;
for (i=0; i < ui->tree->entries_num; i++) {
struct user_info *info = store_read(ui->userinfo_store, i);
userinfo_print(out, info);
store_release(ui->userinfo_store, i, info);
}
}
#endif
if (out != stdout)
fclose(out);
}
/* Implement the pager_read_page callback from the pager library. See
<hurd/pager.h> for the interface description. */
error_t
pager_read_page (struct user_pager_info *pager,
vm_offset_t page,
vm_address_t *buf,
int *writelock)
{
error_t err;
struct rwlock *nplock;
daddr_t addr;
int disksize;
err = find_address (pager, page, &addr, &disksize, &nplock, 1);
if (err)
return err;
if (addr)
{
size_t read = 0;
err = store_read (store, addr << log2_dev_blocks_per_dev_bsize,
disksize, (void **)buf, &read);
if (read != disksize)
err = EIO;
if (!err && disksize != __vm_page_size)
bzero ((void *)(*buf + disksize), __vm_page_size - disksize);
*writelock = 0;
}
else
{
#if 0
printf ("Write-locked pagein Object %#x\tOffset %#x\n", pager, page);
fflush (stdout);
#endif
*buf = (vm_address_t) mmap (0, vm_page_size, PROT_READ|PROT_WRITE,
MAP_ANON, 0, 0);
*writelock = 1;
}
if (nplock)
rwlock_reader_unlock (nplock);
return err;
}
int
main (int argc, char **argv)
{
struct store *store = 0;
store_offset_t addr = -1;
int dumped = 0, use_file_io = 0, block_size = 0;
void dump (store_offset_t addr, ssize_t len)
{
char buf[4096];
void *data = buf;
size_t data_len = sizeof (buf);
/* XXX: store->size can be too big for len. */
error_t err =
store_read (store, addr, len < 0 ? store->size : len,
&data, &data_len);
if (err)
error (5, err, store->name ? "%s" : "<store>", store->name);
if (write (1, data, data_len) < 0)
error (6, errno, "stdout");
if (data != buf)
munmap (data, data_len);
}
int get_next_line(int const fd, char **line)
{
static t_list *files = 0;
t_list *list;
t_file *file;
char *buf;
int ret;
list = files;
while (list && ((t_file*)list->content)->fd != fd)
list = list->next;
if (!list)
{
file = (t_file*)malloc(sizeof(t_file));
file->fd = fd;
file->buf = ft_strnew(BUFF_SIZE + 1);
list = ft_lstnew(file, sizeof(t_file));
ft_lstadd(&files, list);
}
buf = ft_strnew(BUFF_SIZE + 1);
ret = store_read(list->content, line, buf);
free(buf);
return (ret);
}
static store_error_t store_get(
const store_t *store,
uint64_t irecord,
trace_t *trace) {
record_t *record;
uint64_t data_offset;
store_error_t err;
size_t nbytes;
if (irecord >= store->nrecords) {
*trace = ZERO_TRACE;
return INVALID_RECORD;
}
record = &store->records[irecord];
data_offset = xe64toh(record->data_offset);
trace->itmin = xe32toh(record->itmin);
trace->nsamples = xe32toh(record->nsamples);
trace->begin_value = fe32toh(record->begin_value);
trace->end_value = fe32toh(record->end_value);
if (!inlimits(trace->itmin) || !inposlimits(trace->nsamples) ||
data_offset >= UINT64_MAX - SLIMIT * sizeof(gf_dtype)) {
return BAD_RECORD;
}
if (REC_EMPTY == data_offset) {
*trace = ZERO_TRACE;
return EMPTY_RECORD;
}
if (REC_ZERO == data_offset) {
*trace = ZERO_TRACE;
trace->itmin = xe32toh(record->itmin);
return SUCCESS;
}
trace->is_zero = 0;
if (data_offset + trace->nsamples*sizeof(gf_dtype) > store->data_size) {
*trace = ZERO_TRACE;
return BAD_DATA_OFFSET;
}
if (REC_SHORT == data_offset) {
trace->data = &record->begin_value;
} else {
if (NULL != store->data) {
trace->data = &store->data[data_offset/sizeof(gf_dtype)];
} else {
if (NULL == store->memdata[irecord]) {
nbytes = trace->nsamples * sizeof(gf_dtype);
store->memdata[irecord] = (gf_dtype*)malloc(nbytes);
if (NULL == store->memdata[irecord]) {
*trace = ZERO_TRACE;
return ALLOC_FAILED;
}
err = store_read(store, data_offset, nbytes, store->memdata[irecord]);
if (SUCCESS != err) {
free(store->memdata[irecord]);
store->memdata[irecord] = NULL;
*trace = ZERO_TRACE;
return err;
}
}
trace->data = store->memdata[irecord];
}
}
return SUCCESS;
}
/**
* @brief Major file/read parsing function
*
* @param cptr Pointer to calloc initialized scores. Uses samtools API as opposed to seq_density -> faster
* @param databl_start Pointer to genomic start positions of data blocks
* @param databl_end Pointer to genomic end positions of data blocks
* @param user_args Structure holding all user input
* @param cs Pointer to chromosome_size structure holding chromosome dimensions
* @param bam_file Pointer to samfile_t structure holding the file handler
* @param first_call Is this the first call to this function?
* @return Structure seq_block with results of the read scanning such as mapmass and file status
* @details Major function looping through every read in file until a new chromosome starts. Updates cptr scores and returns statistics on reads
* @note Very long. Should be refactored?
* @todo
*/
seq_block_t seq_density(usersize *cptr, uint32_t *databl_start, uint32_t *databl_end, user_arguments_t *user_args,
chromosome_size_t *cs, samfile_t *bam_file, int *first_call)
/*reads all reads from bam until a new chromosome is reached and sets first read infos stored in the struct fr.
* returns a flag about the parse status and writes the scores to cptr*/
{
int abs_gen_end;
uint32_t nindex=0,*ind_start,*ind_end,max_index=cs->min_indexspace;
usersize * cptr_beg=cptr;
seq_block_t bresults={0};
read_metrics_t rm={0};
static buffered_read_t fr={0};
if(*first_call)fr.cigar=(uint32_t *)Calloc(MAX_NCIGAR,uint32_t);
int lpos=fr.pos,last_end=fr.pos+fr.l_seq;
uint32_t BUFFERLIMIT=cs->max_pos+1;//there are no more than this many items calloc'ed
uint32_t INDEXLIMIT=cs->min_indexspace;
ind_start=databl_start;ind_end=databl_end;//note the start of pointers
while(1){//heavy main loop will go through this n=[amount of reads] times!
bam1_t * current_read=bam_init1();
/* ######### SCAN BLOCK ################## */
bresults.file_status=samread(bam_file,current_read);
if (bresults.file_status == -1){// EOF
if (*first_call){// EOF
warning("No compatible read found for this settings!\n");
if(!bresults.paired && user_args->PAIRED)warning("No proper pair [flag 2] found in this file. \nSet 'paired_only' to FALSE\n");
bresults.file_status = -10;
return bresults;
}
#if verbose==1
printf("EOF detected -> %d read(s) screened!\n",bresults.total_reads);
#endif
cs->max_pos=last_end;//set maximal absolut position
*(databl_end++)=cs->max_pos;//completes the missing end entry to be on par with start
cs->min_scorespace+=*(databl_end-1)-*(databl_start-1)+1;
cs->min_indexspace=nindex;
*databl_end=0;*databl_start=0;//end flag
databl_start=ind_start;databl_end=ind_end;//reset pointer to the beginning
free_samio(&fr,current_read);
bresults.file_status = 0;
return bresults;
}else if(bresults.file_status<-1){
warning("File truncated!\n");
if (!*first_call)free_samio(&fr,current_read);
return bresults; // truncated
}
/* MAIN QUALITY CHECKPOINT */
quality_check(&rm,current_read,user_args,&bresults,lpos);
#if verbose==4
print_readinfo(&bresults,current_read,&rm,bam_file);
#endif
if(rm.skip<0){
bresults.file_status = rm.skip;
return bresults;
}else if(rm.skip)goto SKIP_READ;
abs_gen_end=rm.genomic_end+user_args->EXTEND;
/* ######### END SCAN ######################*/
/*################ BOUNDARY CHECK ######################*/
if(fr.chrom_index!=current_read->core.tid || *first_call){//Did we reach the end of the chromosome? If yes save data and return!
if (!*first_call){
cs->max_pos=last_end;//set maximal absolut position
*databl_end++=cs->max_pos;//completes the missing end entry to be on par with start
cs->min_scorespace+=*(databl_end-1)-*(databl_start-1)+1;
cs->min_indexspace=nindex;
*databl_end=0;*databl_start=0;//end flag
databl_start=ind_start;databl_end=ind_end;//reset pointer to the beginning
}
*first_call=0;
bresults.file_status=1;
bresults.chrom_index_next=current_read->core.tid;
store_read(&fr,current_read,&rm);
return bresults;
} else if(lpos>current_read->core.pos){
bresults.file_status=-5;//something wrong with the positions
warning("Last position>current position. File doesn't seem to be sorted!\n");
free_samio(&fr,current_read);
return bresults;
} else if(BUFFERLIMIT<abs_gen_end || abs_gen_end < 0){
//skip read if sequence out of bounce
//possibly bad header with wrong chromosome margins or EXTEND too large!
warning("BUFFER only %d\n But POS: %d cur_seq_len: %d EXTEND: %d -> %d \n GLOBAL %d\n",
BUFFERLIMIT,current_read->core.pos,rm.read_length,user_args->EXTEND,
current_read->core.pos+rm.read_length+user_args->EXTEND,abs_gen_end);
#if pedantic==1
bresults.file_status=-4;
return bresults;
#endif
bam_destroy1(current_read);
continue;
}
/*##################################### WRITE ###############*/
cptr+=current_read->core.pos;//align pointer to current position
if(user_args->READTHROUGH){
write_density_ungapped(cptr,rm.read_length,&bresults.maxScore);
}else{
write_density_gapped(cptr,bam1_cigar(current_read),current_read->core.n_cigar,&bresults.maxScore);//minor speed panelty to ungapped
}
if(user_args->EXTEND>0){
if(rm.revcomp){
abs_gen_end-=user_args->EXTEND;
if(cptr-user_args->EXTEND>cptr_beg)cptr-=user_args->EXTEND;
else goto NOEXTEND;
}else{
cptr=cptr_beg;
cptr+=rm.genomic_end;
}
int k=0;
for(;k<user_args->EXTEND;++k){++*cptr;++cptr;}
}
NOEXTEND:
cptr=cptr_beg;//jump back to the beginning of the chromosome using the helper
if(!fr.written){//flush the first read of the chromosome stored in the struct fr
cptr+=fr.pos;
if(user_args->READTHROUGH){write_density_ungapped(cptr,fr.tlen,&bresults.maxScore);
}else{write_density_gapped(cptr,fr.cigar,fr.n_cigar,&bresults.maxScore);}
if(user_args->EXTEND>0){
if(fr.revcomp){
if(cptr-user_args->EXTEND>cptr_beg)cptr-=user_args->EXTEND;
else goto FRNOEXTEND;
}else{
cptr=cptr_beg;
cptr+=fr.genomic_end;
}
int k=0;
for(;k<user_args->EXTEND;++k){++*cptr;++cptr;}
}
FRNOEXTEND:
cptr=cptr_beg;//jump back to the beginning of the chromosome using the helper
*databl_start++=fr.pos;++nindex;//genomic coordinate where the data block starts
cs->min_pos=fr.pos;
cs->min_scorespace=0;//will only be set based on the index
cs->min_indexspace=nindex;
cs->max_pos=abs_gen_end;
lpos=fr.pos;
last_end=fr.genomic_end;//load last read info of first read
fr.written=1;//indicate that information has been used
}
/*################ INDEXING ######################*/
cptr=cptr_beg;//jump back to the beginning of the chromosome using the helper
if((current_read->core.pos-last_end)>=user_args->COMPRESSION){//check whether there was a large block without any data -> Triggers a jump on the sequence!
nindex++;//add one index
if(max_index<=nindex){//check whether we are already over the allocated index space
printf("Index space found: %d > Index space allocated: %d !!",nindex,max_index);
error("Error in indexing allocation detected!");
}
*databl_end++=last_end;//genomic coordinate where the data block ends | lags one index position behind start in the main loop!
cs->min_scorespace+=*(databl_end-1)-*(databl_start-1)+1;//+1 because start=end is still one Bp!
*databl_start++=current_read->core.pos;//genomic coordinate where the data block starts beginning with the current read
}
lpos=current_read->core.pos;
last_end= user_args->READTHROUGH ? max(current_read->core.pos+rm.read_length,last_end) : max(abs_gen_end,last_end);
SKIP_READ:
if(nindex>=INDEXLIMIT && !first_call)error("Index overflow!\n");
bam_destroy1(current_read);
}//end of bam index parsing skip tag section
bresults.file_status=-4;
return bresults;//can never happen
}
