error_t pager_read_page (struct user_pager_info *pager, vm_offset_t page, vm_address_t *buf, int *write_lock) { int pfn = page / vm_page_size; size_t nread; /* We never request write locks. */ *write_lock = 0; expand_map (pager, page); if (!pager->map[pfn]) vm_allocate (mach_task_self (), buf, vm_page_size, 1); else { store_read (backing_store, pager->map[pfn], vm_page_size, (void **)buf, &nread); if (nread != vm_page_size) { munmap ((caddr_t) *buf, nread); return EIO; } } return 0; }
static void btree_ui_out(struct btree_info *ui, const char *filename) { FILE *out = stdout; if (filename) { printf("output %s\n", filename); out = fopen(filename, "w+"); if (out == NULL) out = stdout; } #if 1 btree_print(ui->tree, (void (*)(void*, void*))userinfo_print, out); #else { int i; for (i=0; i < ui->tree->entries_num; i++) { struct user_info *info = store_read(ui->userinfo_store, i); userinfo_print(out, info); store_release(ui->userinfo_store, i, info); } } #endif if (out != stdout) fclose(out); }
/* Implement the pager_read_page callback from the pager library. See <hurd/pager.h> for the interface description. */ error_t pager_read_page (struct user_pager_info *pager, vm_offset_t page, vm_address_t *buf, int *writelock) { error_t err; struct rwlock *nplock; daddr_t addr; int disksize; err = find_address (pager, page, &addr, &disksize, &nplock, 1); if (err) return err; if (addr) { size_t read = 0; err = store_read (store, addr << log2_dev_blocks_per_dev_bsize, disksize, (void **)buf, &read); if (read != disksize) err = EIO; if (!err && disksize != __vm_page_size) bzero ((void *)(*buf + disksize), __vm_page_size - disksize); *writelock = 0; } else { #if 0 printf ("Write-locked pagein Object %#x\tOffset %#x\n", pager, page); fflush (stdout); #endif *buf = (vm_address_t) mmap (0, vm_page_size, PROT_READ|PROT_WRITE, MAP_ANON, 0, 0); *writelock = 1; } if (nplock) rwlock_reader_unlock (nplock); return err; }
int main (int argc, char **argv) { struct store *store = 0; store_offset_t addr = -1; int dumped = 0, use_file_io = 0, block_size = 0; void dump (store_offset_t addr, ssize_t len) { char buf[4096]; void *data = buf; size_t data_len = sizeof (buf); /* XXX: store->size can be too big for len. */ error_t err = store_read (store, addr, len < 0 ? store->size : len, &data, &data_len); if (err) error (5, err, store->name ? "%s" : "<store>", store->name); if (write (1, data, data_len) < 0) error (6, errno, "stdout"); if (data != buf) munmap (data, data_len); }
int get_next_line(int const fd, char **line) { static t_list *files = 0; t_list *list; t_file *file; char *buf; int ret; list = files; while (list && ((t_file*)list->content)->fd != fd) list = list->next; if (!list) { file = (t_file*)malloc(sizeof(t_file)); file->fd = fd; file->buf = ft_strnew(BUFF_SIZE + 1); list = ft_lstnew(file, sizeof(t_file)); ft_lstadd(&files, list); } buf = ft_strnew(BUFF_SIZE + 1); ret = store_read(list->content, line, buf); free(buf); return (ret); }
static store_error_t store_get( const store_t *store, uint64_t irecord, trace_t *trace) { record_t *record; uint64_t data_offset; store_error_t err; size_t nbytes; if (irecord >= store->nrecords) { *trace = ZERO_TRACE; return INVALID_RECORD; } record = &store->records[irecord]; data_offset = xe64toh(record->data_offset); trace->itmin = xe32toh(record->itmin); trace->nsamples = xe32toh(record->nsamples); trace->begin_value = fe32toh(record->begin_value); trace->end_value = fe32toh(record->end_value); if (!inlimits(trace->itmin) || !inposlimits(trace->nsamples) || data_offset >= UINT64_MAX - SLIMIT * sizeof(gf_dtype)) { return BAD_RECORD; } if (REC_EMPTY == data_offset) { *trace = ZERO_TRACE; return EMPTY_RECORD; } if (REC_ZERO == data_offset) { *trace = ZERO_TRACE; trace->itmin = xe32toh(record->itmin); return SUCCESS; } trace->is_zero = 0; if (data_offset + trace->nsamples*sizeof(gf_dtype) > store->data_size) { *trace = ZERO_TRACE; return BAD_DATA_OFFSET; } if (REC_SHORT == data_offset) { trace->data = &record->begin_value; } else { if (NULL != store->data) { trace->data = &store->data[data_offset/sizeof(gf_dtype)]; } else { if (NULL == store->memdata[irecord]) { nbytes = trace->nsamples * sizeof(gf_dtype); store->memdata[irecord] = (gf_dtype*)malloc(nbytes); if (NULL == store->memdata[irecord]) { *trace = ZERO_TRACE; return ALLOC_FAILED; } err = store_read(store, data_offset, nbytes, store->memdata[irecord]); if (SUCCESS != err) { free(store->memdata[irecord]); store->memdata[irecord] = NULL; *trace = ZERO_TRACE; return err; } } trace->data = store->memdata[irecord]; } } return SUCCESS; }
/** * @brief Major file/read parsing function * * @param cptr Pointer to calloc initialized scores. Uses samtools API as opposed to seq_density -> faster * @param databl_start Pointer to genomic start positions of data blocks * @param databl_end Pointer to genomic end positions of data blocks * @param user_args Structure holding all user input * @param cs Pointer to chromosome_size structure holding chromosome dimensions * @param bam_file Pointer to samfile_t structure holding the file handler * @param first_call Is this the first call to this function? * @return Structure seq_block with results of the read scanning such as mapmass and file status * @details Major function looping through every read in file until a new chromosome starts. Updates cptr scores and returns statistics on reads * @note Very long. Should be refactored? * @todo */ seq_block_t seq_density(usersize *cptr, uint32_t *databl_start, uint32_t *databl_end, user_arguments_t *user_args, chromosome_size_t *cs, samfile_t *bam_file, int *first_call) /*reads all reads from bam until a new chromosome is reached and sets first read infos stored in the struct fr. * returns a flag about the parse status and writes the scores to cptr*/ { int abs_gen_end; uint32_t nindex=0,*ind_start,*ind_end,max_index=cs->min_indexspace; usersize * cptr_beg=cptr; seq_block_t bresults={0}; read_metrics_t rm={0}; static buffered_read_t fr={0}; if(*first_call)fr.cigar=(uint32_t *)Calloc(MAX_NCIGAR,uint32_t); int lpos=fr.pos,last_end=fr.pos+fr.l_seq; uint32_t BUFFERLIMIT=cs->max_pos+1;//there are no more than this many items calloc'ed uint32_t INDEXLIMIT=cs->min_indexspace; ind_start=databl_start;ind_end=databl_end;//note the start of pointers while(1){//heavy main loop will go through this n=[amount of reads] times! bam1_t * current_read=bam_init1(); /* ######### SCAN BLOCK ################## */ bresults.file_status=samread(bam_file,current_read); if (bresults.file_status == -1){// EOF if (*first_call){// EOF warning("No compatible read found for this settings!\n"); if(!bresults.paired && user_args->PAIRED)warning("No proper pair [flag 2] found in this file. \nSet 'paired_only' to FALSE\n"); bresults.file_status = -10; return bresults; } #if verbose==1 printf("EOF detected -> %d read(s) screened!\n",bresults.total_reads); #endif cs->max_pos=last_end;//set maximal absolut position *(databl_end++)=cs->max_pos;//completes the missing end entry to be on par with start cs->min_scorespace+=*(databl_end-1)-*(databl_start-1)+1; cs->min_indexspace=nindex; *databl_end=0;*databl_start=0;//end flag databl_start=ind_start;databl_end=ind_end;//reset pointer to the beginning free_samio(&fr,current_read); bresults.file_status = 0; return bresults; }else if(bresults.file_status<-1){ warning("File truncated!\n"); if (!*first_call)free_samio(&fr,current_read); return bresults; // truncated } /* MAIN QUALITY CHECKPOINT */ quality_check(&rm,current_read,user_args,&bresults,lpos); #if verbose==4 print_readinfo(&bresults,current_read,&rm,bam_file); #endif if(rm.skip<0){ bresults.file_status = rm.skip; return bresults; }else if(rm.skip)goto SKIP_READ; abs_gen_end=rm.genomic_end+user_args->EXTEND; /* ######### END SCAN ######################*/ /*################ BOUNDARY CHECK ######################*/ if(fr.chrom_index!=current_read->core.tid || *first_call){//Did we reach the end of the chromosome? If yes save data and return! if (!*first_call){ cs->max_pos=last_end;//set maximal absolut position *databl_end++=cs->max_pos;//completes the missing end entry to be on par with start cs->min_scorespace+=*(databl_end-1)-*(databl_start-1)+1; cs->min_indexspace=nindex; *databl_end=0;*databl_start=0;//end flag databl_start=ind_start;databl_end=ind_end;//reset pointer to the beginning } *first_call=0; bresults.file_status=1; bresults.chrom_index_next=current_read->core.tid; store_read(&fr,current_read,&rm); return bresults; } else if(lpos>current_read->core.pos){ bresults.file_status=-5;//something wrong with the positions warning("Last position>current position. File doesn't seem to be sorted!\n"); free_samio(&fr,current_read); return bresults; } else if(BUFFERLIMIT<abs_gen_end || abs_gen_end < 0){ //skip read if sequence out of bounce //possibly bad header with wrong chromosome margins or EXTEND too large! warning("BUFFER only %d\n But POS: %d cur_seq_len: %d EXTEND: %d -> %d \n GLOBAL %d\n", BUFFERLIMIT,current_read->core.pos,rm.read_length,user_args->EXTEND, current_read->core.pos+rm.read_length+user_args->EXTEND,abs_gen_end); #if pedantic==1 bresults.file_status=-4; return bresults; #endif bam_destroy1(current_read); continue; } /*##################################### WRITE ###############*/ cptr+=current_read->core.pos;//align pointer to current position if(user_args->READTHROUGH){ write_density_ungapped(cptr,rm.read_length,&bresults.maxScore); }else{ write_density_gapped(cptr,bam1_cigar(current_read),current_read->core.n_cigar,&bresults.maxScore);//minor speed panelty to ungapped } if(user_args->EXTEND>0){ if(rm.revcomp){ abs_gen_end-=user_args->EXTEND; if(cptr-user_args->EXTEND>cptr_beg)cptr-=user_args->EXTEND; else goto NOEXTEND; }else{ cptr=cptr_beg; cptr+=rm.genomic_end; } int k=0; for(;k<user_args->EXTEND;++k){++*cptr;++cptr;} } NOEXTEND: cptr=cptr_beg;//jump back to the beginning of the chromosome using the helper if(!fr.written){//flush the first read of the chromosome stored in the struct fr cptr+=fr.pos; if(user_args->READTHROUGH){write_density_ungapped(cptr,fr.tlen,&bresults.maxScore); }else{write_density_gapped(cptr,fr.cigar,fr.n_cigar,&bresults.maxScore);} if(user_args->EXTEND>0){ if(fr.revcomp){ if(cptr-user_args->EXTEND>cptr_beg)cptr-=user_args->EXTEND; else goto FRNOEXTEND; }else{ cptr=cptr_beg; cptr+=fr.genomic_end; } int k=0; for(;k<user_args->EXTEND;++k){++*cptr;++cptr;} } FRNOEXTEND: cptr=cptr_beg;//jump back to the beginning of the chromosome using the helper *databl_start++=fr.pos;++nindex;//genomic coordinate where the data block starts cs->min_pos=fr.pos; cs->min_scorespace=0;//will only be set based on the index cs->min_indexspace=nindex; cs->max_pos=abs_gen_end; lpos=fr.pos; last_end=fr.genomic_end;//load last read info of first read fr.written=1;//indicate that information has been used } /*################ INDEXING ######################*/ cptr=cptr_beg;//jump back to the beginning of the chromosome using the helper if((current_read->core.pos-last_end)>=user_args->COMPRESSION){//check whether there was a large block without any data -> Triggers a jump on the sequence! nindex++;//add one index if(max_index<=nindex){//check whether we are already over the allocated index space printf("Index space found: %d > Index space allocated: %d !!",nindex,max_index); error("Error in indexing allocation detected!"); } *databl_end++=last_end;//genomic coordinate where the data block ends | lags one index position behind start in the main loop! cs->min_scorespace+=*(databl_end-1)-*(databl_start-1)+1;//+1 because start=end is still one Bp! *databl_start++=current_read->core.pos;//genomic coordinate where the data block starts beginning with the current read } lpos=current_read->core.pos; last_end= user_args->READTHROUGH ? max(current_read->core.pos+rm.read_length,last_end) : max(abs_gen_end,last_end); SKIP_READ: if(nindex>=INDEXLIMIT && !first_call)error("Index overflow!\n"); bam_destroy1(current_read); }//end of bam index parsing skip tag section bresults.file_status=-4; return bresults;//can never happen }