int zigdev_test(void) { char sql[200]; sql_format(sql, 200, "insert into user values('test user', '123456', '*****@*****.**', '15895830938');"); if (cache_opt_item(TBL_USER, sql) < 0) printf("\ncache_opt_item\n"); sql_format(sql, 200, "insert into user values('TihsYloH', '123456', '*****@*****.**', '15895830938');"); if (cache_opt_item(TBL_USER, sql) < 0) printf("\ncache_opt_item\n"); sql_format(sql, 200, "select * from user where user_name = 'TihsYloH';"); if (cache_search(TBL_USER, sql, NULL, NULL, 0) == 0) { printf("\nTihsYloH user find \n"); } else { printf("\nTihsYloH user not find\n"); } sql_format(sql, 200, "select * from user where user_name = 'TihsYloH__';"); if (cache_search(TBL_USER, sql, NULL, NULL, 0) == 0) { printf("\nTihsYloH___ user find \n"); } else { printf("\n\nTihsYloH___ user not find\n"); } return 0; }
/* * Given ranges contained within a bin this makes sure that all sequences * referred to in these ranges have their parent listed as the new bin. * * Returns 0 on success * -1 on failure */ static int break_contig_reparent_seqs(GapIO *io, bin_index_t *bin) { int i, nr = bin->rng ? ArrayMax(bin->rng) : 0; for (i = 0; i < nr; i++) { range_t *r = arrp(range_t, bin->rng, i); if (r->flags & GRANGE_FLAG_UNUSED) continue; if ((r->flags & GRANGE_FLAG_ISMASK) == GRANGE_FLAG_ISANNO) { anno_ele_t *a = (anno_ele_t *)cache_search(io, GT_AnnoEle, r->rec); if (a->bin != bin->rec) { a = cache_rw(io, a); a->bin = bin->rec; } } else { seq_t *seq = (seq_t *)cache_search(io, GT_Seq, r->rec); if (seq->bin != bin->rec) { seq = cache_rw(io, seq); seq->bin = bin->rec; seq->bin_index = i; } } } return 0; }
/* * Adds a contig to a scaffold array. * Gap size, type and evidence refer to the gap between this and the * "previous" contig - ie the last in the scaffold. More complex * scaffold manipulations will be handled elsewhere. * * Set these fields to 0 if you do not know them. * * Returns 0 on success * -1 on failure */ int scaffold_add(GapIO *io, tg_rec scaffold, tg_rec contig, int gap_size, int gap_type, int evidence) { scaffold_t *f; contig_t *c; scaffold_member_t *m; int i; /* Check if this contig is in a scaffold, if so remove now */ c = cache_search(io, GT_Contig, contig); if (c->scaffold) scaffold_remove(io, c->scaffold, contig); if (!(f = cache_search(io, GT_Scaffold, scaffold))) return -1; /* Check if it already exists */ for (i = 0; i < ArrayMax(f->contig); i++) { m = arrp(scaffold_member_t, f->contig, i); if (m->rec == contig) return 0; } /* Append */ f = cache_rw(io, f); m = ArrayRef(f->contig, ArrayMax(f->contig)); // extend m->rec = contig; m->gap_size = ArrayMax(f->contig) > 1 ? gap_size : 0; m->gap_type = gap_type; m->evidence = evidence; /* Update the contig record too */ c = cache_search(io, GT_Contig, contig); c = cache_rw(io, c); c->scaffold = scaffold; #if 0 /* Add a scaffold link to the contig graph too */ if (ArrayMax(f->contig) >= 2) { m = arrp(scaffold_member_t, f->contig, ArrayMax(f->contig)-2); contig_link_t lnk; lnk.rec1 = contig; lnk.rec2 = m->rec; /* Best guess */ lnk.pos1 = 0; lnk.end1 = 1; lnk.pos2 = 0; lnk.end2 = 0; lnk.orientation = 0; lnk.size = 100; lnk.type = CLINK_TYPE_SCAFFOLD; lnk.score = 0; contig_add_link(io, &lnk); } #endif return 0; }
/* * Exports Scaffold information to an AGP file * * Returns 0 on success * -1 on failure */ int scaffold_to_agp(GapIO *io, char *fn) { FILE *fp; int i, j; if (NULL == (fp = fopen(fn, "w+"))) { verror(ERR_WARN, "scaffold_from_agp", "%s: %s", fn, strerror(errno)); return -1; } for (i = 0; io->scaffold && i < ArrayMax(io->scaffold); i++) { scaffold_t *f = cache_search(io, GT_Scaffold, arr(tg_rec, io->scaffold, i)); int start = 1, end = 1; int k = 1; if (!f) { verror(ERR_WARN, "scaffold_from_agp", "Failed to load scaffold\n"); fclose(fp); return -1; } cache_incr(io, f); for (j = 0; f->contig && j < ArrayMax(f->contig); j++) { scaffold_member_t *m = arrp(scaffold_member_t, f->contig, j); contig_t *c = cache_search(io, GT_Contig, m->rec); int ustart, uend; int len; /* Get the unpadded clipped contig length */ consensus_valid_range(io, m->rec, &ustart, &uend); consensus_unpadded_pos(io, m->rec, uend, &uend); len = uend - ustart + 1; if (j) { int gap = m->gap_size; fprintf(fp, "%s\t%d\t%d\t%d\tN\t%d\tfragment\tyes\n", f->name, start, start+gap-1, k++, gap); start += gap; } fprintf(fp, "%s\t%d\t%d\t%d\tW\t%s\t%d\t%d\t+\n", f->name, start, start + len-1, k++, c->name, ustart, uend); start += len; } cache_decr(io, f); } if (0 != fclose(fp)) { verror(ERR_WARN, "scaffold_from_agp", "%s: %s", fn, strerror(errno)); return -1; } return 0; }
static void tag_shift_for_delete(GapIO *io, tg_rec crec, tg_rec srec, int start, int end, int pos, tg_rec brec) { contig_iterator *ci; rangec_t *r; contig_t *c = cache_search(io, GT_Contig, crec);; //printf("< tag in seq %"PRIrec" at %d\n", srec, pos); cache_incr(io, c); ci = contig_iter_new_by_type(io, crec, 0, CITER_FIRST | CITER_ISTART, start+pos, end, GRANGE_FLAG_ISANNO); if (!ci) { cache_decr(io, c); return; } while ((r = contig_iter_next(io, ci))) { range_t r2, *r_out; anno_ele_t *a; bin_index_t *bin; if (r->pair_rec != srec) continue; bin_remove_item(io, &c, GT_AnnoEle, r->rec); r2.start = (r->start > start+pos) ? r->start-1 : r->start; r2.end = r->end-1; r2.mqual = r->mqual; r2.rec = r->rec; r2.pair_rec = r->pair_rec; r2.flags = r->flags; if (r2.end < r2.start) { /* Tag entirely removed now, it must have been on a pad */ a = cache_search(io, GT_AnnoEle, r->rec); a = cache_rw(io, a); cache_deallocate(io, a); continue; } bin = bin_add_to_range(io, &c, brec, &r2, &r_out, NULL, 0); a = cache_search(io, GT_AnnoEle, r->rec); if (a->bin != bin->rec /*|| a->idx != r_out - ArrayBase(range_t, bin->rng)*/) { /* Annotation moved bins */ a = cache_rw(io, a); a->bin = bin->rec; //a->bin_idx = r_out - ArrayBase(range_t, bin->rng); } } cache_decr(io, c); contig_iter_del(ci); }
/* * Removes some or all tags from some or all contigs. * If the contig list or tag list is blank it implies all contigs or all tags. * * Returns 0 on success * -1 on failure */ int delete_tags(GapIO *io, int ncontigs, contig_list_t *contigs, char *tag_list, int verbose) { HashTable *h = NULL; int ret = 0; /* Hash tag types */ if (tag_list && *tag_list) { int i; if (SetActiveTags(tag_list) == -1) { return -1; } h = HashTableCreate(32, 0); for (i = 0; i < number_of_active_tags; i++) { HashData hd; hd.i = 0; HashTableAdd(h, active_tag_types[i], 4, hd, NULL); } } /* Iterate over contig list or all contigs */ if (verbose) vfuncheader("Delete Tags"); if (ncontigs) { int i; for (i = 0; i < ncontigs; i++) { contig_t *c = cache_search(io, GT_Contig, contigs[i].contig); vmessage("Scanning contig %d of %d (%s)\n", i+1, ncontigs, c->name); ret |= delete_tag_single_contig(io, contigs[i].contig, h, verbose); UpdateTextOutput(); cache_flush(io); } } else { int i; tg_rec *order = ArrayBase(tg_rec, io->contig_order); for (i = 0; i < NumContigs(io); i++) { contig_t *c = cache_search(io, GT_Contig, order[i]); vmessage("Scanning contig %d of %d (%s)\n", i+1, NumContigs(io), c->name); ret |= delete_tag_single_contig(io, order[i], h, verbose); UpdateTextOutput(); cache_flush(io); } } SetActiveTags(""); if (h) HashTableDestroy(h, 0); return ret; }
/* * Creates an anno_ele as per anno_ele_new, but also adds it to an object * and creates the bin Range entry too. */ tg_rec anno_ele_add(GapIO *io, int obj_type, tg_rec obj_rec, tg_rec anno_rec, int type, char *comment, int start, int end, char dir) { range_t r; anno_ele_t *e; contig_t *c; tg_rec crec; bin_index_t *bin; tg_rec seq_bin = 0; /* Find contig for obj_rec/obj_type */ if (obj_type == GT_Contig) { crec = obj_rec; } else { int st, en; sequence_get_position2(io, obj_rec, &crec, &st, &en, NULL, &seq_bin, NULL, NULL); start += st; end += st; } c = (contig_t *)cache_search(io, GT_Contig, crec); cache_incr(io, c); r.start = start; r.end = end; r.flags = GRANGE_FLAG_ISANNO; r.mqual = type; r.pair_rec = obj_rec; if (GT_Seq == obj_type) r.flags |= GRANGE_FLAG_TAG_SEQ; r.rec = anno_ele_new(io, 0, obj_type, obj_rec, 0, type, dir, comment); e = (anno_ele_t *)cache_search(io, GT_AnnoEle, r.rec); e = cache_rw(io, e); if (seq_bin) bin = bin_add_to_range(io, &c, seq_bin, &r, NULL, NULL, 0); else bin = bin_add_range(io, &c, &r, NULL, NULL, 0); if (!bin) verror(ERR_FATAL, "anno_ele_add", "bin_add_to_range returned NULL"); e->bin = bin ? bin->rec : 0; cache_decr(io, c); return r.rec; }
/* * Tidies up after break contig or disassemble readings, looking for now * redundant bins. * * This has the following functions (not all implemented yet!) * 1) If a contig is totally empty, remove the contig. * 2) If a bin is empty and all below it, remove the bin. * 3) If a bin is empty and all above it, remove parent bins and link * contig to new root. (TODO) */ static void remove_empty_bins(GapIO *io, tg_rec contig) { contig_t *c = cache_search(io, GT_Contig, contig); tg_rec first = 0; cache_incr(io, c); if (c->bin) { if (remove_empty_bins_r(io, c->bin, &first)) { cache_decr(io, c); contig_destroy(io, contig); return; } if (first != c->bin) { bin_index_t *bin; tg_rec bp, br, cdummy; int offset; /* Cut out the offending waste */ bin = cache_search(io, GT_Bin, first); bin = cache_rw(io, bin); bp = bin->parent; // Find new bin offset bin_get_position(io, bin, &cdummy, &offset); assert(cdummy == contig); bin->pos = offset; bin->parent = contig; bin->parent_type = GT_Contig; bin->flags |= BIN_BIN_UPDATED; c = cache_rw(io, c); br = c->bin; c->bin = first; bin = cache_search(io, GT_Bin, bp); bin = cache_rw(io, bin); if (bin->child[0] == first) bin->child[0] = 0; if (bin->child[1] == first) bin->child[1] = 0; /* Recursively remove the bin tree from old root, br */ bin_destroy_recurse(io, br); } } cache_decr(io, c); }
/* * Creates a new named scaffold. * * Returns scaffold pointer on success. * NULL on failure */ scaffold_t *scaffold_new(GapIO *io, char *name) { tg_rec rec; scaffold_t *f, init_f; if (!io->db->scaffold) return NULL; memset(&init_f, 0, sizeof(scaffold_t)); init_f.name = name; /* Allocate our contig */ rec = cache_item_create(io, GT_Scaffold, &init_f); /* Initialise it */ f = (scaffold_t *)cache_search(io, GT_Scaffold, rec); f = cache_rw(io, f); if (name) scaffold_set_name(io, &f, name); else f->name = NULL; /* Add it to the scaffold order too */ io->scaffold = cache_rw(io, io->scaffold); io->db = cache_rw(io, io->db); ARR(tg_rec, io->scaffold, io->db->Nscaffolds++) = rec; /* Add to the new contigs list */ if (name) add_to_list("new_scaffolds", name); return f; }
void l2_cache_fill(int cpu_num, unsigned long long int addr, int set, int way, int prefetch, unsigned long long int evicted_addr) { // uncomment this line to see the information available to you when there is a cache fill event //printf("0x%llx %d %d %d 0x%llx\n", addr, set, way, prefetch, evicted_addr); cache_insert(addr, prefetch); if (prefetch==1) { prefetch_tot_num++; // Insert Evicted address into pollution filter // only if it is not prefetch int found = cache_search(evicted_addr); if (found == -1) { if (evicted_addr != 0) { printf("ERROR: evicted address not in the cache - %llx \n", evicted_addr); exit(1); } } else if (cache[found].pf != 1) poll_insert(evicted_addr); poll_remove(addr); } cache_remove(evicted_addr); }
// Must be locked over the queue - can be unlocked immediate following the call (safe to use returned request) // Must be synchronized over the "queue get" semaphore // The "queue put" semaphore should be incremened following the call // The returned request is freed from the list and must be destroyed before it goes out of context struct request_bundle queue_getSmallRequest() { fprintf(stderr, "queue_getSmallRequest: Called\n"); struct request* req = q_first(); if (req == NULL) { fprintf(stderr, "queue_getSmallRequest: Error! Queue held no requests!\n"); struct request_bundle bundle = { NULL, NULL }; return bundle; } //assert (queue_size != 0); struct request* small = req; intmax_t smallsize = getFileSize(req->filename); while ( (req = q_nextOf(req)) != NULL ) { intmax_t thissize = getFileSize(req->filename); if (thissize < smallsize) { small = req; smallsize = thissize; } } struct cache_entry* ent = cache_search(small->filename); struct request_bundle bundle = { small, ent }; return bundle; }
/* * Compute the visible end position of a contig. This isn't just the extents * of start_used / end_used in the bins as this can included invisible * data such as cached consensus sequences. */ int contig_visible_end(GapIO *io, tg_rec crec) { rangec_t *r; contig_iterator *ci; ci = contig_iter_new_by_type(io, crec, 1, CITER_LAST | CITER_IEND, CITER_CSTART, CITER_CEND, GRANGE_FLAG_ISANY); if (!ci) { contig_t *c = cache_search(io, GT_Contig, crec); return c->end; } while (r = contig_iter_prev(io, ci)) { int v; if ((r->flags & GRANGE_FLAG_ISMASK) == GRANGE_FLAG_ISCONS) continue; v = r->end; contig_iter_del(ci); return v; } contig_iter_del(ci); return 0; }
/* destroy a cache */ int cache_destroy(const char *file) { Cache *tmp; tmp = cache_search(file); if (tmp == NULL) return(0); cache_free(tmp); return(1); }
/* * Returns the relative orientation of this annotation vs the contig. */ int anno_get_orient(GapIO *io, tg_rec anum) { bin_index_t *bin = NULL; tg_rec bnum; anno_ele_t *a = cache_search(io, GT_AnnoEle, anum); int comp = a->orient; /* Bubble up bins until we hit the root */ for (bnum = a->bin; bnum; bnum = bin->parent) { bin = (bin_index_t *)cache_search(io, GT_Bin, bnum); if (bin->flags & BIN_COMPLEMENTED) comp ^= 1; if (bin->parent_type != GT_Bin) break; } assert(bin && bin->parent_type == GT_Contig); return comp; }
/* * Reads a given GContig number to the GContig pointer. * Here cnum is a contig number from 0 to N-1 indicating the record number * in the cnum-th element of the contig_order array. * * Returns 0 on success. * -1 on failure. */ int gio_read_contig(GapIO *io, int cnum, contig_t **c) { tg_rec crec; if (!io->contig_order) return -1; crec = arr(tg_rec, io->contig_order, cnum); *c = (contig_t *)cache_search(io, GT_Contig, crec); return 0; }
/* * Sets the annotation type, passed in as a string but held in a 4-byte int. * This also attempts to set the cached copy of the type held within the * bin range array. * * Returns 0 on success * -1 on failure */ int anno_ele_set_type(GapIO *io, anno_ele_t **e, char *str) { int type; char stype[5]; anno_ele_t *ae; if (!(ae = cache_rw(io, *e))) return -1; /* Get integer type */ memset(stype, 0, 5); strncpy(stype, str, 4); type = str2type(stype); /* Update annotation */ ae->tag_type = type; /* Also update range_t cached copy of type */ if (ae->bin) { bin_index_t *bin = (bin_index_t *)cache_search(io, GT_Bin, ae->bin); range_t *r = NULL; int i, nranges; if (!bin) return -1; if (!(bin = cache_rw(io, bin))) return -1; /* * Find the index into the bin range. * FIXME: we should add a bin_index element, as seen in seq_t, * to avoid the brute force loop. This doesn't have to be * permanently stored - a cached copy would suffice. */ nranges = bin->rng ? ArrayMax(bin->rng) : 0; for (i = 0; i < nranges; i++) { r = arrp(range_t, bin->rng, i); if (r->flags & GRANGE_FLAG_UNUSED) continue; if (r->rec == ae->rec) break; } if (i == nranges) return -1; bin->flags |= BIN_RANGE_UPDATED; r->mqual = type; } *e = ae; return 0; }
/* * Removes a contig from a scaffold. * * Returns 0 on success * -1 on failure */ int scaffold_remove(GapIO *io, tg_rec scaffold, tg_rec contig) { scaffold_t *f; scaffold_member_t *m, *m2; contig_t *c; int i; c = cache_search(io, GT_Contig, contig); f = cache_search(io, GT_Scaffold, scaffold); if (!c || !f) return -1; if (c->scaffold != scaffold) { verror(ERR_WARN, "scaffold_remove", "Attempted to remove contig #%" PRIrec" from a scaffold #%"PRIrec" it is not a member of", contig, scaffold); return -1; } c = cache_rw(io, c); c->scaffold = 0; f = cache_rw(io, f); for (i = 0; i < ArrayMax(f->contig); i++) { m = arrp(scaffold_member_t, f->contig, i); if (m->rec == contig) { /* Shuffle array down */ for (i++; i < ArrayMax(f->contig); i++) { m2 = arrp(scaffold_member_t, f->contig, i); *m = *m2; m = m2; } ArrayMax(f->contig)--; } } return 0; }
int edview_search_tag_type(edview *xx, int dir, int strand, char *value) { contig_iterator *iter; int start, end; rangec_t *r; rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci); int type = str2type(value); contig_t *c = cache_search(xx->io, GT_Contig, xx->cnum); if (dir) { start = xx->cursor_apos + (dir ? 1 : -1); end = c->end; ifunc = contig_iter_next; } else { start = c->start; end = xx->cursor_apos + (dir ? 1 : -1); ifunc = contig_iter_prev; } iter = contig_iter_new_by_type(xx->io, xx->cnum, 1, dir == 1 ? CITER_FIRST : CITER_LAST, start, end, GRANGE_FLAG_ISANNO); if (!iter) /* Can happen legitimately when we're already at the end of contig */ return -1; while (r = ifunc(xx->io, iter)) { if ((dir && r->start < start) || (!dir && r->start > end)) continue; if (r->mqual == type) break; } if (r) { if (r->flags & GRANGE_FLAG_TAG_SEQ) { int pos; sequence_get_position(xx->io, r->pair_rec, NULL, &pos, NULL, NULL); pos = r->start - pos; edSetCursorPos(xx, GT_Seq, r->pair_rec, pos, 1); } else { edSetCursorPos(xx, GT_Contig, xx->cnum, r->start, 1); } contig_iter_del(iter); return 0; } contig_iter_del(iter); return -1; }
// Must be locked over the queue - can be unlocked immediately following the call // Must be synchronized over the "queue get" semaphore // The "queue put" semaphore should be incremented following the call // The returned request is freed from the list and must be destroyed before it goes out of context struct request_bundle queue_getRequest() { fprintf(stderr, "queue_getRequest: Called\n"); struct request* req = q_shift(); struct cache_entry* ent; if (req == NULL) ent = NULL; else ent = cache_search(req->filename); struct request_bundle bundle = { req, ent }; return bundle; }
int edview_search_tag_indel(edview *xx, int dir, int strand, char *value) { contig_iterator *iter; int start, end; rangec_t *r; rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci); contig_t *c = cache_search(xx->io, GT_Contig, xx->cnum); if (dir) { start = xx->cursor_apos + (dir ? 1 : -1); end = c->end; ifunc = contig_iter_next; } else { start = c->start; end = xx->cursor_apos + (dir ? 1 : -1); ifunc = contig_iter_prev; } iter = contig_iter_new_by_type(xx->io, xx->cnum, 1, dir == 1 ? CITER_FIRST : CITER_LAST, start, end, GRANGE_FLAG_ISREFPOS); if (!iter) return -1; while (r = ifunc(xx->io, iter)) { if ((dir && r->start < start) || (!dir && r->start > end)) continue; break; } if (r) { edSetCursorPos(xx, GT_Contig, xx->cnum, r->start, 1); contig_iter_del(iter); return 0; } contig_iter_del(iter); return -1; }
/* * Removes all tags of specific types (hashed in h, or all if h == NULL) * from a specified contig. * * Returns 0 on success * -1 on failure */ static int delete_tag_single_contig(GapIO *io, tg_rec crec, HashTable *h, int verbose) { contig_iterator *ci; rangec_t *r; contig_t *c; int ret = -1; ci = contig_iter_new_by_type(io, crec, 1, CITER_FIRST, CITER_CSTART, CITER_CEND, GRANGE_FLAG_ISANNO); if (!ci) return -1; if (!(c = cache_search(io, GT_Contig, crec))) { contig_iter_del(ci); return -1; } cache_incr(io, c); while (NULL != (r = contig_iter_next(io, ci))) { char t[5]; (void)type2str(r->mqual, t); if (!h || HashTableSearch(h, t, 4)) { anno_ele_t *e; if (verbose) vmessage("Removing anno %s #%"PRIrec"\tContig %s\t%d..%d\n", t, r->rec, c->name, r->start, r->end); if (bin_remove_item(io, &c, GT_AnnoEle, r->rec)) goto fail; /* FIXME: Need to reclaim the GT_AnnoEle record itself */ } } ret = 0; fail: contig_iter_del(ci); cache_decr(io, c); return ret; }
/* * Removes an anno_ele from the gap database. * FIXME: need to deallocate storage too. (See docs/TODO) * * Returns 0 on success * -1 on failure */ int anno_ele_destroy(GapIO *io, anno_ele_t *e) { bin_index_t *bin; range_t *r; int i; /* Find the bin range pointing to this object */ bin = (bin_index_t *)cache_search(io, GT_Bin, e->bin); if (!bin || !bin->rng || ArrayMax(bin->rng) == 0) return -1; if (!(bin = cache_rw(io, bin))) return -1; for (i = 0; i < ArrayMax(bin->rng); i++) { r = arrp(range_t, bin->rng, i); if (r->flags & GRANGE_FLAG_UNUSED) continue; if (r->rec == e->rec) break; } if (i == ArrayMax(bin->rng)) return -1; /* Mark this bin range as unused */ r->rec = bin->rng_free; r->flags |= GRANGE_FLAG_UNUSED; bin->rng_free = i; bin->flags |= BIN_RANGE_UPDATED | BIN_BIN_UPDATED; bin_incr_nanno(io, bin, -1); if (bin->start_used == r->start || bin->end_used == r->end) bin_set_used_range(io, bin); return 0; }
void l2_prefetcher_operate(int cpu_num, unsigned long long int addr, unsigned long long int ip, int cache_hit) { l2_prefetcher_operate_ampm(cpu_num, addr, ip, cache_hit); acc_tot++; if (!cache_hit) { miss_tot++; if ( poll_search(addr) != -1 ) poll_tot++; } int found = cache_search(addr); if (found != -1) if (cache[found].pf == 1) { prefetch_use_num++; cache[found].pf = 0; } //printf("acc:%d miss:%d pre:%d use_pre:%d poll:%d ___ page miss:%d hit:%d imp:%d\n", acc_tot, miss_tot ,prefetch_tot_num, prefetch_use_num, poll_tot, ampm_page_miss, ampm_page_hit, ampm_imp); }
// Returns request_bundle containing the first request/cache_entry pair if such a pair exists // Otherwise, returns a request_bundle containing only the first request in the queue // request_bundle is returned by value // Must be locked over the request queue - can be unlocked immediately following the call // The returned request is removed from the queue // Must be locked over the cache INCLUDING ALL USAGE OF THE RETURNED CACHE ENTRY // The cache entry REMAINS IN THE CACHE // The returned request is freed from the queue and must be destroyed upon falling out of context // The returned cache entry should NOT be destroyed upon falling out of the caller's context struct request_bundle getCachedRequest() { fprintf(stderr, "getCachedRequest: Called\n"); struct request* req = q_first(); struct cache_entry* ent = NULL; while (req != NULL) { ent = cache_search(req->filename); if (ent != NULL) { q_remove(req); break; } req = q_nextOf(req); } if (req == NULL) req = q_shift(); struct request_bundle bundle = { req, ent }; return bundle; }
//services the request from the queue void * worker(void * arg) { // assign a thread id pthread_mutex_lock(&threadnum_access); int thread_id = thread_num; // thread id (between 0 and n-1) thread_num++; pthread_mutex_unlock(&threadnum_access); unsigned char *buf = NULL;//data for file struct stat file_info;//struct where size is struct timespec t1, t2;//structs for time char time[100];//string for response time long time_diff; bool time_error = false; off_t file_size;//size of file int reqFd;//for the read file char log_entry[MAX_TEXT_SIZE]; char* filename; char full_path[MAX_TEXT_SIZE]; char* file_ending = malloc(sizeof(char) * MAX_REQUEST_LENGTH); char* content_type = malloc(sizeof(char) * MAX_REQUEST_LENGTH); char error_string[251]; bool cache_hit = false;//true for hit false for miss int cache_index = -1; int fd; FILE *read_file; int request_count = 0; // how many requests this worker thread has completed int error_flag = 0; // continue this loop while either dispatchers left or requests in queue while (cur_num_dispatch > 0 || count > 0) { pthread_mutex_lock(&queue_access); // wait only while dispatchers left and no requests in queue while (cur_num_dispatch > 0 && count == 0) { pthread_cond_wait(&queue_full_or_dispatch_exit, &queue_access); } count--; //get time 1 in nano seconds if(clock_gettime(CLOCK_MONOTONIC, &t1) != 0) { perror("can't get time"); strcpy(time,"ERROR"); time_error = true; } filename = queue[r_queue_index].m_szRequest; fd = queue[r_queue_index].m_socket; if (get_type(filename, content_type) == -1) { printf("Failed to get content type.\n"); strcpy(content_type, "Error"); strcpy(error_string, "Could not get content type"); } if (strcmp(content_type, "Error") == 0) error_flag = 1; else error_flag = 0; if (use_cache) cache_index = cache_search(filename, &file_size); if(cache_index >= 0) { if (return_result(fd, content_type, cache[cache_index].data, (int)cache[cache_index].file_size) != 0) { fprintf(stderr, "Failed to return result.\n"); } else { //get time 2 in nano seconds if(clock_gettime(CLOCK_MONOTONIC, &t2) != 0) { perror("can't get time"); strcpy(time,"ERROR"); time_error = true; } else { time_diff = t2.tv_nsec = t1.tv_nsec; time_diff = time_diff / NANO_TO_MILLI; sprintf(time, "%ldms", time_diff); } cache_hit = true; } } else//read from file { sprintf(full_path, "%s%s", path, filename); // open file, copy to buffer if ((read_file = fopen(full_path, "r")) == NULL) { perror("Failed to open file"); strcpy(content_type, "Error"); strcpy(error_string,"could not open file"); error_flag = 1; } else//good read { if(stat(full_path,&file_info) < 0) { perror("Failed to get file size"); strcpy(content_type, "Error"); strcpy(error_string, "could not get file size"); error_flag = 1; } else//good stat { file_size = file_info.st_size; if(buf == NULL) { buf = (unsigned char*)malloc((size_t)file_size); if(buf == NULL) { perror("could not allocate memory"); strcpy(content_type, "Error"); strcpy(error_string,"could not allocate memory"); error_flag = 1; } } else//realloc buf { buf = (unsigned char*)realloc(buf,(size_t)file_size); if(buf == NULL) { perror("could not allocate memory"); strcpy(content_type, "Error"); strcpy(error_string,"could not allocate memory"); error_flag = 1; } }//end realloc }//end good stat }//end good read if(!error_flag) { if (fread(buf, 1, (size_t)file_size, read_file) <= 0)//sizeof(char) { printf("failed to read from file."); strcpy(content_type, "Error"); strcpy(error_string,"could not read file"); error_flag = 1; } else { if (return_result(fd, content_type, buf, (int)file_size) != 0) { fprintf(stderr, "Failed to return result.\n"); } else { if(clock_gettime(CLOCK_MONOTONIC, &t2) != 0) { perror("can't get time"); strcpy(time,"ERROR"); time_error = true; } else { time_diff = t2.tv_nsec = t1.tv_nsec; time_diff = time_diff / NANO_TO_MILLI; sprintf(time,"%ldms",time_diff); } } if(use_cache && cache_insert(buf, filename, file_size) < 0) { fprintf(stderr, "Failed to add file to cache\n"); } } } }//end read file if(error_flag) { if (return_error(fd, error_string) != 0) { fprintf(stderr, "Failed to return error.\n"); } } r_queue_index = (r_queue_index + 1) % queue_length; pthread_mutex_unlock(&queue_access); pthread_cond_broadcast(&queue_empty); // log to file pthread_mutex_lock(&log_access); request_count++; if(cache_hit == true) { sprintf(log_entry,"[%d][%d][%d][%s][%lu][%s][%s]\n",thread_id, request_count, fd, filename,(unsigned long)file_size,time,"HIT"); cache_hit = false; } else { if(!error_flag) { if (use_cache) sprintf(log_entry,"[%d][%d][%d][%s][%lu][%s][%s]\n",thread_id, request_count, fd, filename,(unsigned long)file_size,time,"MISS"); else sprintf(log_entry,"[%d][%d][%d][%s][%lu]\n",thread_id, request_count, fd, filename,(unsigned long)file_size); } else { sprintf(log_entry,"[%d][%d][%d][%s][%s]\n",thread_id, request_count, fd, filename,error_string); } } if(write(logFd,log_entry, strlen(log_entry)) < 0) perror("failed to write to log"); pthread_mutex_unlock(&log_access); } // no more dispatcher threads or requests, work has been completed free(filename); free(buf); free(file_ending); free(content_type); free(read_file); printf("Closing the log\n"); pthread_exit(NULL); return NULL; }
/* * Attempt to find edits. It's not 100% reliable, but works for most cases. * We look for lowercase bases and confidence 100 and 0 (if not N). * We cannot find deleted bases though. */ int edview_search_edit(edview *xx, int dir, int strand, char *value) { int start, end; contig_iterator *iter; rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci); rangec_t *r; int best_pos, found = 0; int fpos; tg_rec fseq; if (dir) { start = xx->cursor_apos + 1; end = CITER_CEND; iter = contig_iter_new(xx->io, xx->cnum, 1, CITER_FIRST | CITER_ISTART, start, end); ifunc = contig_iter_next; best_pos = INT_MAX; } else { start = CITER_CSTART; end = xx->cursor_apos -1; iter = contig_iter_new(xx->io, xx->cnum, 1, CITER_LAST | CITER_IEND, start, end); ifunc = contig_iter_prev; best_pos = INT_MIN; } if (!iter) return -1; while ((r = ifunc(xx->io, iter))) { seq_t *s, *sorig; char *seq, *qual; int seq_len, comp, off = 0, i; if (found && dir && r->start > best_pos) break; if (found && !dir && r->end < best_pos) break; if (NULL == (s = sorig = cache_search(xx->io, GT_Seq, r->rec))) break; if (r->comp ^ (s->len < 0)) { s = dup_seq(s); complement_seq_t(s); } seq = s->seq; qual = s->conf; seq_len = ABS(s->len); if (r->start < start) { off = start - r->start; seq += off; qual += off; seq_len -= off; } for (i = 0; i < seq_len; i++) { if (islower(seq[i]) || qual[i] == 100 || (qual[i] == 0 && seq[i] != 'N' && seq[i] != '-' && seq[i] != '*')) { int pos = r->start + i + off; if (dir) { if (best_pos > pos && pos > xx->cursor_apos) { found = 1; best_pos = pos; fpos = i + off; fseq = r->rec; } break; } else { if (best_pos < pos && pos < xx->cursor_apos) { found = 1; best_pos = pos; fpos = i + off; fseq = r->rec; } } } } if (s != sorig) free(s); } if (found) { edSetCursorPos(xx, fseq == xx->cnum ? GT_Contig : GT_Seq, fseq, fpos, 1); } contig_iter_del(iter); return found ? 0 : -1; }
int edview_search_consensus(edview *xx, int dir, int strand, char *value) { int mismatches = 0; /* exact match */ int where = 2; /* consensus */ char *p; int start, end; char cons[WIN_WIDTH+1]; int patlen; char *uppert, *upperb; int found = 0, at_end = 0; tg_rec fseq; int fpos, i, j; contig_t *c; /* * Parse value search string. It optionally includes two extra params * separated by #. Ie: * <string>#<N.mismatches>#<where>. * <where> is 1 for readings, 2 for consensus, 3 for both. */ if (p = strchr(value, '#')) { mismatches = atoi(p+1); *p = 0; if (p = strchr(p+1, '#')) where = atoi(p+1); } /* uppercase search string, remove pads, and store fwd/rev copies */ patlen = strlen(value); depad_seq(value, &patlen, NULL); if (NULL == (uppert = (char *)xmalloc(patlen + 1))) return 0; if (NULL == (upperb = (char *)xmalloc(patlen + 1))) return 0; uppert[patlen] = upperb[patlen] = 0; for (i = patlen-1; i >= 0; i--) { upperb[i] = uppert[i] = toupper(value[i]); } complement_seq(upperb, patlen); /* Loop */ if (dir) { start = xx->cursor_apos + (dir ? 1 : -1); end = start + (WIN_WIDTH-1); } else { end = xx->cursor_apos + (dir ? 1 : -1); start = end - (WIN_WIDTH-1); } fpos = xx->cursor_apos; c = cache_search(xx->io, GT_Contig, xx->cnum); cache_incr(xx->io, c); do { char *ind, *indt = NULL, *indb = NULL; calculate_consensus_simple(xx->io, xx->cnum, start, end, cons, NULL); cons[WIN_WIDTH] = 0; if (dir) { if (strand == '+' || strand == '=') indt = pstrstr_inexact(cons, uppert, mismatches, NULL); if (strand == '-' || strand == '=') indb = pstrstr_inexact(cons, upperb, mismatches, NULL); } else { if (strand == '+' || strand == '=') indt = prstrstr_inexact(cons, uppert, mismatches, NULL); if (strand == '-' || strand == '=') indb = prstrstr_inexact(cons, upperb, mismatches, NULL); } if (indt && indb) ind = MIN(indt, indb); else if (indt) ind = indt; else if (indb) ind = indb; else ind = NULL; if (ind != NULL) { if (dir) { if (fpos <= start + ind-cons) { found = 1; fpos = start + ind-cons; fseq = xx->cnum; } } else { if (fpos >= start + ind-cons) { found = 1; fpos = start + ind-cons; fseq = xx->cnum; } } break; } /* Next search region - overlapping by patlen+pads */ if (dir) { for (i = WIN_WIDTH-1, j = patlen; j && i; i--) { if (cons[i] != '*') j--; } if (i == 0) break; start += i; end += i; if (start > c->end) at_end = 1; } else { for (i = 0, j = patlen; j && i < WIN_WIDTH; i++) { if (cons[i] != '*') j--; } if (i == WIN_WIDTH) break; start -= WIN_WIDTH-i; end -= WIN_WIDTH-i; if (end < c->start) at_end = 1; } } while (!at_end); cache_decr(xx->io, c); if (found) { edSetCursorPos(xx, fseq == xx->cnum ? GT_Contig : GT_Seq, fseq, fpos, 1); } free(uppert); free(upperb); return found ? 0 : -1; }
int edview_search_tag_anno(edview *xx, int dir, int strand, char *value) { contig_iterator *iter; int start, end; rangec_t *r; rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci); char *r_exp = NULL; contig_t *c = cache_search(xx->io, GT_Contig, xx->cnum); if (value) { if (NULL == (r_exp = REGCMP(xx->interp, value))) { verror(ERR_WARN, "Search by anno", "invalid regular expression"); return -1; } } if (dir) { start = xx->cursor_apos + (dir ? 1 : -1); end = c->end; ifunc = contig_iter_next; } else { start = c->start; end = xx->cursor_apos + (dir ? 1 : -1); ifunc = contig_iter_prev; } iter = contig_iter_new_by_type(xx->io, xx->cnum, 1, dir == 1 ? CITER_FIRST : CITER_LAST, start, end, GRANGE_FLAG_ISANNO); if (!iter) return -1; while (r = ifunc(xx->io, iter)) { anno_ele_t *ae; if ((dir && r->start < start) || (!dir && r->start > end)) continue; if (!r_exp) break; /* blank expr => match all */ ae = cache_search(xx->io, GT_AnnoEle, r->rec); if (!ae->comment) continue; if (REGEX(xx->interp, ae->comment, r_exp)) break; } REGFREE(xx->interp, r_exp); if (r) { if (r->flags & GRANGE_FLAG_TAG_SEQ) { int pos; sequence_get_position(xx->io, r->pair_rec, NULL, &pos, NULL, NULL); pos = r->start - pos; edSetCursorPos(xx, GT_Seq, r->pair_rec, pos, 1); } else { edSetCursorPos(xx, GT_Contig, xx->cnum, r->start, 1); } contig_iter_del(iter); return 0; } contig_iter_del(iter); return -1; }
int edview_search_name(edview *xx, int dir, int strand, char *value) { tg_rec rec, *rp, cnum = -1, best_rec; int best_pos, best_off; int nr, i; rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci); int start, end, cstart; contig_iterator *iter; contig_t *c; /* Check for #num where num is a sequence record in this contig */ if (*value == '#') { char *endp; int64_t v = strtol64(value+1, &endp, 10); rec = v; if (*endp == '\0' && cache_exists(xx->io, GT_Seq, rec)) { sequence_get_clipped_position(xx->io, rec, &cnum, &start, NULL, &cstart, NULL, NULL); if (cnum == xx->cnum) { edSetCursorPos(xx, GT_Seq, rec, cstart - start, 1); return 0; } } } /* Find all hits matching this name */ rp = sequence_index_query_all(xx->io, value, 1, &nr); /* Also get an position-based iterator */ c = cache_search(xx->io, GT_Contig, xx->cnum); if (dir) { start = xx->cursor_apos + 1; end = c->end; ifunc = contig_iter_next; best_pos = end + 1; best_off = 0; } else { start = c->start; end = xx->cursor_apos - 1; ifunc = contig_iter_prev; best_pos = start - 1; best_off = 0; } iter = contig_iter_new_by_type(xx->io, xx->cnum, 1, dir == 1 ? CITER_FIRST : CITER_LAST, start-1, end+1, GRANGE_FLAG_ISSEQ); if (!iter) return -1; /* * The iterator also finds overlapping objects, not just ones beyond this * point. That's fine if we're on the consensus as we probably want to * jump to the first seq-name overlapping this point. * * However if we're on a sequence already, we want the first one * after or before that sequence. So we skip along iterator until we're * at the current record. */ if (xx->cursor_type == GT_Seq) { rangec_t *r; while ((r = ifunc(xx->io, iter))) { if (r->rec == xx->cursor_rec) break; } } /* Alternate between the by-name and by-position scan */ best_rec = -1; for (i = 0; i < nr; i++) { int start, end; rangec_t *r; /* From name index */ rec = rp[i++]; sequence_get_clipped_position(xx->io, rec, &cnum, &start, &end, &cstart, NULL, NULL); if (cnum == xx->cnum) { if ((dir && best_pos > cstart && cstart > xx->cursor_apos) || (!dir && best_pos < cstart && cstart < xx->cursor_apos)) { best_pos = cstart; best_off = cstart - start; best_rec = rec; } } /* From iterator */ if ((r = ifunc(xx->io, iter))) { seq_t *s; if (NULL == (s = cache_search(xx->io, GT_Seq, r->rec))) { /* No match */ best_rec = -1; break; } if (strncmp(s->name, value, strlen(value)) == 0) { /* prefix match */ puts("Found by pos iterator"); best_rec = r->rec; break; } } else { /* End of contig - bail out early */ best_rec = -1; break; } } contig_iter_del(iter); if (rp) free(rp); if (best_rec != -1) { edSetCursorPos(xx, GT_Seq, best_rec, best_off, 1); return 0; } return -1; }
int edview_search_cons_discrep(edview *xx, int dir, int strand, char *value) { int start, end; int found = 0, at_end = 0; int fpos, i; double qval = atof(value); consensus_t cons[WIN_WIDTH+1]; contig_t *c; /* Set initial start positions */ if (dir) { start = xx->cursor_apos + (dir ? 1 : -1); end = start + (WIN_WIDTH-1); } else { end = xx->cursor_apos + (dir ? 1 : -1); start = end - (WIN_WIDTH-1); } fpos = xx->cursor_apos; /* Loop WIN_WIDTH block at a time */ c = cache_search(xx->io, GT_Contig, xx->cnum); cache_incr(xx->io, c); do { calculate_consensus(xx->io, xx->cnum, start, end, cons); if (dir) { for (i = 0; i < WIN_WIDTH; i++) { if (cons[i].discrep >= qval) { found = 1; break; } } } else { for (i = WIN_WIDTH-1; i; i--) { if (cons[i].discrep >= qval) { found = 1; break; } } } if (found) { fpos = start + i; break; } /* Next search region - overlapping by patlen+pads */ if (dir) { start += WIN_WIDTH; end += WIN_WIDTH; if (start > c->end) at_end = 1; } else { start -= WIN_WIDTH; end -= WIN_WIDTH; if (end < c->start) at_end = 1; } } while (!at_end); cache_decr(xx->io, c); if (found) { edSetCursorPos(xx, GT_Contig, xx->cnum, fpos, 1); return 0; } return -1; }