int zigdev_test(void)
{
char sql[200];
sql_format(sql, 200, "insert into user values('test user', '123456', '*****@*****.**', '15895830938');");
if (cache_opt_item(TBL_USER, sql) < 0)
printf("\ncache_opt_item\n");
sql_format(sql, 200, "insert into user values('TihsYloH', '123456', '*****@*****.**', '15895830938');");
if (cache_opt_item(TBL_USER, sql) < 0)
printf("\ncache_opt_item\n");
sql_format(sql, 200, "select * from user where user_name = 'TihsYloH';");
if (cache_search(TBL_USER, sql, NULL, NULL, 0) == 0) {
printf("\nTihsYloH user find \n");
} else {
printf("\nTihsYloH user not find\n");
}
sql_format(sql, 200, "select * from user where user_name = 'TihsYloH__';");
if (cache_search(TBL_USER, sql, NULL, NULL, 0) == 0) {
printf("\nTihsYloH___ user find \n");
} else {
printf("\n\nTihsYloH___ user not find\n");
}
return 0;
}
/*
* Given ranges contained within a bin this makes sure that all sequences
* referred to in these ranges have their parent listed as the new bin.
*
* Returns 0 on success
* -1 on failure
*/
static int break_contig_reparent_seqs(GapIO *io, bin_index_t *bin) {
int i, nr = bin->rng ? ArrayMax(bin->rng) : 0;
for (i = 0; i < nr; i++) {
range_t *r = arrp(range_t, bin->rng, i);
if (r->flags & GRANGE_FLAG_UNUSED)
continue;
if ((r->flags & GRANGE_FLAG_ISMASK) == GRANGE_FLAG_ISANNO) {
anno_ele_t *a = (anno_ele_t *)cache_search(io, GT_AnnoEle, r->rec);
if (a->bin != bin->rec) {
a = cache_rw(io, a);
a->bin = bin->rec;
}
} else {
seq_t *seq = (seq_t *)cache_search(io, GT_Seq, r->rec);
if (seq->bin != bin->rec) {
seq = cache_rw(io, seq);
seq->bin = bin->rec;
seq->bin_index = i;
}
}
}
return 0;
}
/*
* Adds a contig to a scaffold array.
* Gap size, type and evidence refer to the gap between this and the
* "previous" contig - ie the last in the scaffold. More complex
* scaffold manipulations will be handled elsewhere.
*
* Set these fields to 0 if you do not know them.
*
* Returns 0 on success
* -1 on failure
*/
int scaffold_add(GapIO *io, tg_rec scaffold, tg_rec contig,
int gap_size, int gap_type, int evidence) {
scaffold_t *f;
contig_t *c;
scaffold_member_t *m;
int i;
/* Check if this contig is in a scaffold, if so remove now */
c = cache_search(io, GT_Contig, contig);
if (c->scaffold)
scaffold_remove(io, c->scaffold, contig);
if (!(f = cache_search(io, GT_Scaffold, scaffold)))
return -1;
/* Check if it already exists */
for (i = 0; i < ArrayMax(f->contig); i++) {
m = arrp(scaffold_member_t, f->contig, i);
if (m->rec == contig)
return 0;
}
/* Append */
f = cache_rw(io, f);
m = ArrayRef(f->contig, ArrayMax(f->contig)); // extend
m->rec = contig;
m->gap_size = ArrayMax(f->contig) > 1 ? gap_size : 0;
m->gap_type = gap_type;
m->evidence = evidence;
/* Update the contig record too */
c = cache_search(io, GT_Contig, contig);
c = cache_rw(io, c);
c->scaffold = scaffold;
#if 0
/* Add a scaffold link to the contig graph too */
if (ArrayMax(f->contig) >= 2) {
m = arrp(scaffold_member_t, f->contig, ArrayMax(f->contig)-2);
contig_link_t lnk;
lnk.rec1 = contig;
lnk.rec2 = m->rec;
/* Best guess */
lnk.pos1 = 0; lnk.end1 = 1;
lnk.pos2 = 0; lnk.end2 = 0;
lnk.orientation = 0;
lnk.size = 100;
lnk.type = CLINK_TYPE_SCAFFOLD;
lnk.score = 0;
contig_add_link(io, &lnk);
}
#endif
return 0;
}
/*
* Exports Scaffold information to an AGP file
*
* Returns 0 on success
* -1 on failure
*/
int scaffold_to_agp(GapIO *io, char *fn) {
FILE *fp;
int i, j;
if (NULL == (fp = fopen(fn, "w+"))) {
verror(ERR_WARN, "scaffold_from_agp", "%s: %s", fn, strerror(errno));
return -1;
}
for (i = 0; io->scaffold && i < ArrayMax(io->scaffold); i++) {
scaffold_t *f = cache_search(io, GT_Scaffold,
arr(tg_rec, io->scaffold, i));
int start = 1, end = 1;
int k = 1;
if (!f) {
verror(ERR_WARN, "scaffold_from_agp", "Failed to load scaffold\n");
fclose(fp);
return -1;
}
cache_incr(io, f);
for (j = 0; f->contig && j < ArrayMax(f->contig); j++) {
scaffold_member_t *m = arrp(scaffold_member_t, f->contig, j);
contig_t *c = cache_search(io, GT_Contig, m->rec);
int ustart, uend;
int len;
/* Get the unpadded clipped contig length */
consensus_valid_range(io, m->rec, &ustart, &uend);
consensus_unpadded_pos(io, m->rec, uend, &uend);
len = uend - ustart + 1;
if (j) {
int gap = m->gap_size;
fprintf(fp, "%s\t%d\t%d\t%d\tN\t%d\tfragment\tyes\n",
f->name, start, start+gap-1, k++, gap);
start += gap;
}
fprintf(fp, "%s\t%d\t%d\t%d\tW\t%s\t%d\t%d\t+\n",
f->name, start, start + len-1,
k++, c->name, ustart, uend);
start += len;
}
cache_decr(io, f);
}
if (0 != fclose(fp)) {
verror(ERR_WARN, "scaffold_from_agp", "%s: %s", fn, strerror(errno));
return -1;
}
return 0;
}
static void tag_shift_for_delete(GapIO *io, tg_rec crec, tg_rec srec,
int start, int end, int pos, tg_rec brec) {
contig_iterator *ci;
rangec_t *r;
contig_t *c = cache_search(io, GT_Contig, crec);;
//printf("< tag in seq %"PRIrec" at %d\n", srec, pos);
cache_incr(io, c);
ci = contig_iter_new_by_type(io, crec, 0, CITER_FIRST | CITER_ISTART,
start+pos, end, GRANGE_FLAG_ISANNO);
if (!ci) {
cache_decr(io, c);
return;
}
while ((r = contig_iter_next(io, ci))) {
range_t r2, *r_out;
anno_ele_t *a;
bin_index_t *bin;
if (r->pair_rec != srec)
continue;
bin_remove_item(io, &c, GT_AnnoEle, r->rec);
r2.start = (r->start > start+pos) ? r->start-1 : r->start;
r2.end = r->end-1;
r2.mqual = r->mqual;
r2.rec = r->rec;
r2.pair_rec = r->pair_rec;
r2.flags = r->flags;
if (r2.end < r2.start) {
/* Tag entirely removed now, it must have been on a pad */
a = cache_search(io, GT_AnnoEle, r->rec);
a = cache_rw(io, a);
cache_deallocate(io, a);
continue;
}
bin = bin_add_to_range(io, &c, brec, &r2, &r_out, NULL, 0);
a = cache_search(io, GT_AnnoEle, r->rec);
if (a->bin != bin->rec /*||
a->idx != r_out - ArrayBase(range_t, bin->rng)*/) {
/* Annotation moved bins */
a = cache_rw(io, a);
a->bin = bin->rec;
//a->bin_idx = r_out - ArrayBase(range_t, bin->rng);
}
}
cache_decr(io, c);
contig_iter_del(ci);
}
/*
* Removes some or all tags from some or all contigs.
* If the contig list or tag list is blank it implies all contigs or all tags.
*
* Returns 0 on success
* -1 on failure
*/
int delete_tags(GapIO *io, int ncontigs, contig_list_t *contigs,
char *tag_list, int verbose) {
HashTable *h = NULL;
int ret = 0;
/* Hash tag types */
if (tag_list && *tag_list) {
int i;
if (SetActiveTags(tag_list) == -1) {
return -1;
}
h = HashTableCreate(32, 0);
for (i = 0; i < number_of_active_tags; i++) {
HashData hd;
hd.i = 0;
HashTableAdd(h, active_tag_types[i], 4, hd, NULL);
}
}
/* Iterate over contig list or all contigs */
if (verbose)
vfuncheader("Delete Tags");
if (ncontigs) {
int i;
for (i = 0; i < ncontigs; i++) {
contig_t *c = cache_search(io, GT_Contig, contigs[i].contig);
vmessage("Scanning contig %d of %d (%s)\n",
i+1, ncontigs, c->name);
ret |= delete_tag_single_contig(io, contigs[i].contig, h, verbose);
UpdateTextOutput();
cache_flush(io);
}
} else {
int i;
tg_rec *order = ArrayBase(tg_rec, io->contig_order);
for (i = 0; i < NumContigs(io); i++) {
contig_t *c = cache_search(io, GT_Contig, order[i]);
vmessage("Scanning contig %d of %d (%s)\n",
i+1, NumContigs(io), c->name);
ret |= delete_tag_single_contig(io, order[i], h, verbose);
UpdateTextOutput();
cache_flush(io);
}
}
SetActiveTags("");
if (h)
HashTableDestroy(h, 0);
return ret;
}
/*
* Creates an anno_ele as per anno_ele_new, but also adds it to an object
* and creates the bin Range entry too.
*/
tg_rec anno_ele_add(GapIO *io, int obj_type, tg_rec obj_rec, tg_rec anno_rec,
int type, char *comment, int start, int end, char dir) {
range_t r;
anno_ele_t *e;
contig_t *c;
tg_rec crec;
bin_index_t *bin;
tg_rec seq_bin = 0;
/* Find contig for obj_rec/obj_type */
if (obj_type == GT_Contig) {
crec = obj_rec;
} else {
int st, en;
sequence_get_position2(io, obj_rec, &crec, &st, &en, NULL,
&seq_bin, NULL, NULL);
start += st;
end += st;
}
c = (contig_t *)cache_search(io, GT_Contig, crec);
cache_incr(io, c);
r.start = start;
r.end = end;
r.flags = GRANGE_FLAG_ISANNO;
r.mqual = type;
r.pair_rec = obj_rec;
if (GT_Seq == obj_type)
r.flags |= GRANGE_FLAG_TAG_SEQ;
r.rec = anno_ele_new(io, 0, obj_type, obj_rec, 0, type, dir, comment);
e = (anno_ele_t *)cache_search(io, GT_AnnoEle, r.rec);
e = cache_rw(io, e);
if (seq_bin)
bin = bin_add_to_range(io, &c, seq_bin, &r, NULL, NULL, 0);
else
bin = bin_add_range(io, &c, &r, NULL, NULL, 0);
if (!bin)
verror(ERR_FATAL, "anno_ele_add", "bin_add_to_range returned NULL");
e->bin = bin ? bin->rec : 0;
cache_decr(io, c);
return r.rec;
}
/*
* Tidies up after break contig or disassemble readings, looking for now
* redundant bins.
*
* This has the following functions (not all implemented yet!)
* 1) If a contig is totally empty, remove the contig.
* 2) If a bin is empty and all below it, remove the bin.
* 3) If a bin is empty and all above it, remove parent bins and link
* contig to new root. (TODO)
*/
static void remove_empty_bins(GapIO *io, tg_rec contig) {
contig_t *c = cache_search(io, GT_Contig, contig);
tg_rec first = 0;
cache_incr(io, c);
if (c->bin) {
if (remove_empty_bins_r(io, c->bin, &first)) {
cache_decr(io, c);
contig_destroy(io, contig);
return;
}
if (first != c->bin) {
bin_index_t *bin;
tg_rec bp, br, cdummy;
int offset;
/* Cut out the offending waste */
bin = cache_search(io, GT_Bin, first);
bin = cache_rw(io, bin);
bp = bin->parent;
// Find new bin offset
bin_get_position(io, bin, &cdummy, &offset);
assert(cdummy == contig);
bin->pos = offset;
bin->parent = contig;
bin->parent_type = GT_Contig;
bin->flags |= BIN_BIN_UPDATED;
c = cache_rw(io, c);
br = c->bin;
c->bin = first;
bin = cache_search(io, GT_Bin, bp);
bin = cache_rw(io, bin);
if (bin->child[0] == first) bin->child[0] = 0;
if (bin->child[1] == first) bin->child[1] = 0;
/* Recursively remove the bin tree from old root, br */
bin_destroy_recurse(io, br);
}
}
cache_decr(io, c);
}
/*
* Creates a new named scaffold.
*
* Returns scaffold pointer on success.
* NULL on failure
*/
scaffold_t *scaffold_new(GapIO *io, char *name) {
tg_rec rec;
scaffold_t *f, init_f;
if (!io->db->scaffold)
return NULL;
memset(&init_f, 0, sizeof(scaffold_t));
init_f.name = name;
/* Allocate our contig */
rec = cache_item_create(io, GT_Scaffold, &init_f);
/* Initialise it */
f = (scaffold_t *)cache_search(io, GT_Scaffold, rec);
f = cache_rw(io, f);
if (name)
scaffold_set_name(io, &f, name);
else
f->name = NULL;
/* Add it to the scaffold order too */
io->scaffold = cache_rw(io, io->scaffold);
io->db = cache_rw(io, io->db);
ARR(tg_rec, io->scaffold, io->db->Nscaffolds++) = rec;
/* Add to the new contigs list */
if (name)
add_to_list("new_scaffolds", name);
return f;
}
void l2_cache_fill(int cpu_num, unsigned long long int addr, int set, int way, int prefetch, unsigned long long int evicted_addr)
{
// uncomment this line to see the information available to you when there is a cache fill event
//printf("0x%llx %d %d %d 0x%llx\n", addr, set, way, prefetch, evicted_addr);
cache_insert(addr, prefetch);
if (prefetch==1) {
prefetch_tot_num++;
// Insert Evicted address into pollution filter
// only if it is not prefetch
int found = cache_search(evicted_addr);
if (found == -1) {
if (evicted_addr != 0) {
printf("ERROR: evicted address not in the cache - %llx \n", evicted_addr);
exit(1);
}
}
else
if (cache[found].pf != 1)
poll_insert(evicted_addr);
poll_remove(addr);
}
cache_remove(evicted_addr);
}
// Must be locked over the queue - can be unlocked immediate following the call (safe to use returned request)
// Must be synchronized over the "queue get" semaphore
// The "queue put" semaphore should be incremened following the call
// The returned request is freed from the list and must be destroyed before it goes out of context
struct request_bundle queue_getSmallRequest() {
fprintf(stderr, "queue_getSmallRequest: Called\n");
struct request* req = q_first();
if (req == NULL) {
fprintf(stderr, "queue_getSmallRequest: Error! Queue held no requests!\n");
struct request_bundle bundle = { NULL, NULL };
return bundle;
}
//assert (queue_size != 0);
struct request* small = req;
intmax_t smallsize = getFileSize(req->filename);
while ( (req = q_nextOf(req)) != NULL ) {
intmax_t thissize = getFileSize(req->filename);
if (thissize < smallsize) {
small = req;
smallsize = thissize;
}
}
struct cache_entry* ent = cache_search(small->filename);
struct request_bundle bundle = { small, ent };
return bundle;
}
/*
* Compute the visible end position of a contig. This isn't just the extents
* of start_used / end_used in the bins as this can included invisible
* data such as cached consensus sequences.
*/
int contig_visible_end(GapIO *io, tg_rec crec) {
rangec_t *r;
contig_iterator *ci;
ci = contig_iter_new_by_type(io, crec, 1, CITER_LAST | CITER_IEND,
CITER_CSTART, CITER_CEND,
GRANGE_FLAG_ISANY);
if (!ci) {
contig_t *c = cache_search(io, GT_Contig, crec);
return c->end;
}
while (r = contig_iter_prev(io, ci)) {
int v;
if ((r->flags & GRANGE_FLAG_ISMASK) == GRANGE_FLAG_ISCONS)
continue;
v = r->end;
contig_iter_del(ci);
return v;
}
contig_iter_del(ci);
return 0;
}
/* destroy a cache */
int cache_destroy(const char *file) {
Cache *tmp;
tmp = cache_search(file);
if (tmp == NULL)
return(0);
cache_free(tmp);
return(1);
}
/*
* Returns the relative orientation of this annotation vs the contig.
*/
int anno_get_orient(GapIO *io, tg_rec anum) {
bin_index_t *bin = NULL;
tg_rec bnum;
anno_ele_t *a = cache_search(io, GT_AnnoEle, anum);
int comp = a->orient;
/* Bubble up bins until we hit the root */
for (bnum = a->bin; bnum; bnum = bin->parent) {
bin = (bin_index_t *)cache_search(io, GT_Bin, bnum);
if (bin->flags & BIN_COMPLEMENTED)
comp ^= 1;
if (bin->parent_type != GT_Bin)
break;
}
assert(bin && bin->parent_type == GT_Contig);
return comp;
}
/*
* Reads a given GContig number to the GContig pointer.
* Here cnum is a contig number from 0 to N-1 indicating the record number
* in the cnum-th element of the contig_order array.
*
* Returns 0 on success.
* -1 on failure.
*/
int gio_read_contig(GapIO *io, int cnum, contig_t **c) {
tg_rec crec;
if (!io->contig_order)
return -1;
crec = arr(tg_rec, io->contig_order, cnum);
*c = (contig_t *)cache_search(io, GT_Contig, crec);
return 0;
}
/*
* Sets the annotation type, passed in as a string but held in a 4-byte int.
* This also attempts to set the cached copy of the type held within the
* bin range array.
*
* Returns 0 on success
* -1 on failure
*/
int anno_ele_set_type(GapIO *io, anno_ele_t **e, char *str) {
int type;
char stype[5];
anno_ele_t *ae;
if (!(ae = cache_rw(io, *e)))
return -1;
/* Get integer type */
memset(stype, 0, 5);
strncpy(stype, str, 4);
type = str2type(stype);
/* Update annotation */
ae->tag_type = type;
/* Also update range_t cached copy of type */
if (ae->bin) {
bin_index_t *bin = (bin_index_t *)cache_search(io, GT_Bin, ae->bin);
range_t *r = NULL;
int i, nranges;
if (!bin)
return -1;
if (!(bin = cache_rw(io, bin)))
return -1;
/*
* Find the index into the bin range.
* FIXME: we should add a bin_index element, as seen in seq_t,
* to avoid the brute force loop. This doesn't have to be
* permanently stored - a cached copy would suffice.
*/
nranges = bin->rng ? ArrayMax(bin->rng) : 0;
for (i = 0; i < nranges; i++) {
r = arrp(range_t, bin->rng, i);
if (r->flags & GRANGE_FLAG_UNUSED)
continue;
if (r->rec == ae->rec)
break;
}
if (i == nranges)
return -1;
bin->flags |= BIN_RANGE_UPDATED;
r->mqual = type;
}
*e = ae;
return 0;
}
/*
* Removes a contig from a scaffold.
*
* Returns 0 on success
* -1 on failure
*/
int scaffold_remove(GapIO *io, tg_rec scaffold, tg_rec contig) {
scaffold_t *f;
scaffold_member_t *m, *m2;
contig_t *c;
int i;
c = cache_search(io, GT_Contig, contig);
f = cache_search(io, GT_Scaffold, scaffold);
if (!c || !f)
return -1;
if (c->scaffold != scaffold) {
verror(ERR_WARN, "scaffold_remove", "Attempted to remove contig #%"
PRIrec" from a scaffold #%"PRIrec" it is not a member of",
contig, scaffold);
return -1;
}
c = cache_rw(io, c);
c->scaffold = 0;
f = cache_rw(io, f);
for (i = 0; i < ArrayMax(f->contig); i++) {
m = arrp(scaffold_member_t, f->contig, i);
if (m->rec == contig) {
/* Shuffle array down */
for (i++; i < ArrayMax(f->contig); i++) {
m2 = arrp(scaffold_member_t, f->contig, i);
*m = *m2;
m = m2;
}
ArrayMax(f->contig)--;
}
}
return 0;
}
int edview_search_tag_type(edview *xx, int dir, int strand, char *value) {
contig_iterator *iter;
int start, end;
rangec_t *r;
rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci);
int type = str2type(value);
contig_t *c = cache_search(xx->io, GT_Contig, xx->cnum);
if (dir) {
start = xx->cursor_apos + (dir ? 1 : -1);
end = c->end;
ifunc = contig_iter_next;
} else {
start = c->start;
end = xx->cursor_apos + (dir ? 1 : -1);
ifunc = contig_iter_prev;
}
iter = contig_iter_new_by_type(xx->io, xx->cnum, 1,
dir == 1 ? CITER_FIRST : CITER_LAST,
start, end, GRANGE_FLAG_ISANNO);
if (!iter)
/* Can happen legitimately when we're already at the end of contig */
return -1;
while (r = ifunc(xx->io, iter)) {
if ((dir && r->start < start) ||
(!dir && r->start > end))
continue;
if (r->mqual == type)
break;
}
if (r) {
if (r->flags & GRANGE_FLAG_TAG_SEQ) {
int pos;
sequence_get_position(xx->io, r->pair_rec, NULL, &pos, NULL, NULL);
pos = r->start - pos;
edSetCursorPos(xx, GT_Seq, r->pair_rec, pos, 1);
} else {
edSetCursorPos(xx, GT_Contig, xx->cnum, r->start, 1);
}
contig_iter_del(iter);
return 0;
}
contig_iter_del(iter);
return -1;
}
// Must be locked over the queue - can be unlocked immediately following the call
// Must be synchronized over the "queue get" semaphore
// The "queue put" semaphore should be incremented following the call
// The returned request is freed from the list and must be destroyed before it goes out of context
struct request_bundle queue_getRequest() {
fprintf(stderr, "queue_getRequest: Called\n");
struct request* req = q_shift();
struct cache_entry* ent;
if (req == NULL)
ent = NULL;
else
ent = cache_search(req->filename);
struct request_bundle bundle = { req, ent };
return bundle;
}
int edview_search_tag_indel(edview *xx, int dir, int strand, char *value) {
contig_iterator *iter;
int start, end;
rangec_t *r;
rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci);
contig_t *c = cache_search(xx->io, GT_Contig, xx->cnum);
if (dir) {
start = xx->cursor_apos + (dir ? 1 : -1);
end = c->end;
ifunc = contig_iter_next;
} else {
start = c->start;
end = xx->cursor_apos + (dir ? 1 : -1);
ifunc = contig_iter_prev;
}
iter = contig_iter_new_by_type(xx->io, xx->cnum, 1,
dir == 1 ? CITER_FIRST : CITER_LAST,
start, end, GRANGE_FLAG_ISREFPOS);
if (!iter)
return -1;
while (r = ifunc(xx->io, iter)) {
if ((dir && r->start < start) ||
(!dir && r->start > end))
continue;
break;
}
if (r) {
edSetCursorPos(xx, GT_Contig, xx->cnum, r->start, 1);
contig_iter_del(iter);
return 0;
}
contig_iter_del(iter);
return -1;
}
/*
* Removes all tags of specific types (hashed in h, or all if h == NULL)
* from a specified contig.
*
* Returns 0 on success
* -1 on failure
*/
static int delete_tag_single_contig(GapIO *io, tg_rec crec,
HashTable *h, int verbose) {
contig_iterator *ci;
rangec_t *r;
contig_t *c;
int ret = -1;
ci = contig_iter_new_by_type(io, crec, 1, CITER_FIRST,
CITER_CSTART, CITER_CEND,
GRANGE_FLAG_ISANNO);
if (!ci)
return -1;
if (!(c = cache_search(io, GT_Contig, crec))) {
contig_iter_del(ci);
return -1;
}
cache_incr(io, c);
while (NULL != (r = contig_iter_next(io, ci))) {
char t[5];
(void)type2str(r->mqual, t);
if (!h || HashTableSearch(h, t, 4)) {
anno_ele_t *e;
if (verbose)
vmessage("Removing anno %s #%"PRIrec"\tContig %s\t%d..%d\n",
t, r->rec, c->name, r->start, r->end);
if (bin_remove_item(io, &c, GT_AnnoEle, r->rec)) goto fail;
/* FIXME: Need to reclaim the GT_AnnoEle record itself */
}
}
ret = 0;
fail:
contig_iter_del(ci);
cache_decr(io, c);
return ret;
}
/*
* Removes an anno_ele from the gap database.
* FIXME: need to deallocate storage too. (See docs/TODO)
*
* Returns 0 on success
* -1 on failure
*/
int anno_ele_destroy(GapIO *io, anno_ele_t *e) {
bin_index_t *bin;
range_t *r;
int i;
/* Find the bin range pointing to this object */
bin = (bin_index_t *)cache_search(io, GT_Bin, e->bin);
if (!bin || !bin->rng || ArrayMax(bin->rng) == 0)
return -1;
if (!(bin = cache_rw(io, bin)))
return -1;
for (i = 0; i < ArrayMax(bin->rng); i++) {
r = arrp(range_t, bin->rng, i);
if (r->flags & GRANGE_FLAG_UNUSED)
continue;
if (r->rec == e->rec)
break;
}
if (i == ArrayMax(bin->rng))
return -1;
/* Mark this bin range as unused */
r->rec = bin->rng_free;
r->flags |= GRANGE_FLAG_UNUSED;
bin->rng_free = i;
bin->flags |= BIN_RANGE_UPDATED | BIN_BIN_UPDATED;
bin_incr_nanno(io, bin, -1);
if (bin->start_used == r->start || bin->end_used == r->end)
bin_set_used_range(io, bin);
return 0;
}
void l2_prefetcher_operate(int cpu_num, unsigned long long int addr, unsigned long long int ip, int cache_hit)
{
l2_prefetcher_operate_ampm(cpu_num, addr, ip, cache_hit);
acc_tot++;
if (!cache_hit) {
miss_tot++;
if ( poll_search(addr) != -1 )
poll_tot++;
}
int found = cache_search(addr);
if (found != -1)
if (cache[found].pf == 1) {
prefetch_use_num++;
cache[found].pf = 0;
}
//printf("acc:%d miss:%d pre:%d use_pre:%d poll:%d ___ page miss:%d hit:%d imp:%d\n", acc_tot, miss_tot ,prefetch_tot_num, prefetch_use_num, poll_tot, ampm_page_miss, ampm_page_hit, ampm_imp);
}
// Returns request_bundle containing the first request/cache_entry pair if such a pair exists
// Otherwise, returns a request_bundle containing only the first request in the queue
// request_bundle is returned by value
// Must be locked over the request queue - can be unlocked immediately following the call
// The returned request is removed from the queue
// Must be locked over the cache INCLUDING ALL USAGE OF THE RETURNED CACHE ENTRY
// The cache entry REMAINS IN THE CACHE
// The returned request is freed from the queue and must be destroyed upon falling out of context
// The returned cache entry should NOT be destroyed upon falling out of the caller's context
struct request_bundle getCachedRequest() {
fprintf(stderr, "getCachedRequest: Called\n");
struct request* req = q_first();
struct cache_entry* ent = NULL;
while (req != NULL) {
ent = cache_search(req->filename);
if (ent != NULL) {
q_remove(req);
break;
}
req = q_nextOf(req);
}
if (req == NULL)
req = q_shift();
struct request_bundle bundle = { req, ent };
return bundle;
}
//services the request from the queue
void * worker(void * arg)
{
// assign a thread id
pthread_mutex_lock(&threadnum_access);
int thread_id = thread_num; // thread id (between 0 and n-1)
thread_num++;
pthread_mutex_unlock(&threadnum_access);
unsigned char *buf = NULL;//data for file
struct stat file_info;//struct where size is
struct timespec t1, t2;//structs for time
char time[100];//string for response time
long time_diff;
bool time_error = false;
off_t file_size;//size of file
int reqFd;//for the read file
char log_entry[MAX_TEXT_SIZE];
char* filename;
char full_path[MAX_TEXT_SIZE];
char* file_ending = malloc(sizeof(char) * MAX_REQUEST_LENGTH);
char* content_type = malloc(sizeof(char) * MAX_REQUEST_LENGTH);
char error_string[251];
bool cache_hit = false;//true for hit false for miss
int cache_index = -1;
int fd;
FILE *read_file;
int request_count = 0; // how many requests this worker thread has completed
int error_flag = 0;
// continue this loop while either dispatchers left or requests in queue
while (cur_num_dispatch > 0 || count > 0) {
pthread_mutex_lock(&queue_access);
// wait only while dispatchers left and no requests in queue
while (cur_num_dispatch > 0 && count == 0)
{
pthread_cond_wait(&queue_full_or_dispatch_exit, &queue_access);
}
count--;
//get time 1 in nano seconds
if(clock_gettime(CLOCK_MONOTONIC, &t1) != 0)
{
perror("can't get time");
strcpy(time,"ERROR");
time_error = true;
}
filename = queue[r_queue_index].m_szRequest;
fd = queue[r_queue_index].m_socket;
if (get_type(filename, content_type) == -1)
{
printf("Failed to get content type.\n");
strcpy(content_type, "Error");
strcpy(error_string, "Could not get content type");
}
if (strcmp(content_type, "Error") == 0)
error_flag = 1;
else
error_flag = 0;
if (use_cache)
cache_index = cache_search(filename, &file_size);
if(cache_index >= 0)
{
if (return_result(fd, content_type, cache[cache_index].data, (int)cache[cache_index].file_size) != 0)
{
fprintf(stderr, "Failed to return result.\n");
}
else
{
//get time 2 in nano seconds
if(clock_gettime(CLOCK_MONOTONIC, &t2) != 0)
{
perror("can't get time");
strcpy(time,"ERROR");
time_error = true;
}
else
{
time_diff = t2.tv_nsec = t1.tv_nsec;
time_diff = time_diff / NANO_TO_MILLI;
sprintf(time, "%ldms", time_diff);
}
cache_hit = true;
}
}
else//read from file
{
sprintf(full_path, "%s%s", path, filename);
// open file, copy to buffer
if ((read_file = fopen(full_path, "r")) == NULL)
{
perror("Failed to open file");
strcpy(content_type, "Error");
strcpy(error_string,"could not open file");
error_flag = 1;
}
else//good read
{
if(stat(full_path,&file_info) < 0)
{
perror("Failed to get file size");
strcpy(content_type, "Error");
strcpy(error_string, "could not get file size");
error_flag = 1;
}
else//good stat
{
file_size = file_info.st_size;
if(buf == NULL)
{
buf = (unsigned char*)malloc((size_t)file_size);
if(buf == NULL)
{
perror("could not allocate memory");
strcpy(content_type, "Error");
strcpy(error_string,"could not allocate memory");
error_flag = 1;
}
}
else//realloc buf
{
buf = (unsigned char*)realloc(buf,(size_t)file_size);
if(buf == NULL)
{
perror("could not allocate memory");
strcpy(content_type, "Error");
strcpy(error_string,"could not allocate memory");
error_flag = 1;
}
}//end realloc
}//end good stat
}//end good read
if(!error_flag)
{
if (fread(buf, 1, (size_t)file_size, read_file) <= 0)//sizeof(char)
{
printf("failed to read from file.");
strcpy(content_type, "Error");
strcpy(error_string,"could not read file");
error_flag = 1;
}
else
{
if (return_result(fd, content_type, buf, (int)file_size) != 0)
{
fprintf(stderr, "Failed to return result.\n");
}
else
{
if(clock_gettime(CLOCK_MONOTONIC, &t2) != 0)
{
perror("can't get time");
strcpy(time,"ERROR");
time_error = true;
}
else
{
time_diff = t2.tv_nsec = t1.tv_nsec;
time_diff = time_diff / NANO_TO_MILLI;
sprintf(time,"%ldms",time_diff);
}
}
if(use_cache && cache_insert(buf, filename, file_size) < 0)
{
fprintf(stderr, "Failed to add file to cache\n");
}
}
}
}//end read file
if(error_flag)
{
if (return_error(fd, error_string) != 0)
{
fprintf(stderr, "Failed to return error.\n");
}
}
r_queue_index = (r_queue_index + 1) % queue_length;
pthread_mutex_unlock(&queue_access);
pthread_cond_broadcast(&queue_empty);
// log to file
pthread_mutex_lock(&log_access);
request_count++;
if(cache_hit == true)
{
sprintf(log_entry,"[%d][%d][%d][%s][%lu][%s][%s]\n",thread_id, request_count, fd, filename,(unsigned long)file_size,time,"HIT");
cache_hit = false;
}
else
{
if(!error_flag)
{
if (use_cache)
sprintf(log_entry,"[%d][%d][%d][%s][%lu][%s][%s]\n",thread_id, request_count, fd, filename,(unsigned long)file_size,time,"MISS");
else
sprintf(log_entry,"[%d][%d][%d][%s][%lu]\n",thread_id, request_count, fd, filename,(unsigned long)file_size);
}
else
{
sprintf(log_entry,"[%d][%d][%d][%s][%s]\n",thread_id, request_count, fd, filename,error_string);
}
}
if(write(logFd,log_entry, strlen(log_entry)) < 0)
perror("failed to write to log");
pthread_mutex_unlock(&log_access);
}
// no more dispatcher threads or requests, work has been completed
free(filename);
free(buf);
free(file_ending);
free(content_type);
free(read_file);
printf("Closing the log\n");
pthread_exit(NULL);
return NULL;
}
/*
* Attempt to find edits. It's not 100% reliable, but works for most cases.
* We look for lowercase bases and confidence 100 and 0 (if not N).
* We cannot find deleted bases though.
*/
int edview_search_edit(edview *xx, int dir, int strand, char *value) {
int start, end;
contig_iterator *iter;
rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci);
rangec_t *r;
int best_pos, found = 0;
int fpos;
tg_rec fseq;
if (dir) {
start = xx->cursor_apos + 1;
end = CITER_CEND;
iter = contig_iter_new(xx->io, xx->cnum, 1,
CITER_FIRST | CITER_ISTART,
start, end);
ifunc = contig_iter_next;
best_pos = INT_MAX;
} else {
start = CITER_CSTART;
end = xx->cursor_apos -1;
iter = contig_iter_new(xx->io, xx->cnum, 1,
CITER_LAST | CITER_IEND,
start, end);
ifunc = contig_iter_prev;
best_pos = INT_MIN;
}
if (!iter)
return -1;
while ((r = ifunc(xx->io, iter))) {
seq_t *s, *sorig;
char *seq, *qual;
int seq_len, comp, off = 0, i;
if (found && dir && r->start > best_pos)
break;
if (found && !dir && r->end < best_pos)
break;
if (NULL == (s = sorig = cache_search(xx->io, GT_Seq, r->rec)))
break;
if (r->comp ^ (s->len < 0)) {
s = dup_seq(s);
complement_seq_t(s);
}
seq = s->seq;
qual = s->conf;
seq_len = ABS(s->len);
if (r->start < start) {
off = start - r->start;
seq += off;
qual += off;
seq_len -= off;
}
for (i = 0; i < seq_len; i++) {
if (islower(seq[i]) ||
qual[i] == 100 ||
(qual[i] == 0 && seq[i] != 'N' && seq[i] != '-'
&& seq[i] != '*')) {
int pos = r->start + i + off;
if (dir) {
if (best_pos > pos && pos > xx->cursor_apos) {
found = 1;
best_pos = pos;
fpos = i + off;
fseq = r->rec;
}
break;
} else {
if (best_pos < pos && pos < xx->cursor_apos) {
found = 1;
best_pos = pos;
fpos = i + off;
fseq = r->rec;
}
}
}
}
if (s != sorig)
free(s);
}
if (found) {
edSetCursorPos(xx, fseq == xx->cnum ? GT_Contig : GT_Seq,
fseq, fpos, 1);
}
contig_iter_del(iter);
return found ? 0 : -1;
}
int edview_search_consensus(edview *xx, int dir, int strand, char *value) {
int mismatches = 0; /* exact match */
int where = 2; /* consensus */
char *p;
int start, end;
char cons[WIN_WIDTH+1];
int patlen;
char *uppert, *upperb;
int found = 0, at_end = 0;
tg_rec fseq;
int fpos, i, j;
contig_t *c;
/*
* Parse value search string. It optionally includes two extra params
* separated by #. Ie:
* <string>#<N.mismatches>#<where>.
* <where> is 1 for readings, 2 for consensus, 3 for both.
*/
if (p = strchr(value, '#')) {
mismatches = atoi(p+1);
*p = 0;
if (p = strchr(p+1, '#'))
where = atoi(p+1);
}
/* uppercase search string, remove pads, and store fwd/rev copies */
patlen = strlen(value);
depad_seq(value, &patlen, NULL);
if (NULL == (uppert = (char *)xmalloc(patlen + 1)))
return 0;
if (NULL == (upperb = (char *)xmalloc(patlen + 1)))
return 0;
uppert[patlen] = upperb[patlen] = 0;
for (i = patlen-1; i >= 0; i--) {
upperb[i] = uppert[i] = toupper(value[i]);
}
complement_seq(upperb, patlen);
/* Loop */
if (dir) {
start = xx->cursor_apos + (dir ? 1 : -1);
end = start + (WIN_WIDTH-1);
} else {
end = xx->cursor_apos + (dir ? 1 : -1);
start = end - (WIN_WIDTH-1);
}
fpos = xx->cursor_apos;
c = cache_search(xx->io, GT_Contig, xx->cnum);
cache_incr(xx->io, c);
do {
char *ind, *indt = NULL, *indb = NULL;
calculate_consensus_simple(xx->io, xx->cnum, start, end, cons, NULL);
cons[WIN_WIDTH] = 0;
if (dir) {
if (strand == '+' || strand == '=')
indt = pstrstr_inexact(cons, uppert, mismatches, NULL);
if (strand == '-' || strand == '=')
indb = pstrstr_inexact(cons, upperb, mismatches, NULL);
} else {
if (strand == '+' || strand == '=')
indt = prstrstr_inexact(cons, uppert, mismatches, NULL);
if (strand == '-' || strand == '=')
indb = prstrstr_inexact(cons, upperb, mismatches, NULL);
}
if (indt && indb)
ind = MIN(indt, indb);
else if (indt)
ind = indt;
else if (indb)
ind = indb;
else
ind = NULL;
if (ind != NULL) {
if (dir) {
if (fpos <= start + ind-cons) {
found = 1;
fpos = start + ind-cons;
fseq = xx->cnum;
}
} else {
if (fpos >= start + ind-cons) {
found = 1;
fpos = start + ind-cons;
fseq = xx->cnum;
}
}
break;
}
/* Next search region - overlapping by patlen+pads */
if (dir) {
for (i = WIN_WIDTH-1, j = patlen; j && i; i--) {
if (cons[i] != '*')
j--;
}
if (i == 0)
break;
start += i;
end += i;
if (start > c->end)
at_end = 1;
} else {
for (i = 0, j = patlen; j && i < WIN_WIDTH; i++) {
if (cons[i] != '*')
j--;
}
if (i == WIN_WIDTH)
break;
start -= WIN_WIDTH-i;
end -= WIN_WIDTH-i;
if (end < c->start)
at_end = 1;
}
} while (!at_end);
cache_decr(xx->io, c);
if (found) {
edSetCursorPos(xx, fseq == xx->cnum ? GT_Contig : GT_Seq,
fseq, fpos, 1);
}
free(uppert);
free(upperb);
return found ? 0 : -1;
}
int edview_search_tag_anno(edview *xx, int dir, int strand, char *value) {
contig_iterator *iter;
int start, end;
rangec_t *r;
rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci);
char *r_exp = NULL;
contig_t *c = cache_search(xx->io, GT_Contig, xx->cnum);
if (value) {
if (NULL == (r_exp = REGCMP(xx->interp, value))) {
verror(ERR_WARN, "Search by anno", "invalid regular expression");
return -1;
}
}
if (dir) {
start = xx->cursor_apos + (dir ? 1 : -1);
end = c->end;
ifunc = contig_iter_next;
} else {
start = c->start;
end = xx->cursor_apos + (dir ? 1 : -1);
ifunc = contig_iter_prev;
}
iter = contig_iter_new_by_type(xx->io, xx->cnum, 1,
dir == 1 ? CITER_FIRST : CITER_LAST,
start, end, GRANGE_FLAG_ISANNO);
if (!iter)
return -1;
while (r = ifunc(xx->io, iter)) {
anno_ele_t *ae;
if ((dir && r->start < start) ||
(!dir && r->start > end))
continue;
if (!r_exp)
break; /* blank expr => match all */
ae = cache_search(xx->io, GT_AnnoEle, r->rec);
if (!ae->comment)
continue;
if (REGEX(xx->interp, ae->comment, r_exp))
break;
}
REGFREE(xx->interp, r_exp);
if (r) {
if (r->flags & GRANGE_FLAG_TAG_SEQ) {
int pos;
sequence_get_position(xx->io, r->pair_rec, NULL, &pos, NULL, NULL);
pos = r->start - pos;
edSetCursorPos(xx, GT_Seq, r->pair_rec, pos, 1);
} else {
edSetCursorPos(xx, GT_Contig, xx->cnum, r->start, 1);
}
contig_iter_del(iter);
return 0;
}
contig_iter_del(iter);
return -1;
}
int edview_search_name(edview *xx, int dir, int strand, char *value)
{
tg_rec rec, *rp, cnum = -1, best_rec;
int best_pos, best_off;
int nr, i;
rangec_t *(*ifunc)(GapIO *io, contig_iterator *ci);
int start, end, cstart;
contig_iterator *iter;
contig_t *c;
/* Check for #num where num is a sequence record in this contig */
if (*value == '#') {
char *endp;
int64_t v = strtol64(value+1, &endp, 10);
rec = v;
if (*endp == '\0' && cache_exists(xx->io, GT_Seq, rec)) {
sequence_get_clipped_position(xx->io, rec, &cnum,
&start, NULL, &cstart, NULL, NULL);
if (cnum == xx->cnum) {
edSetCursorPos(xx, GT_Seq, rec, cstart - start, 1);
return 0;
}
}
}
/* Find all hits matching this name */
rp = sequence_index_query_all(xx->io, value, 1, &nr);
/* Also get an position-based iterator */
c = cache_search(xx->io, GT_Contig, xx->cnum);
if (dir) {
start = xx->cursor_apos + 1;
end = c->end;
ifunc = contig_iter_next;
best_pos = end + 1;
best_off = 0;
} else {
start = c->start;
end = xx->cursor_apos - 1;
ifunc = contig_iter_prev;
best_pos = start - 1;
best_off = 0;
}
iter = contig_iter_new_by_type(xx->io, xx->cnum, 1,
dir == 1 ? CITER_FIRST : CITER_LAST,
start-1, end+1, GRANGE_FLAG_ISSEQ);
if (!iter)
return -1;
/*
* The iterator also finds overlapping objects, not just ones beyond this
* point. That's fine if we're on the consensus as we probably want to
* jump to the first seq-name overlapping this point.
*
* However if we're on a sequence already, we want the first one
* after or before that sequence. So we skip along iterator until we're
* at the current record.
*/
if (xx->cursor_type == GT_Seq) {
rangec_t *r;
while ((r = ifunc(xx->io, iter))) {
if (r->rec == xx->cursor_rec)
break;
}
}
/* Alternate between the by-name and by-position scan */
best_rec = -1;
for (i = 0; i < nr; i++) {
int start, end;
rangec_t *r;
/* From name index */
rec = rp[i++];
sequence_get_clipped_position(xx->io, rec, &cnum, &start, &end,
&cstart, NULL, NULL);
if (cnum == xx->cnum) {
if ((dir && best_pos > cstart && cstart > xx->cursor_apos) ||
(!dir && best_pos < cstart && cstart < xx->cursor_apos)) {
best_pos = cstart;
best_off = cstart - start;
best_rec = rec;
}
}
/* From iterator */
if ((r = ifunc(xx->io, iter))) {
seq_t *s;
if (NULL == (s = cache_search(xx->io, GT_Seq, r->rec))) {
/* No match */
best_rec = -1;
break;
}
if (strncmp(s->name, value, strlen(value)) == 0) {
/* prefix match */
puts("Found by pos iterator");
best_rec = r->rec;
break;
}
} else {
/* End of contig - bail out early */
best_rec = -1;
break;
}
}
contig_iter_del(iter);
if (rp)
free(rp);
if (best_rec != -1) {
edSetCursorPos(xx, GT_Seq, best_rec, best_off, 1);
return 0;
}
return -1;
}
int edview_search_cons_discrep(edview *xx, int dir, int strand, char *value) {
int start, end;
int found = 0, at_end = 0;
int fpos, i;
double qval = atof(value);
consensus_t cons[WIN_WIDTH+1];
contig_t *c;
/* Set initial start positions */
if (dir) {
start = xx->cursor_apos + (dir ? 1 : -1);
end = start + (WIN_WIDTH-1);
} else {
end = xx->cursor_apos + (dir ? 1 : -1);
start = end - (WIN_WIDTH-1);
}
fpos = xx->cursor_apos;
/* Loop WIN_WIDTH block at a time */
c = cache_search(xx->io, GT_Contig, xx->cnum);
cache_incr(xx->io, c);
do {
calculate_consensus(xx->io, xx->cnum, start, end, cons);
if (dir) {
for (i = 0; i < WIN_WIDTH; i++) {
if (cons[i].discrep >= qval) {
found = 1;
break;
}
}
} else {
for (i = WIN_WIDTH-1; i; i--) {
if (cons[i].discrep >= qval) {
found = 1;
break;
}
}
}
if (found) {
fpos = start + i;
break;
}
/* Next search region - overlapping by patlen+pads */
if (dir) {
start += WIN_WIDTH;
end += WIN_WIDTH;
if (start > c->end)
at_end = 1;
} else {
start -= WIN_WIDTH;
end -= WIN_WIDTH;
if (end < c->start)
at_end = 1;
}
} while (!at_end);
cache_decr(xx->io, c);
if (found) {
edSetCursorPos(xx, GT_Contig, xx->cnum, fpos, 1);
return 0;
}
return -1;
}
