/* free the memory used by the vetting structure */
void
mtsFreeWork (vetting_t ** w)
{
vetting_t *work = (*w);
if (work->desired.X != -SPECIAL || work->desired.Y != -SPECIAL)
{
heap_free (work->untested.h, free);
heap_destroy (&work->untested.h);
heap_free (work->no_fix.h, free);
heap_destroy (&work->no_fix.h);
heap_free (work->no_hi.h, free);
heap_destroy (&work->no_hi.h);
heap_free (work->hi_candidate.h, free);
heap_destroy (&work->hi_candidate.h);
}
else
{
while (!vector_is_empty (work->untested.v))
free (vector_remove_last (work->untested.v));
vector_destroy (&work->untested.v);
while (!vector_is_empty (work->no_fix.v))
free (vector_remove_last (work->no_fix.v));
vector_destroy (&work->no_fix.v);
while (!vector_is_empty (work->no_hi.v))
free (vector_remove_last (work->no_hi.v));
vector_destroy (&work->no_hi.v);
while (!vector_is_empty (work->hi_candidate.v))
free (vector_remove_last (work->hi_candidate.v));
vector_destroy (&work->hi_candidate.v);
}
free (work);
(*w) = NULL;
}
static void router_test2(void)
{
int i;
struct node_t* node;
struct node_t* result[8];
static struct node_t s_nodes[100000];
uint8_t id[N_NODEID] = { 0xAB, 0xCD, 0xEF, 0x89, };
heap_t* heap, *heap2;
struct router_t* router;
router = router_create(id);
heap = heap_create(node_compare_less, (void*)id);
heap_reserve(heap, 8 + 1);
for (i = 0; i < sizeof(s_nodes) / sizeof(s_nodes[0]); i++)
{
int v = rand();
memset(&s_nodes[i], 0, sizeof(s_nodes[i]));
memcpy(s_nodes[i].id, &v, sizeof(v));
s_nodes[i].ref = 1;
if (0 == router_add(router, s_nodes[i].id, &s_nodes[i].addr, &node))
{
heap_push(heap, node);
if (heap_size(heap) > 8)
{
node_release((struct node_t*)heap_top(heap));
heap_pop(heap);
}
}
}
assert(8 == heap_size(heap));
assert(8 == router_nearest(router, id, result, 8));
heap2 = heap_create(node_compare_less, (void*)id);
heap_reserve(heap2, 8);
for (i = 0; i < 8; i++)
{
heap_push(heap2, result[i]);
}
assert(heap_size(heap) == heap_size(heap2));
for (i = 0; i < 8; i++)
{
assert(heap_top(heap2) == heap_top(heap));
heap_pop(heap);
heap_pop(heap2);
}
router_destroy(router);
heap_destroy(heap);
heap_destroy(heap2);
printf("router test ok!\n");
}
glist_t vithist_sort (glist_t vithist_list)
{
heap_t heap;
gnode_t *gn;
vithist_t *h;
glist_t vithist_new;
int32 ret, score;
vithist_new = NULL;
heap = heap_new();
for (gn = vithist_list; gn; gn = gnode_next(gn)) {
h = (vithist_t *) gnode_ptr(gn);
if (heap_insert (heap, (void *) h, h->scr) < 0) {
E_ERROR("Panic: heap_insert() failed\n");
return NULL;
}
}
/*
* Note: The heap returns nodes with ASCENDING values; and glist_add adds new nodes to the
* HEAD of the list. So we get a glist in the desired descending score order.
*/
while ((ret = heap_pop (heap, (void **)(&h), &score)) > 0)
vithist_new = glist_add_ptr (vithist_new, (void *)h);
if (ret < 0) {
E_ERROR("Panic: heap_pop() failed\n");
return NULL;
}
heap_destroy (heap);
return vithist_new;
}
void outputSorted(const Person people[],
int numPeople,
int (* compare)(const void *pKey1, const void *pKey2))
{
Heap heap;
void *data;
int i;
/* Initialize heap */
heap_init(&heap, compare, NULL);
/* Add people to heap */
for (i = 0; i < numPeople; ++i)
if (heap_insert(&heap, &people[i]) != 0)
fatalError("Failed to insert person into heap");
/* Extract and output people from heap */
while (heap_size(&heap) > 0) {
if (heap_extract(&heap, &data) != 0)
fatalError("Failed to extract person from heap");
outputPerson((const Person *)data);
}
/* Destroy heap */
heap_destroy(&heap);
}
void timers_shutdown(void)
{
if (initialized > 1) {
initialized--;
return;
}
/* Stop all timers. */
if (timers->heap->len > 0)
warning(0, "Timers shutting down with %ld active timers.",
timers->heap->len);
while (timers->heap->len > 0)
gwtimer_stop(timers->heap->tab[0]);
/* Kill timer thread */
timers->stopping = 1;
gwthread_wakeup(timers->thread);
gwthread_join(timers->thread);
initialized = 0;
/* Free resources */
heap_destroy(timers->heap);
mutex_destroy(timers->mutex);
gw_free(timers);
}
void test_heapify(void) {
size_t i;
/* initialize data */
int *array = zmalloc(sizeof(int) * LOTS_OF_INTS);
int sum = 0;
for (i = 0; i < LOTS_OF_INTS; i++) {
array[i] = random() % 123456789;
sum += array[i];
}
/* construct a heap from our random array */
heap_t *h = heap_heapify(array, LOTS_OF_INTS, sizeof(int),
NULL, (cmp_func_t)intcmp);
assert(heap_size(h) == LOTS_OF_INTS);
int ctrlsum = 0, maxctrl, *e;
/* peek head */
if (heap_size(h)) maxctrl = *(int*)heap_get(h, 0);
while (heap_size(h)) {
e = heap_pop(h); ctrlsum += *e;
/* check that elements are in heap order */
assert(*e <= maxctrl); maxctrl = *e;
}
assert(ctrlsum == sum);
heap_destroy(h);
free(array);
}
void database_destroy (database_t * db)
{
for (file_t * i = db->files; i != db->files_end; ++i)
free (i->versions);
for (tag_t * i = db->tags; i != db->tags_end; ++i) {
free (i->tag_files);
free (i->branch_versions);
free (i->tags);
free (i->parents);
free (i->changeset.children);
free (i->changeset.merge);
free (i->fixups);
}
for (changeset_t ** i = db->changesets; i != db->changesets_end; ++i) {
free ((*i)->children);
free ((*i)->versions);
free ((*i)->merge);
free (*i);
}
free (db->files);
free (db->tags);
free (db->changesets);
heap_destroy (&db->ready_changesets);
}
void dijkstra(graph* g, unsigned int source) {
unsigned int u, v, edge_count;
node *n, *d;
edge *e;
heap *Q;
g->nodes[source].distance = 0;
Q = heap_make(compare, g);
while(!heap_is_empty(Q)) {
u = heap_delete_min(Q);
n = &g->nodes[u];
edge_count = n->edge_count;
for(v = 0; v < edge_count; v++) {
e = &n->edges[v];
d = &g->nodes[e->destination];
if(d->distance > n->distance + e->weight) {
/*
Relajo los vertices
*/
d->distance = n->distance + e->weight;
/*
Actualizo el nodo con la distancia optima a este
*/
d->previous = u;
/*
Actualizo la cola de prioridad (el vertice solo puede
haber subido en prioridad, entonces solo hago heapify-up
*/
heap_heapify_up(Q, d->heap_index);
}
}
}
heap_destroy(Q);
}
void vithist_prune (vithist_t *vh, dict_t *dict, int32 frm,
int32 maxwpf, int32 maxhist, int32 beam)
{
int32 se, fe, filler_done, th;
vithist_entry_t *ve;
heap_t h;
s3wid_t *wid;
int32 i;
assert (frm >= 0);
se = vh->frame_start[frm];
fe = vh->n_entry - 1;
th = vh->bestscore[frm] + beam;
h = heap_new ();
wid = (s3wid_t *) ckd_calloc (maxwpf+1, sizeof(s3wid_t));
wid[0] = BAD_S3WID;
for (i = se; i <= fe; i++) {
ve = vh->entry[VITHIST_ID2BLK(i)] + VITHIST_ID2BLKOFFSET(i);
heap_insert (h, (void *)ve, -(ve->score));
ve->valid = 0;
}
/* Mark invalid entries: beyond maxwpf words and below threshold */
filler_done = 0;
while ((heap_pop (h, (void **)(&ve), &i) > 0) && (ve->score >= th) && (maxhist > 0)) {
if (dict_filler_word (dict, ve->wid)) {
/* Major HACK!! Keep only one best filler word entry per frame */
if (filler_done)
continue;
filler_done = 1;
}
/* Check if this word already valid (e.g., under a different history) */
for (i = 0; IS_S3WID(wid[i]) && (wid[i] != ve->wid); i++);
if (NOT_S3WID(wid[i])) {
/* New word; keep only if <maxwpf words already entered, even if >= thresh */
if (maxwpf > 0) {
wid[i] = ve->wid;
wid[i+1] = BAD_S3WID;
--maxwpf;
--maxhist;
ve->valid = 1;
}
} else if (! vh->bghist) {
--maxhist;
ve->valid = 1;
}
}
ckd_free ((void *) wid);
heap_destroy (h);
/* Garbage collect invalid entries */
vithist_frame_gc (vh, frm);
}
void etimer_destroy(etimer_t *timer)
{
list_t *queue;
etimer_event_t *event;
etimer_lock(timer);
queue = list_create();
while( (event = heap_shift(timer->queue)) ) {
list_append(queue, event);
}
heap_destroy(timer->queue, NULL);
close(timer->pin);
close(timer->pout);
close(timer->epfd);
etimer_unlock(timer);
pthread_mutex_destroy(&timer->lock);
free(timer);
etimer_events_do(queue, NULL, 1);
list_destroy(queue, etimer_event_destructor);
}
struct Path *road_min(struct Graph *graph, int from, int to){
Heap *heap;
HNode *array;
Edge *edge, *tmp;
struct Path *p, *q;
int i, j, val, n = graph->num;
int end = find_node(graph, to)->adj;
int start = find_node(graph, from)->adj;
struct VNode **path = malloc(sizeof(struct VNode*) * n);
heap = heap_init();
heap->num = n;
array = heap->array;
bzero(path, sizeof(struct VNode*) * n);
for(i = 1; i <= n; i++){
array[i].value = INF;
array[i].vertex = i;
array[i].pos = i;
}
array[start + 1].value = 0;
heap_up(heap, start + 1);
do{
j = array[1].vertex - 1;
if(j == end){
break;
}
val = array[1].value;
for(edge = graph->array[j]->link; edge; tmp = edge->next, edge = tmp){
i = edge->node->adj + 1;
if(array[array[i].pos].value > val + edge->dut){
array[array[i].pos].value = val + edge->dut;
heap_up(heap, array[i].pos);
path[i - 1] = graph->array[j];
}
}
}while(heap_top_del(heap));
heap_destroy(heap);
i = end;
q = NULL;
while(path[i]){
j = path[i]->adj;
p = malloc(sizeof(Path));
p->node = path[i];
p->next = q;
q = p;
i = j;
}
free(path);
return q;
}
// FIXME - we don't cope optimally with the situation where a branch is
// created, files deleted, and then the branch tagged (without rtag). We'll
// never know that the tag was placed on the branch; instead we'll place the tag
// on the trunk.
static void branch_graph (database_t * db,
tag_t *** tree_order, tag_t *** tree_order_end)
{
// First, go through each tag, and put it on all the branches.
for (tag_t * i = db->tags; i != db->tags_end; ++i) {
i->changeset.unready_count = 0;
for (version_t ** j = i->tag_files; j != i->tag_files_end; ++j) {
if ((*j)->branch)
record_branch_tag ((*j)->branch, i);
if (*j != (*j)->file->versions &&
(*j)[-1].implicit_merge &&
(*j)[-1].used &&
(*j)[-1].branch)
record_branch_tag ((*j)[-1].branch, i);
}
}
// Go through each branch and put record it on the tags.
for (tag_t * i = db->tags; i != db->tags_end; ++i)
for (branch_tag_t * j = i->tags; j != i->tags_end; ++j) {
ARRAY_EXTEND (j->tag->parents);
j->tag->parents_end[-1].branch = i;
j->tag->parents_end[-1].weight = j->weight;
++j->tag->changeset.unready_count;
}
// Do a cycle breaking pass of the branches.
heap_t heap;
heap_init (&heap, offsetof (tag_t, changeset.ready_index), tag_compare);
// Release all the tags that are ready right now; also sort the parent
// lists.
for (tag_t * i = db->tags; i != db->tags_end; ++i) {
ARRAY_SORT (i->parents, compare_pb);
if (i->changeset.unready_count == 0) {
i->is_released = true;
heap_insert (&heap, i);
}
}
while (!heap_empty (&heap))
tag_released (&heap, heap_pop (&heap), tree_order, tree_order_end);
for (tag_t * i = db->tags; i != db->tags_end; ++i)
while (!i->is_released) {
break_cycle (&heap, i);
while (!heap_empty (&heap))
tag_released (&heap, heap_pop (&heap),
tree_order, tree_order_end);
}
heap_destroy (&heap);
}
int main(int argc, char * argv[])
{
int length, num, c;
int * array;
heap h;
if (argc != 3) {
fprintf(stderr,"Usage: %s <heap-length> <num-tests>\n", argv[0]);
return 1;
}
length = atoi(argv[1]);
num = atoi(argv[2]);
if (length <= 0) {
fprintf(stderr,"Error: length <= 0\n");
return 1;
}
h = heap_create();
array = ALLOC(int,length);
nusmv_assert(array);
for (c = 1; c <= num; c++) {
int i, i1, i2;
printf("Test %2d:", c);
for (i = 0; i < length; i++) {
array[i] = i+1;
}
for (i = 0; i < 4 * length; i++) {
i1 = utils_random() % length;
i2 = utils_random() % length;
if (i1 == i2) continue;
array[i1] = array[i1] + array[i2];
array[i2] = array[i1] - array[i2];
array[i1] = array[i1] - array[i2];
}
for (i = 0; i < length; i++) {
printf(" %d", array[i]);
heap_add(h, -array[i], (void *)array[i]);
}
printf("\n------->");
for (i = 0; i < length; i++) {
int val = (int)heap_getmax(h);
printf(" %d", val);
nusmv_assert(val == i+1);
}
printf("\n");
assert(heap_isempty(h));
}
heap_destroy(h);
return 0;
}
void generate_test_heap(void) {
size_t n = 0, i, j;
int sum = 0, *e=NULL;
heap_t *h = heap_init(free, (cmp_func_t)intcmp);
for (i = 0; i < 1000; i++) {
switch (random() % 3) {
case 0:
case 2:
e = zmalloc(sizeof(int)); *e = random() % 123456789;
n++; sum += *e;
heap_insert(h, e);
break;
case 1:
e = heap_pop(h);
if (e) { n--; sum -= *e; free(e); }
break;
}
/* check propper auto resize */
assert(heap_capacity(h) >= heap_size(h));
for (j = 0; j < heap_size(h); j++) {
size_t left = heap_child_l(j);
size_t right = heap_child_r(j);
size_t greatest = heap_greatest_child(h, j);
/* check for heap order is preserved and greatest child is correct */
if (heap_size(h) > right) {
assert(*(int*)heap_get(h, j) >= *(int*)heap_get(h, right));
assert(*(int*)heap_get(h, greatest) >= *(int*)heap_get(h, right));
}
if (heap_size(h) > left) {
assert(*(int*)heap_get(h, j) >= *(int*)heap_get(h, left));
assert(*(int*)heap_get(h, greatest) >= *(int*)heap_get(h, left));
}
}
}
assert(heap_size(h) == n);
int ctrlsum = 0, maxctrl;
if (heap_size(h)) maxctrl = *(int*)heap_get(h, 0);
while (heap_size(h)) {
e = heap_pop(h); ctrlsum += *e;
/* check that elements are in heap order */
assert(*e <= maxctrl); maxctrl = *e;
free(e);
}
assert(ctrlsum == sum);
heap_destroy(h);
}
int main (int argc, char* argv[])
{
printf("Test de heap_destroy\n");
heap h=prof_heap_create(f);
int a=1;
h=prof_heap_insert(h, &a);
heap_dump(h);
heap_destroy(h);
return 0;
}
int threadpool_destroy(threadpool_t *pool, int block, int timeout) {
int ret;
assert(pool);
Pthread_mutex_lock(&pool->mutex);
if (!pool->exit) {
/* you should call `threadpool_exit' first */
Pthread_mutex_unlock(&pool->mutex);
return -1;
}
if (pool->threads_num != 0) {
if (!block) {
Pthread_mutex_unlock(&pool->mutex);
return -1;
} else {
struct timespec ts;
struct timeval tv;
gettimeofday(&tv, NULL);
ts.tv_sec = tv.tv_sec + timeout;
ts.tv_nsec = tv.tv_usec * 1000;
while (pool->threads_num != 0) {
if (timeout == 0) {
Pthread_cond_wait(&pool->exit_cond, &pool->mutex);
goto CONT;
} else {
ret = Pthread_cond_timedwait(&pool->exit_cond,
&pool->mutex, &ts);
if (ret == 0) {
goto CONT;
} else if (ret == ETIMEDOUT) {
Pthread_mutex_unlock(&pool->mutex);
return -1;
}
}
}
}
}
CONT:
Pthread_mutex_unlock(&pool->mutex);
heap_destroy(&pool->task_queue);
Pthread_mutex_destroy(&pool->mutex);
Pthread_cond_destroy(&pool->cond);
Pthread_cond_destroy(&pool->exit_cond);
Pthread_cond_destroy(&pool->task_over_cond);
free(pool);
return 0;
}
void gw_timerset_destroy(Timerset *set)
{
if (set == NULL)
return;
/* Stop all timers. */
while (set->heap->len > 0)
gw_timer_stop(set->heap->tab[0]);
/* Kill timer thread */
set->stopping = 1;
gwthread_wakeup(set->thread);
gwthread_join(set->thread);
/* Free resources */
heap_destroy(set->heap);
mutex_destroy(set->mutex);
gw_free(set);
}
static bool
cmd_test_heap()
{
pagetable_t pt;
pagetable_create(&pt, (void *)0x8000000000, PAGE_SIZE * 1024);
pagetable_activate(&pt);
struct heap *heap = heap_create(&pt, (void *)0x9000000000, 1024);
void *ptr1 = heap_alloc(heap, 128);
void *ptr2 = heap_alloc(heap, 0xff00);
void *ptr3 = heap_alloc(heap, 8);
heap_free(heap, ptr1);
heap_free(heap, ptr2);
heap_free(heap, ptr3);
heap_destroy(heap);
pagetable_activate(NULL);
pagetable_destroy(&pt);
return true;
}
/* return a list of nodes representing the shortest path from v
* we can't have negative edges, distances are either an overestimate
* or exact (that's why we don't update downstream nodes */
void graph_explore_dijkstra(graph_t *g, graph_vertex_t *s,
graph_vertex_t *e, visit_func_t visit) {
/* clear graph search assets unless we're exploring the whole graph */
if (g) graph_clear_pathtrace(g);
/* mark starting point's prev as a self reference to avoid looping back */
s->path.prev = s;
/* just a simple queue for breadth first search */
heap_t *prioq = heap_init(NULL, (cmp_func_t)min_path_cmp);
heap_insert(prioq, s);
while ((s = heap_pop(prioq))) {
int clock = 0;
list_node_t *it;
/* break if searching for a specific destination */
if (e && e == s) break;
s->path.visited = 1;
s->path.pre_clock = clock++;
if (visit) visit(s);
for (it = s->edges->first; it != NULL; it = it->next) {
graph_edge_t *ve = (graph_edge_t *)it->data;
graph_vertex_t *v = ve->connection;
if (!v->path.prev ||
v->path.distance > s->path.distance + ve->weight) {
v->path.prev = s;
v->path.distance = s->path.distance + ve->weight;
/* check if destination is already on the heap */
ssize_t idx = heap_find(prioq, v);
if (idx != -1)
heap_bubble_up(prioq, v, idx);
else
heap_insert(prioq, v);
}
}
}
heap_destroy(prioq);
}
huff_code_t *
huff_code_build_int(int32 const *values, int32 const *frequencies, int nvals)
{
huff_code_t *hc;
huff_node_t *root;
heap_t *q;
int i;
hc = ckd_calloc(1, sizeof(*hc));
hc->refcount = 1;
hc->type = HUFF_CODE_INT;
/* Initialize the heap with nodes for each symbol. */
q = heap_new();
for (i = 0; i < nvals; ++i) {
heap_insert(q,
huff_node_new_int(values[i]),
frequencies[i]);
}
/* Now build the tree, which gives us codeword lengths. */
root = huff_code_build_tree(q);
heap_destroy(q);
if (root == NULL || root->nbits > 64) {
E_ERROR("Huffman trees currently limited to 32 bits\n");
huff_node_free_int(root);
huff_code_free(hc);
return NULL;
}
/* Build a canonical codebook. */
hc->maxbits = root->nbits;
huff_code_canonicalize(hc, root);
/* Tree no longer needed. */
huff_node_free_int(root);
return hc;
}
huff_code_t *
huff_code_build_str(char * const *values, int32 const *frequencies, int nvals)
{
huff_code_t *hc;
huff_node_t *root;
heap_t *q;
int i;
hc = (huff_code_t*)ckd_calloc(1, sizeof(*hc));
hc->refcount = 1;
hc->type = HUFF_CODE_STR;
/* Initialize the heap with nodes for each symbol. */
q = heap_new();
for (i = 0; i < nvals; ++i) {
heap_insert(q,
huff_node_new_str(values[i]),
frequencies[i]);
}
/* Now build the tree, which gives us codeword lengths. */
root = huff_code_build_tree(q);
heap_destroy(q);
if (root == 0 || root->nbits > 32) {
E_ERROR("Huffman trees currently limited to 32 bits\n");
huff_node_free_str(root, TRUE);
huff_code_free(hc);
return 0;
}
/* Build a canonical codebook. */
hc->maxbits = root->nbits;
huff_code_canonicalize(hc, root);
/* Tree no longer needed (note we retain pointers to its strings). */
huff_node_free_str(root, FALSE);
return hc;
}
int main (void)
{
char a;
heap_init ();
heap_insert ('0');
heap_insert ('3');
heap_insert ('2');
heap_insert ('1');
heap_insert ('4');
heap_insert ('5');
heap_insert ('6');
heap_insert ('7');
heap_insert ('8');
heap_insert ('9');
heap_print ();
heap_extract_min (&a);
do
{
printf ("\n--------------------\nremove: %c\n", a);
heap_print ();
} while (heap_extract_min (&a));
heap_destroy ();
}
int search(int n) {
struct arg_struct *args = malloc(sizeof (struct arg_struct));
args->topic = n;
args->startidx = 0;
args->endidx = num_docs;
args->base = 0;
heap h;
heap_create(&h,0,NULL);
args->h = &h;
scansearch(args);
float* min_key;
int* min_val;
int rank = TOP_K;
while (heap_delmin(&h, (void**)&min_key, (void**)&min_val)) {
printf("MB%02d Q0 %ld %d %f " SCANNAME "\n", (n+1), tweetids[*min_val], rank, *min_key);
rank--;
}
heap_destroy(&h);
free(args);
}
static int suite (heap_t * (*creator)(size_t))
{
static test_input_t data[] = {
{ 12.0, "hello world" },
{ -13.0, "too much negativity" },
{ 42.9, "byebye universe" },
{ 0.0, NULL }};
heap_t * heap;
if (NULL == (heap = test_create (data, creator))) {
fprintf (stderr, "OOPS: creation failed\n");
return -1;
}
fprintf (stderr, "\nafter creation:\n");
test_enumerate (heap);
fprintf (stderr, "\nlet's modify some existing and bogus elements...\n");
if (0 != heap_change_key (heap, data[0].key, -22.0, data[0].value)) {
fprintf (stderr, "OOPS: that should have succeeded!\n");
return -2;
}
if (0 == heap_change_key (heap, 888.999, -1.0, data[0].value)) {
fprintf (stderr, "OOPS: invalid old_key should have failed!\n");
return -3;
}
if (0 == heap_change_key (heap, data[1].key, 22000.3, "blah")) {
fprintf (stderr, "OOPS: invalid value have failed!\n");
return -4;
}
fprintf (stderr, "\nafter changing a key:\n");
test_enumerate (heap);
heap_destroy (heap);
return 0;
}
/*
* cluster
*
* Check that the relation is a relation in the appropriate user
* ACL. I will use the same security that limits users on the
* renamerel() function.
*
* Check that the index specified is appropriate for the task
* ( ie it's an index over this relation ). This is trickier.
*
* Create a list of all the other indicies on this relation. Because
* the cluster will wreck all the tids, I'll need to destroy bogus
* indicies. The user will have to re-create them. Not nice, but
* I'm not a nice guy. The alternative is to try some kind of post
* destroy re-build. This may be possible. I'll check out what the
* index create functiond want in the way of paramaters. On the other
* hand, re-creating n indicies may blow out the space.
*
* Create new (temporary) relations for the base heap and the new
* index.
*
* Exclusively lock the relations.
*
* Create new clustered index and base heap relation.
*
*/
void
cluster(char oldrelname[], char oldindexname[])
{
Oid OIDOldHeap, OIDOldIndex, OIDNewHeap;
Relation OldHeap, OldIndex;
Relation NewHeap;
char *NewIndexName;
char *szNewHeapName;
/*
*
* I'm going to force all checking back into the commands.c function.
*
* Get the list if indicies for this relation. If the index we want
* is among them, do not add it to the 'kill' list, as it will be
* handled by the 'clean up' code which commits this transaction.
*
* I'm not using the SysCache, because this will happen but
* once, and the slow way is the sure way in this case.
*
*/
/*
* Like vacuum, cluster spans transactions, so I'm going to handle it in
* the same way.
*/
/* matches the StartTransaction in PostgresMain() */
OldHeap = heap_openr(oldrelname);
if (!RelationIsValid(OldHeap)) {
elog(WARN, "cluster: unknown relation: \"%-.*s\"",
NAMEDATALEN, oldrelname);
}
OIDOldHeap = OldHeap->rd_id; /* Get OID for the index scan */
OldIndex=index_openr(oldindexname);/* Open old index relation */
if (!RelationIsValid(OldIndex)) {
elog(WARN, "cluster: unknown index: \"%-.*s\"",
NAMEDATALEN, oldindexname);
}
OIDOldIndex = OldIndex->rd_id; /* OID for the index scan */
heap_close(OldHeap);
index_close(OldIndex);
/*
* I need to build the copies of the heap and the index. The Commit()
* between here is *very* bogus. If someone is appending stuff, they will
* get the lock after being blocked and add rows which won't be present in
* the new table. Bleagh! I'd be best to try and ensure that no-one's
* in the tables for the entire duration of this process with a pg_vlock.
*/
NewHeap = copy_heap(OIDOldHeap);
OIDNewHeap = NewHeap->rd_id;
szNewHeapName = pstrdup(NewHeap->rd_rel->relname.data);
/* Need to do this to make the new heap visible. */
CommandCounterIncrement();
rebuildheap(OIDNewHeap, OIDOldHeap, OIDOldIndex);
/* Need to do this to make the new heap visible. */
CommandCounterIncrement();
/* can't be found in the SysCache. */
copy_index(OIDOldIndex, OIDNewHeap); /* No contention with the old */
/*
* make this really happen. Flush all the buffers.
*/
CommitTransactionCommand();
StartTransactionCommand();
/*
* Questionable bit here. Because the renamerel destroys all trace of the
* pre-existing relation, I'm going to Destroy old, and then rename new
* to old. If this fails, it fails, and you lose your old. Tough - say
* I. Have good backups!
*/
/*
Here lies the bogosity. The RelationNameGetRelation returns a bad
list of TupleDescriptors. Damn. Can't work out why this is.
*/
heap_destroy(oldrelname); /* AAAAAAAAGH!! */
CommandCounterIncrement();
/*
* The Commit flushes all palloced memory, so I have to grab the
* New stuff again. This is annoying, but oh heck!
*/
/*
renamerel(szNewHeapName.data, oldrelname);
TypeRename(&szNewHeapName, &szOldRelName);
sprintf(NewIndexName.data, "temp_%x", OIDOldIndex);
renamerel(NewIndexName.data, szOldIndexName.data);
*/
NewIndexName = palloc(NAMEDATALEN+1); /* XXX */
sprintf(NewIndexName, "temp_%x", OIDOldIndex);
renamerel(NewIndexName, oldindexname);
}
int main(int argc, char **argv) {
Heap heap;
void *data;
int intval[30], i;
/*****************************************************************************
* *
* Initialize the heap. *
* *
*****************************************************************************/
heap_init(&heap, compare_int, NULL);
/*****************************************************************************
* *
* Perform some heap operations. *
* *
*****************************************************************************/
i = 0;
intval[i] = 5;
fprintf(stdout, "Inserting %03d\n", intval[i]);
if (heap_insert(&heap, &intval[i]) != 0)
return 1;
print_heap(&heap);
i++;
intval[i] = 10;
fprintf(stdout, "Inserting %03d\n", intval[i]);
if (heap_insert(&heap, &intval[i]) != 0)
return 1;
print_heap(&heap);
i++;
intval[i] = 20;
fprintf(stdout, "Inserting %03d\n", intval[i]);
if (heap_insert(&heap, &intval[i]) != 0)
return 1;
print_heap(&heap);
i++;
intval[i] = 1;
fprintf(stdout, "Inserting %03d\n", intval[i]);
if (heap_insert(&heap, &intval[i]) != 0)
return 1;
print_heap(&heap);
i++;
intval[i] = 25;
fprintf(stdout, "Inserting %03d\n", intval[i]);
if (heap_insert(&heap, &intval[i]) != 0)
return 1;
print_heap(&heap);
i++;
intval[i] = 22;
fprintf(stdout, "Inserting %03d\n", intval[i]);
if (heap_insert(&heap, &intval[i]) != 0)
return 1;
print_heap(&heap);
i++;
intval[i] = 9;
fprintf(stdout, "Inserting %03d\n", intval[i]);
if (heap_insert(&heap, &intval[i]) != 0)
return 1;
print_heap(&heap);
i++;
while (heap_size(&heap) > 0) {
if (heap_extract(&heap, (void **) &data) != 0)
return 1;
fprintf(stdout, "Extracting %03d\n", *(int *) data);
print_heap(&heap);
}
/*****************************************************************************
* *
* Destroy the heap. *
* *
*****************************************************************************/
fprintf(stdout, "Destroying the heap\n");
heap_destroy(&heap);
return 0;
}
colap
colap_destroy(colap c) {
return heap_destroy(c);
}
void heap_free(heap_t *h) {
heap_destroy(h);
free(h);
}
int main(int argc, const char* argv[]) {
if (argc <= 1) {
printf("PLEASE ENTER THREAD NUMBER!\n");
return 0;
}
int nthreads=atoi(argv[1]);
printf("Number of threads: %d\n", nthreads);
init_pos();
double total = 0;
int N = 3;
int count;
for (count = 1; count <= N; count ++) {
struct timeval begin, end;
double time_spent;
gettimeofday(&begin, NULL);
int n;
for (n=0; n<NUM_TOPICS; n++) {
// printf("Processing topic %d...\n", topics2011[n][0]);
heap h_array[nthreads];
memset(h_array,0,sizeof(h_array));
struct threadpool *pool;
pool = threadpool_init(nthreads);
int i = 0;
for (i=0; i<nthreads; i++) {
struct arg_struct *args = malloc(sizeof *args);
args->topic = n;
args->startidx = i*(int)(ceil((double)NUM_DOCS / nthreads));
if ((i+1)*(int)(ceil((double)NUM_DOCS / nthreads)) > NUM_DOCS) {
args->endidx = NUM_DOCS;
} else {
args->endidx = (i+1)*(int)(ceil((double)NUM_DOCS / nthreads));
}
args->base = termindexes[nthreads-1][i];
heap h;
h_array[i] = h;
args->h = &h_array[i];
threadpool_add_task(pool,search,args,0);
}
threadpool_free(pool,1);
heap h_merge;
heap_create(&h_merge,0,NULL);
float* min_key_merge;
int* min_val_merge;
for (i=0; i<nthreads; i++) {
float* min_key;
int* min_val;
while(heap_delmin(&h_array[i], (void**)&min_key, (void**)&min_val)) {
int size = heap_size(&h_merge);
if ( size < TOP_K ) {
heap_insert(&h_merge, min_key, min_val);
} else {
heap_min(&h_merge, (void**)&min_key_merge, (void**)&min_val_merge);
if (*min_key_merge < *min_key) {
heap_delmin(&h_merge, (void**)&min_key_merge, (void**)&min_val_merge);
heap_insert(&h_merge, min_key, min_val);
}
}
}
heap_destroy(&h_array[i]);
}
int rank = TOP_K;
while (heap_delmin(&h_merge, (void**)&min_key_merge, (void**)&min_val_merge)) {
printf("MB%02d Q0 %ld %d %f Scan1_pos_multithread_intraquery\n", (n+1), tweetids[*min_val_merge], rank, *min_key_merge);
rank--;
}
heap_destroy(&h_merge);
}
gettimeofday(&end, NULL);
time_spent = (double)((end.tv_sec * 1000000 + end.tv_usec) - (begin.tv_sec * 1000000 + begin.tv_usec));
total = total + time_spent / 1000.0;
}
printf("Total time = %f ms\n", total/N);
printf("Time per query = %f ms\n", (total/N)/NUM_TOPICS);
}
void sort(char *inputfile, char *outputfile, int numattrs, int attributes[], int bufsize)
{
// open stream to read data file
printf("Buffer size: %d bytes\n", bufsize);
printf("Block size: %d bytes\n", DISK_BLOCK_SIZE);
printf("Reading from: %s\n", inputfile);
FILE *in = fopen(inputfile, "r");
// read metadata
metadata m;
m.record_size = m.header_size = 0;
fread(&m.numattrs, sizeof(unsigned int), 1, in);
m.header_size += sizeof(unsigned int) + 2 * m.numattrs * sizeof(int);
m.attrlist = (attribute *) malloc(sizeof(attribute) * m.numattrs);
printf("Header size: %d bytes\n", m.header_size);
int i, j;
for (i = 0; i < m.numattrs; i++)
{
fread(&m.attrlist[i].type, sizeof(int), 1, in);
fread(&m.attrlist[i].size, sizeof(int), 1, in);
m.record_size += m.attrlist[i].size;
}
printf("Record size: %d bytes\n", m.record_size);
_meta = m;
_meta.num_comp_attrs = numattrs;
_meta.comparable_attrs = attributes;
// max no. of disk blocks that fit in the buffer
int num_records_in_buffer = bufsize / m.record_size;
int num_records_in_block = DISK_BLOCK_SIZE / m.record_size;
int num_blocks_in_buffer = bufsize / DISK_BLOCK_SIZE;
printf("Records in block: %d\n", num_records_in_block);
printf("Blocks in buffer: %d \n", num_blocks_in_buffer);
// the buffer
char *buffer = (char *) malloc(bufsize);
// create the initial set of runs
size_t records_read, records_written;
i = 0;
char run_name[20];
FILE *out;
unsigned int num_recs_read = 0;
unsigned int num_recs_written = 0;
while (1)
{
records_read = fread(buffer, m.record_size,
num_records_in_buffer, in);
if (records_read == 0)
break;
num_recs_read += records_read;
// sort the chunk
qsort(buffer, records_read, m.record_size, compare_records);
// write sorted chunk to a run file
i++;
sprintf(run_name, "run_%d.bin", i);
out = fopen(run_name, "w");
write_metadata(out, m);
records_written = fwrite(buffer, m.record_size,
records_read, out);
num_recs_written += records_written;
fclose(out);
}
printf("Read %d records from input file.\n", num_recs_read);
// number of generated runs
int init_num_runs = i;
printf("Number of initial runs generated: %d\nMerging...\n", i);
/********************************* MERGE ********************************/
// number of runs in a pass
int num_runs = init_num_runs;
// max no. of runs we can merge at once
int max_merge_runs = (bufsize / DISK_BLOCK_SIZE) - 1;
int start = 1;
int interm_runp = 1;
char temp_run_name[20];
int runs_left_this_pass = num_runs;
// run multiple passes over the runs to merge into one file
while (1)
{
// check if we have to move to the next pass
if (runs_left_this_pass == 0)
{
start = 1;
interm_runp = 1;
int merged_at_once = (num_runs < max_merge_runs ? num_runs : max_merge_runs);
if (num_runs == merged_at_once)
{
// the final merged output of all runs is in run_1.bin
// rename it to 'outputfile'
rename("run_1.bin", outputfile);
printf("Done.\nOutput written to: %s\n", outputfile);
break;
}
num_runs = (num_runs / merged_at_once) + 1 * (num_runs % merged_at_once != 0);
runs_left_this_pass = num_runs;
}
// decide how many runs to merge at once
int merge_at_once = (runs_left_this_pass < max_merge_runs ? runs_left_this_pass : max_merge_runs);
sprintf(temp_run_name, "run_temp.bin");
// how much to read from one run file
int blocks_per_run = (bufsize / (merge_at_once + 1)) / DISK_BLOCK_SIZE;
int records_per_run = blocks_per_run * num_records_in_block;
int bytes_per_run = blocks_per_run * num_records_in_block * m.record_size;
// read from runs into buffer
FILE **runs = (FILE **) malloc(merge_at_once * sizeof(FILE *));
char **run_pointers = (char **) malloc((merge_at_once + 1) * sizeof(char *));
char **run_bp = (char **) malloc((merge_at_once + 1) * sizeof(char *));
char **run_ep = (char **) malloc((merge_at_once + 1) * sizeof(char *));
heap *HEAP = (heap *) malloc(sizeof(heap));
heap_init(HEAP, compare_records);
for (i = 0; i < merge_at_once; i++)
{
sprintf(run_name, "run_%d.bin", i + start);
runs[i] = fopen(run_name, "r");
fseek(runs[i], m.header_size, SEEK_SET);
char *buffer_start_addr = buffer + i * bytes_per_run;
fread(buffer_start_addr, num_records_in_block * m.record_size, blocks_per_run, runs[i]);
run_bp[i] = buffer_start_addr;
heap_push(HEAP, run_bp[i]);
run_pointers[i] = run_bp[i] + m.record_size;
run_ep[i] = run_bp[i] + bytes_per_run;
}
// the output buffer
run_bp[i] = run_pointers[i] = buffer + i * bytes_per_run;
// open a temporary run file for storing merged contents of all the generated runs
FILE *temp_out = fopen(temp_run_name, "w");
write_metadata(temp_out, m);
fclose(temp_out);
// read chunks of data from each run file into a heap
// and write to a temp run file until all the run files become empty
num_recs_read = 0;
int push = 0;
int read_from_runs = 0;
while (1)
{
// get min record from heap, and compute the index of the parent run
void *min_rec = heap_pop(HEAP);
if (!min_rec)
{
// write out whatever is left in the output buffer and exit
if (run_pointers[merge_at_once] > run_bp[merge_at_once])
{
FILE *output_run = fopen(temp_run_name, "a");
size_t check = fwrite(run_bp[merge_at_once],
1, run_pointers[merge_at_once] - run_bp[merge_at_once], output_run);
fclose(output_run);
// printf("Emptying output buffer, writing %d bytes\n", check);
}
break;
}
int min_rec_run = get_run_number(min_rec, run_bp, merge_at_once);
// copy the min record popped from the heap into the output run
memcpy(run_pointers[merge_at_once], min_rec, m.record_size);
run_pointers[merge_at_once] += m.record_size;
// fetch a new record from the run and push into heap,
// adjusting the run pointer
if (run_pointers[min_rec_run - 1] < run_ep[min_rec_run - 1])
{
heap_push(HEAP, run_pointers[min_rec_run - 1]);
push++;
run_pointers[min_rec_run - 1] += m.record_size;
}
else if (run_pointers[min_rec_run - 1] >= run_ep[min_rec_run - 1])
{
size_t check = fread(run_bp[min_rec_run - 1],
m.record_size, records_per_run, runs[min_rec_run - 1]);
read_from_runs += check;
// reset the run pointer
if (check > 0)
{
run_pointers[min_rec_run - 1] = run_bp[min_rec_run - 1];
run_ep[min_rec_run - 1] = run_bp[min_rec_run - 1] + check * m.record_size;
// push first record from fresh lot into heap
heap_push(HEAP, run_pointers[min_rec_run - 1]);
push++;
run_pointers[min_rec_run - 1] += m.record_size;
}
}
// check for overflow of the output run,
// write to output run if an overflow is detected
if (run_pointers[merge_at_once] >= run_bp[merge_at_once] + bytes_per_run)
{
FILE *output_run = fopen(temp_run_name, "a");
size_t check = fwrite(run_bp[merge_at_once],
m.record_size, records_per_run, output_run);
fclose(output_run);
num_recs_read += check;
// reset the run pointer
run_pointers[merge_at_once] = run_bp[merge_at_once];
}
}
// clean up
heap_destroy(HEAP);
free(HEAP);
free(run_ep);
free(run_bp);
free(run_pointers);
for (i = 0; i < merge_at_once; i++)
fclose(runs[i]);
free(runs);
// delete run files used to build this run file
for (i = 0; i < merge_at_once; i++)
{
sprintf(run_name, "run_%d.bin", i + start);
remove(run_name);
}
// compute number of runs left in this pass
runs_left_this_pass -= merge_at_once;
start += merge_at_once;
// rename temp file to run_<index>.bin
sprintf(run_name, "run_%d.bin", interm_runp);
interm_runp++;
rename(temp_run_name, run_name);
// printf("Merged %d - %d into %s\n", start - merge_at_once, start - 1, run_name);
}
/************************* END MERGE *************************************/
// clean up
free(m.attrlist);
free(buffer);
// close the input data file
fclose(in);
}