void possect_add(u32 id, Pvoid_t tokens) { static growing_glist *ixlist; static glist **lists; Word_t *ptr; Word_t idx = 0; uint ix_cnt = 0; uint i = 0; toc_e toc; if (!ixlist){ GGLIST_INIT(ixlist, 100); } /* write TOC entry */ toc.offs = ftello64(data_f); toc.val = id; fwrite(&toc, sizeof(toc_e), 1, toc_f); /* write ixeme list */ ixlist->lst.len = 0; JLF(ptr, tokens, idx); while (ptr != NULL){ ++ix_cnt; GGLIST_APPEND(ixlist, idx); JLN(ptr, tokens, idx); } lists = xmalloc((ix_cnt + 1) * sizeof(glist*)); lists[i++] = &ixlist->lst; /* write ixeme positions */ idx = 0; JLF(ptr, tokens, idx); while (ptr != NULL){ growing_glist *g = (growing_glist*)*ptr; lists[i++] = &g->lst; JLN(ptr, tokens, idx); } encode_poslists(lists, ix_cnt + 1); /* free stuff */ idx = 0; JLF(ptr, tokens, idx); while (ptr != NULL){ growing_glist *g = (growing_glist*)*ptr; free(g); JLN(ptr, tokens, idx); } }
int network_manager_destroy(struct network_manager* nm) { if(nm->listening_sockets != NULL) { size_t num_interfaces = chaind_config(nm->chaind)->network.num_interfaces; for(size_t i = 0; i < num_interfaces; i++) { close(nm->listening_sockets[i]); } free(nm->listening_sockets); } int rc; Word_t wrc; JHSFA(wrc, nm->claimed_invs); JHSFA(wrc, nm->wanted_invs_by_inv); struct inv** pinv = NULL; Word_t index = 0; JLF(pinv, nm->block_inv_list, index); while(pinv != NULL) { free(*pinv); JLN(pinv, nm->block_inv_list, index); } JLFA(wrc, nm->block_inv_list); index = 0; JLF(pinv, nm->tx_inv_list, index); while(pinv != NULL) { free(*pinv); JLN(pinv, nm->tx_inv_list, index); } JLFA(wrc, nm->tx_inv_list); if(nm->peer_discovery != NULL) { peer_discovery_done(nm->peer_discovery); nm->peer_discovery = NULL; } JLFA(rc, nm->poll_socket_by_peer); vector_free(&nm->poll_fds); // TODO free nm->peer_list // TODO free nm->peer_by_address free(nm); return 0; }
pq_overload_iterator pq_overload_new_iterator(const pq_overload *overload) { return (pq_overload_iterator) { .signature_index = -1, .argnum_index = 0, .iterating_non_variadic = 1, }; } static Pvoid_t _pq_overload_next_lookup_table(const pq_overload *overload, pq_overload_iterator *it) { Pvoid_t table = it->iterating_non_variadic ? overload->function_table : overload->variadic_function_table; Word_t *pvalue; if(it->argnum_index == -1) { it->argnum_index = 0; JLF(pvalue, table, it->argnum_index); } else { JLN(pvalue, table, it->argnum_index); } it->signature_index = -1; if(pvalue) return (Pvoid_t)*pvalue; else if(it->iterating_non_variadic) { it->iterating_non_variadic = 0; it->argnum_index = -1; return _pq_overload_next_lookup_table(overload, it); } else return NULL; }
static int get_pollfds(struct dbus* dbus, struct pollfd *pollfds) { int count = 0; Word_t index = 0; DBusWatch** pwatch; JLF(pwatch, dbus->watches, index); while(pwatch != NULL) { DBusWatch* watch = *pwatch; if(dbus_watch_get_enabled(watch)) { pollfds[count].fd = dbus_watch_get_unix_fd(watch); pollfds[count].events = 0; int flags = dbus_watch_get_flags(watch); if(flags & DBUS_WATCH_READABLE) pollfds[count].events |= POLLIN | POLLPRI; if(flags & DBUS_WATCH_WRITABLE) pollfds[count].events |= POLLOUT; count += 1; } JLN(pwatch, dbus->watches, index); } return count; }
static void reverse_lookup(u32 id) { Pvoid_t qexp = judify_qexp(); Word_t idx = 0; Word_t *ptr = NULL; uint i = 0; JLF(ptr, qexp, idx); printf("ID: %d is expanded from: ", id); while (ptr != NULL) { const glist *lst = (const glist*)*ptr; uint j; for (j = 0; j < lst->len; j++) { if (lst->lst[j] == id) { printf("%u ", (u32)idx); ++i; } if (lst->lst[j] >= id) break; } JLN(ptr, qexp, idx); } if (!i) printf("(null)\n"); else printf("\n"); }
int dogma_free_context(dogma_context_t* ctx) { dogma_drone_context_t** value; dogma_key_t index = 0; int ret; if(ctx->fleet != NULL) { bool found; DOGMA_ASSUME_OK(dogma_remove_fleet_member(ctx->fleet, ctx, &found)); assert(found == true && ctx->fleet == NULL); } dogma_free_env(ctx, ctx->character); dogma_free_env(ctx, ctx->gang); dogma_reset_skill_levels(ctx); JLF(value, ctx->drone_map, index); while(value != NULL) { /* The drone environments were freed when char was freed */ free(*value); JLN(value, ctx->drone_map, index); } JLFA(ret, ctx->drone_map); free(ctx); return DOGMA_OK; }
static void save_pjarray (Pvoid_t PJArray) { FILE* dat[2]; int const buf_size = (1 << 20); PWord_t buf[2]; int buf_pos[2] = {0,0}; for (uint8_t d = L; d < N; ++d) { char fname[100 + 1]; snprintf (fname, 100, "hex.%i.%s.dat", SIZE, show_player[d]); dat[d] = fopen (fname, "wb+"); if (!dat[d]) { fprintf ( stderr, "could not open dat file %s: %s\n" , fname, strerror (errno) ); exit (EXIT_FAILURE); } buf[d] = malloc (buf_size * sizeof (Word_t)); if (!buf[d]) { fprintf (stderr, "could not allocate memory\n"); exit (EXIT_FAILURE); } } Word_t Index = 0; PWord_t PValue; JLF (PValue, PJArray, Index); while (PValue) { buf[*PValue][buf_pos[*PValue]++] = Index; if (buf_pos[*PValue] == buf_size) { WRITE (*PValue); } JLN (PValue, PJArray, Index); } for (uint8_t d = L; d < N; ++d) { WRITE (d); free (buf[d]); fclose (dat[d]); } }
int dogma_set_env_state(dogma_context_t* ctx, dogma_env_t* env, dogma_state_t newstate) { dogma_array_t enveffects; dogma_key_t index = 0; const dogma_type_effect_t** te; const dogma_effect_t* e; dogma_expctx_t result; if(env->state == newstate) return DOGMA_OK; DOGMA_ASSUME_OK(dogma_get_type_effects(env->id, &enveffects)); JLF(te, enveffects, index); while(te != NULL) { DOGMA_ASSUME_OK(dogma_get_effect((*te)->effectid, &e)); JLN(te, enveffects, index); if(e->fittingusagechanceattributeid > 0) { /* Effect is chance-based */ if(newstate > 0) { continue; } /* When unplugging the environment, turn off all * chance-based effects as a precautionary measure */ bool* v; int ret; JLG(v, env->chance_effects, e->id); if(v != NULL) { assert(*v == true); JLD(ret, env->chance_effects, e->id); DOGMA_ASSUME_OK(dogma_eval_expression( ctx, env, e->postexpressionid, &result )); } continue; } if((newstate >> e->category) & 1) { if(!((env->state >> e->category) & 1)) { DOGMA_ASSUME_OK(dogma_eval_expression( ctx, env, e->preexpressionid, &result )); } } else if((env->state >> e->category) & 1) { DOGMA_ASSUME_OK(dogma_eval_expression( ctx, env, e->postexpressionid, &result )); } }
switch_status_t switch_api_l3_route_entries_get( switch_l3_table_iterator_fn iterator_fn) { void *temp = NULL; switch_handle_t vrf_handle = 0; switch_status_t status = SWITCH_STATUS_SUCCESS; JLF(temp, switch_vrf_v4_routes, vrf_handle); while (temp) { status = switch_api_l3_v4_route_entries_get_by_vrf(vrf_handle, iterator_fn); JLN(temp, switch_vrf_v4_routes, vrf_handle); } vrf_handle = 0; JLF(temp, switch_vrf_v6_routes, vrf_handle); while (temp) { status = switch_api_l3_v6_route_entries_get_by_vrf(vrf_handle, iterator_fn); JLN(temp, switch_vrf_v6_routes, vrf_handle); } return status; }
void jtableL_iterate(jtableL *table, jtableL_cb f, void *param) { PWord_t PValue; Word_t key = 0UL; JLF(PValue, table->t, key); while (PValue) { f(key, (void *)*PValue, param); JLN(PValue, table->t, key); } }
int dogma_get_location_env(dogma_context_t* ctx, dogma_location_t location, dogma_env_t** env) { dogma_env_t** env1; dogma_env_t** env2; dogma_drone_context_t** drone_env1; dogma_key_t index = 0; switch(location.type) { case DOGMA_LOC_Char: *env = ctx->character; return DOGMA_OK; case DOGMA_LOC_Implant: JLG(env1, ctx->character->children, location.implant_index); if(env1 == NULL) return DOGMA_NOT_FOUND; *env = *env1; return DOGMA_OK; case DOGMA_LOC_Skill: JLG(env1, ctx->character->children, location.skill_typeid); if(env1 == NULL) return DOGMA_NOT_FOUND; *env = *env1; return DOGMA_OK; case DOGMA_LOC_Ship: *env = ctx->ship; return DOGMA_OK; case DOGMA_LOC_Module: JLG(env1, ctx->ship->children, location.module_index); if(env1 == NULL) return DOGMA_NOT_FOUND; *env = *env1; return DOGMA_OK; case DOGMA_LOC_Charge: JLG(env1, ctx->ship->children, location.module_index); if(env1 == NULL) return DOGMA_NOT_FOUND; JLF(env2, (*env1)->children, index); if(env2 == NULL) return DOGMA_NOT_FOUND; *env = *env2; return DOGMA_OK; case DOGMA_LOC_Drone: JLG(drone_env1, ctx->drone_map, location.drone_typeid); if(drone_env1 == NULL) return DOGMA_NOT_FOUND; *env = (*drone_env1)->drone; assert(*env != NULL); return DOGMA_OK; default: return DOGMA_NOT_FOUND; } }
static void update_peers(struct network_manager* nm) { // Perform the select first fd_set read_fds = nm->poll_read_fds, write_fds = nm->poll_write_fds, exception_fds = nm->poll_exception_fds; struct timeval timeout; timeout.tv_sec = 0; timeout.tv_usec = 0; if(select(nm->poll_max_fd + 1, &read_fds, &write_fds, &exception_fds, &timeout) < 0) { // weird error? perror("select"); return; } // Then loop over all peers passing in the select status struct network_peer** ppeer = NULL; Word_t index = 0; struct vector disconnected_peers; vector_init(&disconnected_peers); JLF(ppeer, nm->peer_list, index); while(ppeer != NULL) { int action_flags = 0; int* psock = NULL; uintptr_t p = (uintptr_t)*ppeer; JLG(psock, nm->poll_socket_by_peer, p); if(psock != NULL) { if(FD_ISSET(*psock, &read_fds)) action_flags |= NETWORK_PEER_ACTION_FLAGS_READ; if(FD_ISSET(*psock, &write_fds)) action_flags |= NETWORK_PEER_ACTION_FLAGS_WRITE; if(FD_ISSET(*psock, &exception_fds)) action_flags |= NETWORK_PEER_ACTION_FLAGS_EXCEPTION; } network_peer_update(*ppeer, action_flags); if(network_peer_disconnected(*ppeer) == 1) { vector_add(&disconnected_peers, (uintptr_t)*ppeer); } JLN(ppeer, nm->peer_list, index); } size_t num_disconnected = vector_count(&disconnected_peers); for(size_t i = 0; i < num_disconnected; i++) { struct network_peer* peer = (struct network_peer*)vector_get(&disconnected_peers, i); stop_peer(nm, peer); } vector_free(&disconnected_peers); }
void pq_overload_destroy(pq_context *ctx, pq_overload *overload) { Word_t bytes, index, *pvalue; Pvoid_t table; index = 0; JLF(pvalue, overload->variadic_function_table, index); while(pvalue != NULL) { table = (Pvoid_t)*pvalue; JLFA(bytes, table); JLN(pvalue, overload->variadic_function_table, index); } JLFA(bytes, overload->variadic_function_table); index = 0; JLF(pvalue, overload->function_table, index); while(pvalue != NULL) { table = (Pvoid_t)*pvalue; JLFA(bytes, table); JLN(pvalue, overload->function_table, index); } JLFA(bytes, overload->function_table); }
int dbus_update(struct dbus* dbus) { DBusDispatchStatus status; // handle watches Word_t watch_count = 0; JLC(watch_count, dbus->watches, 0, -1); struct pollfd* pollfds = (struct pollfd*)alloca(sizeof(struct pollfd) * watch_count); int fdcount = get_pollfds(dbus, pollfds); if(poll(pollfds, fdcount, 0) < 0) { return -1; } // process the watches DBusWatch** pwatch; Word_t index = 0; int c = 0; JLF(pwatch, dbus->watches, index); while(pwatch != NULL) { struct pollfd* poll_result = &pollfds[c]; struct DBusWatch* watch = *pwatch; if(dbus_watch_get_enabled(watch)) { assert(poll_result->fd == dbus_watch_get_unix_fd(watch)); int flags = 0; int revents = poll_result->revents; if((revents & POLLIN) != 0) flags |= DBUS_WATCH_READABLE; if((revents & POLLOUT) != 0) flags |= DBUS_WATCH_WRITABLE; if((revents & POLLERR) != 0) flags |= DBUS_WATCH_ERROR; if((revents & POLLHUP) != 0) flags |= DBUS_WATCH_HANGUP; if(flags != 0) dbus_watch_handle(watch, flags); c++; } JLN(pwatch, dbus->watches, index); } // dispatch incoming messages while((status = dbus_connection_get_dispatch_status(dbus->conn)) != DBUS_DISPATCH_COMPLETE) { dbus_connection_dispatch(dbus->conn); } // Send outgoing messages if(dbus_connection_has_messages_to_send(dbus->conn)) { dbus_connection_flush(dbus->conn); } return 0; }
switch_status_t switch_api_l3_v6_routes_print_all(void) { switch_handle_t vrf_handle = 0; void *temp = NULL; switch_status_t status = SWITCH_STATUS_SUCCESS; JLF(temp, switch_vrf_v6_routes, vrf_handle); while (temp) { status = switch_api_l3_v6_routes_print_by_vrf(vrf_handle); JLN(temp, switch_vrf_v6_routes, vrf_handle); } return status; }
static void destroy_node(node_t *node) { Word_t index = 0; Word_t *pnode; Word_t rc_word; JLF(pnode, node->PJLarray_branches, index); while (pnode != NULL) { destroy_node((node_t *)*pnode); JLN(pnode, node->PJLarray_branches, index); } JLFA(rc_word, node->PJLarray_branches); JLFA(rc_word, node->PJLarray_prefixes); switch_free(node); }
static void switch_packet_tx_from_hosts(fd_set read_fds) { switch_hostif_info_t *hostif_info = NULL; void *temp = NULL; Word_t index = 0; JLF(temp, switch_intf_fd_array, index); while (temp) { hostif_info = (switch_hostif_info_t *)(*(unsigned long *)temp); if (FD_ISSET(hostif_info->intf_fd, &read_fds)) { switch_packet_tx_from_host(hostif_info->intf_fd); } JLN(temp, switch_intf_fd_array, index); } }
void free_p_entries(my_int len_sa) { my_int i; Word_t Bytes; idx=0; JLF(pval,judy,idx); while (1) { if (pval==NULL) break; J1FA(Bytes,*pval); JLN(pval,judy,idx); } JLFA(Bytes,judy); }
static uint encode_tokens(const Pvoid_t tokens) { static growing_glist *tok; static char *buf; static uint buf_len; uint i, max_pos = 0; if (!tok){ GGLIST_INIT(tok, 1000); } tok->lst.len = 0; Word_t *ptr; Word_t xid = 0; JLF(ptr, tokens, xid); while (ptr){ u32 x = xid; GGLIST_APPEND(tok, x); const growing_glist *g = (const growing_glist*)*ptr; for (i = 0; i < g->lst.len; i++) if (g->lst.lst[i] > max_pos) max_pos = g->lst.lst[i]; JLN(ptr, tokens, xid); } u32 offs = 0; uint need = rice_encode(NULL, &offs, tok->lst.lst, tok->lst.len); uint need_bytes = BITS_TO_BYTES(need); if (need_bytes > buf_len){ free(buf); buf_len = need_bytes; buf = xmalloc(buf_len + 12); memset(buf, 0, buf_len + 12); } offs = 0; elias_gamma_write(buf, &offs, need); rice_encode(buf, &offs, tok->lst.lst, tok->lst.len); offs = BITS_TO_BYTES(offs); fwrite(buf, offs, 1, data_f); memset(buf, 0, offs); return max_pos; }
static void print_all() { Pvoid_t qexp = judify_qexp(); Word_t idx = 0; Word_t *ptr = NULL; JLF(ptr, qexp, idx); while (ptr != NULL) { const glist *lst = (const glist*)*ptr; printf("ID: %d expands to: ", (u32)idx); print_glist(lst); JLN(ptr, qexp, idx); } }
static void remove_watch(DBusWatch* watch, void *userdata) { struct dbus* dbus = (struct dbus*)userdata; Word_t index = 0; DBusWatch** pwatch; JLF(pwatch, dbus->watches, index); while(pwatch != NULL) { if(*pwatch == watch) { Word_t wrc; JLD(wrc, dbus->watches, index); break; } JLN(pwatch, dbus->watches, index); } }
// returns false if empty vector, else true bool string_first(l3obj* obj, char** val, long* index) { assert(obj->_type == t_str); LIVEO(obj); assert(index); assert(val); Word_t * PValue; // pointer to array element value Word_t Index = 0; JLF(PValue, obj->_judyL, Index); if (PValue) { *index = Index; *val = *(char**)(PValue); return true; } return false; }
my_int p_size(my_int len_sa) { my_int i; Word_t Bytes; Word_t total=0; idx=0; JLF(pval,judy,idx); while (1) { if (pval==NULL) break; J1MU(Bytes,*pval); total+=Bytes; JLN(pval,judy,idx); } JLMU(Bytes,judy); total+=Bytes; return total; }
static int switch_packet_select_fd_get(fd_set *read_fds) { switch_hostif_info_t *hostif_info = NULL; void *temp = NULL; int nfds; Word_t index = 0; nfds = (cpu_sock_fd > pipe_fd[0]) ? cpu_sock_fd : pipe_fd[0]; JLF(temp, switch_intf_fd_array, index); while (temp) { hostif_info = (switch_hostif_info_t *)(*(unsigned long *)temp); FD_SET(hostif_info->intf_fd, read_fds); if (hostif_info->intf_fd > nfds) { nfds = hostif_info->intf_fd; } JLN(temp, switch_intf_fd_array, index); } return nfds + 1; }
pq_value *pq_overload_next_function(const pq_overload *overload, pq_overload_iterator *it) { Word_t *pvalue; Pvoid_t table; table = _pq_overload_current_lookup_table(overload, it); do { if(it->signature_index == -1) { it->signature_index = 0; JLF(pvalue, table, it->signature_index); } else { JLN(pvalue, table, it->signature_index); } if(pvalue) { return (pq_value *)*pvalue; } else { table = _pq_overload_next_lookup_table(overload, it); } } while(table); return NULL; }
static inline pq_value *_pq_overload_for_types_non_variadic(pq_context *ctx, pq_overload *overload, size_t n, pq_type **types) { Word_t *pvalue; pq_value *best_match; int best_score = -1, score; Pvoid_t by_argnum_table = overload->function_table, by_type_table; JLG(pvalue, by_argnum_table, n); if(pvalue != NULL && pvalue != PJERR ) { by_type_table = (Pvoid_t)*pvalue; Word_t index = 0; JLF(pvalue, by_type_table, index); while(pvalue != NULL) { score = _pq_signature_match_score((pq_type *)index, n, types); if(score > best_score) { best_score = score; best_match = (pq_value *)*pvalue; } JLN(pvalue, by_type_table, index); } if(best_score >= 0) return best_match; } return NULL; }
int dogma_init_context(dogma_context_t** ctx) { dogma_context_t* new_ctx = malloc(sizeof(dogma_context_t)); dogma_env_t** value; dogma_key_t index = 0; const dogma_type_t** type; new_ctx->gang = malloc(sizeof(dogma_env_t)); new_ctx->character = malloc(sizeof(dogma_env_t)); new_ctx->ship = malloc(sizeof(dogma_env_t)); new_ctx->area = NULL; new_ctx->fleet = NULL; DOGMA_INIT_ENV(new_ctx->gang, 0, NULL, 0, new_ctx); DOGMA_INIT_ENV(new_ctx->character, 0, NULL, 0, new_ctx); DOGMA_INIT_ENV(new_ctx->ship, 0, new_ctx->character, 0, new_ctx); JLI(value, new_ctx->character->children, 0); *value = new_ctx->ship; new_ctx->default_skill_level = DOGMA_MAX_SKILL_LEVEL; new_ctx->skill_levels = (dogma_array_t)NULL; new_ctx->drone_map = (dogma_array_t)NULL; *ctx = new_ctx; /* Inject all skills. This is somewhat costly, maybe there is a * way to do it lazily? */ JLF(type, types_by_id, index); while(type != NULL) { if((*type)->categoryid == CAT_Skill) { dogma_inject_skill(*ctx, (*type)->id); } JLN(type, types_by_id, index); } return DOGMA_OK; }
//void possect_add_old(u32 key, const Pvoid_t tokens); void possect_add(const Pvoid_t tokens) { static char *posbuf; static char *xidbuf; if (!posbuf){ posbuf = xmalloc(iblock_size * 8 + 8); memset(posbuf, 0, iblock_size * 8 + 8); xidbuf = xmalloc((iblock_size * 2 + 3) * 8); memset(xidbuf, 0, (iblock_size * 2 + 3) * 8); } Word_t *ptr; u32 len, xoffs, poffs, prev_offs, prev_xid, xsum, xmax, osum, omax; Word_t xid = poffs = prev_offs = prev_xid =\ xsum = osum = osum = omax = xmax = len = 0; JLF(ptr, tokens, xid); while (ptr){ growing_glist *g = (growing_glist*)*ptr; *ptr = poffs; rice_encode(posbuf, &poffs, g->lst.lst, g->lst.len); free(g); xsum += xid - prev_xid; osum += poffs - prev_offs; if (xid - prev_xid > xmax) xmax = xid - prev_xid; if (poffs - prev_offs > omax) omax = poffs - prev_offs; prev_xid = xid; prev_offs = poffs; ++len; JLN(ptr, tokens, xid); } poffs = BITS_TO_BYTES(poffs); xid = prev_xid = prev_offs = xoffs = 0; elias_gamma_write(xidbuf, &xoffs, len + 1); if (!len) goto write_buffers; u32 rice_f1 = estimate_rice_f_param(xsum, xmax, len); u32 rice_f2 = estimate_rice_f_param(osum, omax, len); rice_write(xidbuf, &xoffs, rice_f1, 2); rice_write(xidbuf, &xoffs, rice_f2, 2); JLF(ptr, tokens, xid); while (ptr){ rice_write(xidbuf, &xoffs, xid - prev_xid, rice_f1); prev_xid = xid; rice_write(xidbuf, &xoffs, *ptr - prev_offs, rice_f2); prev_offs = *ptr; JLN(ptr, tokens, xid); } write_buffers: xoffs = BITS_TO_BYTES(xoffs); toc_e toc; toc.offs = ftello64(data_f); toc.val = xoffs; fwrite(&toc, sizeof(toc_e), 1, toc_f); fwrite(xidbuf, xoffs, 1, data_f); fwrite(posbuf, poffs, 1, data_f); memset(xidbuf, 0, xoffs); memset(posbuf, 0, poffs); }
/* This functions codes all ixeme occurrences in a segment in the correct * order. The catch is that instead of using global ixeme IDs, we use a * minimal local set of IDs, namely integers [1..|S|] for |S| individual * ixemes. Mapping between the IDs is implictely saved in the ascending * order of ixeme IDs, obtainable from the forward index. */ void possect_add(const Pvoid_t tokens) { static uint *offsets; //static uint offsets_len; static char *encode_buf; //static uint buf_len; if (!offsets){ offsets = xmalloc(segment_size * sizeof(uint)); encode_buf = xmalloc(segment_size * 8); memset(encode_buf, 0, segment_size * 8); } memset(offsets, 0, segment_size * sizeof(uint)); //ENSURE_IBLOCK Word_t max = 0; JLC(max, tokens, 0, -1); /* write TOC entry */ toc_e toc; toc.offs = ftello64(data_f); toc.val = max; fwrite(&toc, sizeof(toc_e), 1, toc_f); /* write token list */ uint max_pos = encode_tokens(tokens); /* if (segment_size > offsets_len){ free(offsets); offsets_len = segment_size; offsets = xmalloc(offsets_len * sizeof(uint)); } memset(offsets, 0, offsets_len * sizeof(uint)); */ /* estimate rice_f */ u32 rice_f, offs = 0; if (max) rice_f = estimate_rice_f((u32*)&max, 1); else rice_f = 2; /* allocate bits to positions */ Word_t *ptr; //uint max_pos = 0; int i, j, c = 0; Word_t xid = 0; JLF(ptr, tokens, xid); while (ptr){ offs = 0; rice_write(NULL, &offs, ++c, rice_f); const growing_glist *g = (const growing_glist*)*ptr; for (i = 0; i < g->lst.len; i++){ offsets[g->lst.lst[i] - 1] = offs; if (g->lst.lst[i] > max_pos) max_pos = g->lst.lst[i]; } JLN(ptr, tokens, xid); } /* compute bit offset for each position */ offs = 0; elias_gamma_write(NULL, &offs, max_pos + 1); rice_write(NULL, &offs, rice_f, 2); for (i = 0; i < max_pos; i++){ uint tmp = offsets[i]; if (!tmp) dub_die("A hole in poslist at %u / %u!", i, max_pos); offsets[i] = offs; offs += tmp; } u32 endpos = offs; //u32 total_bits = offs; //elias_gamma_write(NULL, &offs, total_bits); //rice_write(NULL, &offs, 0, rice_f); u32 total_bytes = BITS_TO_BYTES(offs); if (total_bytes > segment_size * 8) dub_die("Pos buffer too small! %u > %u.", total_bytes, segment_size * 8); /* if (total_bytes > buf_len){ free(encode_buf); buf_len = total_bytes; encode_buf = xmalloc(buf_len + 4); memset(encode_buf, 0, buf_len + 4); } */ /* write codes */ offs = 0; elias_gamma_write(encode_buf, &offs, max_pos + 1); rice_write(encode_buf, &offs, rice_f, 2); //elias_gamma_write(encode_buf, &offs, total_bits); xid = c = j = 0; JLF(ptr, tokens, xid); while (ptr){ ++c; growing_glist *g = (growing_glist*)*ptr; for (i = 0; i < g->lst.len; i++){ //u32 d = offsets[g->lst.lst[i] - 1]; //dub_msg("OFFS %u VAL %u", d, c); rice_write(encode_buf, &offsets[g->lst.lst[i] - 1], c, rice_f); //u32 dd = rice_read(encode_buf, &d, rice_f); //if (c != dd) // dub_die("POKS! %u, %u != %u", key, dd, c); ++j; } free(g); JLN(ptr, tokens, xid); } //rice_write(encode_buf, &endpos, 0, rice_f); fwrite(encode_buf, total_bytes, 1, data_f); memset(encode_buf, 0, total_bytes); }
int main(int argc, char *argv[]) { if (argc != 8) { fprintf(stderr, "Usage: ./merge_N [out_dir_name 1] [item_#_file 2] [length_#_file 3] [count_dist_file 4] [number_of_temp_prefix 5] [from_temp_n] [to_temp_n]\n"); // /root/cx_src/src/merge_N /tmp/data /tmp/result/ino.txt /tmp/result/lno.txt /tmp/result/cdo.txt 3 0 1 return -1; } if ((itf = fopen(argv[2], "a")) == NULL) { fprintf(stderr, "Failed to open file \"%s\" for writing item numbers\n", argv[2]); return -1; } // fixing each temp file fprintf(stdout, "Start fixing temp files\n"); for (i = atoi(argv[6]); i <= atoi(argv[7]); ++i) { //for (i = 0; i < TEMP_N; ++i) { // fix the temp file i int temp_prefix_num = 0;// num of concurrent threads while (temp_prefix_num < atoi(argv[5])){ sprintf(buffer, "%s/%s%d-%d.txt", argv[1], TEMP_PREFIX, i, temp_prefix_num); fprintf(stdout, "\rWorking on temp file: \"%s\" \n", buffer); //fflush(stdout); if ((tsf[i] = fopen(buffer, "r")) == NULL) { fprintf(stderr, "Failed to open file \"%s\" for reading temp strings\n", buffer); //continue; //break; return -1; } while (fscanf(tsf[i], "%"PRId64"\t", &itemn) != EOF) { fgets(Index, BUFFER_SIZE, tsf[i]); for (Len = strlen(Index) - 1; Index[Len] == '\n' || Index[Len] == '\r'; Len--) Index[Len] = 0; ++Len; JSLI(PValNgramS, PJSLNgram, (uint8_t *)Index); if (PValNgramS == PJERR) { fprintf(stderr, "Malloc failed for \"PJSLNgram\"\n"); //return -1; } (*PValNgramS) += itemn; JLI(PValTotC, PJLTotCount, Len); if (PValTotC == PJERR) { fprintf(stderr, "Malloc failed for \"PJLTotCount\"\n"); //return -1; } *PValTotC += itemn; if (*PValNgramS == itemn) { JLI(PValNgramC, PJLNgramCount, Len); if (PValNgramC == PJERR) { fprintf(stderr, "Malloc failed for \"PJLNgramCount\"\n"); //return -1; } ++*PValNgramC; } } sprintf(buffer, "rm %s/%s%d-%d.txt", argv[1], TEMP_PREFIX, i, temp_prefix_num++); if (system(buffer) == -1) { fprintf(stderr, "Failed to execute command: \"%s\"\n", buffer); //return -1; } fclose(tsf[i]); } // write the final temp file sprintf(buffer, "%s/%s%d.txt", argv[1], TEMP_PREFIX, i); if ((tsf[i] = fopen(buffer, "w")) == NULL) { fprintf(stderr, "Failed to open file \"%s\" for writing temp strings\n", buffer); return -1; } Index[0] = '\0'; JSLF(PValNgramS, PJSLNgram, (uint8_t *)Index); while (PValNgramS != NULL) { fprintf(tsf[i], "%lu\t%s\n", *PValNgramS, Index); Count = *PValNgramS; JLI(PValCountC, PJLCountCount, Count); if (PValCountC == PJERR) { fprintf(stderr, "Malloc failed for \"PJLCountCount\"\n"); return -1; } ++*PValCountC; JSLN(PValNgramS, PJSLNgram, (uint8_t *)Index); } JSLFA(Bytes, PJSLNgram); fflush(tsf[i]); fclose(tsf[i]); fprintf(itf, "Temp file \"%s/%s%d\" uses %lu Bytes of memory\n", argv[1], TEMP_PREFIX, i, Bytes); fflush(itf); } fclose(itf); if ((lef = fopen(argv[3], "a")) == NULL) { fprintf(stderr, "Failed to open file \"%s\" for writing length number\n", argv[3]); return -1; } Total = NgramN = 0; JLF(PValTotC, PJLTotCount, Total); JLF(PValNgramC, PJLNgramCount, NgramN); while (PValTotC != NULL) { fprintf(lef, "%lu\t%lu\t%lu\n", Total, *PValNgramC, *PValTotC); JLN(PValTotC, PJLTotCount, Total); JLN(PValNgramC, PJLNgramCount, NgramN); } JLFA(Bytes, PJLTotCount); JLFA(Bytes, PJLNgramCount); fflush(lef); fclose(lef); if ((cdf = fopen(argv[4], "a")) == NULL) { fprintf(stderr, "Failed to open file \"%s\" for writing count distribuction\n", argv[4]); return -1; } Count = 0; JLF(PValCountC, PJLCountCount, Count); while (PValCountC != NULL) { fprintf(cdf, "%lu\t%lu\n", Count, *PValCountC); JLN(PValCountC, PJLCountCount, Count); } JLFA(Bytes, PJLCountCount); fflush(cdf); fclose(cdf); return 0; }