void possect_add(u32 id, Pvoid_t tokens)
{
static growing_glist *ixlist;
static glist **lists;
Word_t *ptr;
Word_t idx = 0;
uint ix_cnt = 0;
uint i = 0;
toc_e toc;
if (!ixlist){
GGLIST_INIT(ixlist, 100);
}
/* write TOC entry */
toc.offs = ftello64(data_f);
toc.val = id;
fwrite(&toc, sizeof(toc_e), 1, toc_f);
/* write ixeme list */
ixlist->lst.len = 0;
JLF(ptr, tokens, idx);
while (ptr != NULL){
++ix_cnt;
GGLIST_APPEND(ixlist, idx);
JLN(ptr, tokens, idx);
}
lists = xmalloc((ix_cnt + 1) * sizeof(glist*));
lists[i++] = &ixlist->lst;
/* write ixeme positions */
idx = 0;
JLF(ptr, tokens, idx);
while (ptr != NULL){
growing_glist *g = (growing_glist*)*ptr;
lists[i++] = &g->lst;
JLN(ptr, tokens, idx);
}
encode_poslists(lists, ix_cnt + 1);
/* free stuff */
idx = 0;
JLF(ptr, tokens, idx);
while (ptr != NULL){
growing_glist *g = (growing_glist*)*ptr;
free(g);
JLN(ptr, tokens, idx);
}
}
int network_manager_destroy(struct network_manager* nm)
{
if(nm->listening_sockets != NULL) {
size_t num_interfaces = chaind_config(nm->chaind)->network.num_interfaces;
for(size_t i = 0; i < num_interfaces; i++) {
close(nm->listening_sockets[i]);
}
free(nm->listening_sockets);
}
int rc;
Word_t wrc;
JHSFA(wrc, nm->claimed_invs);
JHSFA(wrc, nm->wanted_invs_by_inv);
struct inv** pinv = NULL;
Word_t index = 0;
JLF(pinv, nm->block_inv_list, index);
while(pinv != NULL) {
free(*pinv);
JLN(pinv, nm->block_inv_list, index);
}
JLFA(wrc, nm->block_inv_list);
index = 0;
JLF(pinv, nm->tx_inv_list, index);
while(pinv != NULL) {
free(*pinv);
JLN(pinv, nm->tx_inv_list, index);
}
JLFA(wrc, nm->tx_inv_list);
if(nm->peer_discovery != NULL) {
peer_discovery_done(nm->peer_discovery);
nm->peer_discovery = NULL;
}
JLFA(rc, nm->poll_socket_by_peer);
vector_free(&nm->poll_fds);
// TODO free nm->peer_list
// TODO free nm->peer_by_address
free(nm);
return 0;
}
pq_overload_iterator pq_overload_new_iterator(const pq_overload *overload) {
return (pq_overload_iterator) {
.signature_index = -1,
.argnum_index = 0,
.iterating_non_variadic = 1,
};
}
static Pvoid_t _pq_overload_next_lookup_table(const pq_overload *overload, pq_overload_iterator *it) {
Pvoid_t table = it->iterating_non_variadic ? overload->function_table : overload->variadic_function_table;
Word_t *pvalue;
if(it->argnum_index == -1) {
it->argnum_index = 0;
JLF(pvalue, table, it->argnum_index);
}
else {
JLN(pvalue, table, it->argnum_index);
}
it->signature_index = -1;
if(pvalue) return (Pvoid_t)*pvalue;
else if(it->iterating_non_variadic) {
it->iterating_non_variadic = 0;
it->argnum_index = -1;
return _pq_overload_next_lookup_table(overload, it);
}
else return NULL;
}
static int get_pollfds(struct dbus* dbus, struct pollfd *pollfds)
{
int count = 0;
Word_t index = 0;
DBusWatch** pwatch;
JLF(pwatch, dbus->watches, index);
while(pwatch != NULL) {
DBusWatch* watch = *pwatch;
if(dbus_watch_get_enabled(watch)) {
pollfds[count].fd = dbus_watch_get_unix_fd(watch);
pollfds[count].events = 0;
int flags = dbus_watch_get_flags(watch);
if(flags & DBUS_WATCH_READABLE) pollfds[count].events |= POLLIN | POLLPRI;
if(flags & DBUS_WATCH_WRITABLE) pollfds[count].events |= POLLOUT;
count += 1;
}
JLN(pwatch, dbus->watches, index);
}
return count;
}
static void reverse_lookup(u32 id)
{
Pvoid_t qexp = judify_qexp();
Word_t idx = 0;
Word_t *ptr = NULL;
uint i = 0;
JLF(ptr, qexp, idx);
printf("ID: %d is expanded from: ", id);
while (ptr != NULL) {
const glist *lst = (const glist*)*ptr;
uint j;
for (j = 0; j < lst->len; j++) {
if (lst->lst[j] == id) {
printf("%u ", (u32)idx);
++i;
}
if (lst->lst[j] >= id)
break;
}
JLN(ptr, qexp, idx);
}
if (!i)
printf("(null)\n");
else
printf("\n");
}
int dogma_free_context(dogma_context_t* ctx) {
dogma_drone_context_t** value;
dogma_key_t index = 0;
int ret;
if(ctx->fleet != NULL) {
bool found;
DOGMA_ASSUME_OK(dogma_remove_fleet_member(ctx->fleet, ctx, &found));
assert(found == true && ctx->fleet == NULL);
}
dogma_free_env(ctx, ctx->character);
dogma_free_env(ctx, ctx->gang);
dogma_reset_skill_levels(ctx);
JLF(value, ctx->drone_map, index);
while(value != NULL) {
/* The drone environments were freed when char was freed */
free(*value);
JLN(value, ctx->drone_map, index);
}
JLFA(ret, ctx->drone_map);
free(ctx);
return DOGMA_OK;
}
static void save_pjarray (Pvoid_t PJArray)
{
FILE* dat[2];
int const buf_size = (1 << 20);
PWord_t buf[2];
int buf_pos[2] = {0,0};
for (uint8_t d = L; d < N; ++d)
{
char fname[100 + 1];
snprintf (fname, 100, "hex.%i.%s.dat", SIZE, show_player[d]);
dat[d] = fopen (fname, "wb+");
if (!dat[d])
{
fprintf ( stderr, "could not open dat file %s: %s\n"
, fname, strerror (errno)
);
exit (EXIT_FAILURE);
}
buf[d] = malloc (buf_size * sizeof (Word_t));
if (!buf[d])
{
fprintf (stderr, "could not allocate memory\n");
exit (EXIT_FAILURE);
}
}
Word_t Index = 0;
PWord_t PValue;
JLF (PValue, PJArray, Index);
while (PValue)
{
buf[*PValue][buf_pos[*PValue]++] = Index;
if (buf_pos[*PValue] == buf_size)
{
WRITE (*PValue);
}
JLN (PValue, PJArray, Index);
}
for (uint8_t d = L; d < N; ++d)
{
WRITE (d);
free (buf[d]);
fclose (dat[d]);
}
}
int dogma_set_env_state(dogma_context_t* ctx, dogma_env_t* env, dogma_state_t newstate) {
dogma_array_t enveffects;
dogma_key_t index = 0;
const dogma_type_effect_t** te;
const dogma_effect_t* e;
dogma_expctx_t result;
if(env->state == newstate) return DOGMA_OK;
DOGMA_ASSUME_OK(dogma_get_type_effects(env->id, &enveffects));
JLF(te, enveffects, index);
while(te != NULL) {
DOGMA_ASSUME_OK(dogma_get_effect((*te)->effectid, &e));
JLN(te, enveffects, index);
if(e->fittingusagechanceattributeid > 0) {
/* Effect is chance-based */
if(newstate > 0) {
continue;
}
/* When unplugging the environment, turn off all
* chance-based effects as a precautionary measure */
bool* v;
int ret;
JLG(v, env->chance_effects, e->id);
if(v != NULL) {
assert(*v == true);
JLD(ret, env->chance_effects, e->id);
DOGMA_ASSUME_OK(dogma_eval_expression(
ctx, env,
e->postexpressionid,
&result
));
}
continue;
}
if((newstate >> e->category) & 1) {
if(!((env->state >> e->category) & 1)) {
DOGMA_ASSUME_OK(dogma_eval_expression(
ctx, env,
e->preexpressionid,
&result
));
}
} else if((env->state >> e->category) & 1) {
DOGMA_ASSUME_OK(dogma_eval_expression(
ctx, env,
e->postexpressionid,
&result
));
}
}
switch_status_t switch_api_l3_route_entries_get(
switch_l3_table_iterator_fn iterator_fn) {
void *temp = NULL;
switch_handle_t vrf_handle = 0;
switch_status_t status = SWITCH_STATUS_SUCCESS;
JLF(temp, switch_vrf_v4_routes, vrf_handle);
while (temp) {
status = switch_api_l3_v4_route_entries_get_by_vrf(vrf_handle, iterator_fn);
JLN(temp, switch_vrf_v4_routes, vrf_handle);
}
vrf_handle = 0;
JLF(temp, switch_vrf_v6_routes, vrf_handle);
while (temp) {
status = switch_api_l3_v6_route_entries_get_by_vrf(vrf_handle, iterator_fn);
JLN(temp, switch_vrf_v6_routes, vrf_handle);
}
return status;
}
void jtableL_iterate(jtableL *table, jtableL_cb f, void *param) {
PWord_t PValue;
Word_t key = 0UL;
JLF(PValue, table->t, key);
while (PValue) {
f(key, (void *)*PValue, param);
JLN(PValue, table->t, key);
}
}
int dogma_get_location_env(dogma_context_t* ctx, dogma_location_t location, dogma_env_t** env) {
dogma_env_t** env1;
dogma_env_t** env2;
dogma_drone_context_t** drone_env1;
dogma_key_t index = 0;
switch(location.type) {
case DOGMA_LOC_Char:
*env = ctx->character;
return DOGMA_OK;
case DOGMA_LOC_Implant:
JLG(env1, ctx->character->children, location.implant_index);
if(env1 == NULL) return DOGMA_NOT_FOUND;
*env = *env1;
return DOGMA_OK;
case DOGMA_LOC_Skill:
JLG(env1, ctx->character->children, location.skill_typeid);
if(env1 == NULL) return DOGMA_NOT_FOUND;
*env = *env1;
return DOGMA_OK;
case DOGMA_LOC_Ship:
*env = ctx->ship;
return DOGMA_OK;
case DOGMA_LOC_Module:
JLG(env1, ctx->ship->children, location.module_index);
if(env1 == NULL) return DOGMA_NOT_FOUND;
*env = *env1;
return DOGMA_OK;
case DOGMA_LOC_Charge:
JLG(env1, ctx->ship->children, location.module_index);
if(env1 == NULL) return DOGMA_NOT_FOUND;
JLF(env2, (*env1)->children, index);
if(env2 == NULL) return DOGMA_NOT_FOUND;
*env = *env2;
return DOGMA_OK;
case DOGMA_LOC_Drone:
JLG(drone_env1, ctx->drone_map, location.drone_typeid);
if(drone_env1 == NULL) return DOGMA_NOT_FOUND;
*env = (*drone_env1)->drone;
assert(*env != NULL);
return DOGMA_OK;
default:
return DOGMA_NOT_FOUND;
}
}
static void update_peers(struct network_manager* nm)
{
// Perform the select first
fd_set read_fds = nm->poll_read_fds,
write_fds = nm->poll_write_fds,
exception_fds = nm->poll_exception_fds;
struct timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = 0;
if(select(nm->poll_max_fd + 1, &read_fds, &write_fds, &exception_fds, &timeout) < 0) {
// weird error?
perror("select");
return;
}
// Then loop over all peers passing in the select status
struct network_peer** ppeer = NULL;
Word_t index = 0;
struct vector disconnected_peers;
vector_init(&disconnected_peers);
JLF(ppeer, nm->peer_list, index);
while(ppeer != NULL) {
int action_flags = 0;
int* psock = NULL;
uintptr_t p = (uintptr_t)*ppeer;
JLG(psock, nm->poll_socket_by_peer, p);
if(psock != NULL) {
if(FD_ISSET(*psock, &read_fds)) action_flags |= NETWORK_PEER_ACTION_FLAGS_READ;
if(FD_ISSET(*psock, &write_fds)) action_flags |= NETWORK_PEER_ACTION_FLAGS_WRITE;
if(FD_ISSET(*psock, &exception_fds)) action_flags |= NETWORK_PEER_ACTION_FLAGS_EXCEPTION;
}
network_peer_update(*ppeer, action_flags);
if(network_peer_disconnected(*ppeer) == 1) {
vector_add(&disconnected_peers, (uintptr_t)*ppeer);
}
JLN(ppeer, nm->peer_list, index);
}
size_t num_disconnected = vector_count(&disconnected_peers);
for(size_t i = 0; i < num_disconnected; i++) {
struct network_peer* peer = (struct network_peer*)vector_get(&disconnected_peers, i);
stop_peer(nm, peer);
}
vector_free(&disconnected_peers);
}
void pq_overload_destroy(pq_context *ctx, pq_overload *overload) {
Word_t bytes, index, *pvalue;
Pvoid_t table;
index = 0;
JLF(pvalue, overload->variadic_function_table, index);
while(pvalue != NULL) {
table = (Pvoid_t)*pvalue;
JLFA(bytes, table);
JLN(pvalue, overload->variadic_function_table, index);
}
JLFA(bytes, overload->variadic_function_table);
index = 0;
JLF(pvalue, overload->function_table, index);
while(pvalue != NULL) {
table = (Pvoid_t)*pvalue;
JLFA(bytes, table);
JLN(pvalue, overload->function_table, index);
}
JLFA(bytes, overload->function_table);
}
int dbus_update(struct dbus* dbus)
{
DBusDispatchStatus status;
// handle watches
Word_t watch_count = 0;
JLC(watch_count, dbus->watches, 0, -1);
struct pollfd* pollfds = (struct pollfd*)alloca(sizeof(struct pollfd) * watch_count);
int fdcount = get_pollfds(dbus, pollfds);
if(poll(pollfds, fdcount, 0) < 0) {
return -1;
}
// process the watches
DBusWatch** pwatch;
Word_t index = 0;
int c = 0;
JLF(pwatch, dbus->watches, index);
while(pwatch != NULL) {
struct pollfd* poll_result = &pollfds[c];
struct DBusWatch* watch = *pwatch;
if(dbus_watch_get_enabled(watch)) {
assert(poll_result->fd == dbus_watch_get_unix_fd(watch));
int flags = 0;
int revents = poll_result->revents;
if((revents & POLLIN) != 0) flags |= DBUS_WATCH_READABLE;
if((revents & POLLOUT) != 0) flags |= DBUS_WATCH_WRITABLE;
if((revents & POLLERR) != 0) flags |= DBUS_WATCH_ERROR;
if((revents & POLLHUP) != 0) flags |= DBUS_WATCH_HANGUP;
if(flags != 0) dbus_watch_handle(watch, flags);
c++;
}
JLN(pwatch, dbus->watches, index);
}
// dispatch incoming messages
while((status = dbus_connection_get_dispatch_status(dbus->conn)) != DBUS_DISPATCH_COMPLETE) {
dbus_connection_dispatch(dbus->conn);
}
// Send outgoing messages
if(dbus_connection_has_messages_to_send(dbus->conn)) {
dbus_connection_flush(dbus->conn);
}
return 0;
}
switch_status_t switch_api_l3_v6_routes_print_all(void) {
switch_handle_t vrf_handle = 0;
void *temp = NULL;
switch_status_t status = SWITCH_STATUS_SUCCESS;
JLF(temp, switch_vrf_v6_routes, vrf_handle);
while (temp) {
status = switch_api_l3_v6_routes_print_by_vrf(vrf_handle);
JLN(temp, switch_vrf_v6_routes, vrf_handle);
}
return status;
}
static void destroy_node(node_t *node) {
Word_t index = 0;
Word_t *pnode;
Word_t rc_word;
JLF(pnode, node->PJLarray_branches, index);
while (pnode != NULL) {
destroy_node((node_t *)*pnode);
JLN(pnode, node->PJLarray_branches, index);
}
JLFA(rc_word, node->PJLarray_branches);
JLFA(rc_word, node->PJLarray_prefixes);
switch_free(node);
}
static void switch_packet_tx_from_hosts(fd_set read_fds) {
switch_hostif_info_t *hostif_info = NULL;
void *temp = NULL;
Word_t index = 0;
JLF(temp, switch_intf_fd_array, index);
while (temp) {
hostif_info = (switch_hostif_info_t *)(*(unsigned long *)temp);
if (FD_ISSET(hostif_info->intf_fd, &read_fds)) {
switch_packet_tx_from_host(hostif_info->intf_fd);
}
JLN(temp, switch_intf_fd_array, index);
}
}
void free_p_entries(my_int len_sa)
{
my_int i;
Word_t Bytes;
idx=0;
JLF(pval,judy,idx);
while (1)
{
if (pval==NULL) break;
J1FA(Bytes,*pval);
JLN(pval,judy,idx);
}
JLFA(Bytes,judy);
}
static uint encode_tokens(const Pvoid_t tokens)
{
static growing_glist *tok;
static char *buf;
static uint buf_len;
uint i, max_pos = 0;
if (!tok){
GGLIST_INIT(tok, 1000);
}
tok->lst.len = 0;
Word_t *ptr;
Word_t xid = 0;
JLF(ptr, tokens, xid);
while (ptr){
u32 x = xid;
GGLIST_APPEND(tok, x);
const growing_glist *g = (const growing_glist*)*ptr;
for (i = 0; i < g->lst.len; i++)
if (g->lst.lst[i] > max_pos)
max_pos = g->lst.lst[i];
JLN(ptr, tokens, xid);
}
u32 offs = 0;
uint need = rice_encode(NULL,
&offs, tok->lst.lst, tok->lst.len);
uint need_bytes = BITS_TO_BYTES(need);
if (need_bytes > buf_len){
free(buf);
buf_len = need_bytes;
buf = xmalloc(buf_len + 12);
memset(buf, 0, buf_len + 12);
}
offs = 0;
elias_gamma_write(buf, &offs, need);
rice_encode(buf, &offs, tok->lst.lst, tok->lst.len);
offs = BITS_TO_BYTES(offs);
fwrite(buf, offs, 1, data_f);
memset(buf, 0, offs);
return max_pos;
}
static void print_all()
{
Pvoid_t qexp = judify_qexp();
Word_t idx = 0;
Word_t *ptr = NULL;
JLF(ptr, qexp, idx);
while (ptr != NULL) {
const glist *lst = (const glist*)*ptr;
printf("ID: %d expands to: ", (u32)idx);
print_glist(lst);
JLN(ptr, qexp, idx);
}
}
static void remove_watch(DBusWatch* watch, void *userdata)
{
struct dbus* dbus = (struct dbus*)userdata;
Word_t index = 0;
DBusWatch** pwatch;
JLF(pwatch, dbus->watches, index);
while(pwatch != NULL) {
if(*pwatch == watch) {
Word_t wrc;
JLD(wrc, dbus->watches, index);
break;
}
JLN(pwatch, dbus->watches, index);
}
}
// returns false if empty vector, else true
bool string_first(l3obj* obj, char** val, long* index) {
assert(obj->_type == t_str);
LIVEO(obj);
assert(index);
assert(val);
Word_t * PValue; // pointer to array element value
Word_t Index = 0;
JLF(PValue, obj->_judyL, Index);
if (PValue) {
*index = Index;
*val = *(char**)(PValue);
return true;
}
return false;
}
my_int p_size(my_int len_sa)
{
my_int i;
Word_t Bytes;
Word_t total=0;
idx=0;
JLF(pval,judy,idx);
while (1)
{
if (pval==NULL) break;
J1MU(Bytes,*pval);
total+=Bytes;
JLN(pval,judy,idx);
}
JLMU(Bytes,judy);
total+=Bytes;
return total;
}
static int switch_packet_select_fd_get(fd_set *read_fds) {
switch_hostif_info_t *hostif_info = NULL;
void *temp = NULL;
int nfds;
Word_t index = 0;
nfds = (cpu_sock_fd > pipe_fd[0]) ? cpu_sock_fd : pipe_fd[0];
JLF(temp, switch_intf_fd_array, index);
while (temp) {
hostif_info = (switch_hostif_info_t *)(*(unsigned long *)temp);
FD_SET(hostif_info->intf_fd, read_fds);
if (hostif_info->intf_fd > nfds) {
nfds = hostif_info->intf_fd;
}
JLN(temp, switch_intf_fd_array, index);
}
return nfds + 1;
}
pq_value *pq_overload_next_function(const pq_overload *overload, pq_overload_iterator *it) {
Word_t *pvalue;
Pvoid_t table;
table = _pq_overload_current_lookup_table(overload, it);
do {
if(it->signature_index == -1) {
it->signature_index = 0;
JLF(pvalue, table, it->signature_index);
}
else {
JLN(pvalue, table, it->signature_index);
}
if(pvalue) {
return (pq_value *)*pvalue;
}
else {
table = _pq_overload_next_lookup_table(overload, it);
}
} while(table);
return NULL;
}
static inline pq_value *_pq_overload_for_types_non_variadic(pq_context *ctx, pq_overload *overload, size_t n, pq_type **types) {
Word_t *pvalue;
pq_value *best_match;
int best_score = -1, score;
Pvoid_t by_argnum_table = overload->function_table, by_type_table;
JLG(pvalue, by_argnum_table, n);
if(pvalue != NULL && pvalue != PJERR ) {
by_type_table = (Pvoid_t)*pvalue;
Word_t index = 0;
JLF(pvalue, by_type_table, index);
while(pvalue != NULL) {
score = _pq_signature_match_score((pq_type *)index, n, types);
if(score > best_score) {
best_score = score;
best_match = (pq_value *)*pvalue;
}
JLN(pvalue, by_type_table, index);
}
if(best_score >= 0) return best_match;
}
return NULL;
}
int dogma_init_context(dogma_context_t** ctx) {
dogma_context_t* new_ctx = malloc(sizeof(dogma_context_t));
dogma_env_t** value;
dogma_key_t index = 0;
const dogma_type_t** type;
new_ctx->gang = malloc(sizeof(dogma_env_t));
new_ctx->character = malloc(sizeof(dogma_env_t));
new_ctx->ship = malloc(sizeof(dogma_env_t));
new_ctx->area = NULL;
new_ctx->fleet = NULL;
DOGMA_INIT_ENV(new_ctx->gang, 0, NULL, 0, new_ctx);
DOGMA_INIT_ENV(new_ctx->character, 0, NULL, 0, new_ctx);
DOGMA_INIT_ENV(new_ctx->ship, 0, new_ctx->character, 0, new_ctx);
JLI(value, new_ctx->character->children, 0);
*value = new_ctx->ship;
new_ctx->default_skill_level = DOGMA_MAX_SKILL_LEVEL;
new_ctx->skill_levels = (dogma_array_t)NULL;
new_ctx->drone_map = (dogma_array_t)NULL;
*ctx = new_ctx;
/* Inject all skills. This is somewhat costly, maybe there is a
* way to do it lazily? */
JLF(type, types_by_id, index);
while(type != NULL) {
if((*type)->categoryid == CAT_Skill) {
dogma_inject_skill(*ctx, (*type)->id);
}
JLN(type, types_by_id, index);
}
return DOGMA_OK;
}
//void possect_add_old(u32 key, const Pvoid_t tokens);
void possect_add(const Pvoid_t tokens)
{
static char *posbuf;
static char *xidbuf;
if (!posbuf){
posbuf = xmalloc(iblock_size * 8 + 8);
memset(posbuf, 0, iblock_size * 8 + 8);
xidbuf = xmalloc((iblock_size * 2 + 3) * 8);
memset(xidbuf, 0, (iblock_size * 2 + 3) * 8);
}
Word_t *ptr;
u32 len, xoffs, poffs, prev_offs, prev_xid,
xsum, xmax, osum, omax;
Word_t xid = poffs = prev_offs = prev_xid =\
xsum = osum = osum = omax = xmax = len = 0;
JLF(ptr, tokens, xid);
while (ptr){
growing_glist *g = (growing_glist*)*ptr;
*ptr = poffs;
rice_encode(posbuf, &poffs, g->lst.lst, g->lst.len);
free(g);
xsum += xid - prev_xid;
osum += poffs - prev_offs;
if (xid - prev_xid > xmax)
xmax = xid - prev_xid;
if (poffs - prev_offs > omax)
omax = poffs - prev_offs;
prev_xid = xid;
prev_offs = poffs;
++len;
JLN(ptr, tokens, xid);
}
poffs = BITS_TO_BYTES(poffs);
xid = prev_xid = prev_offs = xoffs = 0;
elias_gamma_write(xidbuf, &xoffs, len + 1);
if (!len)
goto write_buffers;
u32 rice_f1 = estimate_rice_f_param(xsum, xmax, len);
u32 rice_f2 = estimate_rice_f_param(osum, omax, len);
rice_write(xidbuf, &xoffs, rice_f1, 2);
rice_write(xidbuf, &xoffs, rice_f2, 2);
JLF(ptr, tokens, xid);
while (ptr){
rice_write(xidbuf, &xoffs, xid - prev_xid, rice_f1);
prev_xid = xid;
rice_write(xidbuf, &xoffs, *ptr - prev_offs, rice_f2);
prev_offs = *ptr;
JLN(ptr, tokens, xid);
}
write_buffers:
xoffs = BITS_TO_BYTES(xoffs);
toc_e toc;
toc.offs = ftello64(data_f);
toc.val = xoffs;
fwrite(&toc, sizeof(toc_e), 1, toc_f);
fwrite(xidbuf, xoffs, 1, data_f);
fwrite(posbuf, poffs, 1, data_f);
memset(xidbuf, 0, xoffs);
memset(posbuf, 0, poffs);
}
/* This functions codes all ixeme occurrences in a segment in the correct
* order. The catch is that instead of using global ixeme IDs, we use a
* minimal local set of IDs, namely integers [1..|S|] for |S| individual
* ixemes. Mapping between the IDs is implictely saved in the ascending
* order of ixeme IDs, obtainable from the forward index. */
void possect_add(const Pvoid_t tokens)
{
static uint *offsets;
//static uint offsets_len;
static char *encode_buf;
//static uint buf_len;
if (!offsets){
offsets = xmalloc(segment_size * sizeof(uint));
encode_buf = xmalloc(segment_size * 8);
memset(encode_buf, 0, segment_size * 8);
}
memset(offsets, 0, segment_size * sizeof(uint));
//ENSURE_IBLOCK
Word_t max = 0;
JLC(max, tokens, 0, -1);
/* write TOC entry */
toc_e toc;
toc.offs = ftello64(data_f);
toc.val = max;
fwrite(&toc, sizeof(toc_e), 1, toc_f);
/* write token list */
uint max_pos = encode_tokens(tokens);
/*
if (segment_size > offsets_len){
free(offsets);
offsets_len = segment_size;
offsets = xmalloc(offsets_len * sizeof(uint));
}
memset(offsets, 0, offsets_len * sizeof(uint));
*/
/* estimate rice_f */
u32 rice_f, offs = 0;
if (max)
rice_f = estimate_rice_f((u32*)&max, 1);
else
rice_f = 2;
/* allocate bits to positions */
Word_t *ptr;
//uint max_pos = 0;
int i, j, c = 0;
Word_t xid = 0;
JLF(ptr, tokens, xid);
while (ptr){
offs = 0;
rice_write(NULL, &offs, ++c, rice_f);
const growing_glist *g = (const growing_glist*)*ptr;
for (i = 0; i < g->lst.len; i++){
offsets[g->lst.lst[i] - 1] = offs;
if (g->lst.lst[i] > max_pos)
max_pos = g->lst.lst[i];
}
JLN(ptr, tokens, xid);
}
/* compute bit offset for each position */
offs = 0;
elias_gamma_write(NULL, &offs, max_pos + 1);
rice_write(NULL, &offs, rice_f, 2);
for (i = 0; i < max_pos; i++){
uint tmp = offsets[i];
if (!tmp)
dub_die("A hole in poslist at %u / %u!", i, max_pos);
offsets[i] = offs;
offs += tmp;
}
u32 endpos = offs;
//u32 total_bits = offs;
//elias_gamma_write(NULL, &offs, total_bits);
//rice_write(NULL, &offs, 0, rice_f);
u32 total_bytes = BITS_TO_BYTES(offs);
if (total_bytes > segment_size * 8)
dub_die("Pos buffer too small! %u > %u.",
total_bytes, segment_size * 8);
/*
if (total_bytes > buf_len){
free(encode_buf);
buf_len = total_bytes;
encode_buf = xmalloc(buf_len + 4);
memset(encode_buf, 0, buf_len + 4);
}
*/
/* write codes */
offs = 0;
elias_gamma_write(encode_buf, &offs, max_pos + 1);
rice_write(encode_buf, &offs, rice_f, 2);
//elias_gamma_write(encode_buf, &offs, total_bits);
xid = c = j = 0;
JLF(ptr, tokens, xid);
while (ptr){
++c;
growing_glist *g = (growing_glist*)*ptr;
for (i = 0; i < g->lst.len; i++){
//u32 d = offsets[g->lst.lst[i] - 1];
//dub_msg("OFFS %u VAL %u", d, c);
rice_write(encode_buf,
&offsets[g->lst.lst[i] - 1], c, rice_f);
//u32 dd = rice_read(encode_buf, &d, rice_f);
//if (c != dd)
// dub_die("POKS! %u, %u != %u", key, dd, c);
++j;
}
free(g);
JLN(ptr, tokens, xid);
}
//rice_write(encode_buf, &endpos, 0, rice_f);
fwrite(encode_buf, total_bytes, 1, data_f);
memset(encode_buf, 0, total_bytes);
}
int
main(int argc, char *argv[]) {
if (argc != 8) {
fprintf(stderr, "Usage: ./merge_N [out_dir_name 1] [item_#_file 2] [length_#_file 3] [count_dist_file 4] [number_of_temp_prefix 5] [from_temp_n] [to_temp_n]\n");
// /root/cx_src/src/merge_N /tmp/data /tmp/result/ino.txt /tmp/result/lno.txt /tmp/result/cdo.txt 3 0 1
return -1;
}
if ((itf = fopen(argv[2], "a")) == NULL) {
fprintf(stderr, "Failed to open file \"%s\" for writing item numbers\n", argv[2]);
return -1;
}
// fixing each temp file
fprintf(stdout, "Start fixing temp files\n");
for (i = atoi(argv[6]); i <= atoi(argv[7]); ++i) {
//for (i = 0; i < TEMP_N; ++i) {
// fix the temp file i
int temp_prefix_num = 0;// num of concurrent threads
while (temp_prefix_num < atoi(argv[5])){
sprintf(buffer, "%s/%s%d-%d.txt", argv[1], TEMP_PREFIX, i, temp_prefix_num);
fprintf(stdout, "\rWorking on temp file: \"%s\" \n", buffer);
//fflush(stdout);
if ((tsf[i] = fopen(buffer, "r")) == NULL) {
fprintf(stderr, "Failed to open file \"%s\" for reading temp strings\n", buffer);
//continue;
//break;
return -1;
}
while (fscanf(tsf[i], "%"PRId64"\t", &itemn) != EOF) {
fgets(Index, BUFFER_SIZE, tsf[i]);
for (Len = strlen(Index) - 1; Index[Len] == '\n' || Index[Len] == '\r'; Len--)
Index[Len] = 0;
++Len;
JSLI(PValNgramS, PJSLNgram, (uint8_t *)Index);
if (PValNgramS == PJERR) {
fprintf(stderr, "Malloc failed for \"PJSLNgram\"\n");
//return -1;
}
(*PValNgramS) += itemn;
JLI(PValTotC, PJLTotCount, Len);
if (PValTotC == PJERR) {
fprintf(stderr, "Malloc failed for \"PJLTotCount\"\n");
//return -1;
}
*PValTotC += itemn;
if (*PValNgramS == itemn) {
JLI(PValNgramC, PJLNgramCount, Len);
if (PValNgramC == PJERR) {
fprintf(stderr, "Malloc failed for \"PJLNgramCount\"\n");
//return -1;
}
++*PValNgramC;
}
}
sprintf(buffer, "rm %s/%s%d-%d.txt", argv[1], TEMP_PREFIX, i, temp_prefix_num++);
if (system(buffer) == -1) {
fprintf(stderr, "Failed to execute command: \"%s\"\n", buffer);
//return -1;
}
fclose(tsf[i]);
}
// write the final temp file
sprintf(buffer, "%s/%s%d.txt", argv[1], TEMP_PREFIX, i);
if ((tsf[i] = fopen(buffer, "w")) == NULL) {
fprintf(stderr, "Failed to open file \"%s\" for writing temp strings\n", buffer);
return -1;
}
Index[0] = '\0';
JSLF(PValNgramS, PJSLNgram, (uint8_t *)Index);
while (PValNgramS != NULL) {
fprintf(tsf[i], "%lu\t%s\n", *PValNgramS, Index);
Count = *PValNgramS;
JLI(PValCountC, PJLCountCount, Count);
if (PValCountC == PJERR) {
fprintf(stderr, "Malloc failed for \"PJLCountCount\"\n");
return -1;
}
++*PValCountC;
JSLN(PValNgramS, PJSLNgram, (uint8_t *)Index);
}
JSLFA(Bytes, PJSLNgram);
fflush(tsf[i]);
fclose(tsf[i]);
fprintf(itf, "Temp file \"%s/%s%d\" uses %lu Bytes of memory\n", argv[1], TEMP_PREFIX, i, Bytes);
fflush(itf);
}
fclose(itf);
if ((lef = fopen(argv[3], "a")) == NULL) {
fprintf(stderr, "Failed to open file \"%s\" for writing length number\n", argv[3]);
return -1;
}
Total = NgramN = 0;
JLF(PValTotC, PJLTotCount, Total);
JLF(PValNgramC, PJLNgramCount, NgramN);
while (PValTotC != NULL) {
fprintf(lef, "%lu\t%lu\t%lu\n", Total, *PValNgramC, *PValTotC);
JLN(PValTotC, PJLTotCount, Total);
JLN(PValNgramC, PJLNgramCount, NgramN);
}
JLFA(Bytes, PJLTotCount);
JLFA(Bytes, PJLNgramCount);
fflush(lef);
fclose(lef);
if ((cdf = fopen(argv[4], "a")) == NULL) {
fprintf(stderr, "Failed to open file \"%s\" for writing count distribuction\n", argv[4]);
return -1;
}
Count = 0;
JLF(PValCountC, PJLCountCount, Count);
while (PValCountC != NULL) {
fprintf(cdf, "%lu\t%lu\n", Count, *PValCountC);
JLN(PValCountC, PJLCountCount, Count);
}
JLFA(Bytes, PJLCountCount);
fflush(cdf);
fclose(cdf);
return 0;
}
