int ddb_cons_add(struct ddb_cons *db,
const struct ddb_entry *key,
const struct ddb_entry *value)
{
uintptr_t *val_ptr;
uintptr_t *key_ptr;
valueid_t value_id;
struct ddb_deltalist *value_list;
if (!(key_ptr = ddb_map_insert_str(db->keys_map, key)))
return -1;
if (!*key_ptr && !(*key_ptr = (uintptr_t)ddb_deltalist_new()))
return -1;
value_list = (struct ddb_deltalist*)*key_ptr;
if (value){
if (!(val_ptr = ddb_map_insert_str(db->values_map, value)))
return -1;
if (!*val_ptr){
*val_ptr = ddb_map_num_items(db->values_map);
db->uvalues_total_size += value->length;
}
value_id = *val_ptr;
if (ddb_deltalist_append(value_list, value_id))
return -1;
}
#ifdef DDB_PROFILE
if (!(++db->counter & 1048575))
print_mem_usage(db);
#endif
return 0;
}
char *ddb_cons_finalize(struct ddb_cons *cons, uint64_t *length, uint64_t flags)
{
struct ddb_packed *pack = NULL;
struct ddb_entry *order = NULL;
char *buf = NULL;
int disable_compression, err = 1;
DDB_TIMER_DEF
if (!(pack = buffer_init()))
goto err;
if (pack_header(pack, cons))
goto err;
#ifdef DDB_PROFILE
print_mem_usage(cons);
#endif
DDB_TIMER_START
pack->head->hash_offs = pack->offs;
if (!(order = pack_hash(pack, cons->keys_map)))
goto err;
DDB_TIMER_END("hash")
DDB_TIMER_START
pack->head->key2values_offs = pack->offs;
if (pack_key2values(pack, order, cons->keys_map,
flags & DDB_OPT_UNIQUE_ITEMS))
goto err;
DDB_TIMER_END("key2values")
DDB_TIMER_START
flags |= maybe_disable_compression(cons);
disable_compression = flags & DDB_OPT_DISABLE_COMPRESSION;
pack->head->id2value_offs = pack->offs;
if (pack_id2value(pack, cons->values_map, disable_compression))
goto err;
else{
ddb_map_free(cons->values_map);
cons->values_map = NULL;
}
DDB_TIMER_END("id2values")
if (!disable_compression){
DDB_TIMER_START
pack->head->codebook_offs = pack->offs;
if (pack_codebook(pack))
goto err;
DDB_TIMER_END("save_codebook")
}
PROCESS_THREAD(memory_debugger_process, ev, data)
{
static struct etimer periodic;
PROCESS_BEGIN();
PROCESS_PAUSE();
memory_debugger_init();
syslog(LOG_DEBUG, "process started");
etimer_set(&periodic, MEMORY_DEBUGGER_INTERVAL*CLOCK_SECOND);
while(1) {
PROCESS_YIELD();
if(etimer_expired(&periodic)) {
etimer_reset(&periodic);
print_mem_usage();
}
}
PROCESS_END();
}
ri_t reduce_gbla_matrix(mat_t * mat, int verbose, int nthreads)
{
/* timing structs */
struct timeval t_load_start;
struct timeval t_complete;
if (verbose > 2)
gettimeofday(&t_complete, NULL);
/* A^-1 * B */
if (verbose > 2) {
printf("---------------------------------------------------------------------------\n");
printf("GBLA Matrix Reduction\n");
printf("---------------------------------------------------------------------------\n");
gettimeofday(&t_load_start, NULL);
printf("%-38s","Reducing A ...");
fflush(stdout);
}
if (mat->A->blocks != NULL) {
if (elim_fl_A_sparse_dense_block(&(mat->A), mat->B, mat->mod, nthreads)) {
printf("Error while reducing A.\n");
return 1;
}
}
if (verbose > 2) {
printf("%9.3f sec\n",
walltime(t_load_start) / (1000000));
}
if (verbose > 3) {
print_mem_usage();
}
/* reducing submatrix C to zero using methods of Faugère & Lachartre */
if (verbose > 2) {
gettimeofday(&t_load_start, NULL);
printf("%-38s","Reducing C ...");
fflush(stdout);
}
if (mat->C->blocks != NULL) {
if (elim_fl_C_sparse_dense_block(mat->B, &(mat->C), mat->D, mat->mod, nthreads)) {
printf("Error while reducing C.\n");
return 1;
}
}
if (verbose > 2) {
printf("%9.3f sec\n",
walltime(t_load_start) / (1000000));
}
if (verbose > 3) {
print_mem_usage();
}
/* copy block D to dense wide (re_l_t) representation */
mat->DR = copy_block_to_dense_matrix(&(mat->D), nthreads, 1);
mat->DR->mod = mat->mod;
#if 0
printf("number of rows of DR %u\n", mat->DR->nrows);
for (int ii=0; ii<mat->DR->nrows; ++ii) {
printf("ROW %d\n",ii);
if (mat->DR->row[ii]->init_val == NULL)
printf("NULL!");
else {
printf("%u || ", mat->DR->row[ii]->lead);
for (int jj=0; jj<mat->DR->ncols; ++jj)
#if defined(GBLA_USE_UINT16) || defined(GBLA_USE_UINT32)
printf("%u (%u) ", mat->DR->row[ii]->init_val[jj], jj+mat->ncl);
#else
printf("%.0f ", mat->DR->row[ii]->init_val[jj]);
#endif
}
printf("\n");
}
#endif
/* eliminate mat->DR using a structured Gaussian Elimination process on the rows */
nelts_t rank_D = 0;
/* echelonizing D to zero using methods of Faugère & Lachartre */
if (verbose > 2) {
gettimeofday(&t_load_start, NULL);
printf("%-38s","Reducing D ...");
fflush(stdout);
}
if (mat->DR->nrows > 0) {
if (nthreads == 1) {
rank_D = elim_fl_dense_D_completely(mat->DR, nthreads);
} else {
rank_D = elim_fl_dense_D(mat->DR, nthreads);
nelts_t l;
for (l=1; l<mat->DR->rank; ++l) {
/* for (l=(int)(mat->DR->rank-1); l>0; --l) { */
copy_piv_to_val(mat->DR, mat->DR->rank-l-1);
completely_reduce_D(mat->DR, mat->DR->rank-l-1);
}
}
}
if (verbose > 2) {
printf("%9.3f sec %5d %5d %5d\n",
walltime(t_load_start) / (1000000), rank_D, mat->DR->nrows - rank_D, mat->DR->nrows);
}
/* if we simplify, then copy B to dense row representation */
if (mat->sl > 0 && mat->B->blocks != NULL) {
/* first copy B to BR (dense row format) */
mat->BR = copy_block_to_dense_matrix(&(mat->B), nthreads, 0);
mat->BR->mod = mat->mod;
}
if (verbose > 3) {
print_mem_usage();
}
if (verbose > 2) {
printf("---------------------------------------------------------------------------\n");
printf("%-38s","Reduction completed ...");
fflush(stdout);
printf("%9.3f sec\n",
walltime(t_complete) / (1000000));
if (verbose > 3)
print_mem_usage();
}
return rank_D;
}
ri_t reduce_gbla_matrix_keep_A(mat_t *mat, int verbose, int nthreads)
{
/* timing structs */
struct timeval t_load_start;
struct timeval t_complete;
if (verbose > 2)
gettimeofday(&t_complete, NULL);
/* A^-1 * B */
if (verbose > 2) {
printf("---------------------------------------------------------------------------\n");
printf("GBLA Matrix Reduction\n");
printf("---------------------------------------------------------------------------\n");
gettimeofday(&t_load_start, NULL);
printf("%-38s","Storing A in C ...");
fflush(stdout);
}
if (mat->AR->row != NULL) {
if (elim_fl_C_sparse_dense_keep_A(mat->CR, &(mat->AR), mat->mod, nthreads)) {
printf("Error while reducing A.\n");
return 1;
}
}
if (verbose > 2) {
printf("%9.3f sec\n",
walltime(t_load_start) / (1000000));
}
if (verbose > 3) {
print_mem_usage();
}
/* reducing submatrix C to zero using methods of Faugère & Lachartre */
if (verbose > 2) {
gettimeofday(&t_load_start, NULL);
printf("%-38s","Copying C to sparse block representation ...");
fflush(stdout);
}
if (mat->CR->row != NULL) {
mat->C = copy_sparse_to_block_matrix(mat->CR, nthreads);
free_sparse_matrix(&(mat->CR), nthreads);
}
if (verbose > 2) {
printf("%9.3f sec\n",
walltime(t_load_start) / (1000000));
}
if (verbose > 3) {
print_mem_usage();
}
if (verbose > 2) {
printf("%-38s","Reducing C to zero ...");
fflush(stdout);
}
if (mat->C != NULL) {
if (elim_fl_C_sparse_dense_block(mat->B, &(mat->C), mat->D, mat->mod, nthreads)) {
printf("Error while reducing A.\n");
return 1;
}
}
if (verbose > 2) {
printf("%9.3f sec\n",
walltime(t_load_start) / (1000000));
}
if (verbose > 3) {
print_mem_usage();
}
/* copy block D to dense wide (re_l_t) representation */
mat->DR = copy_block_to_dense_matrix(&(mat->D), nthreads, 1);
mat->DR->mod = mat->mod;
/* eliminate mat->DR using a structured Gaussian Elimination process on the rows */
nelts_t rank_D = 0;
/* echelonizing D to zero using methods of Faugère & Lachartre */
if (verbose > 2) {
gettimeofday(&t_load_start, NULL);
printf("%-38s","Reducing D ...");
fflush(stdout);
}
if (mat->DR->nrows > 0)
/* rank_D = elim_fl_dense_D(mat->DR, nthreads); */
rank_D = elim_fl_dense_D_completely(mat->DR, nthreads);
if (verbose > 2) {
printf("%9.3f sec %5d %5d %5d\n",
walltime(t_load_start) / (1000000), rank_D, mat->DR->nrows - rank_D, mat->DR->nrows);
}
if (verbose > 3) {
print_mem_usage();
}
if (verbose > 2) {
printf("---------------------------------------------------------------------------\n");
printf("%-38s","Reduction completed ...");
fflush(stdout);
printf("%9.3f sec\n",
walltime(t_complete) / (1000000));
if (verbose > 3)
print_mem_usage();
}
return rank_D;
}
void
memory_debugger_init(void)
{
syslog(LOG_DEBUG, "INIT");
print_mem_usage();
}
void
memory_debugger_init(void)
{
PRINTF("-- MEMDBG: INIT\r\n");
print_mem_usage();
}