uint64_t mix_hashed_nums(uint8_t *hashed_nums, const uint8_t *unhashedData, size_t unhashed_sz, uint8_t **mixed_hash, uint8_t *hash_digest) { uint32_t i, index = 0; const uint32_t hashed_nums_len = SHA256_LEN; uint64_t count; uint8_t tmp_val, tmp_array[SHA256_LEN + 2]; //initialize the class for the extend hash Extend_Array new_hash; Extend_Array_init(&new_hash); //set the first hash length in the temp array to all 0xff memset(tmp_array, 0xff, SHA256_LEN); //set the last two bytes to \000 *(tmp_array + SHA256_LEN) = *(tmp_array + SHA256_LEN + 1) = 0; for(count = 0;; count++) { //+1 to keeps a 0 value of *(hashed_nums + index) moving on i = hex_char_to_int(*(hashed_nums + index)) + 1; index += i; //if we hit the end of the hash, rehash it if(index >= hashed_nums_len) { index = index % hashed_nums_len; sha256_to_str(hashed_nums, hashed_nums_len, hashed_nums, hash_digest); //rescramble } tmp_val = *(hashed_nums + index); join_to_array(tmp_array, tmp_val); //plop tmp_val at the end of tmp_array sha256_to_str(tmp_array, SHA256_LEN + 1, tmp_array, hash_digest); //extend the expanded hash to the array extend_array(&new_hash, count * SHA256_LEN, tmp_array, SHA256_LEN, false); //check if the last value of hashed_nums is the same as the last value in tmp_array if(index == hashed_nums_len - 1) if(tmp_val == *(tmp_array + SHA256_LEN - 1)) { //add to count since we extended the array, but break will exit the for loop and count // will not get incremenented by the for loop count++; break; } } //extend the unhashed data to the end and add the \000 to the end extend_array(&new_hash, count * SHA256_LEN, (unsigned char*)unhashedData, unhashed_sz, true); //assign the address of new_hash's array to mixed_hash *mixed_hash = new_hash.array; return count * SHA256_LEN + unhashed_sz; }
dary * dary_new(void) { dary *da; int i = 0; da = dary_allocator(sizeof(dary)); da->length = 0; da->base = NULL; da->check = NULL; da->values = NULL; da->free_head = -1; extend_array(da, UCHAR_MAX + 2); da->free_head = UCHAR_MAX + 1; for (i =0 ; i < UCHAR_MAX + 1; i++) { da->base[i] = 0; da->check[i] = 0; } dprint("%d",i); da->check[i] = -i; da->can_free = 1; return da; }
void ra_append(roaring_array_t *ra, uint16_t key, void *container, uint8_t typecode) { extend_array(ra, 1); const int32_t pos = ra->size; ra->keys[pos] = key; ra->containers[pos] = container; ra->typecodes[pos] = typecode; ra->size++; }
void ra_append_move_range(roaring_array_t *ra, roaring_array_t *sa, uint16_t start_index, uint16_t end_index) { extend_array(ra, end_index - start_index); for (uint16_t i = start_index; i < end_index; ++i) { const int32_t pos = ra->size; ra->keys[pos] = sa->keys[i]; ra->containers[pos] = sa->containers[i]; ra->typecodes[pos] = sa->typecodes[i]; ra->size++; } }
template <class elem_t, class holder_t> void adlist_tos_base<elem_t, holder_t>::insert(size_t elem_num, elem_t the_elem) { assert(elem_num >= 0); if (elem_num == 0) { prepend(the_elem); return; } if (elem_num >= _cached_size) { while (_cached_size < elem_num) append(zero((elem_t *)0)); append(the_elem); return; } extend_array(); holder_t *follow; if (elem_num - 1 < _index_size) { follow = _index[elem_num - 1]; } else { if (_index_size == 0) { _index[0] = _head_e; typedef adlist_tos_base<elem_t, holder_t> *this_type; ++(CONST_CAST(this_type, this)->_index_size); } size_t total = _index_size - 1; follow = _index[total]; while (total < elem_num - 1) { ++total; follow = follow->next; _index[total] = follow; } } holder_t *new_e = new holder_t(the_elem, follow->next, follow); follow->next = new_e; if (_tail_e == follow) _tail_e = new_e; else new_e->next->previous = new_e; _index[elem_num] = new_e; _index_size = elem_num + 1; ++_cached_size; }
void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key, void *container, uint8_t typecode) { extend_array(ra, 1); // May be an optimization opportunity with DIY memmove memmove(&(ra->keys[i + 1]), &(ra->keys[i]), sizeof(uint16_t) * (ra->size - i)); memmove(&(ra->containers[i + 1]), &(ra->containers[i]), sizeof(void *) * (ra->size - i)); memmove(&(ra->typecodes[i + 1]), &(ra->typecodes[i]), sizeof(uint8_t) * (ra->size - i)); ra->keys[i] = key; ra->containers[i] = container; ra->typecodes[i] = typecode; ra->size++; }
template <class elem_t, class holder_t> void adlist_tos_base<elem_t, holder_t>::insert_after( tos_handle<elem_t> the_handle, elem_t the_elem) { holder_t *elem_ptr = (holder_t *)(from_handle(the_handle)); if (elem_ptr == 0) return; extend_array(); holder_t *new_e = new holder_t(the_elem, elem_ptr->next); elem_ptr->next = new_e; if (_tail_e == elem_ptr) _tail_e = new_e; else new_e->next->previous = new_e; _index_size = 0; ++_cached_size; }
static dary_status move_conflicts(dary *da, int want_to_be_free_index, int *n) { int conflicts[256+1], move_to; /* MAX conflicts 256, +1 is for termination */ int retcode; int parent = get_parent_index(da, want_to_be_free_index); int i = 0; dprint("want_to_be_free is %d and its parent is %d", want_to_be_free_index, parent); get_children(da, parent, conflicts); /* coflicts are want_to_be_free and its siblings. (For non-English speaker like me: Siblings means brothers and sisters.) */ retcode = find_movable_location(da, conflicts, &move_to); if (retcode != DARY_STATUS_SUCCESS) { dprint("move_conflicts:Could not found movable location. Retrying...\n"); retcode = extend_array(da, da->length + 256); if (retcode != DARY_STATUS_SUCCESS) { return retcode; } /* This find_movable_loc must be success, but ensure retcode to make sure. */ retcode = find_movable_location(da, conflicts, &move_to); if (retcode != DARY_STATUS_SUCCESS) { return retcode; } } while (conflicts[i] != -1) { if (conflicts[i] == *n) { dprint("parent %d moved to %d\n", parent, parent + (move_to - conflicts[0])); *n = *n + (move_to - conflicts[0]); break; } dprint("confl[%d], %d ", i, conflicts[i]); i++; } move_conflicts_real(da, conflicts, move_to); return DARY_STATUS_SUCCESS; }
void ra_append_copy(roaring_array_t *ra, roaring_array_t *sa, uint16_t index, bool copy_on_write) { extend_array(ra, 1); const int32_t pos = ra->size; // old contents is junk not needing freeing ra->keys[pos] = sa->keys[index]; // the shared container will be in two bitmaps if(copy_on_write) { sa->containers[index] = get_copy_of_container(sa->containers[index], &sa->typecodes[index],copy_on_write); ra->containers[pos] = sa->containers[index]; ra->typecodes[pos] = sa->typecodes[index]; } else { ra->containers[pos] = container_clone(sa->containers[index], sa->typecodes[index]); ra->typecodes[pos] = sa->typecodes[index]; } ra->size++; }
template <class elem_t, class holder_t> void adlist_tos_base<elem_t, holder_t>::prepend(elem_t the_elem) { extend_array(); if (_head_e == 0) { _head_e = new holder_t(the_elem); _tail_e = _head_e; } else { _head_e->previous = new holder_t(the_elem, _head_e); _head_e = _head_e->previous; } _index_size = 1; _index[0] = _head_e; ++_cached_size; }
void ra_append_range(roaring_array_t *ra, roaring_array_t *sa, uint16_t start_index, uint16_t end_index, bool copy_on_write) { extend_array(ra, end_index - start_index); for (uint16_t i = start_index; i < end_index; ++i) { const int32_t pos = ra->size; ra->keys[pos] = sa->keys[i]; if(copy_on_write) { sa->containers[i] = get_copy_of_container(sa->containers[i], &sa->typecodes[i],copy_on_write); ra->containers[pos] = sa->containers[i]; ra->typecodes[pos] = sa->typecodes[i]; } else { ra->containers[pos] = container_clone(sa->containers[i], sa->typecodes[i]); ra->typecodes[pos] = sa->typecodes[i]; } ra->size++; } }
template <class elem_t, class holder_t> void adlist_tos_base<elem_t, holder_t>::append(elem_t the_elem) { extend_array(); if (_tail_e == 0) { _head_e = new holder_t(the_elem); _tail_e = _head_e; } else { _tail_e->next = new holder_t(the_elem, 0, _tail_e); _tail_e = _tail_e->next; } if (_index_size == _cached_size) { ++_index_size; _index[_cached_size] = _tail_e; } ++_cached_size; }
void stack_push(stack s, void *object){ if (full(s)) extend_array(s); ++s->top; s->stack_array[s->top] = object; }
/** * @brief Main function for oaextendmpi and oaextendsingle * @param argc * @param argv[] * @return */ int main (int argc, char *argv[]) { #ifdef OAEXTEND_SINGLECORE const int n_processors = 1; const int this_rank = 0; #else MPI::Init (argc, argv); const int n_processors = MPI::COMM_WORLD.Get_size (); const int this_rank = MPI::COMM_WORLD.Get_rank (); #endif // printf("MPI: this_rank %d\n", this_rank); /*----------SET STARTING ARRAY(S)-----------*/ if (this_rank != MASTER) { #ifdef OAEXTEND_MULTICORE slave_print (QUIET, "M: running core %d/%d\n", this_rank, n_processors); #endif algorithm_t algorithm = MODE_INVALID; AnyOption *opt = parseOptions (argc, argv, algorithm); OAextend oaextend; oaextend.setAlgorithm (algorithm); #ifdef OAEXTEND_MULTICORE log_print (NORMAL, "slave: receiving algorithm int\n"); algorithm = (algorithm_t)receive_int_slave (); oaextend.setAlgorithm (algorithm); // printf("slave %d: receive algorithm %d\n", this_rank, algorithm); #endif extend_slave_code (this_rank, oaextend); delete opt; } else { double Tstart = get_time_ms (); int nr_extensions; arraylist_t solutions, extensions; algorithm_t algorithm = MODE_INVALID; AnyOption *opt = parseOptions (argc, argv, algorithm); print_copyright (); int loglevel = opt->getIntValue ('l', NORMAL); setloglevel (loglevel); int dosort = opt->getIntValue ('s', 1); int userowsymm = opt->getIntValue ("rowsymmetry", 1); int maxk = opt->getIntValue ("maxk", 100000); int initcolprev = opt->getIntValue ("initcolprev", 1); const bool streaming = opt->getFlag ("streaming"); bool restart = false; if (opt->getValue ("restart") != NULL || opt->getValue ('r') != NULL) { restart = true; } const char *oaconfigfile = opt->getStringValue ('c', "oaconfig.txt"); const char *resultprefix = opt->getStringValue ('o', "result"); arrayfile::arrayfilemode_t mode = arrayfile_t::parseModeString (opt->getStringValue ('f', "T")); OAextend oaextend; oaextend.setAlgorithm (algorithm); if (streaming) { logstream (SYSTEM) << "operating in streaming mode, sorting of arrays will not work " << std::endl; oaextend.extendarraymode = OAextend::STOREARRAY; } // J5_45 int xx = opt->getFlag ('x'); if (xx) { oaextend.j5structure = J5_45; } if (userowsymm == 0) { oaextend.use_row_symmetry = userowsymm; printf ("use row symmetry -> %d\n", oaextend.use_row_symmetry); } if (opt->getFlag ('g')) { std::cout << "only generating arrays (no LMC check)" << endl; oaextend.checkarrays = 0; } if (opt->getFlag ("help") || (opt->getValue ("coptions") != NULL)) { if (opt->getFlag ("help")) { opt->printUsage (); } if (opt->getValue ("coptions") != NULL) { print_options (cout); } } else { logstream (QUIET) << "#time start: " << currenttime () << std::endl; arraydata_t *ad; ad = readConfigFile (oaconfigfile); ad->lmc_overflow_check (); if (ad == 0) { return 1; } log_print (NORMAL, "Using design file: %s (runs %d, strength %d)\n", oaconfigfile, ad->N, ad->strength); if (oaextend.getAlgorithm () == MODE_AUTOSELECT) { oaextend.setAlgorithmAuto (ad); } #ifdef OAEXTEND_SINGLECORE if (initcolprev == 0) { log_print (NORMAL, "setting oaextend.init_column_previous to %d\n", INITCOLUMN_ZERO); oaextend.init_column_previous = INITCOLUMN_ZERO; } #endif #ifdef OAEXTEND_MULTICORE int alg = oaextend.getAlgorithm (); for (int i = 0; i < n_processors; i++) { if (i == MASTER) { continue; } log_print (NORMAL, "MASTER: sending algorithm %d to slave %d\n", alg, i); send_int (alg, i); } #endif colindex_t col_start; log_print (SYSTEM, "using algorithm %d (%s)\n", oaextend.getAlgorithm (), oaextend.getAlgorithmName ().c_str ()); if (log_print (NORMAL, "")) { cout << oaextend.__repr__ (); } if (restart) { // start from result file int initres = init_restart (opt->getValue ('r'), col_start, solutions); if (initres == 1) { // check if restarting went OK logstream (SYSTEM) << "Problem with restart from " << opt->getValue ('r') << "" << endl; #ifdef OAEXTEND_MPI MPI_Abort (MPI_COMM_WORLD, 1); #endif exit (1); } if (dosort) { double Ttmp = get_time_ms (); sort (solutions.begin (), solutions.end ()); // solutions.sort(); log_print (QUIET, " sorting of initial solutions: %.3f [s]\n", get_time_ms () - Ttmp); } // check that oaconfig agrees with loaded arrays if (solutions.size () > 0) { if (ad->N != solutions[0].n_rows) { printf ("Problem: oaconfig does not agree with loaded arrays!\n"); // free_sols(solutions); ?? solutions.clear (); } } } else { // starting with root if (check_divisibility (ad) == false) { log_print (SYSTEM, "ERROR: Failed divisibility test!\n"); #ifdef OAEXTEND_MPI MPI_Abort (MPI_COMM_WORLD, 1); #endif exit (1); } create_root (ad, solutions); col_start = ad->strength; } /*-----------MAIN EXTENSION LOOP-------------*/ log_print (SYSTEM, "M: running with %d procs\n", n_processors); maxk = std::min (maxk, ad->ncols); time_t seconds; for (colindex_t current_col = col_start; current_col < maxk; current_col++) { fflush (stdout); arraydata_t *adcol = new arraydata_t (ad, current_col + 1); if (streaming) { string fname = resultprefix; fname += "-streaming"; fname += "-" + oafilestring (ad); logstream (NORMAL) << "oaextend: streaming mode: create file " << fname << std::endl; int nb = arrayfile_t::arrayNbits (*ad); oaextend.storefile.createfile (fname, adcol->N, ad->ncols, -1, ABINARY, nb); } log_print (SYSTEM, "Starting with column %d (%d, total time: %.2f [s])\n", current_col + 1, (int)solutions.size (), get_time_ms () - Tstart); nr_extensions = 0; arraylist_t::const_iterator cur_extension; int csol = 0; for (cur_extension = solutions.begin (); cur_extension != solutions.end (); cur_extension++) { print_progress (csol, solutions, extensions, Tstart, current_col); logstream (NORMAL) << cur_extension[0]; if (n_processors == 1) { nr_extensions += extend_array (*cur_extension, adcol, current_col, extensions, oaextend); } else { #ifdef OAEXTEND_MPI throw_runtime_exception("mpi version of oextend is no longer supported"); double Ttmp = get_time_ms (); int slave = collect_solutions_single (extensions, adcol); log_print (DEBUG, " time: %.2f, collect time %.2f\n", (get_time_ms () - Tstart), (get_time_ms () - Ttmp)); /* OPTIMIZE: send multiple arrays to slave 1 once step */ extend_array_mpi (cur_extension->array, 1, adcol, current_col, slave); #endif } #ifdef OAEXTEND_MPI if ((csol + 1) % nsolsync == 0) { collect_solutions_all (extensions, adcol); } #endif // OPTIMIZE: periodically write part of the solutions to disk csol++; /* increase current solution */ } #ifdef OAEXTEND_MPI collect_solutions_all (extensions, adcol); #endif if (checkloglevel (NORMAL)) { csol = 0; for (cur_extension = extensions.begin (); cur_extension != extensions.end (); cur_extension++) { log_print (DEBUG, "%i: -----\n", ++csol); logstream (DEBUG) << cur_extension[0]; } } if (dosort) { log_print (DEBUG, "Sorting solutions, necessary for multi-processor code\n"); double Ttmp = get_time_ms (); sort (extensions.begin (), extensions.end ()); log_print (DEBUG, " sorting time: %.3f [s]\n", get_time_ms () - Ttmp); } save_arrays (extensions, adcol, resultprefix, mode); solutions.swap (extensions); // swap old and newly found solutions extensions.clear (); // clear old to free up space for new extensions log_print (SYSTEM, "Done with column %i, total of %i solutions (time %.2f s))\n", current_col + 1, (int)solutions.size (), get_time_ms () - Tstart); delete adcol; #ifdef OAANALYZE_DISCR logstream (NORMAL) << "Discriminant: " << endl; print_discriminant (ad->N, current_col + 1); #endif } /*------------------------*/ time (&seconds); struct tm *tminfo; tminfo = localtime (&seconds); log_print (SYSTEM, "TIME: %.2f s, %s", get_time_ms () - Tstart, asctime (tminfo)); logstream (QUIET) << "#time end: " << currenttime () << std::endl; logstream (QUIET) << "#time total: " << printfstring ("%.1f", get_time_ms () - Tstart) << " [s]" << std::endl; solutions.clear (); delete ad; #ifdef OAEXTEND_MPI stop_slaves (); #endif } delete opt; } /* end of master code */ #ifdef OAEXTEND_MPI MPI_Finalize (); #endif return 0; }
dary_status dary_insert(dary *da, const uchar *key, int value) { int i, n, m; n = key[0]; dprint("key:%s", key); dprint("free_head:%d", da->free_head); // validate_array(da); for (i = 1; key[i] != '\0';i++) { if (is_leaf_node(da, n)) { m = assign_base(da, n, key[i]); } else { m = da->base[n] + key[i]; } dprint("m = %d (base[%d](%d) + key[%d](%d))", m, n, da->base[n], i, key[i]); if (m >= da->length) { int retcode = extend_array(da, m + 256); if (retcode != DARY_STATUS_SUCCESS) return retcode; } if (is_free_node(da, m)) { dprint("%d is free node, insert", m); dprint("key[%d] = %d", i, key[i]); insert_node_to(da, m, n, key[i]); } else if (is_no_conflict(da, m, n, key[i])) { dprint("no conflict for key[%d]", i); } else { /* FIXME: Here should be added a check for which is easy to move. */ if (should_move_m(da, m, n)) { dprint("move m"); int retcode = move_conflicts(da, m, &n); if (retcode != DARY_STATUS_SUCCESS) { dprint("failed to move conflicted node %d\n",m); return retcode; } insert_node_to(da, m, n, key[i]); } else { dprint("move n, n=%d, base[n] = %d", n, da->base[n]); int retcode = move_conflicts2(da, n, key[i]); dprint("current n=%d, base[n] = %d", n, da->base[n]); if (retcode != DARY_STATUS_SUCCESS) { dprint("failed to move conflicted node %d\n",m); return retcode; } m = da->base[n] + key[i]; insert_node_to(da, m, n, key[i]); } } // fprintf(stderr, "\n"); n = m; } da->values[n] = value; // validate_array(da); // dary_dump(da); return DARY_STATUS_SUCCESS; }
static dary_status move_conflicts2(dary *da, int n, uchar k) { int conflicts[256+1], move_to; /* MAX conflicts 256, +1 is for termination */ int retcode; int i = 0; get_children(da, n, conflicts); dprint("da->base[%d] =%d, k = %d", n, da->base[n], k); // ここはなんとか関数化できないか? if (conflicts[0] < da->base[n] + k) { while (conflicts[i] != -1) { i++; } dprint("last node is k"); conflicts[i] = da->base[n] + k; conflicts[i+1] = -1; } else { int len = length(conflicts); memmove(&conflicts[1], conflicts, (len + 1) * sizeof(int)); conflicts[0] = da->base[n] + k; dprint("first node is k"); } i = 0; while (conflicts[i] != -1) { dprint("conflicts[%d] = %d", i, conflicts[i]); i++; } retcode = find_movable_location(da, conflicts, &move_to); if (retcode != DARY_STATUS_SUCCESS) { dprint("move_conflicts:Could not found movable location. Retrying...\n"); retcode = extend_array(da, da->length + 256); if (retcode != DARY_STATUS_SUCCESS) { return retcode; } /* This find_movable_loc must be success, but ensure return value to make sure. */ retcode = find_movable_location(da, conflicts, &move_to); if (retcode != DARY_STATUS_SUCCESS) { return retcode; } } i = 0; int offsets[257]; calc_offsets(conflicts, offsets); for (i = 0; conflicts[i] != -1; i++) { if (conflicts[i] != da->base[n] + k) { move_node(da, conflicts[i], move_to + offsets[i]); } } da->base[n] = move_to - (conflicts[0] - da->base[n]); dprint("da->base[%d] = da->base[%d] (%d) + move_to %d - %d = %d", n, n, da->base[n], move_to, conflicts[0], da->base[n] + (move_to - conflicts[0])); return DARY_STATUS_SUCCESS; }