uint64_t mix_hashed_nums(uint8_t *hashed_nums, const uint8_t *unhashedData, size_t unhashed_sz,
uint8_t **mixed_hash, uint8_t *hash_digest)
{
uint32_t i, index = 0;
const uint32_t hashed_nums_len = SHA256_LEN;
uint64_t count;
uint8_t tmp_val, tmp_array[SHA256_LEN + 2];
//initialize the class for the extend hash
Extend_Array new_hash;
Extend_Array_init(&new_hash);
//set the first hash length in the temp array to all 0xff
memset(tmp_array, 0xff, SHA256_LEN);
//set the last two bytes to \000
*(tmp_array + SHA256_LEN) = *(tmp_array + SHA256_LEN + 1) = 0;
for(count = 0;; count++)
{
//+1 to keeps a 0 value of *(hashed_nums + index) moving on
i = hex_char_to_int(*(hashed_nums + index)) + 1;
index += i;
//if we hit the end of the hash, rehash it
if(index >= hashed_nums_len)
{
index = index % hashed_nums_len;
sha256_to_str(hashed_nums, hashed_nums_len, hashed_nums, hash_digest); //rescramble
}
tmp_val = *(hashed_nums + index);
join_to_array(tmp_array, tmp_val); //plop tmp_val at the end of tmp_array
sha256_to_str(tmp_array, SHA256_LEN + 1, tmp_array, hash_digest);
//extend the expanded hash to the array
extend_array(&new_hash, count * SHA256_LEN, tmp_array, SHA256_LEN, false);
//check if the last value of hashed_nums is the same as the last value in tmp_array
if(index == hashed_nums_len - 1)
if(tmp_val == *(tmp_array + SHA256_LEN - 1))
{
//add to count since we extended the array, but break will exit the for loop and count
// will not get incremenented by the for loop
count++;
break;
}
}
//extend the unhashed data to the end and add the \000 to the end
extend_array(&new_hash, count * SHA256_LEN, (unsigned char*)unhashedData, unhashed_sz, true);
//assign the address of new_hash's array to mixed_hash
*mixed_hash = new_hash.array;
return count * SHA256_LEN + unhashed_sz;
}
dary *
dary_new(void)
{
dary *da;
int i = 0;
da = dary_allocator(sizeof(dary));
da->length = 0;
da->base = NULL;
da->check = NULL;
da->values = NULL;
da->free_head = -1;
extend_array(da, UCHAR_MAX + 2);
da->free_head = UCHAR_MAX + 1;
for (i =0 ; i < UCHAR_MAX + 1; i++) {
da->base[i] = 0;
da->check[i] = 0;
}
dprint("%d",i);
da->check[i] = -i;
da->can_free = 1;
return da;
}
void ra_append(roaring_array_t *ra, uint16_t key, void *container,
uint8_t typecode) {
extend_array(ra, 1);
const int32_t pos = ra->size;
ra->keys[pos] = key;
ra->containers[pos] = container;
ra->typecodes[pos] = typecode;
ra->size++;
}
void ra_append_move_range(roaring_array_t *ra, roaring_array_t *sa,
uint16_t start_index, uint16_t end_index) {
extend_array(ra, end_index - start_index);
for (uint16_t i = start_index; i < end_index; ++i) {
const int32_t pos = ra->size;
ra->keys[pos] = sa->keys[i];
ra->containers[pos] = sa->containers[i];
ra->typecodes[pos] = sa->typecodes[i];
ra->size++;
}
}
template <class elem_t, class holder_t> void
adlist_tos_base<elem_t, holder_t>::insert(size_t elem_num,
elem_t the_elem)
{
assert(elem_num >= 0);
if (elem_num == 0)
{
prepend(the_elem);
return;
}
if (elem_num >= _cached_size)
{
while (_cached_size < elem_num)
append(zero((elem_t *)0));
append(the_elem);
return;
}
extend_array();
holder_t *follow;
if (elem_num - 1 < _index_size)
{
follow = _index[elem_num - 1];
}
else
{
if (_index_size == 0)
{
_index[0] = _head_e;
typedef adlist_tos_base<elem_t, holder_t> *this_type;
++(CONST_CAST(this_type, this)->_index_size);
}
size_t total = _index_size - 1;
follow = _index[total];
while (total < elem_num - 1)
{
++total;
follow = follow->next;
_index[total] = follow;
}
}
holder_t *new_e = new holder_t(the_elem, follow->next, follow);
follow->next = new_e;
if (_tail_e == follow)
_tail_e = new_e;
else
new_e->next->previous = new_e;
_index[elem_num] = new_e;
_index_size = elem_num + 1;
++_cached_size;
}
void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key,
void *container, uint8_t typecode) {
extend_array(ra, 1);
// May be an optimization opportunity with DIY memmove
memmove(&(ra->keys[i + 1]), &(ra->keys[i]),
sizeof(uint16_t) * (ra->size - i));
memmove(&(ra->containers[i + 1]), &(ra->containers[i]),
sizeof(void *) * (ra->size - i));
memmove(&(ra->typecodes[i + 1]), &(ra->typecodes[i]),
sizeof(uint8_t) * (ra->size - i));
ra->keys[i] = key;
ra->containers[i] = container;
ra->typecodes[i] = typecode;
ra->size++;
}
template <class elem_t, class holder_t> void
adlist_tos_base<elem_t, holder_t>::insert_after(
tos_handle<elem_t> the_handle, elem_t the_elem)
{
holder_t *elem_ptr = (holder_t *)(from_handle(the_handle));
if (elem_ptr == 0)
return;
extend_array();
holder_t *new_e = new holder_t(the_elem, elem_ptr->next);
elem_ptr->next = new_e;
if (_tail_e == elem_ptr)
_tail_e = new_e;
else
new_e->next->previous = new_e;
_index_size = 0;
++_cached_size;
}
static dary_status
move_conflicts(dary *da, int want_to_be_free_index, int *n)
{
int conflicts[256+1], move_to; /* MAX conflicts 256, +1 is for termination */
int retcode;
int parent = get_parent_index(da, want_to_be_free_index);
int i = 0;
dprint("want_to_be_free is %d and its parent is %d", want_to_be_free_index, parent);
get_children(da, parent, conflicts);
/* coflicts are want_to_be_free and its siblings.
(For non-English speaker like me: Siblings means brothers and sisters.) */
retcode = find_movable_location(da, conflicts, &move_to);
if (retcode != DARY_STATUS_SUCCESS) {
dprint("move_conflicts:Could not found movable location. Retrying...\n");
retcode = extend_array(da, da->length + 256);
if (retcode != DARY_STATUS_SUCCESS) {
return retcode;
}
/* This find_movable_loc must be success, but ensure retcode to make sure. */
retcode = find_movable_location(da, conflicts, &move_to);
if (retcode != DARY_STATUS_SUCCESS) {
return retcode;
}
}
while (conflicts[i] != -1) {
if (conflicts[i] == *n) {
dprint("parent %d moved to %d\n", parent, parent + (move_to - conflicts[0]));
*n = *n + (move_to - conflicts[0]);
break;
}
dprint("confl[%d], %d ", i, conflicts[i]);
i++;
}
move_conflicts_real(da, conflicts, move_to);
return DARY_STATUS_SUCCESS;
}
void ra_append_copy(roaring_array_t *ra, roaring_array_t *sa, uint16_t index, bool copy_on_write) {
extend_array(ra, 1);
const int32_t pos = ra->size;
// old contents is junk not needing freeing
ra->keys[pos] = sa->keys[index];
// the shared container will be in two bitmaps
if(copy_on_write) {
sa->containers[index] = get_copy_of_container(sa->containers[index], &sa->typecodes[index],copy_on_write);
ra->containers[pos] = sa->containers[index];
ra->typecodes[pos] = sa->typecodes[index];
} else {
ra->containers[pos] =
container_clone(sa->containers[index], sa->typecodes[index]);
ra->typecodes[pos] = sa->typecodes[index];
}
ra->size++;
}
template <class elem_t, class holder_t> void
adlist_tos_base<elem_t, holder_t>::prepend(elem_t the_elem)
{
extend_array();
if (_head_e == 0)
{
_head_e = new holder_t(the_elem);
_tail_e = _head_e;
}
else
{
_head_e->previous = new holder_t(the_elem, _head_e);
_head_e = _head_e->previous;
}
_index_size = 1;
_index[0] = _head_e;
++_cached_size;
}
void ra_append_range(roaring_array_t *ra, roaring_array_t *sa,
uint16_t start_index, uint16_t end_index, bool copy_on_write) {
extend_array(ra, end_index - start_index);
for (uint16_t i = start_index; i < end_index; ++i) {
const int32_t pos = ra->size;
ra->keys[pos] = sa->keys[i];
if(copy_on_write) {
sa->containers[i] = get_copy_of_container(sa->containers[i], &sa->typecodes[i],copy_on_write);
ra->containers[pos] = sa->containers[i];
ra->typecodes[pos] = sa->typecodes[i];
} else {
ra->containers[pos] =
container_clone(sa->containers[i], sa->typecodes[i]);
ra->typecodes[pos] = sa->typecodes[i];
}
ra->size++;
}
}
template <class elem_t, class holder_t> void
adlist_tos_base<elem_t, holder_t>::append(elem_t the_elem)
{
extend_array();
if (_tail_e == 0)
{
_head_e = new holder_t(the_elem);
_tail_e = _head_e;
}
else
{
_tail_e->next = new holder_t(the_elem, 0, _tail_e);
_tail_e = _tail_e->next;
}
if (_index_size == _cached_size)
{
++_index_size;
_index[_cached_size] = _tail_e;
}
++_cached_size;
}
void stack_push(stack s, void *object){
if (full(s))
extend_array(s);
++s->top;
s->stack_array[s->top] = object;
}
/**
* @brief Main function for oaextendmpi and oaextendsingle
* @param argc
* @param argv[]
* @return
*/
int main (int argc, char *argv[]) {
#ifdef OAEXTEND_SINGLECORE
const int n_processors = 1;
const int this_rank = 0;
#else
MPI::Init (argc, argv);
const int n_processors = MPI::COMM_WORLD.Get_size ();
const int this_rank = MPI::COMM_WORLD.Get_rank ();
#endif
// printf("MPI: this_rank %d\n", this_rank);
/*----------SET STARTING ARRAY(S)-----------*/
if (this_rank != MASTER) {
#ifdef OAEXTEND_MULTICORE
slave_print (QUIET, "M: running core %d/%d\n", this_rank, n_processors);
#endif
algorithm_t algorithm = MODE_INVALID;
AnyOption *opt = parseOptions (argc, argv, algorithm);
OAextend oaextend;
oaextend.setAlgorithm (algorithm);
#ifdef OAEXTEND_MULTICORE
log_print (NORMAL, "slave: receiving algorithm int\n");
algorithm = (algorithm_t)receive_int_slave ();
oaextend.setAlgorithm (algorithm);
// printf("slave %d: receive algorithm %d\n", this_rank, algorithm);
#endif
extend_slave_code (this_rank, oaextend);
delete opt;
} else {
double Tstart = get_time_ms ();
int nr_extensions;
arraylist_t solutions, extensions;
algorithm_t algorithm = MODE_INVALID;
AnyOption *opt = parseOptions (argc, argv, algorithm);
print_copyright ();
int loglevel = opt->getIntValue ('l', NORMAL);
setloglevel (loglevel);
int dosort = opt->getIntValue ('s', 1);
int userowsymm = opt->getIntValue ("rowsymmetry", 1);
int maxk = opt->getIntValue ("maxk", 100000);
int initcolprev = opt->getIntValue ("initcolprev", 1);
const bool streaming = opt->getFlag ("streaming");
bool restart = false;
if (opt->getValue ("restart") != NULL || opt->getValue ('r') != NULL) {
restart = true;
}
const char *oaconfigfile = opt->getStringValue ('c', "oaconfig.txt");
const char *resultprefix = opt->getStringValue ('o', "result");
arrayfile::arrayfilemode_t mode = arrayfile_t::parseModeString (opt->getStringValue ('f', "T"));
OAextend oaextend;
oaextend.setAlgorithm (algorithm);
if (streaming) {
logstream (SYSTEM) << "operating in streaming mode, sorting of arrays will not work "
<< std::endl;
oaextend.extendarraymode = OAextend::STOREARRAY;
}
// J5_45
int xx = opt->getFlag ('x');
if (xx) {
oaextend.j5structure = J5_45;
}
if (userowsymm == 0) {
oaextend.use_row_symmetry = userowsymm;
printf ("use row symmetry -> %d\n", oaextend.use_row_symmetry);
}
if (opt->getFlag ('g')) {
std::cout << "only generating arrays (no LMC check)" << endl;
oaextend.checkarrays = 0;
}
if (opt->getFlag ("help") || (opt->getValue ("coptions") != NULL)) {
if (opt->getFlag ("help")) {
opt->printUsage ();
}
if (opt->getValue ("coptions") != NULL) {
print_options (cout);
}
} else {
logstream (QUIET) << "#time start: " << currenttime () << std::endl;
arraydata_t *ad;
ad = readConfigFile (oaconfigfile);
ad->lmc_overflow_check ();
if (ad == 0) {
return 1;
}
log_print (NORMAL, "Using design file: %s (runs %d, strength %d)\n", oaconfigfile, ad->N,
ad->strength);
if (oaextend.getAlgorithm () == MODE_AUTOSELECT) {
oaextend.setAlgorithmAuto (ad);
}
#ifdef OAEXTEND_SINGLECORE
if (initcolprev == 0) {
log_print (NORMAL, "setting oaextend.init_column_previous to %d\n", INITCOLUMN_ZERO);
oaextend.init_column_previous = INITCOLUMN_ZERO;
}
#endif
#ifdef OAEXTEND_MULTICORE
int alg = oaextend.getAlgorithm ();
for (int i = 0; i < n_processors; i++) {
if (i == MASTER) {
continue;
}
log_print (NORMAL, "MASTER: sending algorithm %d to slave %d\n", alg, i);
send_int (alg, i);
}
#endif
colindex_t col_start;
log_print (SYSTEM, "using algorithm %d (%s)\n", oaextend.getAlgorithm (),
oaextend.getAlgorithmName ().c_str ());
if (log_print (NORMAL, "")) {
cout << oaextend.__repr__ ();
}
if (restart) { // start from result file
int initres = init_restart (opt->getValue ('r'), col_start, solutions);
if (initres == 1) { // check if restarting went OK
logstream (SYSTEM) << "Problem with restart from " << opt->getValue ('r') << ""
<< endl;
#ifdef OAEXTEND_MPI
MPI_Abort (MPI_COMM_WORLD, 1);
#endif
exit (1);
}
if (dosort) {
double Ttmp = get_time_ms ();
sort (solutions.begin (), solutions.end ()); // solutions.sort();
log_print (QUIET, " sorting of initial solutions: %.3f [s]\n",
get_time_ms () - Ttmp);
}
// check that oaconfig agrees with loaded arrays
if (solutions.size () > 0) {
if (ad->N != solutions[0].n_rows) {
printf ("Problem: oaconfig does not agree with loaded arrays!\n");
// free_sols(solutions); ??
solutions.clear ();
}
}
} else {
// starting with root
if (check_divisibility (ad) == false) {
log_print (SYSTEM, "ERROR: Failed divisibility test!\n");
#ifdef OAEXTEND_MPI
MPI_Abort (MPI_COMM_WORLD, 1);
#endif
exit (1);
}
create_root (ad, solutions);
col_start = ad->strength;
}
/*-----------MAIN EXTENSION LOOP-------------*/
log_print (SYSTEM, "M: running with %d procs\n", n_processors);
maxk = std::min (maxk, ad->ncols);
time_t seconds;
for (colindex_t current_col = col_start; current_col < maxk; current_col++) {
fflush (stdout);
arraydata_t *adcol = new arraydata_t (ad, current_col + 1);
if (streaming) {
string fname = resultprefix;
fname += "-streaming";
fname += "-" + oafilestring (ad);
logstream (NORMAL) << "oaextend: streaming mode: create file " << fname
<< std::endl;
int nb = arrayfile_t::arrayNbits (*ad);
oaextend.storefile.createfile (fname, adcol->N, ad->ncols, -1, ABINARY, nb);
}
log_print (SYSTEM, "Starting with column %d (%d, total time: %.2f [s])\n",
current_col + 1, (int)solutions.size (), get_time_ms () - Tstart);
nr_extensions = 0;
arraylist_t::const_iterator cur_extension;
int csol = 0;
for (cur_extension = solutions.begin (); cur_extension != solutions.end ();
cur_extension++) {
print_progress (csol, solutions, extensions, Tstart, current_col);
logstream (NORMAL) << cur_extension[0];
if (n_processors == 1) {
nr_extensions += extend_array (*cur_extension, adcol,
current_col, extensions, oaextend);
} else {
#ifdef OAEXTEND_MPI
throw_runtime_exception("mpi version of oextend is no longer supported");
double Ttmp = get_time_ms ();
int slave = collect_solutions_single (extensions, adcol);
log_print (DEBUG, " time: %.2f, collect time %.2f\n",
(get_time_ms () - Tstart), (get_time_ms () - Ttmp));
/* OPTIMIZE: send multiple arrays to slave 1 once step */
extend_array_mpi (cur_extension->array, 1, adcol, current_col, slave);
#endif
}
#ifdef OAEXTEND_MPI
if ((csol + 1) % nsolsync == 0) {
collect_solutions_all (extensions, adcol);
}
#endif
// OPTIMIZE: periodically write part of the solutions to disk
csol++; /* increase current solution */
}
#ifdef OAEXTEND_MPI
collect_solutions_all (extensions, adcol);
#endif
if (checkloglevel (NORMAL)) {
csol = 0;
for (cur_extension = extensions.begin (); cur_extension != extensions.end ();
cur_extension++) {
log_print (DEBUG, "%i: -----\n", ++csol);
logstream (DEBUG) << cur_extension[0];
}
}
if (dosort) {
log_print (DEBUG, "Sorting solutions, necessary for multi-processor code\n");
double Ttmp = get_time_ms ();
sort (extensions.begin (), extensions.end ());
log_print (DEBUG, " sorting time: %.3f [s]\n", get_time_ms () - Ttmp);
}
save_arrays (extensions, adcol, resultprefix, mode);
solutions.swap (extensions); // swap old and newly found solutions
extensions.clear (); // clear old to free up space for new extensions
log_print (SYSTEM, "Done with column %i, total of %i solutions (time %.2f s))\n",
current_col + 1, (int)solutions.size (), get_time_ms () - Tstart);
delete adcol;
#ifdef OAANALYZE_DISCR
logstream (NORMAL) << "Discriminant: " << endl;
print_discriminant (ad->N, current_col + 1);
#endif
}
/*------------------------*/
time (&seconds);
struct tm *tminfo;
tminfo = localtime (&seconds);
log_print (SYSTEM, "TIME: %.2f s, %s", get_time_ms () - Tstart, asctime (tminfo));
logstream (QUIET) << "#time end: " << currenttime () << std::endl;
logstream (QUIET) << "#time total: " << printfstring ("%.1f", get_time_ms () - Tstart)
<< " [s]" << std::endl;
solutions.clear ();
delete ad;
#ifdef OAEXTEND_MPI
stop_slaves ();
#endif
}
delete opt;
} /* end of master code */
#ifdef OAEXTEND_MPI
MPI_Finalize ();
#endif
return 0;
}
dary_status
dary_insert(dary *da, const uchar *key, int value)
{
int i, n, m;
n = key[0];
dprint("key:%s", key);
dprint("free_head:%d", da->free_head);
// validate_array(da);
for (i = 1; key[i] != '\0';i++) {
if (is_leaf_node(da, n)) {
m = assign_base(da, n, key[i]);
} else {
m = da->base[n] + key[i];
}
dprint("m = %d (base[%d](%d) + key[%d](%d))", m, n, da->base[n], i, key[i]);
if (m >= da->length) {
int retcode = extend_array(da, m + 256);
if (retcode != DARY_STATUS_SUCCESS)
return retcode;
}
if (is_free_node(da, m)) {
dprint("%d is free node, insert", m);
dprint("key[%d] = %d", i, key[i]);
insert_node_to(da, m, n, key[i]);
} else if (is_no_conflict(da, m, n, key[i])) {
dprint("no conflict for key[%d]", i);
} else {
/* FIXME: Here should be added a check for which is easy to move. */
if (should_move_m(da, m, n)) {
dprint("move m");
int retcode = move_conflicts(da, m, &n);
if (retcode != DARY_STATUS_SUCCESS) {
dprint("failed to move conflicted node %d\n",m);
return retcode;
}
insert_node_to(da, m, n, key[i]);
} else {
dprint("move n, n=%d, base[n] = %d", n, da->base[n]);
int retcode = move_conflicts2(da, n, key[i]);
dprint("current n=%d, base[n] = %d", n, da->base[n]);
if (retcode != DARY_STATUS_SUCCESS) {
dprint("failed to move conflicted node %d\n",m);
return retcode;
}
m = da->base[n] + key[i];
insert_node_to(da, m, n, key[i]);
}
}
// fprintf(stderr, "\n");
n = m;
}
da->values[n] = value;
// validate_array(da);
// dary_dump(da);
return DARY_STATUS_SUCCESS;
}
static dary_status
move_conflicts2(dary *da, int n, uchar k)
{
int conflicts[256+1], move_to; /* MAX conflicts 256, +1 is for termination */
int retcode;
int i = 0;
get_children(da, n, conflicts);
dprint("da->base[%d] =%d, k = %d", n, da->base[n], k);
// ここはなんとか関数化できないか？
if (conflicts[0] < da->base[n] + k) {
while (conflicts[i] != -1) {
i++;
}
dprint("last node is k");
conflicts[i] = da->base[n] + k;
conflicts[i+1] = -1;
} else {
int len = length(conflicts);
memmove(&conflicts[1], conflicts, (len + 1) * sizeof(int));
conflicts[0] = da->base[n] + k;
dprint("first node is k");
}
i = 0;
while (conflicts[i] != -1) {
dprint("conflicts[%d] = %d", i, conflicts[i]);
i++;
}
retcode = find_movable_location(da, conflicts, &move_to);
if (retcode != DARY_STATUS_SUCCESS) {
dprint("move_conflicts:Could not found movable location. Retrying...\n");
retcode = extend_array(da, da->length + 256);
if (retcode != DARY_STATUS_SUCCESS) {
return retcode;
}
/* This find_movable_loc must be success, but ensure return value to make sure. */
retcode = find_movable_location(da, conflicts, &move_to);
if (retcode != DARY_STATUS_SUCCESS) {
return retcode;
}
}
i = 0;
int offsets[257];
calc_offsets(conflicts, offsets);
for (i = 0; conflicts[i] != -1; i++) {
if (conflicts[i] != da->base[n] + k) {
move_node(da, conflicts[i], move_to + offsets[i]);
}
}
da->base[n] = move_to - (conflicts[0] - da->base[n]);
dprint("da->base[%d] = da->base[%d] (%d) + move_to %d - %d = %d",
n, n, da->base[n], move_to, conflicts[0],
da->base[n] + (move_to - conflicts[0]));
return DARY_STATUS_SUCCESS;
}
