/*!
* l_dnaHashCreateFromPta()
*
* Input: pta
* Return: dahash, or null on error
*/
L_DNAHASH *
l_dnaHashCreateFromPta(PTA *pta)
{
l_int32 i, n, x, y;
l_uint32 nsize;
l_uint64 key;
L_DNAHASH *dahash;
PROCNAME("l_dnaHashCreateFromPta");
/* Build up dnaHash of indices, hashed by a key that is
* a large linear combination of x and y values designed to
* randomize the key. Having about 20 pts in each bucket is
* roughly optimal for speed for large sets. */
n = ptaGetCount(pta);
findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */
/* fprintf(stderr, "Prime used: %d\n", nsize); */
/* Add each point, using the hash as key and the index into
* @ptas as the value. Storing the index enables operations
* that check for duplicates. */
dahash = l_dnaHashCreate(nsize, 8);
for (i = 0; i < n; i++) {
ptaGetIPt(pta, i, &x, &y);
l_hashPtToUint64Fast(nsize, x, y, &key);
l_dnaHashAdd(dahash, key, (l_float64)i);
}
return dahash;
}
/*!
* \brief l_dnaHashCreateFromDna()
*
* \param[in] da
* \return dahash if OK; 1 on error
*
* <pre>
* Notes:
* (1) The values stored in the %dahash are indices into %da;
* %dahash has no use without %da.
* </pre>
*/
L_DNAHASH *
l_dnaHashCreateFromDna(L_DNA *da)
{
l_int32 i, n;
l_uint32 nsize;
l_uint64 key;
l_float64 val;
L_DNAHASH *dahash;
PROCNAME("l_dnaHashCreateFromDna");
if (!da)
return (L_DNAHASH *)ERROR_PTR("da not defined", procName, NULL);
n = l_dnaGetCount(da);
findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */
dahash = l_dnaHashCreate(nsize, 8);
for (i = 0; i < n; i++) {
l_dnaGetDValue(da, i, &val);
l_hashFloat64ToUint64(nsize, val, &key);
l_dnaHashAdd(dahash, key, (l_float64)i);
}
return dahash;
}
/*!
* \brief l_dnaHashCreateFromSarray()
*
* \param[in] sa
* \return dahash, or NULL on error
*/
L_DNAHASH *
l_dnaHashCreateFromSarray(SARRAY *sa)
{
char *str;
l_int32 i, n;
l_uint32 nsize;
l_uint64 key;
L_DNAHASH *dahash;
/* Build up dnaHash of indices, hashed by a 64-bit key that
* should randomize the lower bits used in bucket selection.
* Having about 20 pts in each bucket is roughly optimal. */
n = sarrayGetCount(sa);
findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */
/* fprintf(stderr, "Prime used: %d\n", nsize); */
/* Add each string, using the hash as key and the index into %sa
* as the value. Storing the index enables operations that check
* for duplicates. */
dahash = l_dnaHashCreate(nsize, 8);
for (i = 0; i < n; i++) {
str = sarrayGetString(sa, i, L_NOCOPY);
l_hashStringToUint64(str, &key);
l_dnaHashAdd(dahash, key, (l_float64)i);
}
return dahash;
}
/*!
* \brief l_dnaIntersectionByHash()
*
* \param[in] da1, da2
* \return dad intersection of the number arrays, or NULL on error
*
* <pre>
* Notes:
* (1) This uses the same method for building the intersection set
* as ptaIntersectionByHash() and sarrayIntersectionByHash().
* </pre>
*/
L_DNA *
l_dnaIntersectionByHash(L_DNA *da1,
L_DNA *da2)
{
l_int32 n1, n2, nsmall, nbuckets, i, index1, index2;
l_uint32 nsize2;
l_uint64 key;
l_float64 val;
L_DNAHASH *dahash1, *dahash2;
L_DNA *da_small, *da_big, *dad;
PROCNAME("l_dnaIntersectionByHash");
if (!da1)
return (L_DNA *)ERROR_PTR("da1 not defined", procName, NULL);
if (!da2)
return (L_DNA *)ERROR_PTR("da2 not defined", procName, NULL);
/* Put the elements of the biggest array into a dnahash */
n1 = l_dnaGetCount(da1);
n2 = l_dnaGetCount(da2);
da_small = (n1 < n2) ? da1 : da2; /* do not destroy da_small */
da_big = (n1 < n2) ? da2 : da1; /* do not destroy da_big */
dahash1 = l_dnaHashCreateFromDna(da_big);
/* Build up the intersection of numbers. Add to %dad
* if the number is in da_big (using dahash1) but hasn't
* yet been seen in the traversal of da_small (using dahash2). */
dad = l_dnaCreate(0);
nsmall = l_dnaGetCount(da_small);
findNextLargerPrime(nsmall / 20, &nsize2); /* buckets in hash table */
dahash2 = l_dnaHashCreate(nsize2, 0);
nbuckets = l_dnaHashGetCount(dahash2);
for (i = 0; i < nsmall; i++) {
l_dnaGetDValue(da_small, i, &val);
l_dnaFindValByHash(da_big, dahash1, val, &index1);
if (index1 >= 0) { /* found */
l_dnaFindValByHash(da_small, dahash2, val, &index2);
if (index2 == -1) { /* not found */
l_dnaAddNumber(dad, val);
l_hashFloat64ToUint64(nbuckets, val, &key);
l_dnaHashAdd(dahash2, key, (l_float64)i);
}
}
}
l_dnaHashDestroy(&dahash1);
l_dnaHashDestroy(&dahash2);
return dad;
}
/*!
* \brief l_dnaMakeHistoByHash()
*
* \param[in] das
* \param[out] pdahash hash map: val --> index
* \param[out] pdav array of values: index --> val
* \param[out] pdac histo array of counts: index --> count
* \return 0 if OK; 1 on error
*
* <pre>
* Notes:
* (1) Generates and returns a dna of occurrences (histogram),
* an aligned dna of values, and an associated hashmap.
* The hashmap takes %dav and a value, and points into the
* histogram in %dac.
* (2) The dna of values, %dav, is aligned with the histogram %dac,
* and is needed for fast lookup. It is a hash set, because
* the values are unique.
* (3) Lookup is simple:
* l_dnaFindValByHash(dav, dahash, val, &index);
* if (index >= 0)
* l_dnaGetIValue(dac, index, &icount);
* else
* icount = 0;
* </pre>
*/
l_ok
l_dnaMakeHistoByHash(L_DNA *das,
L_DNAHASH **pdahash,
L_DNA **pdav,
L_DNA **pdac)
{
l_int32 i, n, nitems, index, count;
l_uint32 nsize;
l_uint64 key;
l_float64 val;
L_DNA *dac, *dav;
L_DNAHASH *dahash;
PROCNAME("l_dnaMakeHistoByHash");
if (pdahash) *pdahash = NULL;
if (pdac) *pdac = NULL;
if (pdav) *pdav = NULL;
if (!pdahash || !pdac || !pdav)
return ERROR_INT("&dahash, &dac, &dav not all defined", procName, 1);
if (!das)
return ERROR_INT("das not defined", procName, 1);
if ((n = l_dnaGetCount(das)) == 0)
return ERROR_INT("no data in das", procName, 1);
findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */
dahash = l_dnaHashCreate(nsize, 8);
dac = l_dnaCreate(n); /* histogram */
dav = l_dnaCreate(n); /* the values */
for (i = 0, nitems = 0; i < n; i++) {
l_dnaGetDValue(das, i, &val);
/* Is this value already stored in dav? */
l_dnaFindValByHash(dav, dahash, val, &index);
if (index >= 0) { /* found */
l_dnaGetIValue(dac, (l_float64)index, &count);
l_dnaSetValue(dac, (l_float64)index, count + 1);
} else { /* not found */
l_hashFloat64ToUint64(nsize, val, &key);
l_dnaHashAdd(dahash, key, (l_float64)nitems);
l_dnaAddNumber(dav, val);
l_dnaAddNumber(dac, 1);
nitems++;
}
}
*pdahash = dahash;
*pdac = dac;
*pdav = dav;
return 0;
}
/*!
* \brief sarrayIntersectionByHash()
*
* \param[in] sa1, sa2
* \return sad intersection of the strings, or NULL on error
*
* <pre>
* Notes:
* (1) This is faster than sarrayIntersectionByAset(), because the
* bucket lookup is O(n).
* </pre>
*/
SARRAY *
sarrayIntersectionByHash(SARRAY *sa1,
SARRAY *sa2)
{
char *str;
l_int32 n1, n2, nsmall, i, index1, index2;
l_uint32 nsize2;
l_uint64 key;
L_DNAHASH *dahash1, *dahash2;
SARRAY *sa_small, *sa_big, *sad;
PROCNAME("sarrayIntersectionByHash");
if (!sa1)
return (SARRAY *)ERROR_PTR("sa1 not defined", procName, NULL);
if (!sa2)
return (SARRAY *)ERROR_PTR("sa2 not defined", procName, NULL);
/* Put the elements of the biggest sarray into a dnahash */
n1 = sarrayGetCount(sa1);
n2 = sarrayGetCount(sa2);
sa_small = (n1 < n2) ? sa1 : sa2; /* do not destroy sa_small */
sa_big = (n1 < n2) ? sa2 : sa1; /* do not destroy sa_big */
dahash1 = l_dnaHashCreateFromSarray(sa_big);
/* Build up the intersection of strings. Add to %sad
* if the string is in sa_big (using dahash1) but hasn't
* yet been seen in the traversal of sa_small (using dahash2). */
sad = sarrayCreate(0);
nsmall = sarrayGetCount(sa_small);
findNextLargerPrime(nsmall / 20, &nsize2); /* buckets in hash table */
dahash2 = l_dnaHashCreate(nsize2, 0);
for (i = 0; i < nsmall; i++) {
str = sarrayGetString(sa_small, i, L_NOCOPY);
sarrayFindStringByHash(sa_big, dahash1, str, &index1);
if (index1 >= 0) {
sarrayFindStringByHash(sa_small, dahash2, str, &index2);
if (index2 == -1) {
sarrayAddString(sad, str, L_COPY);
l_hashStringToUint64(str, &key);
l_dnaHashAdd(dahash2, key, (l_float64)i);
}
}
}
l_dnaHashDestroy(&dahash1);
l_dnaHashDestroy(&dahash2);
return sad;
}
/*!
* ptaIntersectionByHash()
*
* Input: pta1, pta2
* Return: ptad (intersection of the point sets), or null on error
*
* Notes:
* (1) This is faster than ptaIntersectionByAset(), because the
* bucket lookup is O(n). It should be used if the pts are
* integers (e.g., representing pixel positions).
*/
PTA *
ptaIntersectionByHash(PTA *pta1,
PTA *pta2)
{
l_int32 n1, n2, nsmall, i, x, y, index1, index2;
l_uint32 nsize2;
l_uint64 key;
L_DNAHASH *dahash1, *dahash2;
PTA *pta_small, *pta_big, *ptad;
PROCNAME("ptaIntersectionByHash");
if (!pta1)
return (PTA *)ERROR_PTR("pta1 not defined", procName, NULL);
if (!pta2)
return (PTA *)ERROR_PTR("pta2 not defined", procName, NULL);
/* Put the elements of the biggest pta into a dnahash */
n1 = ptaGetCount(pta1);
n2 = ptaGetCount(pta2);
pta_small = (n1 < n2) ? pta1 : pta2; /* do not destroy pta_small */
pta_big = (n1 < n2) ? pta2 : pta1; /* do not destroy pta_big */
dahash1 = l_dnaHashCreateFromPta(pta_big);
/* Build up the intersection of points. Add to ptad
* if the point is in pta_big (using dahash1) but hasn't
* yet been seen in the traversal of pta_small (using dahash2). */
ptad = ptaCreate(0);
nsmall = ptaGetCount(pta_small);
findNextLargerPrime(nsmall / 20, &nsize2); /* buckets in hash table */
dahash2 = l_dnaHashCreate(nsize2, 0);
for (i = 0; i < nsmall; i++) {
ptaGetIPt(pta_small, i, &x, &y);
ptaFindPtByHash(pta_big, dahash1, x, y, &index1);
if (index1 >= 0) { /* found */
ptaFindPtByHash(pta_small, dahash2, x, y, &index2);
if (index2 == -1) { /* not found */
ptaAddPt(ptad, x, y);
l_hashPtToUint64Fast(nsize2, x, y, &key);
l_dnaHashAdd(dahash2, key, (l_float64)i);
}
}
}
l_dnaHashDestroy(&dahash1);
l_dnaHashDestroy(&dahash2);
return ptad;
}
/*!
* \brief l_dnaRemoveDupsByHash()
*
* \param[in] das
* \param[out] pdad hash set
* \param[out] pdahash [optional] dnahash used for lookup
* \return 0 if OK; 1 on error
*
* <pre>
* Notes:
* (1) Generates a dna with unique values.
* (2) The dnahash is built up with dad to assure uniqueness.
* It can be used to find if an element is in the set:
* l_dnaFindValByHash(dad, dahash, val, &index)
* </pre>
*/
l_ok
l_dnaRemoveDupsByHash(L_DNA *das,
L_DNA **pdad,
L_DNAHASH **pdahash)
{
l_int32 i, n, index, items;
l_uint32 nsize;
l_uint64 key;
l_float64 val;
L_DNA *dad;
L_DNAHASH *dahash;
PROCNAME("l_dnaRemoveDupsByHash");
if (pdahash) *pdahash = NULL;
if (!pdad)
return ERROR_INT("&dad not defined", procName, 1);
*pdad = NULL;
if (!das)
return ERROR_INT("das not defined", procName, 1);
n = l_dnaGetCount(das);
findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */
dahash = l_dnaHashCreate(nsize, 8);
dad = l_dnaCreate(n);
*pdad = dad;
for (i = 0, items = 0; i < n; i++) {
l_dnaGetDValue(das, i, &val);
l_dnaFindValByHash(dad, dahash, val, &index);
if (index < 0) { /* not found */
l_hashFloat64ToUint64(nsize, val, &key);
l_dnaHashAdd(dahash, key, (l_float64)items);
l_dnaAddNumber(dad, val);
items++;
}
}
if (pdahash)
*pdahash = dahash;
else
l_dnaHashDestroy(&dahash);
return 0;
}
/*!
* ptaRemoveDupsByHash()
*
* Input: ptas (assumed to be integer values)
* &ptad (<return> unique set of pts; duplicates removed)
* &dahash (<optional return> dnahash used for lookup)
* Return: 0 if OK, 1 on error
*
* Notes:
* (1) Generates a pta with unique values.
* (2) The dnahash is built up with ptad to assure uniqueness.
* It can be used to find if a point is in the set:
* ptaFindPtByHash(ptad, dahash, x, y, &index)
* (3) The hash of the (x,y) location is simple and fast. It scales
* up with the number of buckets to insure a fairly random
* bucket selection for adjacent points.
* (4) A Dna is used rather than a Numa because we need accurate
* representation of 32-bit integers that are indices into ptas.
* Integer --> float --> integer conversion makes errors for
* integers larger than 10M.
* (5) This is faster than ptaRemoveDupsByAset(), because the
* bucket lookup is O(n), although there is a double-loop
* lookup within the dna in each bucket.
*/
l_int32
ptaRemoveDupsByHash(PTA *ptas,
PTA **pptad,
L_DNAHASH **pdahash)
{
l_int32 i, n, index, items, x, y;
l_uint32 nsize;
l_uint64 key;
l_float64 val;
PTA *ptad;
L_DNAHASH *dahash;
PROCNAME("ptaRemoveDupsByHash");
if (pdahash) *pdahash = NULL;
if (!pptad)
return ERROR_INT("&ptad not defined", procName, 1);
*pptad = NULL;
if (!ptas)
return ERROR_INT("ptas not defined", procName, 1);
n = ptaGetCount(ptas);
findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */
dahash = l_dnaHashCreate(nsize, 8);
ptad = ptaCreate(n);
*pptad = ptad;
for (i = 0, items = 0; i < n; i++) {
ptaGetIPt(ptas, i, &x, &y);
ptaFindPtByHash(ptad, dahash, x, y, &index);
if (index < 0) { /* not found */
l_hashPtToUint64Fast(nsize, x, y, &key);
l_dnaHashAdd(dahash, key, (l_float64)items);
ptaAddPt(ptad, x, y);
items++;
}
}
if (pdahash)
*pdahash = dahash;
else
l_dnaHashDestroy(&dahash);
return 0;
}
/*!
* \brief sarrayRemoveDupsByHash()
*
* \param[in] sas
* \param[out] psad unique set of strings; duplicates removed
* \param[out] pdahash [optional] dnahash used for lookup
* \return 0 if OK, 1 on error
*
* <pre>
* Notes:
* (1) Generates a sarray with unique values.
* (2) The dnahash is built up with sad to assure uniqueness.
* It can be used to find if a string is in the set:
* sarrayFindValByHash(sad, dahash, str, \&index)
* (3) The hash of the string location is simple and fast. It scales
* up with the number of buckets to insure a fairly random
* bucket selection input strings.
* (4) This is faster than sarrayRemoveDupsByAset(), because the
* bucket lookup is O(n), although there is a double-loop
* lookup within the dna in each bucket.
* </pre>
*/
l_int32
sarrayRemoveDupsByHash(SARRAY *sas,
SARRAY **psad,
L_DNAHASH **pdahash)
{
char *str;
l_int32 i, n, index, items;
l_uint32 nsize;
l_uint64 key;
SARRAY *sad;
L_DNAHASH *dahash;
PROCNAME("sarrayRemoveDupsByHash");
if (pdahash) *pdahash = NULL;
if (!psad)
return ERROR_INT("&sad not defined", procName, 1);
*psad = NULL;
if (!sas)
return ERROR_INT("sas not defined", procName, 1);
n = sarrayGetCount(sas);
findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */
dahash = l_dnaHashCreate(nsize, 8);
sad = sarrayCreate(n);
*psad = sad;
for (i = 0, items = 0; i < n; i++) {
str = sarrayGetString(sas, i, L_NOCOPY);
sarrayFindStringByHash(sad, dahash, str, &index);
if (index < 0) { /* not found */
l_hashStringToUint64(str, &key);
l_dnaHashAdd(dahash, key, (l_float64)items);
sarrayAddString(sad, str, L_COPY);
items++;
}
}
if (pdahash)
*pdahash = dahash;
else
l_dnaHashDestroy(&dahash);
return 0;
}