/*! * l_dnaHashCreateFromPta() * * Input: pta * Return: dahash, or null on error */ L_DNAHASH * l_dnaHashCreateFromPta(PTA *pta) { l_int32 i, n, x, y; l_uint32 nsize; l_uint64 key; L_DNAHASH *dahash; PROCNAME("l_dnaHashCreateFromPta"); /* Build up dnaHash of indices, hashed by a key that is * a large linear combination of x and y values designed to * randomize the key. Having about 20 pts in each bucket is * roughly optimal for speed for large sets. */ n = ptaGetCount(pta); findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */ /* fprintf(stderr, "Prime used: %d\n", nsize); */ /* Add each point, using the hash as key and the index into * @ptas as the value. Storing the index enables operations * that check for duplicates. */ dahash = l_dnaHashCreate(nsize, 8); for (i = 0; i < n; i++) { ptaGetIPt(pta, i, &x, &y); l_hashPtToUint64Fast(nsize, x, y, &key); l_dnaHashAdd(dahash, key, (l_float64)i); } return dahash; }
/*! * \brief l_dnaHashCreateFromDna() * * \param[in] da * \return dahash if OK; 1 on error * * <pre> * Notes: * (1) The values stored in the %dahash are indices into %da; * %dahash has no use without %da. * </pre> */ L_DNAHASH * l_dnaHashCreateFromDna(L_DNA *da) { l_int32 i, n; l_uint32 nsize; l_uint64 key; l_float64 val; L_DNAHASH *dahash; PROCNAME("l_dnaHashCreateFromDna"); if (!da) return (L_DNAHASH *)ERROR_PTR("da not defined", procName, NULL); n = l_dnaGetCount(da); findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */ dahash = l_dnaHashCreate(nsize, 8); for (i = 0; i < n; i++) { l_dnaGetDValue(da, i, &val); l_hashFloat64ToUint64(nsize, val, &key); l_dnaHashAdd(dahash, key, (l_float64)i); } return dahash; }
/*! * \brief l_dnaHashCreateFromSarray() * * \param[in] sa * \return dahash, or NULL on error */ L_DNAHASH * l_dnaHashCreateFromSarray(SARRAY *sa) { char *str; l_int32 i, n; l_uint32 nsize; l_uint64 key; L_DNAHASH *dahash; /* Build up dnaHash of indices, hashed by a 64-bit key that * should randomize the lower bits used in bucket selection. * Having about 20 pts in each bucket is roughly optimal. */ n = sarrayGetCount(sa); findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */ /* fprintf(stderr, "Prime used: %d\n", nsize); */ /* Add each string, using the hash as key and the index into %sa * as the value. Storing the index enables operations that check * for duplicates. */ dahash = l_dnaHashCreate(nsize, 8); for (i = 0; i < n; i++) { str = sarrayGetString(sa, i, L_NOCOPY); l_hashStringToUint64(str, &key); l_dnaHashAdd(dahash, key, (l_float64)i); } return dahash; }
/*! * \brief l_dnaIntersectionByHash() * * \param[in] da1, da2 * \return dad intersection of the number arrays, or NULL on error * * <pre> * Notes: * (1) This uses the same method for building the intersection set * as ptaIntersectionByHash() and sarrayIntersectionByHash(). * </pre> */ L_DNA * l_dnaIntersectionByHash(L_DNA *da1, L_DNA *da2) { l_int32 n1, n2, nsmall, nbuckets, i, index1, index2; l_uint32 nsize2; l_uint64 key; l_float64 val; L_DNAHASH *dahash1, *dahash2; L_DNA *da_small, *da_big, *dad; PROCNAME("l_dnaIntersectionByHash"); if (!da1) return (L_DNA *)ERROR_PTR("da1 not defined", procName, NULL); if (!da2) return (L_DNA *)ERROR_PTR("da2 not defined", procName, NULL); /* Put the elements of the biggest array into a dnahash */ n1 = l_dnaGetCount(da1); n2 = l_dnaGetCount(da2); da_small = (n1 < n2) ? da1 : da2; /* do not destroy da_small */ da_big = (n1 < n2) ? da2 : da1; /* do not destroy da_big */ dahash1 = l_dnaHashCreateFromDna(da_big); /* Build up the intersection of numbers. Add to %dad * if the number is in da_big (using dahash1) but hasn't * yet been seen in the traversal of da_small (using dahash2). */ dad = l_dnaCreate(0); nsmall = l_dnaGetCount(da_small); findNextLargerPrime(nsmall / 20, &nsize2); /* buckets in hash table */ dahash2 = l_dnaHashCreate(nsize2, 0); nbuckets = l_dnaHashGetCount(dahash2); for (i = 0; i < nsmall; i++) { l_dnaGetDValue(da_small, i, &val); l_dnaFindValByHash(da_big, dahash1, val, &index1); if (index1 >= 0) { /* found */ l_dnaFindValByHash(da_small, dahash2, val, &index2); if (index2 == -1) { /* not found */ l_dnaAddNumber(dad, val); l_hashFloat64ToUint64(nbuckets, val, &key); l_dnaHashAdd(dahash2, key, (l_float64)i); } } } l_dnaHashDestroy(&dahash1); l_dnaHashDestroy(&dahash2); return dad; }
/*! * \brief l_dnaMakeHistoByHash() * * \param[in] das * \param[out] pdahash hash map: val --> index * \param[out] pdav array of values: index --> val * \param[out] pdac histo array of counts: index --> count * \return 0 if OK; 1 on error * * <pre> * Notes: * (1) Generates and returns a dna of occurrences (histogram), * an aligned dna of values, and an associated hashmap. * The hashmap takes %dav and a value, and points into the * histogram in %dac. * (2) The dna of values, %dav, is aligned with the histogram %dac, * and is needed for fast lookup. It is a hash set, because * the values are unique. * (3) Lookup is simple: * l_dnaFindValByHash(dav, dahash, val, &index); * if (index >= 0) * l_dnaGetIValue(dac, index, &icount); * else * icount = 0; * </pre> */ l_ok l_dnaMakeHistoByHash(L_DNA *das, L_DNAHASH **pdahash, L_DNA **pdav, L_DNA **pdac) { l_int32 i, n, nitems, index, count; l_uint32 nsize; l_uint64 key; l_float64 val; L_DNA *dac, *dav; L_DNAHASH *dahash; PROCNAME("l_dnaMakeHistoByHash"); if (pdahash) *pdahash = NULL; if (pdac) *pdac = NULL; if (pdav) *pdav = NULL; if (!pdahash || !pdac || !pdav) return ERROR_INT("&dahash, &dac, &dav not all defined", procName, 1); if (!das) return ERROR_INT("das not defined", procName, 1); if ((n = l_dnaGetCount(das)) == 0) return ERROR_INT("no data in das", procName, 1); findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */ dahash = l_dnaHashCreate(nsize, 8); dac = l_dnaCreate(n); /* histogram */ dav = l_dnaCreate(n); /* the values */ for (i = 0, nitems = 0; i < n; i++) { l_dnaGetDValue(das, i, &val); /* Is this value already stored in dav? */ l_dnaFindValByHash(dav, dahash, val, &index); if (index >= 0) { /* found */ l_dnaGetIValue(dac, (l_float64)index, &count); l_dnaSetValue(dac, (l_float64)index, count + 1); } else { /* not found */ l_hashFloat64ToUint64(nsize, val, &key); l_dnaHashAdd(dahash, key, (l_float64)nitems); l_dnaAddNumber(dav, val); l_dnaAddNumber(dac, 1); nitems++; } } *pdahash = dahash; *pdac = dac; *pdav = dav; return 0; }
/*! * \brief sarrayIntersectionByHash() * * \param[in] sa1, sa2 * \return sad intersection of the strings, or NULL on error * * <pre> * Notes: * (1) This is faster than sarrayIntersectionByAset(), because the * bucket lookup is O(n). * </pre> */ SARRAY * sarrayIntersectionByHash(SARRAY *sa1, SARRAY *sa2) { char *str; l_int32 n1, n2, nsmall, i, index1, index2; l_uint32 nsize2; l_uint64 key; L_DNAHASH *dahash1, *dahash2; SARRAY *sa_small, *sa_big, *sad; PROCNAME("sarrayIntersectionByHash"); if (!sa1) return (SARRAY *)ERROR_PTR("sa1 not defined", procName, NULL); if (!sa2) return (SARRAY *)ERROR_PTR("sa2 not defined", procName, NULL); /* Put the elements of the biggest sarray into a dnahash */ n1 = sarrayGetCount(sa1); n2 = sarrayGetCount(sa2); sa_small = (n1 < n2) ? sa1 : sa2; /* do not destroy sa_small */ sa_big = (n1 < n2) ? sa2 : sa1; /* do not destroy sa_big */ dahash1 = l_dnaHashCreateFromSarray(sa_big); /* Build up the intersection of strings. Add to %sad * if the string is in sa_big (using dahash1) but hasn't * yet been seen in the traversal of sa_small (using dahash2). */ sad = sarrayCreate(0); nsmall = sarrayGetCount(sa_small); findNextLargerPrime(nsmall / 20, &nsize2); /* buckets in hash table */ dahash2 = l_dnaHashCreate(nsize2, 0); for (i = 0; i < nsmall; i++) { str = sarrayGetString(sa_small, i, L_NOCOPY); sarrayFindStringByHash(sa_big, dahash1, str, &index1); if (index1 >= 0) { sarrayFindStringByHash(sa_small, dahash2, str, &index2); if (index2 == -1) { sarrayAddString(sad, str, L_COPY); l_hashStringToUint64(str, &key); l_dnaHashAdd(dahash2, key, (l_float64)i); } } } l_dnaHashDestroy(&dahash1); l_dnaHashDestroy(&dahash2); return sad; }
/*! * ptaIntersectionByHash() * * Input: pta1, pta2 * Return: ptad (intersection of the point sets), or null on error * * Notes: * (1) This is faster than ptaIntersectionByAset(), because the * bucket lookup is O(n). It should be used if the pts are * integers (e.g., representing pixel positions). */ PTA * ptaIntersectionByHash(PTA *pta1, PTA *pta2) { l_int32 n1, n2, nsmall, i, x, y, index1, index2; l_uint32 nsize2; l_uint64 key; L_DNAHASH *dahash1, *dahash2; PTA *pta_small, *pta_big, *ptad; PROCNAME("ptaIntersectionByHash"); if (!pta1) return (PTA *)ERROR_PTR("pta1 not defined", procName, NULL); if (!pta2) return (PTA *)ERROR_PTR("pta2 not defined", procName, NULL); /* Put the elements of the biggest pta into a dnahash */ n1 = ptaGetCount(pta1); n2 = ptaGetCount(pta2); pta_small = (n1 < n2) ? pta1 : pta2; /* do not destroy pta_small */ pta_big = (n1 < n2) ? pta2 : pta1; /* do not destroy pta_big */ dahash1 = l_dnaHashCreateFromPta(pta_big); /* Build up the intersection of points. Add to ptad * if the point is in pta_big (using dahash1) but hasn't * yet been seen in the traversal of pta_small (using dahash2). */ ptad = ptaCreate(0); nsmall = ptaGetCount(pta_small); findNextLargerPrime(nsmall / 20, &nsize2); /* buckets in hash table */ dahash2 = l_dnaHashCreate(nsize2, 0); for (i = 0; i < nsmall; i++) { ptaGetIPt(pta_small, i, &x, &y); ptaFindPtByHash(pta_big, dahash1, x, y, &index1); if (index1 >= 0) { /* found */ ptaFindPtByHash(pta_small, dahash2, x, y, &index2); if (index2 == -1) { /* not found */ ptaAddPt(ptad, x, y); l_hashPtToUint64Fast(nsize2, x, y, &key); l_dnaHashAdd(dahash2, key, (l_float64)i); } } } l_dnaHashDestroy(&dahash1); l_dnaHashDestroy(&dahash2); return ptad; }
/*! * \brief l_dnaRemoveDupsByHash() * * \param[in] das * \param[out] pdad hash set * \param[out] pdahash [optional] dnahash used for lookup * \return 0 if OK; 1 on error * * <pre> * Notes: * (1) Generates a dna with unique values. * (2) The dnahash is built up with dad to assure uniqueness. * It can be used to find if an element is in the set: * l_dnaFindValByHash(dad, dahash, val, &index) * </pre> */ l_ok l_dnaRemoveDupsByHash(L_DNA *das, L_DNA **pdad, L_DNAHASH **pdahash) { l_int32 i, n, index, items; l_uint32 nsize; l_uint64 key; l_float64 val; L_DNA *dad; L_DNAHASH *dahash; PROCNAME("l_dnaRemoveDupsByHash"); if (pdahash) *pdahash = NULL; if (!pdad) return ERROR_INT("&dad not defined", procName, 1); *pdad = NULL; if (!das) return ERROR_INT("das not defined", procName, 1); n = l_dnaGetCount(das); findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */ dahash = l_dnaHashCreate(nsize, 8); dad = l_dnaCreate(n); *pdad = dad; for (i = 0, items = 0; i < n; i++) { l_dnaGetDValue(das, i, &val); l_dnaFindValByHash(dad, dahash, val, &index); if (index < 0) { /* not found */ l_hashFloat64ToUint64(nsize, val, &key); l_dnaHashAdd(dahash, key, (l_float64)items); l_dnaAddNumber(dad, val); items++; } } if (pdahash) *pdahash = dahash; else l_dnaHashDestroy(&dahash); return 0; }
/*! * ptaRemoveDupsByHash() * * Input: ptas (assumed to be integer values) * &ptad (<return> unique set of pts; duplicates removed) * &dahash (<optional return> dnahash used for lookup) * Return: 0 if OK, 1 on error * * Notes: * (1) Generates a pta with unique values. * (2) The dnahash is built up with ptad to assure uniqueness. * It can be used to find if a point is in the set: * ptaFindPtByHash(ptad, dahash, x, y, &index) * (3) The hash of the (x,y) location is simple and fast. It scales * up with the number of buckets to insure a fairly random * bucket selection for adjacent points. * (4) A Dna is used rather than a Numa because we need accurate * representation of 32-bit integers that are indices into ptas. * Integer --> float --> integer conversion makes errors for * integers larger than 10M. * (5) This is faster than ptaRemoveDupsByAset(), because the * bucket lookup is O(n), although there is a double-loop * lookup within the dna in each bucket. */ l_int32 ptaRemoveDupsByHash(PTA *ptas, PTA **pptad, L_DNAHASH **pdahash) { l_int32 i, n, index, items, x, y; l_uint32 nsize; l_uint64 key; l_float64 val; PTA *ptad; L_DNAHASH *dahash; PROCNAME("ptaRemoveDupsByHash"); if (pdahash) *pdahash = NULL; if (!pptad) return ERROR_INT("&ptad not defined", procName, 1); *pptad = NULL; if (!ptas) return ERROR_INT("ptas not defined", procName, 1); n = ptaGetCount(ptas); findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */ dahash = l_dnaHashCreate(nsize, 8); ptad = ptaCreate(n); *pptad = ptad; for (i = 0, items = 0; i < n; i++) { ptaGetIPt(ptas, i, &x, &y); ptaFindPtByHash(ptad, dahash, x, y, &index); if (index < 0) { /* not found */ l_hashPtToUint64Fast(nsize, x, y, &key); l_dnaHashAdd(dahash, key, (l_float64)items); ptaAddPt(ptad, x, y); items++; } } if (pdahash) *pdahash = dahash; else l_dnaHashDestroy(&dahash); return 0; }
/*! * \brief sarrayRemoveDupsByHash() * * \param[in] sas * \param[out] psad unique set of strings; duplicates removed * \param[out] pdahash [optional] dnahash used for lookup * \return 0 if OK, 1 on error * * <pre> * Notes: * (1) Generates a sarray with unique values. * (2) The dnahash is built up with sad to assure uniqueness. * It can be used to find if a string is in the set: * sarrayFindValByHash(sad, dahash, str, \&index) * (3) The hash of the string location is simple and fast. It scales * up with the number of buckets to insure a fairly random * bucket selection input strings. * (4) This is faster than sarrayRemoveDupsByAset(), because the * bucket lookup is O(n), although there is a double-loop * lookup within the dna in each bucket. * </pre> */ l_int32 sarrayRemoveDupsByHash(SARRAY *sas, SARRAY **psad, L_DNAHASH **pdahash) { char *str; l_int32 i, n, index, items; l_uint32 nsize; l_uint64 key; SARRAY *sad; L_DNAHASH *dahash; PROCNAME("sarrayRemoveDupsByHash"); if (pdahash) *pdahash = NULL; if (!psad) return ERROR_INT("&sad not defined", procName, 1); *psad = NULL; if (!sas) return ERROR_INT("sas not defined", procName, 1); n = sarrayGetCount(sas); findNextLargerPrime(n / 20, &nsize); /* buckets in hash table */ dahash = l_dnaHashCreate(nsize, 8); sad = sarrayCreate(n); *psad = sad; for (i = 0, items = 0; i < n; i++) { str = sarrayGetString(sas, i, L_NOCOPY); sarrayFindStringByHash(sad, dahash, str, &index); if (index < 0) { /* not found */ l_hashStringToUint64(str, &key); l_dnaHashAdd(dahash, key, (l_float64)items); sarrayAddString(sad, str, L_COPY); items++; } } if (pdahash) *pdahash = dahash; else l_dnaHashDestroy(&dahash); return 0; }