int main(int argc, char **argv)
{
/* Use only one hash function */
assert(USE_32 + USE_64 + USE_128 == 1);
printf("Starting computing hashes\n");
struct timespec start, end;
clock_gettime(CLOCK_REALTIME, &start);
long long sum = 0;
int i = 0, j = 0;
long long A[NUM_LONGS];
for(i = 0; i < NUM_PKTS; i ++) {
for(j = 0; j < NUM_LONGS; j ++) {
A[j] = 0xffffffffffffffffL + i + j;
}
sum += CityHash64((char *) A, NUM_LONGS * sizeof(long long));
}
clock_gettime(CLOCK_REALTIME, &end);
double seconds = (end.tv_sec - start.tv_sec) +
(double) (end.tv_nsec - start.tv_nsec) / 1000000000;
double nanoseconds = (end.tv_sec - start.tv_sec) * 1000000000 +
(end.tv_nsec - start.tv_nsec);
printf("Time = %f, time per hash = %f ns, sum = %lld\n",
seconds, nanoseconds / NUM_PKTS, sum);
return 0;
}
static int fsal_acl_hash_both(hash_parameter_t *hparam,
struct gsh_buffdesc *key, uint32_t *index,
uint64_t *rbthash)
{
*rbthash = CityHash64(key->addr, key->len);
*index = *rbthash % hparam->index_size;
return 1;
}
uint64_t
cch_cityhash64 (const char *buf, size_t len)
{
#ifdef HAVE_CITYHASH64
return CityHash64(buf, len);
#else
feature_failure(__func__);
#endif
}
/**
* @brief Creates a hashtable key for a pseudofs node given the fullpath.
*
* Creates a hashtable key for a pseudofs node given the fullpath.
*
* @param[in] pseudopath Full path of the pseudofs node
* @param[in] len Length of the full path
*
* @return the key
*/
struct gsh_buffdesc create_pseudo_handle_key(char *pseudopath, ushort len)
{
struct gsh_buffdesc key;
uint64 hashkey;
hashkey = CityHash64(pseudopath, len);
key.addr = package_pseudo_handle(pseudopath, len, hashkey);
key.len = V4_FH_OPAQUE_SIZE;
return key;
}
LogTuple buildKeyAndValue(const std::string &log) {
// Current timestamp in ISO-8601 format
// Message timestamp converted to ISO-8601 format
// username associated with the event (if any, e.g. src@ladmin)
// node name (e.g. src@ladmin)
// The line’s log payload
static const RE2 valueRegex(
"-\\s(\\d+)(?:\\s\\S+){5}\\s(\\w+)(.)(\\w+)\\s(.*)$");
int key, timestamp; // value is in seconds
char nodeChar;
std::string username, nodename, msg, value;
if(RE2::FullMatch(log, valueRegex, ×tamp, &username, &nodeChar,
&nodename, &msg)) {
key = static_cast<int64_t>(CityHash64(log.data(), log.size()) + timestamp);
value = bolt::timeInMillisAsIso8601(bolt::timeNowMilli()) + ":"
+ std::to_string(timestamp * 1000) + ":" + username + nodeChar
+ nodename + ":" + msg;
}
return LogTuple(key, value);
}
CITYHASH_API
void GetCityHash64(HWND hwndParent, int string_size, char *variables, stack_t **stacktop)
{
TCHAR hashString[MAX_STRLEN];
TCHAR hexResult[18];
g_stacktop = stacktop;
g_variables = variables;
memset(hashString, 0, sizeof(hashString));
memset(hexResult, 0, sizeof(hexResult));
if (!popString(hashString)) {
pushString(L"error");
return;
}
uint64 result = CityHash64((const char*)&hashString[0], wcslen(hashString)*sizeof(TCHAR));
swprintf_s(hexResult, 17, L"%16I64X", result);
pushString(hexResult);
}
void GetCityHash64(HWND hwndParent, int string_size, char *variables, stack_t **stacktop)
{
TCHAR hashString[MAX_STRLEN];
TCHAR hexResult[18] = { _T('\0') };
g_stacktop = stacktop;
g_variables = variables;
memset(hashString, 0, sizeof(hashString));
memset(hexResult, 0, sizeof(hexResult));
if (!popString(hashString)) {
pushString(L"error");
return;
}
uint64 result = CityHash64((const char*)&hashString[0], wcslen(hashString)*sizeof(TCHAR));
// If the hash happens to work out to less than 16 hash digits it will just
// use less of the buffer.
swprintf(hexResult, L"%I64X", result);
pushString(hexResult);
}
std::uint64_t Document::getIdHash(const char* id) {
return id != nullptr ? CityHash64(id, std::strlen(id)) : 0;
}
static int
city_hash(const char *buf, int len)
{
return (int)CityHash64(buf, len);
}
uint64_t SkCityHash::Compute64(const char *data, size_t size) {
return CityHash64(data, size);
}
foreach(batch_index, BATCH_SIZE) {
char *name = name_lo[batch_index].name;
if(batch_index != BATCH_SIZE - 1) {
__builtin_prefetch(name_lo[batch_index + 1].name, 0, 3);
}
int c_i, i; /**< URL char iterator and slot iterator */
int bkt_num, bkt_1, bkt_2;
int terminate = 0; /**< Stop processing this URL? */
int prefix_match_found = 0; /**< Stop this hash-table lookup ? */
/**< For names that we cannot find, dst_port is -1 */
dst_ports[batch_index] = -1;
for(c_i = 0; name[c_i] != 0; c_i ++) {
if(name[c_i] == '/') {
break;
}
}
c_i ++;
for(; name[c_i] != 0; c_i ++) {
if(name[c_i] != '/') {
continue;
}
uint64_t prefix_hash = CityHash64WithSeed(name, c_i + 1, NDN_SEED);
uint16_t tag = prefix_hash >> 48;
struct ndn_slot *slots;
/**< name[0] -> name[c_i] is a prefix of length c_i + 1 */
for(bkt_num = 1; bkt_num <= 2; bkt_num ++) {
if(bkt_num == 1) {
bkt_1 = prefix_hash & NDN_NUM_BKT_;
FPP_EXPENSIVE(&ht[bkt_1]);
slots = ht[bkt_1].slots;
} else {
bkt_2 = (bkt_1 ^ CityHash64((char *) &tag, 2)) & NDN_NUM_BKT_;
FPP_EXPENSIVE(&ht[bkt_2]);
slots = ht[bkt_2].slots;
}
/**< Now, "slots" points to an ndn_bucket. Find a valid slot
* with a matching tag. */
for(i = 0; i < NDN_NUM_SLOTS; i ++) {
int8_t _dst_port = slots[i].dst_port;
uint64_t _hash = slots[i].cityhash;
if(_dst_port >= 0 && _hash == prefix_hash) {
/**< Record the dst port: this may get overwritten by
* longer prefix matches later */
dst_ports[batch_index] = slots[i].dst_port;
if(slots[i].is_terminal == 1) {
/**< A terminal FIB entry: we're done! */
terminate = 1;
}
prefix_match_found = 1;
break;
}
}
/**< Stop the hash-table lookup for name[0 ... c_i] */
if(prefix_match_found == 1) {
break;
}
}
/**< Stop processing the name if we found a terminal FIB entry */
if(terminate == 1) {
break;
}
} /**< Loop over URL characters ends here */
} /**< Loop over batch ends here */
// Make local hash out of arbitrary data.
void MakeLocalHash(const void* data, uint32_t size, LocalHash* hash) {
*hash = CityHash64(data, size);
}
static size_t
compute(const char* buffer, size_t length)
{
return static_cast<size_t>(CityHash64(buffer, length));
}
int main(int argc, char ** argv)
{
using Strings = std::vector<std::string>;
using Hashes = std::vector<char>;
Strings strings;
size_t rows = 0;
size_t bytes = 0;
{
Stopwatch watch;
DB::ReadBufferFromFileDescriptor in(STDIN_FILENO);
while (!in.eof())
{
strings.push_back(std::string());
DB::readEscapedString(strings.back(), in);
DB::assertChar('\n', in);
bytes += strings.back().size() + 1;
}
watch.stop();
rows = strings.size();
std::cerr << std::fixed << std::setprecision(2)
<< "Read " << rows << " rows, " << bytes / 1000000.0 << " MB"
<< ", elapsed: " << watch.elapsedSeconds()
<< " (" << rows / watch.elapsedSeconds() << " rows/sec., " << bytes / 1000000.0 / watch.elapsedSeconds() << " MB/sec.)"
<< std::endl;
}
Hashes hashes(16 * rows);
{
Stopwatch watch;
for (size_t i = 0; i < rows; ++i)
{
*reinterpret_cast<UInt64*>(&hashes[i * 16]) = CityHash64(strings[i].data(), strings[i].size());
}
watch.stop();
UInt64 check = CityHash64(&hashes[0], hashes.size());
std::cerr << std::fixed << std::setprecision(2)
<< "CityHash64 (check = " << check << ")"
<< ", elapsed: " << watch.elapsedSeconds()
<< " (" << rows / watch.elapsedSeconds() << " rows/sec., " << bytes / 1000000.0 / watch.elapsedSeconds() << " MB/sec.)"
<< std::endl;
}
/* {
Stopwatch watch;
std::vector<char> seed(16);
for (size_t i = 0; i < rows; ++i)
{
sipHash(
reinterpret_cast<unsigned char *>(&hashes[i * 16]),
reinterpret_cast<const unsigned char *>(strings[i].data()),
strings[i].size(),
reinterpret_cast<const unsigned char *>(&seed[0]));
}
watch.stop();
UInt64 check = CityHash64(&hashes[0], hashes.size());
std::cerr << std::fixed << std::setprecision(2)
<< "SipHash (check = " << check << ")"
<< ", elapsed: " << watch.elapsedSeconds()
<< " (" << rows / watch.elapsedSeconds() << " rows/sec., " << bytes / 1000000.0 / watch.elapsedSeconds() << " MB/sec.)"
<< std::endl;
}*/
{
Stopwatch watch;
for (size_t i = 0; i < rows; ++i)
{
SipHash hash;
hash.update(strings[i].data(), strings[i].size());
hash.get128(&hashes[i * 16]);
}
watch.stop();
UInt64 check = CityHash64(&hashes[0], hashes.size());
std::cerr << std::fixed << std::setprecision(2)
<< "SipHash, stream (check = " << check << ")"
<< ", elapsed: " << watch.elapsedSeconds()
<< " (" << rows / watch.elapsedSeconds() << " rows/sec., " << bytes / 1000000.0 / watch.elapsedSeconds() << " MB/sec.)"
<< std::endl;
}
{
Stopwatch watch;
for (size_t i = 0; i < rows; ++i)
{
MD5_CTX state;
MD5_Init(&state);
MD5_Update(&state, reinterpret_cast<const unsigned char *>(strings[i].data()), strings[i].size());
MD5_Final(reinterpret_cast<unsigned char *>(&hashes[i * 16]), &state);
}
watch.stop();
UInt64 check = CityHash64(&hashes[0], hashes.size());
std::cerr << std::fixed << std::setprecision(2)
<< "MD5 (check = " << check << ")"
<< ", elapsed: " << watch.elapsedSeconds()
<< " (" << rows / watch.elapsedSeconds() << " rows/sec., " << bytes / 1000000.0 / watch.elapsedSeconds() << " MB/sec.)"
<< std::endl;
}
return 0;
}
inline mtn::index_address_t
hash()
{
return CityHash64(reinterpret_cast<char*>(this), sizeof(uint32_t) * 3);
}
int CalcCatFeatureHash(const TStringBuf& feature) {
return CityHash64(feature) & 0xffffffff;
}
uint64 CityHash64CXX(string str) {
return CityHash64(str.c_str(), str.size());
}
void process_batch(struct ndn_name *name_lo, int *dst_ports,
struct ndn_bucket *ht)
{
char *name[BATCH_SIZE];
int i[BATCH_SIZE];
int c_i[BATCH_SIZE];
int bkt_2[BATCH_SIZE];
int bkt_1[BATCH_SIZE];
int bkt_num[BATCH_SIZE];
int terminate[BATCH_SIZE];
int prefix_match_found[BATCH_SIZE];
uint64_t prefix_hash[BATCH_SIZE];
uint16_t tag[BATCH_SIZE];
struct ndn_slot *slots[BATCH_SIZE];
int8_t _dst_port[BATCH_SIZE];
uint64_t _hash[BATCH_SIZE];
int I = 0; // batch index
void *batch_rips[BATCH_SIZE]; // goto targets
int iMask = 0; // No packet is done yet
int temp_index;
for(temp_index = 0; temp_index < BATCH_SIZE; temp_index ++) {
batch_rips[temp_index] = &&fpp_start;
}
fpp_start:
name[I] = name_lo[I].name;
FPP_PSS(name[I], fpp_label_1);
fpp_label_1:
/**< URL char iterator and slot iterator */
terminate[I] = 0; /**< Stop processing this URL? */
prefix_match_found[I] = 0; /**< Stop this hash-table lookup ? */
/**< For names that we cannot find, dst_port is -1 */
dst_ports[I] = -1;
for(c_i[I] = 0; name[I][c_i[I]] != 0; c_i[I] ++) {
if(name[I][c_i[I]] == '/') {
break;
}
}
c_i[I] ++;
for(; name[I][c_i[I]] != 0; c_i[I] ++) {
if(name[I][c_i[I]] != '/') {
continue;
}
prefix_hash[I] = CityHash64WithSeed(name[I], c_i[I] + 1, NDN_SEED);
tag[I] = prefix_hash[I] >> 48;
/**< name[0] -> name[c_i] is a prefix of length c_i + 1 */
for(bkt_num[I] = 1; bkt_num[I] <= 2; bkt_num[I] ++) {
if(bkt_num[I] == 1) {
bkt_1[I] = prefix_hash[I] & NDN_NUM_BKT_;
FPP_PSS(&ht[bkt_1[I]], fpp_label_2);
fpp_label_2:
slots[I] = ht[bkt_1[I]].slots;
} else {
bkt_2[I] = (bkt_1[I] ^ CityHash64((char *) &tag[I], 2)) & NDN_NUM_BKT_;
FPP_PSS(&ht[bkt_2[I]], fpp_label_3);
fpp_label_3:
slots[I] = ht[bkt_2[I]].slots;
}
/**< Now, "slots" points to an ndn_bucket. Find a valid slot
* with a matching tag. */
for(i[I] = 0; i[I] < NDN_NUM_SLOTS; i[I] ++) {
_dst_port[I] = slots[I][i[I]].dst_port;
_hash[I] = slots[I][i[I]].cityhash;
if(_dst_port[I] >= 0 && _hash[I] == prefix_hash[I]) {
/**< Record the dst port: this may get overwritten by
* longer prefix matches later */
dst_ports[I] = slots[I][i[I]].dst_port;
if(slots[I][i[I]].is_terminal == 1) {
/**< A terminal FIB entry: we're done! */
terminate[I] = 1;
}
prefix_match_found[I] = 1;
break;
}
}
/**< Stop the hash-table lookup for name[0 ... c_i] */
if(prefix_match_found[I] == 1) {
break;
}
}
/**< Stop processing the name if we found a terminal FIB entry */
if(terminate[I] == 1) {
break;
}
} /**< Loop over URL characters ends here */
/**< Loop over batch ends here */
fpp_end:
batch_rips[I] = &&fpp_end;
iMask = FPP_SET(iMask, I);
if(iMask == (1 << BATCH_SIZE) - 1) {
return;
}
I = (I + 1) & BATCH_SIZE_;
goto *batch_rips[I];
}