int main(int argc, char *argv[]) { int fail = 0; uint32_t hash_test[SHA1_DIGEST_WORDS], hash_base[SHA1_DIGEST_WORDS]; uint32_t murmur3_test[MURMUR3_x64_128_DIGEST_WORDS], murmur3_base[MURMUR3_x64_128_DIGEST_WORDS]; uint8_t *buff = NULL; int size, offset; struct mh_sha1_murmur3_x64_128_ctx *update_ctx = NULL; printf(" " xstr(TEST_UPDATE_FUNCTION) "_test:"); srand(TEST_SEED); buff = malloc(TEST_LEN); update_ctx = malloc(sizeof(*update_ctx)); if (buff == NULL || update_ctx == NULL) { printf("malloc failed test aborted\n"); return -1; } // Rand test1 rand_buffer(buff, TEST_LEN); mh_sha1_murmur3_x64_128_base(buff, TEST_LEN, TEST_SEED, hash_base, murmur3_base); CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff, TEST_LEN)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("fail rand1 test\n"); return -1; } else putchar('.'); // Test various size messages for (size = TEST_LEN; size >= 0; size--) { // Fill with rand data rand_buffer(buff, size); mh_sha1_murmur3_x64_128_base(buff, size, TEST_SEED, hash_base, murmur3_base); CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff, size)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("Fail size=%d\n", size); return -1; } if ((size & 0xff) == 0) { putchar('.'); fflush(0); } } // Test various buffer offsets and sizes printf("offset tests"); for (size = TEST_LEN - 256; size > 256; size -= 11) { for (offset = 0; offset < 256; offset++) { mh_sha1_murmur3_x64_128_base(buff + offset, size, TEST_SEED, hash_base, murmur3_base); CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff + offset, size)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("Fail size=%d offset=%d\n", size, offset); return -1; } } if ((size & 0xf) == 0) { putchar('.'); fflush(0); } } // Run efence tests printf("efence tests"); for (size = TEST_SIZE; size > 0; size--) { offset = TEST_LEN - size; mh_sha1_murmur3_x64_128_base(buff + offset, size, TEST_SEED, hash_base, murmur3_base); CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff + offset, size)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("Fail size=%d offset=%d\n", size, offset); return -1; } if ((size & 0xf) == 0) { putchar('.'); fflush(0); } } printf("\n" xstr(TEST_UPDATE_FUNCTION) "_test: %s\n", fail == 0 ? "Pass" : "Fail"); return fail; }
/* return the fd of a local copy, to operate on */ int dfs_get_local_copy(pentry_t *pe, const char * const remote, int flags) { int fd; dpl_dict_t *metadata = NULL; dpl_dict_t *headers = NULL; struct get_data get_data = { .fd = -1, .buf = NULL }; dpl_status_t rc = DPL_FAILURE; char *local = NULL; unsigned encryption = 0; local = tmpstr_printf("%s%s", conf->cache_dir, remote); LOG(LOG_DEBUG, "bucket=%s, path=%s, local=%s", ctx->cur_bucket, remote, local); if (-1 == download_headers((char *)remote, &headers)) { LOG(LOG_NOTICE, "%s: can't download headers", remote); fd = -1; goto end; } metadata = dpl_dict_new(13); if (! metadata) { LOG(LOG_ERR, "dpl_dict_new: can't allocate memory"); fd = -1; goto end; } if (DPL_FAILURE == dpl_get_metadata_from_headers( #if defined(DPL_VERSION_MAJOR) && defined(DPL_VERSION_MINOR) && (DPL_VERSION_MAJOR == 0 && DPL_VERSION_MINOR >= 2) || (DPL_VERSION_MAJOR > 0) ctx, #endif headers, metadata)) { LOG(LOG_ERR, "%s: metadata extraction failed", remote); fd = -1; goto end; } if (-1 == pentry_set_metadata(pe, metadata)) { LOG(LOG_ERR, "can't update metadata"); fd = -1; goto end; } if (-1 == check_permissions(pe, metadata)) { LOG(LOG_NOTICE, "permission denied"); fd = -1; goto end; } /* If the remote MD5 matches a cache file, we don't have to download * it again, just return the (open) file descriptor of the cache file */ if (0 == compare_digests(pe, headers)) { fd = pentry_get_fd(pe); goto end; } /* a cache file already exists, its MD5 digest is different, so * just remove it */ if (0 == access(local, F_OK)) { LOG(LOG_DEBUG, "removing cache file '%s'", local); if (-1 == unlink(local)) LOG(LOG_ERR, "unlink(%s): %s", local, strerror(errno)); } get_data.fd = open(local, O_RDWR|O_CREAT|O_TRUNC, 0600); if (-1 == get_data.fd) { LOG(LOG_ERR, "open: %s: %s (%d)", local, strerror(errno), errno); fd = -1; goto end; } encryption = check_encryption_flag(metadata); rc = dpl_openread(ctx, (char *)remote, encryption, NULL, cb_get_buffered, &get_data, &metadata); if (DPL_SUCCESS != rc) { LOG(LOG_ERR, "dpl_openread: %s", dpl_status_str(rc)); close(get_data.fd); fd = -1; goto end; } /* If the file is compressed, uncompress it! */ if(-1 == handle_compression(remote, local, &get_data, metadata)) { fd = -1; goto end; } if (-1 == close(get_data.fd)) { LOG(LOG_ERR, "close(path=%s, fd=%d): %s", local, get_data.fd, strerror(errno)); fd = -1; goto end; } fd = open(local, flags, 0600); if (-1 == fd) { LOG(LOG_ERR, "open(path=%s, fd=%d): %s", local, fd, strerror(errno)); fd = -1; goto end; } end: if (metadata) dpl_dict_free(metadata); if (headers) dpl_dict_free(headers); return fd; }
int main(int argc, char *argv[]) { int fail = 0, i; uint32_t hash_test[SHA1_DIGEST_WORDS], hash_base[SHA1_DIGEST_WORDS]; uint32_t murmur3_test[MURMUR3_x64_128_DIGEST_WORDS], murmur3_base[MURMUR3_x64_128_DIGEST_WORDS]; uint8_t *buff = NULL; int update_count; int size1, size2, offset, addr_offset; struct mh_sha1_murmur3_x64_128_ctx *update_ctx = NULL; uint8_t *mem_addr = NULL; printf(" " xstr(TEST_UPDATE_FUNCTION) "_test:"); srand(TEST_SEED); buff = malloc(TEST_LEN); update_ctx = malloc(sizeof(*update_ctx)); if (buff == NULL || update_ctx == NULL) { printf("malloc failed test aborted\n"); return -1; } // Rand test1 rand_buffer(buff, TEST_LEN); mh_sha1_murmur3_x64_128_base(buff, TEST_LEN, TEST_SEED, hash_base, murmur3_base); CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff, TEST_LEN)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("fail rand1 test\n"); return -1; } else putchar('.'); // Test various size messages by update twice. printf("\n various size messages by update twice tests"); for (size1 = TEST_LEN; size1 >= 0; size1--) { // Fill with rand data rand_buffer(buff, TEST_LEN); mh_sha1_murmur3_x64_128_base(buff, TEST_LEN, TEST_SEED, hash_base, murmur3_base); // subsequent update size2 = TEST_LEN - size1; // size2 is different with the former CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff, size1)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff + size1, size2)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("Fail size1=%d\n", size1); return -1; } if ((size2 & 0xff) == 0) { putchar('.'); fflush(0); } } // Test various update count printf("\n various update count tests"); for (update_count = 1; update_count <= TEST_LEN; update_count++) { // Fill with rand data rand_buffer(buff, TEST_LEN); mh_sha1_murmur3_x64_128_base(buff, TEST_LEN, TEST_SEED, hash_base, murmur3_base); // subsequent update size1 = TEST_LEN / update_count; size2 = TEST_LEN - size1 * (update_count - 1); // size2 is different with the former CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); for (i = 1, offset = 0; i < update_count; i++) { CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff + offset, size1)); offset += size1; } CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff + offset, size2)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("Fail size1=%d\n", size1); return -1; } if ((size2 & 0xff) == 0) { putchar('.'); fflush(0); } } // test various start address of ctx. printf("\n various start address of ctx test"); free(update_ctx); mem_addr = (uint8_t *) malloc(sizeof(*update_ctx) + AVX512_ALIGNED * 10); for (addr_offset = AVX512_ALIGNED * 10; addr_offset >= 0; addr_offset--) { // Fill with rand data rand_buffer(buff, TEST_LEN); mh_sha1_murmur3_x64_128_base(buff, TEST_LEN, TEST_SEED, hash_base, murmur3_base); // a unaligned offset update_ctx = (struct mh_sha1_murmur3_x64_128_ctx *)(mem_addr + addr_offset); CHECK_RETURN(mh_sha1_murmur3_x64_128_init(update_ctx, TEST_SEED)); CHECK_RETURN(TEST_UPDATE_FUNCTION(update_ctx, buff, TEST_LEN)); CHECK_RETURN(TEST_FINAL_FUNCTION(update_ctx, hash_test, murmur3_test)); fail = compare_digests(hash_base, hash_test, murmur3_base, murmur3_test); if (fail) { printf("Fail addr_offset=%d\n", addr_offset); return -1; } if ((addr_offset & 0xf) == 0) { putchar('.'); fflush(0); } } printf("\n" xstr(TEST_UPDATE_FUNCTION) "_test: %s\n", fail == 0 ? "Pass" : "Fail"); return fail; }