ifq_codes_t
ifq_open_index(char *fastq_path, char *index_prefix, ifq_index_t *index)
{
char *hash_path = concatenate( index_prefix, ".hsh" );
char *lookup_path = concatenate( index_prefix, ".lup" );
ifq_codes_t ret = IFQ_OK;
index->fastq_file = bgzf_open( fastq_path , "r" );
if( index->fastq_file == NULL )
{
ret = IFQ_BAD_FASTQ;
goto index_error;
}
index->hash_file = fopen( hash_path , "r" );
if( index->hash_file == NULL )
{
ret = IFQ_BAD_PREFIX;
goto index_error;
}
index->hash = cmph_load( index->hash_file );
if( index->hash == NULL )
{
ret = IFQ_BAD_HASH;
goto index_error;
}
index->lookup_fd = open( lookup_path, O_RDWR );
if( index->lookup_fd == -1 )
{
ret = IFQ_BAD_PREFIX;
goto index_error;
}
struct stat sb;
fstat( index->lookup_fd, &sb );
index->lookup_size = sb.st_size;
index->table = (uint64_t *) mmap( NULL, index->lookup_size, PROT_READ, MAP_FILE | MAP_SHARED, index->lookup_fd, 0 );
if( index->table == MAP_FAILED )
{
ret = IFQ_BAD_INDEX;
goto index_error;
}
index_error:
free( hash_path );
free( lookup_path );
return ret;
}
int open_disk_hash(struct cmph_hash_obj *o) {
char mphfile[1024];
FILE *mph;
sprintf(mphfile,"%s.mph",o->keyfile);
o->keys_fd = fopen(mphfile, "r");
if (o->keys_fd == NULL)
{
fprintf(stderr, "File %s not found\n", mphfile);
return(1);
}
o->hash = cmph_load(o->keys_fd);
return (0);
}
int test(cmph_uint32* items_to_hash, cmph_uint32 items_len, CMPH_ALGO alg_n)
{
cmph_t *hash;
cmph_config_t *config;
cmph_io_adapter_t *source;
cmph_uint32 i;
char filename[256];
FILE* mphf_fd = NULL;
printf("%s (%u)\n", cmph_names[alg_n], alg_n);
source = cmph_io_struct_vector_adapter(items_to_hash,
(cmph_uint32)sizeof(cmph_uint32),
0,
(cmph_uint32)sizeof(cmph_uint32),
items_len);
config = cmph_config_new(source);
cmph_config_set_algo(config, alg_n);
if (alg_n == CMPH_BRZ) {
sprintf(filename, "%s_%u.mph", cmph_names[alg_n], items_len);
mphf_fd = fopen(filename, "w");
cmph_config_set_mphf_fd(config, mphf_fd);
}
hash = cmph_new(config);
cmph_config_destroy(config);
if (alg_n == CMPH_BRZ) {
cmph_dump(hash, mphf_fd);
cmph_destroy(hash);
fclose(mphf_fd);
mphf_fd = fopen(filename, "r");
hash = cmph_load(mphf_fd);
}
printf("packed_size %u\n",cmph_packed_size(hash));
for (i=0; i<items_len; ++i)
printf("%d -> %u\n",
items_to_hash[i],
cmph_search(hash,
(char*)(items_to_hash+i),
(cmph_uint32)sizeof(cmph_uint32)));
printf("\n");
cmph_io_vector_adapter_destroy(source);
cmph_destroy(hash);
if (alg_n == CMPH_BRZ) {
fclose(mphf_fd);
}
return 0;
}
void BlockHashIndex::LoadRange(size_t i)
{
#ifdef HAVE_CMPH
std::fseek(m_fileHandle, m_fileHandleStart + m_seekIndex[i], SEEK_SET);
cmph_t* hash = cmph_load(m_fileHandle);
m_arrays[i] = new PairedPackedArray<>(0, m_orderBits,
m_fingerPrintBits);
m_arrays[i]->Load(m_fileHandle);
m_hashes[i] = (void*)hash;
m_clocks[i] = clock();
m_numLoadedRanges++;
#endif
}
// Create minimal perfect hash function from in-memory vector
int main(int argc, char **argv)
{
// Creating a filled vector
unsigned int i = 0;
const char *vector[] = {"aaaaaaaaaa", "bbbbbbbbbb", "cccccccccc", "dddddddddd", "eeeeeeeeee",
"ffffffffff", "gggggggggg", "hhhhhhhhhh", "iiiiiiiiii", "jjjjjjjjjj"};
unsigned int nkeys = 10;
FILE* mphf_fd = fopen("temp.mph", "w");
// Source of keys
cmph_io_adapter_t *source = cmph_io_vector_adapter((char **)vector, nkeys);
//Create minimal perfect hash function using the brz algorithm.
cmph_config_t *config = cmph_config_new(source);
cmph_config_set_algo(config, CMPH_BRZ);
cmph_config_set_mphf_fd(config, mphf_fd);
cmph_t *hash = cmph_new(config);
cmph_config_destroy(config);
cmph_dump(hash, mphf_fd);
cmph_destroy(hash);
fclose(mphf_fd);
//Find key
mphf_fd = fopen("temp.mph", "r");
hash = cmph_load(mphf_fd);
while (i < nkeys) {
const char *key = vector[i];
unsigned int id = cmph_search(hash, key, (cmph_uint32)strlen(key));
fprintf(stderr, "key:%s -- hash:%u\n", key, id);
i++;
}
//Destroy hash
cmph_destroy(hash);
cmph_io_vector_adapter_destroy(source);
fclose(mphf_fd);
return 0;
}
int main(int argc, char **argv)
{
char verbosity = 0;
char *mphf_file = NULL;
const char *keys_file = NULL;
FILE *mphf_fd = stdout;
FILE *keys_fd;
cmph_uint32 nkeys = UINT_MAX;
cmph_uint32 i = 0;
cmph_t *mphf = NULL;
cmph_io_adapter_t *source;
while (1)
{
char ch = (char)getopt(argc, argv, "hVvk:m:");
if (ch == -1) break;
switch (ch)
{
case 'k':
{
char *endptr;
nkeys = (cmph_uint32) strtoul(optarg, &endptr, 10);
if(*endptr != 0) {
fprintf(stderr, "Invalid number of keys %s\n", optarg);
exit(1);
}
}
break;
case 'm':
mphf_file = strdup(optarg);
break;
case 'v':
++verbosity;
break;
case 'V':
printf("%s\n", VERSION);
return 0;
case 'h':
usage_long(argv[0]);
return 0;
default:
usage(argv[0]);
return 1;
}
}
if (optind != argc - 1)
{
usage(argv[0]);
return 1;
}
keys_file = argv[optind];
int ret = 0;
if (mphf_file == NULL)
{
mphf_file = (char *)malloc(strlen(keys_file) + 5);
memcpy(mphf_file, keys_file, strlen(keys_file));
memcpy(mphf_file + strlen(keys_file), ".mph\0", (size_t)5);
}
keys_fd = fopen(keys_file, "r");
if (keys_fd == NULL)
{
fprintf(stderr, "Unable to open file %s: %s\n", keys_file, strerror(errno));
return -1;
}
if(nkeys == UINT_MAX) source = cmph_io_nlfile_adapter(keys_fd);
else source = cmph_io_nlnkfile_adapter(keys_fd, nkeys);
cmph_uint8 * hashtable = NULL;
mphf_fd = fopen(mphf_file, "r");
if (mphf_fd == NULL)
{
fprintf(stderr, "Unable to open input file %s: %s\n", mphf_file, strerror(errno));
free(mphf_file);
return -1;
}
mphf = cmph_load(mphf_fd);
fclose(mphf_fd);
if (!mphf)
{
fprintf(stderr, "Unable to parser input file %s\n", mphf_file);
free(mphf_file);
return -1;
}
cmph_uint32 siz = cmph_size(mphf);
hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8));
memset(hashtable, 0, (size_t)siz);
//check all keys
for (i = 0; i < source->nkeys; ++i)
{
cmph_uint32 h;
char *buf;
cmph_uint32 buflen = 0;
source->read(source->data, &buf, &buflen);
h = cmph_search(mphf, buf, buflen);
if (!(h < siz))
{
fprintf(stderr, "Unknown key %*s in the input.\n", buflen, buf);
ret = 1;
} else if(hashtable[h])
{
fprintf(stderr, "Duplicated or unknown key %*s in the input\n", buflen, buf);
ret = 1;
} else hashtable[h] = 1;
if (verbosity)
{
printf("%s -> %u\n", buf, h);
}
source->dispose(source->data, buf, buflen);
}
cmph_destroy(mphf);
free(hashtable);
fclose(keys_fd);
free(mphf_file);
cmph_io_nlfile_adapter_destroy(source);
return ret;
}
int main(int argc, char **argv)
{
cmph_uint32 verbosity = 0;
char generate = 0;
char *mphf_file = NULL;
FILE *mphf_fd = stdout;
const char *keys_file = NULL;
FILE *keys_fd;
cmph_uint32 nkeys = UINT_MAX;
cmph_uint32 seed = UINT_MAX;
CMPH_HASH *hashes = NULL;
cmph_uint32 nhashes = 0;
cmph_uint32 i;
CMPH_ALGO mph_algo = CMPH_CHM;
double c = 0;
cmph_config_t *config = NULL;
cmph_t *mphf = NULL;
char * tmp_dir = NULL;
cmph_io_adapter_t *source;
cmph_uint32 memory_availability = 0;
cmph_uint32 b = 0;
cmph_uint32 keys_per_bin = 1;
while (1)
{
char ch = (char)getopt(argc, argv, "hVvgc:k:a:M:b:t:f:m:d:s:");
if (ch == -1) break;
switch (ch)
{
case 's':
{
char *cptr;
seed = (cmph_uint32)strtoul(optarg, &cptr, 10);
if(*cptr != 0) {
fprintf(stderr, "Invalid seed %s\n", optarg);
exit(1);
}
}
break;
case 'c':
{
char *endptr;
c = strtod(optarg, &endptr);
if(*endptr != 0) {
fprintf(stderr, "Invalid c value %s\n", optarg);
exit(1);
}
}
break;
case 'g':
generate = 1;
break;
case 'k':
{
char *endptr;
nkeys = (cmph_uint32)strtoul(optarg, &endptr, 10);
if(*endptr != 0) {
fprintf(stderr, "Invalid number of keys %s\n", optarg);
exit(1);
}
}
break;
case 'm':
mphf_file = strdup(optarg);
break;
case 'd':
tmp_dir = strdup(optarg);
break;
case 'M':
{
char *cptr;
memory_availability = (cmph_uint32)strtoul(optarg, &cptr, 10);
if(*cptr != 0) {
fprintf(stderr, "Invalid memory availability %s\n", optarg);
exit(1);
}
}
break;
case 'b':
{
char *cptr;
b = (cmph_uint32)strtoul(optarg, &cptr, 10);
if(*cptr != 0) {
fprintf(stderr, "Parameter b was not found: %s\n", optarg);
exit(1);
}
}
break;
case 't':
{
char *cptr;
keys_per_bin = (cmph_uint32)strtoul(optarg, &cptr, 10);
if(*cptr != 0) {
fprintf(stderr, "Parameter t was not found: %s\n", optarg);
exit(1);
}
}
break;
case 'v':
++verbosity;
break;
case 'V':
printf("%s\n", VERSION);
return 0;
case 'h':
usage_long(argv[0]);
return 0;
case 'a':
{
char valid = 0;
for (i = 0; i < CMPH_COUNT; ++i)
{
if (strcmp(cmph_names[i], optarg) == 0)
{
mph_algo = (CMPH_ALGO)i;
valid = 1;
break;
}
}
if (!valid)
{
fprintf(stderr, "Invalid mph algorithm: %s. It is not available in version %s\n", optarg, VERSION);
return -1;
}
}
break;
case 'f':
{
char valid = 0;
for (i = 0; i < CMPH_HASH_COUNT; ++i)
{
if (strcmp(cmph_hash_names[i], optarg) == 0)
{
hashes = (CMPH_HASH *)realloc(hashes, sizeof(CMPH_HASH) * ( nhashes + 2 ));
hashes[nhashes] = (CMPH_HASH)i;
hashes[nhashes + 1] = CMPH_HASH_COUNT;
++nhashes;
valid = 1;
break;
}
}
if (!valid)
{
fprintf(stderr, "Invalid hash function: %s\n", optarg);
return -1;
}
}
break;
default:
usage(argv[0]);
return 1;
}
}
if (optind != argc - 1)
{
usage(argv[0]);
return 1;
}
keys_file = argv[optind];
if (seed == UINT_MAX) seed = (cmph_uint32)time(NULL);
srand(seed);
int ret = 0;
if (mphf_file == NULL)
{
mphf_file = (char *)malloc(strlen(keys_file) + 5);
memcpy(mphf_file, keys_file, strlen(keys_file));
memcpy(mphf_file + strlen(keys_file), ".mph\0", (size_t)5);
}
keys_fd = fopen(keys_file, "r");
if (keys_fd == NULL)
{
fprintf(stderr, "Unable to open file %s: %s\n", keys_file, strerror(errno));
return -1;
}
if (seed == UINT_MAX) seed = (cmph_uint32)time(NULL);
if(nkeys == UINT_MAX) source = cmph_io_nlfile_adapter(keys_fd);
else source = cmph_io_nlnkfile_adapter(keys_fd, nkeys);
if (generate)
{
//Create mphf
mphf_fd = fopen(mphf_file, "w");
config = cmph_config_new(source);
cmph_config_set_algo(config, mph_algo);
if (nhashes) cmph_config_set_hashfuncs(config, hashes);
cmph_config_set_verbosity(config, verbosity);
cmph_config_set_tmp_dir(config, (cmph_uint8 *) tmp_dir);
cmph_config_set_mphf_fd(config, mphf_fd);
cmph_config_set_memory_availability(config, memory_availability);
cmph_config_set_b(config, b);
cmph_config_set_keys_per_bin(config, keys_per_bin);
//if((mph_algo == CMPH_BMZ || mph_algo == CMPH_BRZ) && c >= 2.0) c=1.15;
if(mph_algo == CMPH_BMZ && c >= 2.0) c=1.15;
if (c != 0) cmph_config_set_graphsize(config, c);
mphf = cmph_new(config);
cmph_config_destroy(config);
if (mphf == NULL)
{
fprintf(stderr, "Unable to create minimum perfect hashing function\n");
//cmph_config_destroy(config);
free(mphf_file);
return -1;
}
if (mphf_fd == NULL)
{
fprintf(stderr, "Unable to open output file %s: %s\n", mphf_file, strerror(errno));
free(mphf_file);
return -1;
}
cmph_dump(mphf, mphf_fd);
cmph_destroy(mphf);
fclose(mphf_fd);
}
else
{
cmph_uint8 * hashtable = NULL;
mphf_fd = fopen(mphf_file, "r");
if (mphf_fd == NULL)
{
fprintf(stderr, "Unable to open input file %s: %s\n", mphf_file, strerror(errno));
free(mphf_file);
return -1;
}
mphf = cmph_load(mphf_fd);
fclose(mphf_fd);
if (!mphf)
{
fprintf(stderr, "Unable to parser input file %s\n", mphf_file);
free(mphf_file);
return -1;
}
cmph_uint32 siz = cmph_size(mphf);
hashtable = (cmph_uint8*)calloc(siz, sizeof(cmph_uint8));
memset(hashtable, 0,(size_t) siz);
//check all keys
for (i = 0; i < source->nkeys; ++i)
{
cmph_uint32 h;
char *buf;
cmph_uint32 buflen = 0;
source->read(source->data, &buf, &buflen);
h = cmph_search(mphf, buf, buflen);
if (!(h < siz))
{
fprintf(stderr, "Unknown key %*s in the input.\n", buflen, buf);
ret = 1;
} else if(hashtable[h] >= keys_per_bin)
{
fprintf(stderr, "More than %u keys were mapped to bin %u\n", keys_per_bin, h);
fprintf(stderr, "Duplicated or unknown key %*s in the input\n", buflen, buf);
ret = 1;
} else hashtable[h]++;
if (verbosity)
{
printf("%s -> %u\n", buf, h);
}
source->dispose(source->data, buf, buflen);
}
cmph_destroy(mphf);
free(hashtable);
}
fclose(keys_fd);
free(mphf_file);
free(tmp_dir);
cmph_io_nlfile_adapter_destroy(source);
return ret;
}
char *cmph_op_build(bot_t * bot, cmphx_t ** cmphx, char *string)
{
cmphx_t *cmphx_ptr = NULL;
char *str = NULL;
char **keys = NULL;
int nkeys = 0;
debug(NULL, "cmph_op_build: Entered: %p %p %p\n", bot, cmphx, string);
if (!bot || !cmphx || !_sNULL(string))
return NULL;
cmphx_ptr = *cmphx;
if (!cmphx_ptr) {
cmphx_ptr = (cmphx_t *) calloc(1, sizeof(cmphx_t));
if (!cmphx_ptr)
return NULL;
*cmphx = cmphx_ptr;
}
if (cmphx_ptr->hash) {
cmph_op_clear(bot, cmphx, string);
}
keys =
tokenize_array(NULL, string,
TOKENIZE_NORMAL | TOKENIZE_EATWHITESPACE, " ",
&nkeys);
if (!keys)
goto cleanup;
tokenize_sort_strings(keys, &nkeys,
TOKENIZE_SORT_STRINGS_FORWARD |
TOKENIZE_SORT_STRINGS_UNIQ);
cmphx_ptr->fp = fopen("/tmp/cmph.mph", "w");
if (!cmphx_ptr->fp)
goto cleanup;
cmphx_ptr->source = cmph_io_vector_adapter((char **)keys, nkeys);
if (!cmphx_ptr->source)
goto cleanup;
cmphx_ptr->config = cmph_config_new(cmphx_ptr->source);
if (!cmphx_ptr->config)
goto cleanup;
cmph_config_set_algo(cmphx_ptr->config, CMPH_BRZ);
cmph_config_set_tmp_dir(cmphx_ptr->config, (cmph_uint8 *) "/tmp/");
cmph_config_set_mphf_fd(cmphx_ptr->config, cmphx_ptr->fp);
cmphx_ptr->hash = cmph_new(cmphx_ptr->config);
if (!cmphx_ptr->hash)
goto cleanup;
cmph_config_destroy(cmphx_ptr->config);
cmphx_ptr->config = NULL;
cmph_dump(cmphx_ptr->hash, cmphx_ptr->fp);
cmph_destroy(cmphx_ptr->hash);
fclose(cmphx_ptr->fp);
cmphx_ptr->fp = fopen("/tmp/cmph.mph", "r");
tokenize_destroy_array(NULL, keys);
debug(NULL, "cmph_op_build: Success\n");
cmphx_ptr->hash = cmph_load(cmphx_ptr->fp);
return str;
/* error */
cleanup:
cmph_op_clear(bot, cmphx, string);
debug(NULL, "cmph_op_build: Failure\n");
return str;
}
