static int getfastastreamkmers(const GtStrArray *filenametab,
unsigned int numofchars,
unsigned int kmersize,
const GtUchar *symbolmap,
bool plainformat,
GtArrayGtCodetype *codeliststream,
GtError *err)
{
GtKmercodeiterator *kmercodeiterator;
const GtKmercode *kmercodeptr;
bool haserr = false;
kmercodeiterator = gt_kmercodeiterator_filetab_new(
filenametab,
numofchars,
kmersize,
symbolmap,
plainformat,
err);
if (!gt_kmercodeiterator_inputexhausted(kmercodeiterator))
{
while (!haserr)
{
int retval = gt_kmercodeiterator_filetab_next(&kmercodeptr,
kmercodeiterator,
err);
if (retval < 0)
{
haserr = true;
} else
{
if (kmercodeptr != NULL)
{
outkmeroccurrence(codeliststream,kmercodeptr);
} else
{
break;
}
}
}
}
gt_kmercodeiterator_delete(kmercodeiterator);
return haserr ? -1 : 0;
}
GtKmercodeiterator *gt_kmercodeiterator_filetab_new(
const GtStrArray *filenametab,
unsigned int numofchars,
unsigned int kmersize,
const GtUchar *symbolmap,
bool plainformat,
GtError *err)
{
GtKmercodeiterator *kmercodeiterator;
GtUchar charcode;
bool haserr = false;
int retval;
gt_error_check(err);
kmercodeiterator = gt_malloc(sizeof (*kmercodeiterator));
kmercodeiterator->esr = NULL;
kmercodeiterator->hasprocessedfirst = false;
kmercodeiterator->inputexhausted = false;
kmercodeiterator->spwp = kmerstream_new(numofchars,kmersize);
kmercodeiterator->totallength = 0;
if (plainformat)
{
kmercodeiterator->fb = gt_sequence_buffer_plain_new(filenametab);
} else
{
kmercodeiterator->fb = gt_sequence_buffer_new_guess_type(filenametab, err);
}
if (kmercodeiterator->fb == NULL)
{
haserr = true;
}
if (!haserr)
{
gt_sequence_buffer_set_symbolmap(kmercodeiterator->fb, symbolmap);
for (kmercodeiterator->currentposition = 0;
kmercodeiterator->currentposition < (unsigned long) kmersize;
kmercodeiterator->currentposition++)
{
retval = gt_sequence_buffer_next(kmercodeiterator->fb,&charcode,err);
if (retval < 0)
{
haserr = true;
break;
}
if (retval == 0)
{
kmercodeiterator->inputexhausted = true;
break;
}
kmercodeiterator->spwp->windowwidth++;
updatespecialpositions(kmercodeiterator->spwp,charcode,false,0);
kmercodeiterator->spwp->cyclicwindow[kmercodeiterator->
spwp->windowwidth-1] = charcode;
}
}
if (haserr)
{
gt_kmercodeiterator_delete(kmercodeiterator);
return NULL;
}
return kmercodeiterator;
}
static int gt_kmer_database_runner(GT_UNUSED int argc, const char **argv,
int parsed_args, void *tool_arguments,
GtError *err)
{
GtKmerDatabaseArguments *arguments = tool_arguments;
int had_err = 0;
GtEncseq *es;
GtUword es_length,
nu_kmer_codes = 0;
GtKmerDatabase *compare_db = NULL,
*db = NULL;
GtLogger *logger;
FILE *fp = NULL;
GtHashmap *kmer_hash = NULL;
GtTimer *timer = NULL;
gt_error_check(err);
gt_assert(arguments);
if (arguments->use_hash)
kmer_hash = gt_hashmap_new(GT_HASH_DIRECT, NULL,
(GtFree) gt_kmer_database_delete_hash_value);
if (arguments->bench)
timer = gt_timer_new_with_progress_description("loading encoded sequence");
logger = gt_logger_new(arguments->verbose, GT_LOGGER_DEFLT_PREFIX, stderr);
if (arguments->verbose && gt_str_length(arguments->print_filename) > 0UL) {
fp = gt_fa_fopen(gt_str_get(arguments->print_filename), "w", err);
gt_logger_set_target(logger, fp);
}
if (!had_err) {
GtEncseqLoader *es_l;
if (arguments->bench)
gt_timer_start(timer);
es_l = gt_encseq_loader_new();
es = gt_encseq_loader_load(es_l, argv[parsed_args], err);
if (arguments->bench)
gt_timer_show_progress(timer, "saving kmers (+iterating over file)",
stdout);
if (es == NULL) {
had_err = -1;
}
gt_encseq_loader_delete(es_l);
}
if (!had_err) {
es_length = gt_encseq_total_length(es);
if (es_length < (GtUword) arguments->kmersize) {
gt_error_set(err, "Input is too short for used kmersize. File length: "
GT_WU " kmersize: %u", es_length, arguments->kmersize);
had_err = -1;
}
}
if (!had_err) {
GtAlphabet *alphabet;
alphabet = gt_encseq_alphabet(es);
if (arguments->bench)
nu_kmer_codes = gt_power_for_small_exponents(
gt_alphabet_num_of_chars(alphabet),
arguments->kmersize);
if (!arguments->merge_only && !arguments->use_hash && !arguments->bench) {
compare_db = gt_kmer_database_new(gt_alphabet_num_of_chars(alphabet),
arguments->kmersize, arguments->sb_size, es);
}
if (!arguments->use_hash) {
db = gt_kmer_database_new(gt_alphabet_num_of_chars(alphabet),
arguments->kmersize,
arguments->sb_size, es);
if (arguments->cutoff) {
if (arguments->mean_cutoff)
gt_kmer_database_use_mean_cutoff(db, (GtUword) 2,
arguments->cutoff_value);
else
gt_kmer_database_set_cutoff(db, arguments->cutoff_value);
if (!arguments->prune)
gt_kmer_database_set_prune(db);
}
}
}
if (!had_err) {
GtUword startpos = 0,
endpos;
GtKmercodeiterator *iter;
const GtKmercode *kmercode = NULL;
iter = gt_kmercodeiterator_encseq_new(es, GT_READMODE_FORWARD,
arguments->kmersize, 0);
while (!had_err && startpos < es_length - (arguments->kmersize - 1)) {
GtUword startpos_add_kmer = startpos;
if (arguments->merge_only) {
endpos = startpos + (arguments->kmersize - 1) +
(gt_rand_max((arguments->sb_size - 1) * 2));
if (endpos > es_length)
endpos = es_length;
}
else {
endpos = startpos + (arguments->kmersize - 1) +
(gt_rand_max(arguments->sb_size - 1));
}
gt_kmercodeiterator_reset(iter, GT_READMODE_FORWARD, startpos);
while ((kmercode = gt_kmercodeiterator_encseq_next(iter)) != NULL &&
startpos_add_kmer <= endpos - (arguments->kmersize - 1)) {
if (!arguments->merge_only && !arguments->use_hash &&
!kmercode->definedspecialposition && !arguments->bench) {
gt_kmer_database_add_kmer(compare_db, kmercode->code,
startpos_add_kmer);
}
if (arguments->use_hash && !kmercode->definedspecialposition) {
gt_kmer_database_add_to_hash(kmer_hash, kmercode->code,
startpos_add_kmer);
}
startpos_add_kmer++;
}
if (!arguments->use_hash) {
gt_kmer_database_add_interval(db, startpos, endpos);
gt_kmer_database_print_buffer(db, logger);
if (!arguments->bench)
had_err = gt_kmer_database_check_consistency(db, err);
}
startpos = endpos + 1;
}
if (!arguments->use_hash) {
gt_kmer_database_flush(db);
gt_kmer_database_print_buffer(db, logger);
if (!had_err && !arguments->bench)
had_err = gt_kmer_database_check_consistency(db, err);
if (!arguments->merge_only && !had_err && !arguments->bench)
had_err = gt_kmer_database_check_consistency(compare_db, err);
if (!arguments->merge_only && !arguments->bench)
gt_kmer_database_print(compare_db, logger, true);
if (!arguments->merge_only && !had_err && !arguments->bench)
had_err = gt_kmer_database_compare(compare_db, db, err);
gt_kmer_database_print(db, logger, true);
}
gt_kmercodeiterator_delete(iter);
}
if (arguments->bench) {
GtKmerStartpos pos;
GtArrayGtUword *pos_hash;
GtUword rand_access = (GtUword) 50000000,
rand_code,
i,
sum = 0;
gt_timer_show_progress(timer, "random access", stdout);
for (i = 0; i < rand_access; i++) {
rand_code = gt_rand_max(nu_kmer_codes - 1);
if (arguments->use_hash) {
pos_hash = gt_hashmap_get(kmer_hash, (const void *) rand_code);
if (pos_hash != NULL)
sum += pos_hash->spaceGtUword[pos_hash->nextfreeGtUword - 1];
}
else {
pos = gt_kmer_database_get_startpos(db, rand_code);
if (pos.no_positions > 0)
sum += pos.startpos[pos.no_positions - 1];
}
}
printf("sum: " GT_WU "\n", sum);
gt_timer_show_progress(timer, "", stdout);
gt_timer_stop(timer);
gt_timer_delete(timer);
}
if (arguments->use_hash)
gt_hashmap_delete(kmer_hash);
gt_encseq_delete(es);
if (!arguments->use_hash)
gt_kmer_database_delete(db);
if (!arguments->merge_only && !arguments->bench)
gt_kmer_database_delete(compare_db);
gt_logger_delete(logger);
gt_fa_fclose(fp);
return had_err;
}