hash_fcn_t * hash_fcn_new( int m )
{
hash_fcn_t * p;
static int one=1;
if( one ) /* one time init */
{
srand( (unsigned) time(0) );
one = 0;
}
// This can make all of the hashing static for testing.
//#define rand() 0
p = (hash_fcn_t*) calloc( 1,sizeof(hash_fcn_t) );
if( !p )
return 0;
p->seed = nearest_prime( (rand()%m)+3191 );
p->scale = nearest_prime( (rand()%m)+709 );
p->hardener = (rand()*rand()) + 133824503;
p->hash_fcn = &hash_fcn_hash;
p->keycmp_fcn = &memcmp;
return p;
}
Hashdh::Hashdh(uint _tsize)
{
tsize = nearest_prime(_tsize);
n = 0;
hashtable = new size_t[tsize];
enclength = new size_t[tsize];
uint i;
for(i=0 ; i<tsize ; i++)
{
hashtable[i] = (size_t)-1;
enclength[i] = 0;
}
}
void ZCE_LIB::hash_prime(const size_t node_num, size_t &real_num)
{
real_num = 0;
#if SUPPORT_BIGBIG_WORLD == 0
if (node_num > 4294967291ul - 100000000ul)
{
real_num = 4294967291ul;
return;
}
#else
#endif
//做一定的放大处理
size_t zoomin_num = ZCE_LIB::zoomin_nodenum(node_num);
real_num = nearest_prime(zoomin_num);
return;
}
void ZCE_LIB::hash_prime_ary(const size_t num_node,size_t &real_num, size_t row, size_t prime_ary[])
{
//做一定的放大处理
size_t zoomin_num = ZCE_LIB::zoomin_nodenum(num_node);
size_t per_row_num = zoomin_num/row + 1;
//制造一个质数队列，
size_t test_num = per_row_num;
for (size_t i = 0; i < row; ++i)
{
prime_ary[i] = nearest_prime(test_num);
test_num = prime_ary[i];
}
//总计放入最后一个列
real_num = 0;
for (size_t i = 0; i < row; ++i)
{
real_num += prime_ary[i];
}
}
int text2wfreq_impl(FILE* infp, FILE* outfp, int init_nwords, int verbosity)
{
int hash_size, scanrc;
struct hash_table vocab;
char word[MAX_STRING_LENGTH];
hash_size = nearest_prime( init_nwords );
new_hashtable( &vocab, hash_size );
while( (scanrc = fscanf(infp, "%500s", word )) == 1 ) {
if ( strlen( word ) >= MAX_STRING_LENGTH ) {
pc_message(verbosity,1,"text2wfreq : WARNING: word too long, will be split: %s...\n",word);
}
if (strlen(word)) {
update( &vocab, word ,verbosity);
}
}
if ( scanrc != EOF ) {
quit(-1,"Error reading input\n");
}
print( outfp, &vocab );
return 0;
}
int GVERB::init(double pfs[], int n_args)
{
if (rtsetoutput(pfs[0], pfs[2]+pfs[11], this) == -1)
return DONT_SCHEDULE;
if (outputChannels() != 2)
return die("GVERB", "stereo output required");
if (rtsetinput(pfs[1], this) == -1)
return DONT_SCHEDULE; // no input
inputframes = pfs[2] * SR;
inputchan = pfs[12];
amp = pfs[3];
float maxroomsize = 300.0f;
float roomsize = 50.0f;
float revtime = 7.0f;
float damping = 0.5f;
float spread = 15.0f;
float inputbandwidth = 0.5f;
float drylevel = 0.0f; //-1.9832f;
float earlylevel = 0.0f; //-1.9832f;
float taillevel = 0.0f;
float ga,gb,gt;
unsigned int i;
int n;
float r;
float diffscale;
int a,b,c,cc,d,dd,e;
float spread1,spread2;
// BGG max/msp heritage, params/etc. stored in this "p" struct (ty_gverb)
p = &realp;
// zero out the struct, to be careful
bzero((void *)p, sizeof (ty_gverb));
p->rate = SR;
p->fdndamping = damping;
p->maxroomsize = maxroomsize;
p->roomsize = CLIP(roomsize, 0.1f, maxroomsize);
p->revtime = revtime;
p->drylevel = drylevel;
p->earlylevel = earlylevel;
p->taillevel = taillevel;
p->maxdelay = p->rate*p->maxroomsize/340.0;
p->largestdelay = p->rate*p->roomsize/340.0;
/* Input damper */
p->inputbandwidth = inputbandwidth;
p->inputdamper = damper_make(1.0 - p->inputbandwidth);
/* FDN section */
p->fdndels = (ty_fixeddelay **)malloc(FDNORDER*sizeof(ty_fixeddelay *));
if(!p->fdndels)
return die("GVERB", "out of memory for fixeddelay ptrs");
for(i = 0; i < FDNORDER; i++)
{
p->fdndels[i] = fixeddelay_make((int)p->maxdelay+1000);
if(!p->fdndels[i])
return die("GVERB", "out of memory for fixeddelays");
}
p->fdngains = (float *)malloc(FDNORDER*sizeof(float));
p->fdnlens = (int *)malloc(FDNORDER*sizeof(int));
if(!p->fdngains || !p->fdnlens)
return die("GVERB", "out of memory for delay gains and lengths");
p->fdndamps = (ty_damper **)malloc(FDNORDER*sizeof(ty_damper *));
if(!p->fdndamps)
return die("GVERB", "out of memory for delay amps");
for(i = 0; i < FDNORDER; i++)
{
p->fdndamps[i] = damper_make(p->fdndamping);
if(!p->fdndamps[i])
return die("GVERB", "out of memory for delay amps 2");
}
ga = 60.0;
gt = p->revtime;
ga = pow(10.0,-ga/20.0);
n = (int)(p->rate*gt);
p->alpha = pow((double)ga,(double)1.0/(double)n);
gb = 0.0;
for(i = 0; i < FDNORDER; i++)
{
if (i == 0) gb = 1.000000*p->largestdelay;
if (i == 1) gb = 0.816490*p->largestdelay;
if (i == 2) gb = 0.707100*p->largestdelay;
if (i == 3) gb = 0.632450*p->largestdelay;
#if 0
p->fdnlens[i] = nearest_prime((int)gb, 0.5);
#else
p->fdnlens[i] = (int)gb;
#endif
// p->fdngains[i] = -pow(p->alpha,(double)p->fdnlens[i]);
p->fdngains[i] = -powf((float)p->alpha,p->fdnlens[i]);
}
p->d = (float *)malloc(FDNORDER*sizeof(float));
p->u = (float *)malloc(FDNORDER*sizeof(float));
p->f = (float *)malloc(FDNORDER*sizeof(float));
if(!p->d || !p->u || !p->f)
return die("GVERB", "out of memory for other delay stuff");
/* Diffuser section */
diffscale = (float)p->fdnlens[3]/(210+159+562+410);
spread1 = spread;
spread2 = 3.0*spread;
b = 210;
r = 0.125541f;
a = (int)(spread1*r);
c = 210+159+a;
cc = c-b;
r = 0.854046f;
a = (int)(spread2*r);
d = 210+159+562+a;
dd = d-c;
e = 1341-d;
p->ldifs = (ty_diffuser **)malloc(4*sizeof(ty_diffuser *));
if(!p->ldifs)
return die("GVERB", "out of memory for diffuser left structs");
p->ldifs[0] = diffuser_make((int)(diffscale*b),0.75);
p->ldifs[1] = diffuser_make((int)(diffscale*cc),0.75);
p->ldifs[2] = diffuser_make((int)(diffscale*dd),0.625);
p->ldifs[3] = diffuser_make((int)(diffscale*e),0.625);
if(!p->ldifs[0] || !p->ldifs[1] || !p->ldifs[2] || !p->ldifs[3])
return die("GVERB", "out of memory for diffuser left makes");
b = 210;
r = -0.568366f;
a = (int)(spread1*r);
c = 210+159+a;
cc = c-b;
r = -0.126815f;
a = (int)(spread2*r);
d = 210+159+562+a;
dd = d-c;
e = 1341-d;
p->rdifs = (ty_diffuser **)malloc(4*sizeof(ty_diffuser *));
if(!p->rdifs)
return die("GVERB", "out of memory for diffuser right structs");
p->rdifs[0] = diffuser_make((int)(diffscale*b),0.75);
p->rdifs[1] = diffuser_make((int)(diffscale*cc),0.75);
p->rdifs[2] = diffuser_make((int)(diffscale*dd),0.625);
p->rdifs[3] = diffuser_make((int)(diffscale*e),0.625);
if(!p->rdifs[0] || !p->rdifs[1] || !p->rdifs[2] || !p->rdifs[3])
return die("GVERB", "out of memory for diffuser right makes");
/* Tapped delay section */
p->tapdelay = fixeddelay_make(44000);
p->taps = (int *)malloc(FDNORDER*sizeof(int));
p->tapgains = (float *)malloc(FDNORDER*sizeof(float));
if(!p->tapdelay || !p->taps || !p->tapgains)
return die("GVERB", "out of memory for taps");
p->taps[0] = (int)(5+0.410*p->largestdelay);
p->taps[1] = (int)(5+0.300*p->largestdelay);
p->taps[2] = (int)(5+0.155*p->largestdelay);
p->taps[3] = (int)(5+0.000*p->largestdelay);
for(i = 0; i < FDNORDER; i++)
{
p->tapgains[i] = pow(p->alpha,(double)p->taps[i]);
}
// these values get set after all the init stuff
if (pfs[4] < 1.0 || pfs[4] > maxroomsize)
return die("GVERB", "bogus roomsize: %f\n", pfs[4]);
gverb_set_roomsize(p, pfs[4]); // sets p->roomsize
if (pfs[5] < 0.1 || pfs[5] > 360.0)
return die("GVERB", "bad revtime: %f\n", pfs[5]);
gverb_set_revtime(p, pfs[5]);
if (pfs[6] < 0.0 || pfs[6] > 1.0)
return die("GVERB", "incorrect damping: %f\n", pfs[6]);
gverb_set_damping(p, pfs[6]);
if (pfs[7] < 0.0 || pfs[7] > 1.0)
return die("GVERB", "input bandwith problem: %f\n", pfs[7]);
gverb_set_inputbandwidth(p, pfs[7]);
if (pfs[8] < -90.0 || pfs[8] > 0.0)
return die("GVERB", "dry level wrong: %f\n", pfs[8]);
gverb_set_drylevel(p, pfs[8]);
if (pfs[9] < -90.0 || pfs[9] > 0.0)
return die("GVERB", "problem with early reflection level: %f\n", pfs[9]);
gverb_set_earlylevel(p, pfs[9]);
if (pfs[10] < -90.0 || pfs[10] > 0.0)
return die("GVERB", "bogus tail level: %f\n", pfs[10]);
gverb_set_taillevel(p, pfs[10]);
branch = 0;
return nSamps();
}
void main(int argc, char *argv[]) {
int i,j;
char *vocab_filename;
FILE *tempfile;
char tempfiles_directory[1000];
int vocab_size;
FILE *vocab_file;
int verbosity;
int buffer_size;
int position_in_buffer;
int number_of_tempfiles;
int max_files;
int fof_size;
unsigned short *buffer;
unsigned short *placeholder;
unsigned short *temp_ngram;
int temp_count;
char temp_word[500];
char temp_word2[500];
char *temp_file_root;
char *temp_file_ext;
char *host_name;
int proc_id;
struct utsname uname_info;
flag write_ascii;
/* Vocab hash table things */
struct hash_table vocabulary;
unsigned long hash_size;
unsigned long M;
tempfile = NULL; /* Just to prevent compilation warnings. */
report_version(&argc,argv);
verbosity = pc_intarg(&argc,argv,"-verbosity",DEFAULT_VERBOSITY);
/* Process command line */
if (pc_flagarg( &argc, argv,"-help") || argc==1) {
fprintf(stderr,"text2idngram - Convert a text stream to an id n-gram stream.\n");
fprintf(stderr,"Usage : text2idngram -vocab .vocab \n");
fprintf(stderr," [ -buffer 100 ]\n");
fprintf(stderr," [ -hash %d ]\n",DEFAULT_HASH_SIZE);
fprintf(stderr," [ -temp %s ]\n",DEFAULT_TEMP);
fprintf(stderr," [ -files %d ]\n",DEFAULT_MAX_FILES);
fprintf(stderr," [ -gzip | -compress ]\n");
fprintf(stderr," [ -verbosity %d ]\n",
DEFAULT_VERBOSITY);
fprintf(stderr," [ -n 3 ]\n");
fprintf(stderr," [ -write_ascii ]\n");
fprintf(stderr," [ -fof_size 10 ]\n");
exit(1);
}
pc_message(verbosity,2,"text2idngram\n");
n = pc_intarg( &argc, argv, "-n",DEFAULT_N);
placeholder = (unsigned short *) rr_malloc(sizeof(unsigned short)*n);
temp_ngram = (unsigned short *) rr_malloc(sizeof(unsigned short)*n);
hash_size = pc_intarg( &argc, argv, "-hash",DEFAULT_HASH_SIZE);
buffer_size = pc_intarg( &argc, argv, "-buffer",STD_MEM);
write_ascii = pc_flagarg(&argc,argv,"-write_ascii");
fof_size = pc_intarg(&argc,argv,"-fof_size",10);
max_files = pc_intarg( &argc, argv, "-files",DEFAULT_MAX_FILES);
vocab_filename = salloc(pc_stringarg( &argc, argv, "-vocab", "" ));
if (!strcmp("",vocab_filename)) {
quit(-1,"text2idngram : Error : Must specify a vocabulary file.\n");
}
strcpy(tempfiles_directory,pc_stringarg( &argc, argv, "-temp",
DEFAULT_TEMP));
if (pc_flagarg(&argc,argv,"-compress")) {
temp_file_ext = salloc(".Z");
}
else {
if (pc_flagarg(&argc,argv,"-gzip")) {
temp_file_ext = salloc(".gz");
}
else {
temp_file_ext = salloc("");
}
}
uname(&uname_info);
host_name = salloc(uname_info.nodename);
proc_id = getpid();
sprintf(temp_word,"%s%s.%d.",TEMP_FILE_ROOT,host_name,proc_id);
temp_file_root = salloc(temp_word);
pc_report_unk_args(&argc,argv,verbosity);
/* If the last charactor in the directory name isn't a / then add one. */
if (tempfiles_directory[strlen(tempfiles_directory)-1] != '/') {
strcat(tempfiles_directory,"/");
}
pc_message(verbosity,2,"Vocab : %s\n",vocab_filename);
pc_message(verbosity,2,"N-gram buffer size : %d\n",buffer_size);
pc_message(verbosity,2,"Hash table size : %d\n",hash_size);
pc_message(verbosity,2,"Temp directory : %s\n",tempfiles_directory);
pc_message(verbosity,2,"Max open files : %d\n",max_files);
pc_message(verbosity,2,"FOF size : %d\n",fof_size);
pc_message(verbosity,2,"n : %d\n",n);
buffer_size *= (1000000/(sizeof(unsigned short)*n));
/* Allocate memory for hash table */
fprintf(stderr,"Initialising hash table...\n");
M = nearest_prime(hash_size);
new_hashtable(&vocabulary,M);
/* Read in the vocabulary */
vocab_size = 0;
vocab_file = rr_iopen(vocab_filename);
pc_message(verbosity,2,"Reading vocabulary...\n");
while (fgets (temp_word, sizeof(temp_word),vocab_file)) {
if (strncmp(temp_word,"##",2)==0) continue;
sscanf (temp_word, "%s ",temp_word2);
/* Check for repeated words in the vocabulary */
if (index2(&vocabulary,temp_word2) != 0) {
fprintf(stderr,"======================================================\n");
fprintf(stderr,"WARNING: word %s is repeated in the vocabulary.\n",temp_word);
fprintf(stderr,"=======================================================\n");
}
if (strncmp(temp_word,"#",1)==0) {
fprintf(stderr,"\n\n===========================================================\n");
fprintf(stderr,":\nWARNING: line assumed NOT a comment:\n");
fprintf(stderr, ">>> %s <<<\n",temp_word);
fprintf(stderr, " '%s' will be included in the vocabulary.\n",temp_word2);
fprintf(stderr, " (comments must start with '##')\n");
fprintf(stderr,"===========================================================\n\n");
}
vocab_size++;
add_to_hashtable(&vocabulary,hash(temp_word2,M),temp_word2,vocab_size);
}
if (vocab_size > MAX_VOCAB_SIZE) {
quit(-1,"text2idngram : Error : Vocabulary size exceeds maximum.\n");
}
pc_message(verbosity,2,"Allocating memory for the n-gram buffer...\n");
buffer=(unsigned short*) rr_malloc(n*(buffer_size+1)*sizeof(unsigned short));
number_of_tempfiles = 0;
/* Read text into buffer */
/* Read in the first ngram */
position_in_buffer = 0;
for (i=0;i<=n-1;i++) {
get_word(stdin,temp_word);
add_to_buffer(index2(&vocabulary,temp_word),0,i,buffer);
}
while (!rr_feof(stdin)) {
/* Fill up the buffer */
pc_message(verbosity,2,"Reading text into the n-gram buffer...\n");
pc_message(verbosity,2,"20,000 n-grams processed for each \".\", 1,000,000 for each line.\n");
while ((position_in_buffer<buffer_size) && (!rr_feof(stdin))) {
position_in_buffer++;
if (position_in_buffer % 20000 == 0) {
if (position_in_buffer % 1000000 == 0) {
pc_message(verbosity,2,".\n");
}
else {
pc_message(verbosity,2,".");
}
}
for (i=1;i<=n-1;i++) {
add_to_buffer(buffer_contents(position_in_buffer-1,i,buffer),
position_in_buffer,i-1,buffer);
}
if (get_word(stdin,temp_word) == 1) {
add_to_buffer(index2(&vocabulary,temp_word),position_in_buffer,
n-1,buffer);
}
}
for (i=0;i<=n-1;i++) {
placeholder[i] = buffer_contents(position_in_buffer,i,buffer);
}
/* Sort buffer */
pc_message(verbosity,2,"\nSorting n-grams...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
n*sizeof(unsigned short),compare_ngrams);
/* Output the buffer to temporary BINARY file */
number_of_tempfiles++;
sprintf(temp_word,"%s%s%hu%s",tempfiles_directory,temp_file_root,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,"Writing sorted n-grams to temporary file %s\n",
temp_word);
tempfile = rr_oopen(temp_word);
for (i=0;i<=n-1;i++) {
temp_ngram[i] = buffer_contents(0,i,buffer);
if (temp_ngram[i] > MAX_VOCAB_SIZE) {
quit(-1,"Invalid trigram in buffer.\nAborting");
}
}
temp_count = 1;
for (i=1;i<=position_in_buffer;i++) {
if (!compare_ngrams(temp_ngram,&buffer[i*n])) {
temp_count++;
}
else {
for (j=0;j<=n-1;j++) {
rr_fwrite(&temp_ngram[j],sizeof(unsigned short),1,
tempfile,"temporary n-gram ids");
temp_ngram[j] = buffer_contents(i,j,buffer);
}
rr_fwrite(&temp_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
temp_count = 1;
}
}
rr_oclose(tempfile);
for (i=0;i<=n-1;i++) {
add_to_buffer(placeholder[i],0,i,buffer);
}
position_in_buffer = 0;
}
/* Merge the temporary files, and output the result to standard output */
pc_message(verbosity,2,"Merging temporary files...\n");
merge_tempfiles(1,
number_of_tempfiles,
temp_file_root,
temp_file_ext,
max_files,
tempfiles_directory,
stdout,
write_ascii,
fof_size);
pc_message(verbosity,0,"text2idngram : Done.\n");
exit(0);
}
int main(int argc, char *argv[]) {
int verbosity;
int vocab_size;
FILE *vocab_file;
int buffer_size;
flag write_ascii;
int max_files;
int number_of_tempfiles;
char *vocab_filename;
char *idngram_filename;
char temp_word[MAX_WORD_LENGTH];
char temp_word2[MAX_WORD_LENGTH];
char temp_word3[MAX_WORD_LENGTH];
flag contains_unks;
int position_in_buffer;
FILE *outfile;
FILE *tempfile;
FILE *non_unk_fp;
ngram_rec *buffer;
flag same_ngram;
int i;
int j;
int fof_size;
int size_of_rec;
char temp_directory[1000];
char *temp_file_ext;
/* Vocab hash table things */
struct idngram_hash_table vocabulary;
unsigned long hash_size;
unsigned long M;
wordid_t *current_ngram;
int current_count;
wordid_t *sort_ngram;
int sort_count;
/* Process command line */
report_version(&argc,argv);
if (argc == 1 || pc_flagarg(&argc, argv,"-help")) {
/* Display help message */
help_message();
exit(1);
}
n = pc_intarg( &argc, argv, "-n",DEFAULT_N);
hash_size = pc_intarg( &argc, argv, "-hash",DEFAULT_HASH_SIZE);
buffer_size = pc_intarg( &argc, argv, "-buffer",STD_MEM);
write_ascii = pc_flagarg(&argc,argv,"-write_ascii");
verbosity = pc_intarg(&argc,argv,"-verbosity",DEFAULT_VERBOSITY);
max_files = pc_intarg( &argc, argv, "-files",DEFAULT_MAX_FILES);
fof_size = pc_intarg(&argc,argv,"-fof_size",10);
vocab_filename = salloc(pc_stringarg( &argc, argv, "-vocab", "" ));
idngram_filename = salloc(pc_stringarg( &argc, argv, "-idngram", "" ));
if (!strcmp("",vocab_filename))
quit(-1,"Error : Must specify a vocabulary file.\n");
if (!strcmp("",idngram_filename))
quit(-1,"text2idngram : Error : Must specify idngram file.\n");
if (pc_flagarg(&argc,argv,"-compress"))
temp_file_ext = salloc(".Z");
else {
if (pc_flagarg(&argc,argv,"-gzip"))
temp_file_ext = salloc(".gz");
else
temp_file_ext = salloc("");
}
strcpy(temp_directory, "cmuclmtk-XXXXXX");
if (mkdtemp(temp_directory) == NULL) {
quit(-1, "Failed to create temporary folder: %s\n", strerror(errno));
}
pc_report_unk_args(&argc,argv,verbosity);
outfile = rr_fopen(idngram_filename,"wb");
pc_message(verbosity,2,"Vocab : %s\n",vocab_filename);
pc_message(verbosity,2,"Output idngram : %s\n",idngram_filename);
pc_message(verbosity,2,"Buffer size : %d\n",buffer_size);
pc_message(verbosity,2,"Hash table size : %d\n",hash_size);
pc_message(verbosity,2,"Max open files : %d\n",max_files);
pc_message(verbosity,2,"n : %d\n",n);
pc_message(verbosity,2,"FOF size : %d\n",fof_size);
size_of_rec = (sizeof(wordid_t) * n) + 16 - (( n* sizeof(wordid_t)) % 16);
buffer_size *= (1000000/((sizeof(ngram_rec) + size_of_rec)));
fprintf(stderr,"buffer size = %d\n",buffer_size);
/* Allocate memory for hash table */
fprintf(stderr,"Initialising hash table...\n");
M = nearest_prime(hash_size);
new_idngram_hashtable(&vocabulary,M);
/* Read in the vocabulary */
vocab_size = 0;
vocab_file = rr_iopen(vocab_filename);
pc_message(verbosity,2,"Reading vocabulary...\n");
while (fgets (temp_word, sizeof(temp_word),vocab_file)) {
if (strncmp(temp_word,"##",2)==0) continue;
sscanf (temp_word, "%s ",temp_word2);
/* Check for vocabulary order */
if (vocab_size > 0 && strcmp(temp_word2,temp_word3)<0)
quit(-1,"wngram2idngram : Error : Vocabulary is not alphabetically ordered.\n");
/* Check for repeated words in the vocabulary */
if (index2(&vocabulary,temp_word2) != 0)
warn_on_repeated_words(temp_word);
warn_on_wrong_vocab_comments(temp_word);
vocab_size++;
add_to_idngram_hashtable(&vocabulary,idngram_hash(temp_word2,M),temp_word2,vocab_size);
strcpy(temp_word3,temp_word2);
}
if (vocab_size > MAX_VOCAB_SIZE)
quit(-1,"Error : Vocabulary size exceeds maximum.\n");
pc_message(verbosity,2,"Allocating memory for the buffer...\n");
buffer=(ngram_rec *) rr_malloc((buffer_size+1)*sizeof(ngram_rec));
for (i=0;i<=buffer_size;i++)
buffer[i].word = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
/* Open the "non-OOV" tempfile */
sprintf(temp_word, "%s/1%s", temp_directory, temp_file_ext);
non_unk_fp = rr_fopen(temp_word,"w");
pc_message(verbosity,2,"Writing non-OOV counts to temporary file %s\n",
temp_word);
number_of_tempfiles = 1;
current_ngram = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
sort_ngram = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
/* Read text into buffer */
position_in_buffer = 0;
while (!rr_feof(stdin)) {
for (i=0;i<=n-1;i++) {
get_word(stdin,temp_word);
current_ngram[i]=index2(&vocabulary,temp_word);
}
if (scanf("%d",¤t_count) != 1)
if (!rr_feof(stdin))
quit(-1,"Error reading n-gram count from stdin.\n");
if (!rr_feof(stdin)) {
contains_unks = 0;
for (i=0;i<=n-1;i++) {
if (!current_ngram[i])
contains_unks = 1;
}
if (contains_unks) {
/* Write to buffer */
position_in_buffer++;
if (position_in_buffer >= buffer_size) {
/* Sort buffer */
pc_message(verbosity,2,
"Sorting n-grams which include an OOV word...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
sizeof(ngram_rec),compare_ngrams2);
pc_message(verbosity,2,"Done.\n");
/* Write buffer to temporary file */
number_of_tempfiles++;
sprintf(temp_word,"%s/%hu%s", temp_directory,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,
"Writing sorted OOV-counts buffer to temporary file %s\n",
temp_word);
tempfile = rr_fopen(temp_word,"w");
for (i=0;i<=n-1;i++)
sort_ngram[i] = buffer[0].word[i];
sort_count = buffer[0].count;
for (i=0;i<=position_in_buffer-2;i++) {
same_ngram = 1;
for (j=n-1;j>=0;j--) {
if (buffer[i].word[j] != sort_ngram[j]) {
same_ngram = 0;
j = -1;
}
}
if (same_ngram)
sort_count += buffer[i].count;
else {
for (j=0;j<=n-1;j++) {
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
sort_ngram[j] = buffer[i].word[j];
}
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
sort_count = buffer[i].count;
}
}
for (j=0;j<=n-1;j++)
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
rr_oclose(tempfile);
position_in_buffer = 1;
}
for (i=0;i<=n-1;i++)
buffer[position_in_buffer-1].word[i] = current_ngram[i];
buffer[position_in_buffer-1].count = current_count;
}else {
/* Write to temporary file */
for (i=0;i<=n-1;i++)
rr_fwrite((char*)¤t_ngram[i],sizeof(wordid_t),1,
non_unk_fp,"temporary n-gram ids");
rr_fwrite((char*)¤t_count,sizeof(int),1,non_unk_fp,
"temporary n-gram counts");
}
}
}
if (position_in_buffer > 0) {
/* Only do this bit if we have actually seen some OOVs */
/* Sort final buffer */
pc_message(verbosity,2,"Sorting final buffer...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
sizeof(ngram_rec),compare_ngrams2);
/* Write final buffer */
number_of_tempfiles++;
sprintf(temp_word,"%s/%hu%s", temp_directory,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,"Writing sorted buffer to temporary file %s\n", temp_word);
tempfile = rr_fopen(temp_word,"w");
for (i=0;i<=n-1;i++)
sort_ngram[i] = buffer[0].word[i];
sort_count = buffer[0].count;
for (i=1;i<=position_in_buffer-1;i++) {
same_ngram = 1;
for (j=n-1;j>=0;j--) {
if (buffer[i].word[j] != sort_ngram[j]) {
same_ngram = 0;
j = -1;
}
}
if (same_ngram)
sort_count += buffer[i].count;
else {
for (j=0;j<=n-1;j++) {
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
sort_ngram[j] = buffer[i].word[j];
}
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
sort_count = buffer[i].count;
}
}
for (j=0;j<=n-1;j++)
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
fclose(tempfile);
}
/* Merge the temporary files, and output the result */
fclose(non_unk_fp);
pc_message(verbosity,2,"Merging temporary files...\n");
merge_idngramfiles(1,
number_of_tempfiles,
temp_directory,
temp_file_ext,
max_files,
outfile,
write_ascii,
fof_size,
n);
fclose(outfile);
rmdir(temp_directory);
pc_message(verbosity,0,"wngram2idngram : Done.\n");
return 0;
}
ty_gverb *gverb_new(int srate, float maxroomsize, float roomsize,
float revtime,
float damping, float spread,
float inputbandwidth, float earlylevel,
float taillevel)
{
ty_gverb *p;
float ga,gb,gt;
int i,n;
float r;
float diffscale;
int a,b,c,cc,d,dd,e;
float spread1,spread2;
p = (ty_gverb *)malloc(sizeof(ty_gverb));
p->rate = srate;
p->fdndamping = damping;
p->maxroomsize = maxroomsize;
p->roomsize = roomsize;
p->revtime = revtime;
p->earlylevel = earlylevel;
p->taillevel = taillevel;
p->maxdelay = p->rate*p->maxroomsize/340.0;
p->largestdelay = p->rate*p->roomsize/340.0;
/* Input damper */
p->inputbandwidth = inputbandwidth;
p->inputdamper = damper_make(1.0 - p->inputbandwidth);
/* FDN section */
p->fdndels = (ty_fixeddelay **)calloc(FDNORDER, sizeof(ty_fixeddelay *));
for(i = 0; i < FDNORDER; i++) {
p->fdndels[i] = fixeddelay_make((int)p->maxdelay+1000);
}
p->fdngains = (float *)calloc(FDNORDER, sizeof(float));
p->fdnlens = (int *)calloc(FDNORDER, sizeof(int));
p->fdndamps = (ty_damper **)calloc(FDNORDER, sizeof(ty_damper *));
for(i = 0; i < FDNORDER; i++) {
p->fdndamps[i] = damper_make(p->fdndamping);
}
ga = 60.0;
gt = p->revtime;
ga = powf(10.0f,-ga/20.0f);
n = p->rate*gt;
p->alpha = pow((double)ga, 1.0/(double)n);
gb = 0.0;
for(i = 0; i < FDNORDER; i++) {
if (i == 0) gb = 1.000000*p->largestdelay;
if (i == 1) gb = 0.816490*p->largestdelay;
if (i == 2) gb = 0.707100*p->largestdelay;
if (i == 3) gb = 0.632450*p->largestdelay;
#if 0
p->fdnlens[i] = nearest_prime((int)gb, 0.5);
#else
p->fdnlens[i] = f_round(gb);
#endif
p->fdngains[i] = -powf((float)p->alpha,p->fdnlens[i]);
}
p->d = (float *)calloc(FDNORDER, sizeof(float));
p->u = (float *)calloc(FDNORDER, sizeof(float));
p->f = (float *)calloc(FDNORDER, sizeof(float));
/* Diffuser section */
diffscale = (float)p->fdnlens[3]/(210+159+562+410);
spread1 = spread;
spread2 = 3.0*spread;
b = 210;
r = 0.125541;
a = spread1*r;
c = 210+159+a;
cc = c-b;
r = 0.854046;
a = spread2*r;
d = 210+159+562+a;
dd = d-c;
e = 1341-d;
p->ldifs = (ty_diffuser **)calloc(4, sizeof(ty_diffuser *));
p->ldifs[0] = diffuser_make((int)(diffscale*b),0.75);
p->ldifs[1] = diffuser_make((int)(diffscale*cc),0.75);
p->ldifs[2] = diffuser_make((int)(diffscale*dd),0.625);
p->ldifs[3] = diffuser_make((int)(diffscale*e),0.625);
b = 210;
r = -0.568366;
a = spread1*r;
c = 210+159+a;
cc = c-b;
r = -0.126815;
a = spread2*r;
d = 210+159+562+a;
dd = d-c;
e = 1341-d;
p->rdifs = (ty_diffuser **)calloc(4, sizeof(ty_diffuser *));
p->rdifs[0] = diffuser_make((int)(diffscale*b),0.75);
p->rdifs[1] = diffuser_make((int)(diffscale*cc),0.75);
p->rdifs[2] = diffuser_make((int)(diffscale*dd),0.625);
p->rdifs[3] = diffuser_make((int)(diffscale*e),0.625);
/* Tapped delay section */
p->tapdelay = fixeddelay_make(44000);
p->taps = (int *)calloc(FDNORDER, sizeof(int));
p->tapgains = (float *)calloc(FDNORDER, sizeof(float));
p->taps[0] = 5+0.410*p->largestdelay;
p->taps[1] = 5+0.300*p->largestdelay;
p->taps[2] = 5+0.155*p->largestdelay;
p->taps[3] = 5+0.000*p->largestdelay;
for(i = 0; i < FDNORDER; i++) {
p->tapgains[i] = pow(p->alpha,(double)p->taps[i]);
}
return(p);
}
// constructor
GVerb( t_CKFLOAT fs)
{
const float maxroomsize = 300.0f;
float roomsize = 30.0f;
float revtime = 5.0f;
float damping = 0.8f;
float spread = 15.0f;
float inputbandwidth = 0.5f;
float drylevel = 0.6f; //-1.9832f;
float earlylevel = 0.4f; //-1.9832f;
float taillevel = 0.5f;
float ga,gb,gt;
unsigned int i;
int n;
float r;
float diffscale;
int a,b,c,cc,d,dd,e;
float spread1,spread2;
p = &realp;
memset((void *)p, 0, sizeof (ty_gverb));
p->rate = fs;
p->fdndamping = damping;
p->maxroomsize = maxroomsize;
p->roomsize = CLIP(roomsize, 0.1f, maxroomsize);
p->revtime = revtime;
p->drylevel = drylevel;
p->earlylevel = earlylevel;
p->taillevel = taillevel;
p->maxdelay = p->rate*p->maxroomsize/340.0;
p->largestdelay = p->rate*p->roomsize/340.0;
/* Input damper */
p->inputbandwidth = inputbandwidth;
p->inputdamper = damper_make(1.0 - p->inputbandwidth);
/* FDN section */
p->fdndels = (ty_fixeddelay **)malloc(FDNORDER*sizeof(ty_fixeddelay *));
for(i = 0; i < FDNORDER; i++)
{
p->fdndels[i] = fixeddelay_make((int)p->maxdelay+1000);
}
p->fdngains = (float *)malloc(FDNORDER*sizeof(float));
p->fdnlens = (int *)malloc(FDNORDER*sizeof(int));
p->fdndamps = (ty_damper **)malloc(FDNORDER*sizeof(ty_damper *));
for(i = 0; i < FDNORDER; i++)
{
p->fdndamps[i] = damper_make(p->fdndamping);
}
ga = 60.0;
gt = p->revtime;
ga = pow(10.0,-ga/20.0);
n = (int)(p->rate*gt);
p->alpha = pow((double)ga,(double)1.0/(double)n);
gb = 0.0;
for(i = 0; i < FDNORDER; i++)
{
if (i == 0) gb = 1.000000*p->largestdelay;
if (i == 1) gb = 0.816490*p->largestdelay;
if (i == 2) gb = 0.707100*p->largestdelay;
if (i == 3) gb = 0.632450*p->largestdelay;
#if 0
p->fdnlens[i] = nearest_prime((int)gb, 0.5);
#else
p->fdnlens[i] = (int)gb;
#endif
// p->fdngains[i] = -pow(p->alpha,(double)p->fdnlens[i]);
p->fdngains[i] = -powf((float)p->alpha,p->fdnlens[i]);
}
p->d = (float *)malloc(FDNORDER*sizeof(float));
p->u = (float *)malloc(FDNORDER*sizeof(float));
p->f = (float *)malloc(FDNORDER*sizeof(float));
/* Diffuser section */
diffscale = (float)p->fdnlens[3]/(210+159+562+410);
spread1 = spread;
spread2 = 3.0*spread;
b = 210;
r = 0.125541f;
a = (int)(spread1*r);
c = 210+159+a;
cc = c-b;
r = 0.854046f;
a = (int)(spread2*r);
d = 210+159+562+a;
dd = d-c;
e = 1341-d;
p->ldifs = (ty_diffuser **)malloc(4*sizeof(ty_diffuser *));
p->ldifs[0] = diffuser_make((int)(diffscale*b),0.75);
p->ldifs[1] = diffuser_make((int)(diffscale*cc),0.75);
p->ldifs[2] = diffuser_make((int)(diffscale*dd),0.625);
p->ldifs[3] = diffuser_make((int)(diffscale*e),0.625);
b = 210;
r = -0.568366f;
a = (int)(spread1*r);
c = 210+159+a;
cc = c-b;
r = -0.126815f;
a = (int)(spread2*r);
d = 210+159+562+a;
dd = d-c;
e = 1341-d;
p->rdifs = (ty_diffuser **)malloc(4*sizeof(ty_diffuser *));
p->rdifs[0] = diffuser_make((int)(diffscale*b),0.75);
p->rdifs[1] = diffuser_make((int)(diffscale*cc),0.75);
p->rdifs[2] = diffuser_make((int)(diffscale*dd),0.625);
p->rdifs[3] = diffuser_make((int)(diffscale*e),0.625);
/* Tapped delay section */
p->tapdelay = fixeddelay_make(44000);
p->taps = (int *)malloc(FDNORDER*sizeof(int));
p->tapgains = (float *)malloc(FDNORDER*sizeof(float));
p->taps[0] = (int)(5+0.410*p->largestdelay);
p->taps[1] = (int)(5+0.300*p->largestdelay);
p->taps[2] = (int)(5+0.155*p->largestdelay);
p->taps[3] = (int)(5+0.000*p->largestdelay);
for(i = 0; i < FDNORDER; i++)
{
p->tapgains[i] = pow(p->alpha,(double)p->taps[i]);
}
}
