/* Search and remove an element */
static int dictDelete(dict *ht, const void *key) {
unsigned int h;
dictEntry *de, *prevde;
if (ht->size == 0)
return DICT_ERR;
h = dictHashKey(ht, key) & ht->sizemask;
de = ht->table[h];
prevde = NULL;
while(de) {
if (dictCompareHashKeys(ht,key,de->key)) {
/* Unlink the element from the list */
if (prevde)
prevde->next = de->next;
else
ht->table[h] = de->next;
dictFreeEntryKey(ht,de);
dictFreeEntryVal(ht,de);
TFREE(de);
ht->used--;
return DICT_OK;
}
prevde = de;
de = de->next;
}
return DICT_ERR; /* not found */
}
/* Destroy an entire hash table */
static int _dictClear(dict *ht) {
unsigned long i;
/* Free all the elements */
for (i = 0; i < ht->size && ht->used > 0; i++) {
dictEntry *he, *nextHe;
if ((he = ht->table[i]) == NULL) continue;
while(he) {
nextHe = he->next;
dictFreeEntryKey(ht, he);
dictFreeEntryVal(ht, he);
TFREE(he);
ht->used--;
he = nextHe;
}
}
/* Free the table and the allocated cache structure */
TFREE(ht->table);
/* Re-initialize the table */
_dictReset(ht);
return DICT_OK; /* never fails */
}
/* Expand or create the hashtable */
static int dictExpand(dict *ht, unsigned long size) {
dict n; /* the NEW hashtable */
unsigned long realsize = _dictNextPower(size), i;
/* the size is invalid if it is smaller than the number of
* elements already inside the hashtable */
if (ht->used > size)
return DICT_ERR;
_dictInit(&n, ht->type, ht->privdata);
n.size = realsize;
n.sizemask = realsize-1;
n.table = calloc(realsize,sizeof(dictEntry*));
/* Copy all the elements from the old to the NEW table:
* note that if the old hash table is empty ht->size is zero,
* so dictExpand just creates an hash table. */
n.used = ht->used;
for (i = 0; i < ht->size && ht->used > 0; i++) {
dictEntry *he, *nextHe;
if (ht->table[i] == NULL) continue;
/* For each hash entry on this slot... */
he = ht->table[i];
while(he) {
unsigned int h;
nextHe = he->next;
/* Get the NEW element index */
h = dictHashKey(ht, he->key) & n.sizemask;
he->next = n.table[h];
n.table[h] = he;
ht->used--;
/* Pass to the next element */
he = nextHe;
}
}
assert(ht->used == 0);
TFREE(ht->table);
/* Remap the NEW hashtable in the old */
*ht = n;
return DICT_OK;
}
void DnarchModule::Bin2Dnarch(const std::string &inBinFile_, const std::string &outDnarchFile_, const CompressorParams& params_,
uint32 threadsNum_, bool verboseMode_)
{
BinModuleConfig conf;
BinFileExtractor* extractor = new BinFileExtractor(params_.minBinSize);
extractor->StartDecompress(inBinFile_, conf);
DnarchFileWriter* dnarch = new DnarchFileWriter();
dnarch->StartCompress(outDnarchFile_, conf.minimizer, params_);
if (threadsNum_ > 1)
{
const uint32 partNum = threadsNum_ + (threadsNum_ >> 1);//threadsNum_ * 2;
const uint64 dnaBufferSize = 1 << 20;
const uint64 outBufferSize = 1 << 20;
MinimizerPartsPool* inPool = new MinimizerPartsPool(partNum, dnaBufferSize);
MinimizerPartsQueue* inQueue = new MinimizerPartsQueue(partNum, 1);
CompressedDnaPartsPool* outPool = new CompressedDnaPartsPool(partNum, outBufferSize);
CompressedDnaPartsQueue* outQueue = new CompressedDnaPartsQueue(partNum, threadsNum_);
BinPartsExtractor* inReader = new BinPartsExtractor(extractor, inQueue, inPool);
DnarchPartsWriter* outWriter = new DnarchPartsWriter(dnarch, outQueue, outPool);
// preprocess small bins and N bin <-- this should be done internally
//
{
DnaCompressor compressor(conf.minimizer, params_);
DnaPacker packer(conf.minimizer);
uint32 signatureId = 0;
CompressedDnaBlock compBin;
BinaryBinBlock binBin;
std::vector<const BinFileExtractor::BlockDescriptor*> descriptors = extractor->GetSmallBlockDescriptors();
uint64 totalDnaBufferSize = 0;
uint64 totalRecords = 0;
for (uint32 i = 0; i < descriptors.size(); ++i)
{
totalDnaBufferSize += descriptors[i]->rawDnaSize;
totalRecords += descriptors[i]->recordsCount;
}
const BinFileExtractor::BlockDescriptor* nd = extractor->GetNBlockDescriptor();
totalDnaBufferSize += nd->rawDnaSize;
totalRecords += nd->recordsCount;
if (compBin.workBuffers.dnaBuffer.data.Size() < totalDnaBufferSize)
compBin.workBuffers.dnaBuffer.data.Extend(totalDnaBufferSize);
// extract and unpack small bins
//
while (extractor->ExtractNextSmallBin(binBin, signatureId))
{
ASSERT(binBin.metaSize != 0);
packer.UnpackFromBin(binBin, compBin.workBuffers.dnaBin, signatureId, compBin.workBuffers.dnaBuffer, true);
}
if (extractor->ExtractNBin(binBin, signatureId) && binBin.metaSize > 0)
packer.UnpackFromBin(binBin, compBin.workBuffers.dnaBin, signatureId, compBin.workBuffers.dnaBuffer, true);
// un-reverse-compliment records
//
{
char rcBuf[DnaRecord::MaxDnaLen];
DnaRecord rcRec;
rcRec.dna = rcBuf;
rcRec.reverse = true;
for (uint64 i = 0; i < compBin.workBuffers.dnaBin.Size(); ++i)
{
DnaRecord& r = compBin.workBuffers.dnaBin[i];
if (r.reverse)
{
r.ComputeRC(rcRec);
std::copy(rcRec.dna, rcRec.dna + r.len, r.dna);
r.reverse = false;
r.minimizerPos = 0;
}
}
}
// compress all bins together
//
const uint32 nSignature = conf.minimizer.TotalMinimizersCount();
compressor.CompressDna(compBin.workBuffers.dnaBin, nSignature, totalDnaBufferSize, compBin.workBuffers.dnaWorkBin, compBin);
dnarch->WriteNextBin(&compBin);
}
// launch stuff
//
mt::thread readerThread(mt::ref(*inReader));
std::vector<IOperator*> operators;
operators.resize(threadsNum_);
#ifdef USE_BOOST_THREAD
boost::thread_group opThreadGroup;
for (uint32 i = 0; i < threadsNum_; ++i)
{
operators[i] = new BinPartsCompressor(conf.minimizer, params_,
inQueue, inPool,
outQueue, outPool);
opThreadGroup.create_thread(mt::ref(*operators[i]));
}
(*outWriter)();
readerThread.join();
opThreadGroup.join_all();
#else
std::vector<mt::thread> opThreadGroup;
for (uint32 i = 0; i < threadsNum_; ++i)
{
operators[i] = new BinPartsCompressor(conf.minimizer, params_,
inQueue, inPool,
outQueue, outPool);
opThreadGroup.push_back(mt::thread(mt::ref(*operators[i])));
}
(*outWriter)();
readerThread.join();
for (mt::thread& t : opThreadGroup)
{
t.join();
}
#endif
for (uint32 i = 0; i < threadsNum_; ++i)
{
delete operators[i];
}
TFREE(outWriter);
TFREE(inReader);
TFREE(outQueue);
TFREE(outPool);
TFREE(inQueue);
TFREE(inPool);
}
else
{
void BinModule::Fastq2Bin(const std::vector<std::string> &inFastqFiles_, const std::string &outBinFile_,
uint32 threadNum_, bool compressedInput_, bool verboseMode_)
{
// TODO: try/catch to free resources
//
IFastqStreamReader* fastqFile = NULL;
if (compressedInput_)
fastqFile = new MultiFastqFileReaderGz(inFastqFiles_);
else
fastqFile = new MultiFastqFileReader(inFastqFiles_);
BinFileWriter binFile;
binFile.StartCompress(outBinFile_, config);
const uint32 minimizersCount = config.minimizer.TotalMinimizersCount();
if (threadNum_ > 1)
{
FastqChunkPool* fastqPool = NULL;
FastqChunkQueue* fastqQueue = NULL;
BinaryPartsPool* binPool = NULL;
BinaryPartsQueue* binQueue = NULL;
FastqChunkReader* fastqReader = NULL;
BinChunkWriter* binWriter = NULL;
const uint32 partNum = threadNum_ * 4;
fastqPool = new FastqChunkPool(partNum, config.fastqBlockSize);
fastqQueue = new FastqChunkQueue(partNum, 1);
binPool = new BinaryPartsPool(partNum, minimizersCount);
binQueue = new BinaryPartsQueue(partNum, threadNum_);
fastqReader = new FastqChunkReader(fastqFile, fastqQueue, fastqPool);
binWriter = new BinChunkWriter(&binFile, binQueue, binPool);
// launch stuff
//
mt::thread readerThread(mt::ref(*fastqReader));
std::vector<IOperator*> operators;
operators.resize(threadNum_);
#ifdef USE_BOOST_THREAD
boost::thread_group opThreadGroup;
for (uint32 i = 0; i < threadNum_; ++i)
{
operators[i] = new BinEncoder(config.minimizer, config.catParams,
fastqQueue, fastqPool,
binQueue, binPool);
opThreadGroup.create_thread(mt::ref(*operators[i]));
}
(*binWriter)();
readerThread.join();
opThreadGroup.join_all();
#else
std::vector<mt::thread> opThreadGroup;
for (uint32 i = 0; i < threadNum_; ++i)
{
operators[i] = new BinEncoder(config.minimizer, config.catParams,
fastqQueue, fastqPool, binQueue, binPool);
opThreadGroup.push_back(mt::thread(mt::ref(*operators[i])));
}
(*binWriter)();
readerThread.join();
for (mt::thread& t : opThreadGroup)
{
t.join();
}
#endif
for (uint32 i = 0; i < threadNum_; ++i)
{
delete operators[i];
}
TFREE(binWriter);
TFREE(fastqReader);
TFREE(binQueue);
TFREE(binPool);
TFREE(fastqQueue);
TFREE(fastqPool);
}
else
{
DnaParser parser;
DnaCategorizer categorizer(config.minimizer, config.catParams);
DnaPacker packer(config.minimizer);
DataChunk fastqChunk(config.fastqBlockSize);
std::vector<DnaRecord> records;
records.resize(1 << 10);
DnaBinBlock dnaBins(minimizersCount);
BinaryBinBlock binBins;
DataChunk dnaBuffer;
while (fastqFile->ReadNextChunk(&fastqChunk))
{
uint64 recordsCount = 0;
parser.ParseFrom(fastqChunk, dnaBuffer, records, recordsCount);
ASSERT(recordsCount > 0);
categorizer.Categorize(records, recordsCount, dnaBins);
packer.PackToBins(dnaBins, binBins);
binFile.WriteNextBlock(&binBins);
}
}
binFile.FinishCompress();
if (verboseMode_)
{
std::vector<uint64> recordCounts;
binFile.GetBinStats(recordCounts);
std::cout << "Signatures count: " << recordCounts.size() << std::endl;
std::cout << "Records distribution in bins by signature:\n";
for (uint32 i = 0; i < recordCounts.size(); ++i)
{
if (recordCounts[i] > 0)
std::cout << i << " : " << recordCounts[i] << '\n';
}
std::cout << std::endl;
}
delete fastqFile;
}
/*
* Create a spoiler file for monsters -BEN-
*/
static void spoil_mon_desc(void)
{
int i, n = 0;
C_TNEW(who, MAX_R_IDX, s16b);
char nam[80];
char lev[80];
char rar[80];
char spd[80];
char ac[80];
char hp[80];
char exp[80];
/* Dump the header */
fprintf(fff, "Monster Spoilers for %s Version %s\n", GAME_NAME,
GAME_VERSION);
fprintf(fff, "------------------------------------------\n\n");
/* Dump the header */
fprintf(fff, "%-40.40s%4s%4s%6s%8s%4s %11.11s\n",
"Name", "Lev", "Rar", "Spd", "Hp", "Ac", "Visual Info");
fprintf(fff, "%-40.40s%4s%4s%6s%8s%4s %11.11s\n",
"----", "---", "---", "---", "--", "--", "-----------");
/* Scan the monsters (except the ghost) */
for (i = 1; i < MAX_R_IDX; i++)
{
monster_race *r_ptr = &r_info[i];
/* Hack - skip "fake" monsters. */
if (is_fake_monster(r_ptr)) continue;
/* Use that monster */
if (r_ptr->name) who[n++] = i;
}
/* Scan again */
for (i = 0; i < n; i++)
{
monster_race *r_ptr = &r_info[who[i]];
cptr pre;
/* Get the "name" */
if (r_ptr->flags1 & (RF1_GUARDIAN)) pre = "[G]";
else if (r_ptr->flags1 & (RF1_UNIQUE)) pre = "[U]";
else pre = "The";
strnfmt(nam, N_ELEMENTS(nam), "%s %.*v", pre,
N_ELEMENTS(nam)-strlen(pre)-1, monster_desc_aux_f3, r_ptr, 1, 0);
/* Level */
sprintf(lev, "%d", r_ptr->level);
/* Rarity */
sprintf(rar, "%d", r_ptr->rarity);
/* Speed */
if (r_ptr->speed >= 110)
{
sprintf(spd, "+%d", (r_ptr->speed - 110));
}
else
{
sprintf(spd, "-%d", (110 - r_ptr->speed));
}
/* Armor Class */
sprintf(ac, "%d", r_ptr->ac);
/* Hitpoints */
if ((r_ptr->flags1 & (RF1_FORCE_MAXHP)) || (r_ptr->hside == 1))
{
sprintf(hp, "%d", r_ptr->hdice * r_ptr->hside);
}
else
{
sprintf(hp, "%dd%d", r_ptr->hdice, r_ptr->hside);
}
/* Power */
sprintf(exp, "%ld", (long)(r_ptr->mexp));
/* Hack -- use visual instead */
sprintf(exp, "%s '%c'", attr_to_text(r_ptr->gfx.da), r_ptr->gfx.dc);
/* Dump the info */
fprintf(fff, "%-40.40s%4s%4s%6s%8s%4s %11.11s\n",
nam, lev, rar, spd, hp, ac, exp);
}
/* End it */
fprintf(fff, "\n");
/* Free the "who" array */
TFREE(who);
}
/*
* Create a spoiler file for items
*/
static void spoil_obj_desc(void)
{
int i, k, s, t, n = 0;
u16b who[200];
C_TNEW(o_name, ONAME_MAX, char);
char wgt[80];
char dam[80];
/* Header */
fprintf(fff, "Spoiler File -- Basic Items (2.?.?)\n\n\n");
/* More Header */
fprintf(fff, "%-45s %8s%7s%5s%9s\n",
"Description", "Dam/AC", "Wgt", "Lev", "Cost");
fprintf(fff, "%-45s %8s%7s%5s%9s\n",
"----------------------------------------",
"------", "---", "---", "----");
/* List the groups */
for (i = 0; TRUE; i++)
{
/* Write out the group title */
if (group_item[i].str)
{
/* Hack -- bubble-sort by cost and then level */
for (s = 0; s < n - 1; s++)
{
for (t = 0; t < n - 1; t++)
{
int i1 = t;
int i2 = t + 1;
int e1;
int e2;
s32b t1;
s32b t2;
kind_info(NULL, NULL, NULL, &e1, &t1, who[i1]);
kind_info(NULL, NULL, NULL, &e2, &t2, who[i2]);
if ((t1 > t2) || ((t1 == t2) && (e1 > e2)))
{
int tmp = who[i1];
who[i1] = who[i2];
who[i2] = tmp;
}
}
}
/* Spoil each item */
for (s = 0; s < n; s++)
{
int e;
s32b v;
/* Describe the kind */
kind_info(o_name, dam, wgt, &e, &v, who[s]);
/* Dump it */
fprintf(fff, " %-45s%8s%7s%5d%9ld\n",
o_name, dam, wgt, e, (long)(v));
}
/* Start a new set */
n = 0;
/* Notice the end */
if (!group_item[i].idx) break;
/* Start a new set */
fprintf(fff, "\n\n%s\n\n", group_item[i].str);
}
/* Acquire legal item types */
for (k = 1; k < MAX_K_IDX; k++)
{
object_kind *k_ptr = &k_info[k];
/* Skip wrong tval's */
if (k_ptr->tval != group_item[i].idx) continue;
/* Hack -- skip items which only have special generation methods. */
if (!kind_created_p(k_ptr)) continue;
/* Save the index */
who[n++] = k;
}
}
TFREE(o_name);
}
static void dictReleaseIterator(dictIterator *iter) {
TFREE(iter);
}
/* Clear & Release the hash table */
static void dictRelease(dict *ht) {
_dictClear(ht);
TFREE(ht);
}