void SyntroStoreBlocksRaw::processQueue()
{
if (m_stream->needRotation())
m_stream->doRotation();
writeBlocks();
}
/*
* Write out file system.
* Clears boot block, writes superblock, bitmaps, inodes, and root directory.
*/
void writeFS(void)
{
/* zero out the boot sector (and associated block) */
__u8 blk[WUFS_BLOCKSIZE];
memset(blk,0x00,WUFS_BLOCKSIZE);
writeBlocks(Disk, 0, blk, 1);
/* write the superblock */
writeBlocks(Disk, 1, SB, 1);
/* write the inode bitmap */
writeBlocks(Disk, 2, IMap, SB->sb_imap_bcnt);
/* write the data block bit map */
int lba = 2 + SB->sb_imap_bcnt;
writeBlocks(Disk, lba, BMap, SB->sb_bmap_bcnt);
/* write the inodes */
lba += SB->sb_bmap_bcnt;
writeBlocks(Disk, lba, Inode, SB->sb_inodes/WUFS_INODES_PER_BLOCK);
/* write the root directory */
int rootDirLBA = Inode[0].in_block[0];
writeBlocks(Disk, rootDirLBA, RootDir, 1);
/* write the badblock indirect block */
if (indirect_addr) {
writeBlocks(Disk, indirect_addr, indirect, 1);
}
/* free at last, free at last */
if (Verbose) {
fprintf(stderr,"file system written.\n");
}
}
void OggVorbisEncoder::close()
{
if (mClosed)
return;
// Mark end of data and flush any remaining data in the buffers
vorbis_analysis_wrote(&mVorbisState, 0);
writeBlocks();
flush();
ogg_stream_clear(&mOggState);
vorbis_block_clear(&mVorbisBlock);
vorbis_dsp_clear(&mVorbisState);
vorbis_info_clear(&mVorbisInfo);
mClosed = true;
}
int QgsDxfExport::writeToFile( QIODevice* d )
{
if ( !d )
{
return 1;
}
if ( !d->isOpen() && !d->open( QIODevice::WriteOnly ) )
{
return 2;
}
mTextStream.setDevice( d );
writeHeader();
writeTables();
writeBlocks();
writeEntities();
writeEndFile();
return 0;
}
void LC_MakerCamSVG::write(RS_Graphic* graphic) {
RS_DEBUG->print("RS_MakerCamSVG::write: Writing root node ...");
graphic->calculateBorders();
min = graphic->getMin();
max = graphic->getMax();
RS2::Unit raw_unit = graphic->getUnit();
switch (raw_unit) {
case RS2::Millimeter:
unit = "mm";
break;
case RS2::Centimeter:
unit = "cm";
break;
case RS2::Inch:
unit = "in";
break;
default:
unit = "";
break;
}
xmlWriter->createRootElement("svg", NAMESPACE_URI_SVG);
xmlWriter->addNamespaceDeclaration("lc", NAMESPACE_URI_LC);
xmlWriter->addNamespaceDeclaration("xlink", NAMESPACE_URI_XLINK);
xmlWriter->addAttribute("width", numXml(max.x - min.x) + unit);
xmlWriter->addAttribute("height", numXml(max.y - min.y) + unit);
xmlWriter->addAttribute("viewBox", "0 0 "+ numXml(max.x - min.x) + " " + numXml(max.y - min.y));
writeBlocks(graphic);
writeLayers(graphic);
}
void LC_MakerCamSVG::write(RS_Graphic* graphic) {
RS_DEBUG->print("RS_MakerCamSVG::write: Writing root node ...");
graphic->calculateBorders();
min = graphic->getMin();
max = graphic->getMax();
RS2::Unit raw_unit = graphic->getUnit();
lengthFactor=1.;
switch (raw_unit) {
case RS2::Centimeter:
unit = "cm";
break;
case RS2::Inch:
unit = "in";
break;
default:
lengthFactor=RS_Units::convert(1., raw_unit, RS2::Millimeter);
//do not break here, default will use mm and convert length to mm
case RS2::Millimeter:
unit = "mm";
break;
}
xmlWriter->createRootElement("svg", NAMESPACE_URI_SVG);
xmlWriter->addNamespaceDeclaration("lc", NAMESPACE_URI_LC);
xmlWriter->addNamespaceDeclaration("xlink", NAMESPACE_URI_XLINK);
xmlWriter->addAttribute("width", lengthXml(max.x - min.x) + unit);
xmlWriter->addAttribute("height", lengthXml(max.y - min.y) + unit);
xmlWriter->addAttribute("viewBox", "0 0 "+ lengthXml(max.x - min.x) + " " + lengthXml(max.y - min.y));
writeBlocks(graphic);
writeLayers(graphic);
}
void bbFileCreate(
char *inName, /* Input file in a tabular bed format <chrom><start><end> + whatever. */
char *chromSizes, /* Two column tab-separated file: <chromosome> <size>. */
int blockSize, /* Number of items to bundle in r-tree. 1024 is good. */
int itemsPerSlot, /* Number of items in lowest level of tree. 64 is good. */
char *asText, /* Field definitions in a string */
struct asObject *as, /* Field definitions parsed out */
boolean doCompress, /* If TRUE then compress data. */
struct slName *extraIndexList, /* List of extra indexes to add */
char *outName) /* BigBed output file name. */
/* Convert tab-separated bed file to binary indexed, zoomed bigBed version. */
{
/* Set up timing measures. */
verboseTimeInit();
struct lineFile *lf = lineFileOpen(inName, TRUE);
bits16 fieldCount = slCount(as->columnList);
bits16 extraIndexCount = slCount(extraIndexList);
struct bbExIndexMaker *eim = NULL;
if (extraIndexList != NULL)
eim = bbExIndexMakerNew(extraIndexList, as);
/* Load in chromosome sizes. */
struct hash *chromSizesHash = NULL;
if (sizesIs2Bit)
chromSizesHash = twoBitChromHash(chromSizes);
else
chromSizesHash = bbiChromSizesFromFile(chromSizes);
verbose(2, "Read %d chromosomes and sizes from %s\n", chromSizesHash->elCount, chromSizes);
/* Do first pass, mostly just scanning file and counting hits per chromosome. */
int minDiff = 0;
double aveSize = 0;
bits64 bedCount = 0;
bits32 uncompressBufSize = 0;
struct bbiChromUsage *usageList = bbiChromUsageFromBedFile(lf, chromSizesHash, eim,
&minDiff, &aveSize, &bedCount, tabSep);
verboseTime(1, "pass1 - making usageList (%d chroms)", slCount(usageList));
verbose(2, "%d chroms in %s. Average span of beds %f\n", slCount(usageList), inName, aveSize);
/* Open output file and write dummy header. */
FILE *f = mustOpen(outName, "wb");
bbiWriteDummyHeader(f);
bbiWriteDummyZooms(f);
/* Write out autoSql string */
bits64 asOffset = ftell(f);
mustWrite(f, asText, strlen(asText) + 1);
verbose(2, "as definition has %d columns\n", fieldCount);
/* Write out dummy total summary. */
struct bbiSummaryElement totalSum;
ZeroVar(&totalSum);
bits64 totalSummaryOffset = ftell(f);
bbiSummaryElementWrite(f, &totalSum);
/* Write out dummy header extension */
bits64 extHeaderOffset = ftell(f);
bits16 extHeaderSize = 64;
repeatCharOut(f, 0, extHeaderSize);
/* Write out extra index stuff if need be. */
bits64 extraIndexListOffset = 0;
bits64 extraIndexListEndOffset = 0;
if (extraIndexList != NULL)
{
extraIndexListOffset = ftell(f);
int extraIndexSize = 16 + 4*1; // Fixed record size 16, plus 1 times field size of 4
repeatCharOut(f, 0, extraIndexSize*extraIndexCount);
extraIndexListEndOffset = ftell(f);
}
/* Write out chromosome/size database. */
bits64 chromTreeOffset = ftell(f);
bbiWriteChromInfo(usageList, blockSize, f);
/* Set up to keep track of possible initial reduction levels. */
int resScales[bbiMaxZoomLevels], resSizes[bbiMaxZoomLevels];
int resTryCount = bbiCalcResScalesAndSizes(aveSize, resScales, resSizes);
/* Write out primary full resolution data in sections, collect stats to use for reductions. */
bits64 dataOffset = ftell(f);
bits32 blockCount = 0;
bits32 maxBlockSize = 0;
struct bbiBoundsArray *boundsArray = NULL;
writeOne(f, bedCount);
if (bedCount > 0)
{
blockCount = bbiCountSectionsNeeded(usageList, itemsPerSlot);
AllocArray(boundsArray, blockCount);
lineFileRewind(lf);
if (eim)
bbExIndexMakerAllocChunkArrays(eim, bedCount);
writeBlocks(usageList, lf, as, itemsPerSlot, boundsArray, blockCount, doCompress,
f, resTryCount, resScales, resSizes, eim, bedCount, fieldCount, &maxBlockSize);
}
verboseTime(1, "pass2 - checking and writing primary data (%lld records, %d fields)",
(long long)bedCount, fieldCount);
/* Write out primary data index. */
bits64 indexOffset = ftell(f);
cirTreeFileBulkIndexToOpenFile(boundsArray, sizeof(boundsArray[0]), blockCount,
blockSize, 1, NULL, bbiBoundsArrayFetchKey, bbiBoundsArrayFetchOffset,
indexOffset, f);
freez(&boundsArray);
verboseTime(2, "index write");
/* Declare arrays and vars that track the zoom levels we actually output. */
bits32 zoomAmounts[bbiMaxZoomLevels];
bits64 zoomDataOffsets[bbiMaxZoomLevels];
bits64 zoomIndexOffsets[bbiMaxZoomLevels];
/* Call monster zoom maker library function that bedGraphToBigWig also uses. */
int zoomLevels = 0;
if (bedCount > 0)
{
zoomLevels = bbiWriteZoomLevels(lf, f, blockSize, itemsPerSlot,
bedWriteReducedOnceReturnReducedTwice, fieldCount,
doCompress, indexOffset - dataOffset,
usageList, resTryCount, resScales, resSizes,
zoomAmounts, zoomDataOffsets, zoomIndexOffsets, &totalSum);
}
/* Write out extra indexes if need be. */
if (eim)
{
int i;
for (i=0; i < eim->indexCount; ++i)
{
eim->fileOffsets[i] = ftell(f);
maxBedNameSize = eim->maxFieldSize[i];
qsort(eim->chunkArrayArray[i], bedCount,
sizeof(struct bbNamedFileChunk), bbNamedFileChunkCmpByName);
assert(sizeof(struct bbNamedFileChunk) == sizeof(eim->chunkArrayArray[i][0]));
bptFileBulkIndexToOpenFile(eim->chunkArrayArray[i], sizeof(eim->chunkArrayArray[i][0]),
bedCount, blockSize, bbNamedFileChunkKey, maxBedNameSize, bbNamedFileChunkVal,
sizeof(bits64) + sizeof(bits64), f);
verboseTime(1, "Sorting and writing extra index %d", i);
}
}
/* Figure out buffer size needed for uncompression if need be. */
if (doCompress)
{
int maxZoomUncompSize = itemsPerSlot * sizeof(struct bbiSummaryOnDisk);
uncompressBufSize = max(maxBlockSize, maxZoomUncompSize);
}
/* Go back and rewrite header. */
rewind(f);
bits32 sig = bigBedSig;
bits16 version = bbiCurrentVersion;
bits16 summaryCount = zoomLevels;
bits32 reserved32 = 0;
bits64 reserved64 = 0;
bits16 definedFieldCount = bedN;
/* Write fixed header */
writeOne(f, sig);
writeOne(f, version);
writeOne(f, summaryCount);
writeOne(f, chromTreeOffset);
writeOne(f, dataOffset);
writeOne(f, indexOffset);
writeOne(f, fieldCount);
writeOne(f, definedFieldCount);
writeOne(f, asOffset);
writeOne(f, totalSummaryOffset);
writeOne(f, uncompressBufSize);
writeOne(f, extHeaderOffset);
assert(ftell(f) == 64);
/* Write summary headers with data. */
int i;
verbose(2, "Writing %d levels of zoom\n", zoomLevels);
for (i=0; i<zoomLevels; ++i)
{
verbose(3, "zoomAmounts[%d] = %d\n", i, (int)zoomAmounts[i]);
writeOne(f, zoomAmounts[i]);
writeOne(f, reserved32);
writeOne(f, zoomDataOffsets[i]);
writeOne(f, zoomIndexOffsets[i]);
}
/* Write rest of summary headers with no data. */
for (i=zoomLevels; i<bbiMaxZoomLevels; ++i)
{
writeOne(f, reserved32);
writeOne(f, reserved32);
writeOne(f, reserved64);
writeOne(f, reserved64);
}
/* Write total summary. */
fseek(f, totalSummaryOffset, SEEK_SET);
bbiSummaryElementWrite(f, &totalSum);
/* Write extended header */
fseek(f, extHeaderOffset, SEEK_SET);
writeOne(f, extHeaderSize);
writeOne(f, extraIndexCount);
writeOne(f, extraIndexListOffset);
repeatCharOut(f, 0, 52); // reserved
assert(ftell(f) - extHeaderOffset == extHeaderSize);
/* Write extra index offsets if need be. */
if (extraIndexCount != 0)
{
fseek(f, extraIndexListOffset, SEEK_SET);
int i;
for (i=0; i<extraIndexCount; ++i)
{
// Write out fixed part of index info
bits16 type = 0; // bPlusTree type
bits16 indexFieldCount = 1;
writeOne(f, type);
writeOne(f, indexFieldCount);
writeOne(f, eim->fileOffsets[i]);
repeatCharOut(f, 0, 4); // reserved
// Write out field list - easy this time because for now always only one field.
bits16 fieldId = eim->indexFields[i];
writeOne(f, fieldId);
repeatCharOut(f, 0, 2); // reserved
}
assert(ftell(f) == extraIndexListEndOffset);
}
/* Write end signature. */
fseek(f, 0L, SEEK_END);
writeOne(f, sig);
/* Clean up. */
lineFileClose(&lf);
carefulClose(&f);
freeHash(&chromSizesHash);
bbiChromUsageFreeList(&usageList);
asObjectFreeList(&as);
}
/*
* int (*alowpc)(), (*ahighpc)(); boundaries of text to be monitored
* WORD *buffer; ptr to space for monitor data(WORDs)
* size_t bufsize; size of above space(in WORDs)
* size_t nfunc; max no. of functions whose calls are counted
* (default nfunc is 300 on PDP11, 600 on others)
*/
void
monitor(int (*alowpc)(void), int (*ahighpc)(void), WORD *buffer,
size_t bufsize, size_t nfunc)
{
uint_t scale;
long text;
char *s;
struct hdr *hdrp;
ANCHOR *newanchp;
size_t ssiz;
int error;
char *lowpc = (char *)alowpc;
char *highpc = (char *)ahighpc;
lmutex_lock(&mon_lock);
if (lowpc == NULL) { /* true only at the end */
error = 0;
if (curAnchor != NULL) { /* if anything was collected!.. */
profil(NULL, 0, 0, 0);
if (writeBlocks() == 0)
error = errno;
}
lmutex_unlock(&mon_lock);
if (error) {
errno = error;
perror(mon_out);
}
return;
}
/*
* Ok - they want to submit a block for immediate use, for
* function call count consumption, and execution profile
* histogram computation.
* If the block fails sanity tests, just bag it.
* Next thing - get name to use. If PROFDIR is NULL, let's
* get out now - they want No Profiling done.
*
* Otherwise:
* Set the block hdr cells.
* Get an anchor for the block, and link the anchor+block onto
* the end of the chain.
* Init the grabba-cell externs (countbase/limit) for this block.
* Finally, call profil and return.
*/
ssiz = ((sizeof (struct hdr) + nfunc * sizeof (struct cnt)) /
sizeof (WORD));
if (ssiz >= bufsize || lowpc >= highpc) {
lmutex_unlock(&mon_lock);
return;
}
if ((s = getenv(PROFDIR)) == NULL) { /* PROFDIR not in environment */
mon_out = MON_OUT; /* use default "mon.out" */
} else if (*s == '\0') { /* value of PROFDIR is NULL */
lmutex_unlock(&mon_lock);
return; /* no profiling on this run */
} else { /* construct "PROFDIR/pid.progname" */
int n;
pid_t pid;
char *name;
size_t len;
len = strlen(s);
/* 15 is space for /pid.mon.out\0, if necessary */
if ((mon_out = libc_malloc(len + strlen(___Argv[0]) + 15))
== NULL) {
lmutex_unlock(&mon_lock);
perror("");
return;
}
(void) strcpy(mon_out, s);
name = mon_out + len;
*name++ = '/'; /* two slashes won't hurt */
if ((pid = getpid()) <= 0) /* extra test just in case */
pid = 1; /* getpid returns something inappropriate */
/* suppress leading zeros */
for (n = 10000; n > pid; n /= 10)
;
for (; ; n /= 10) {
*name++ = pid/n + '0';
if (n == 1)
break;
pid %= n;
}
*name++ = '.';
if (___Argv != NULL) { /* mcrt0.s executed */
if ((s = strrchr(___Argv[0], '/')) != NULL)
(void) strcpy(name, s + 1);
else
(void) strcpy(name, ___Argv[0]);
} else {
(void) strcpy(name, MON_OUT);
}
}
hdrp = (struct hdr *)(uintptr_t)buffer; /* initialize 1st region */
hdrp->lpc = lowpc;
hdrp->hpc = highpc;
hdrp->nfns = nfunc;
/* get an anchor for the block */
newanchp = (curAnchor == NULL) ? &firstAnchor :
(ANCHOR *)libc_malloc(sizeof (ANCHOR));
if (newanchp == NULL) {
lmutex_unlock(&mon_lock);
perror("monitor");
return;
}
/* link anchor+block into chain */
newanchp->monBuffer = hdrp; /* new, down. */
newanchp->next = NULL; /* new, forward to NULL. */
newanchp->prior = curAnchor; /* new, backward. */
if (curAnchor != NULL)
curAnchor->next = newanchp; /* old, forward to new. */
newanchp->flags = HAS_HISTOGRAM; /* note it has a histgm area */
/* got it - enable use by mcount() */
countbase = (char *)buffer + sizeof (struct hdr);
_countlimit = countbase + (nfunc * sizeof (struct cnt));
/* (set size of region 3) */
newanchp->histSize = (int)
(bufsize * sizeof (WORD) - (_countlimit - (char *)buffer));
/* done w/regions 1 + 2: setup 3 to activate profil processing. */
buffer += ssiz; /* move ptr past 2'nd region */
bufsize -= ssiz; /* no. WORDs in third region */
/* no. WORDs of text */
text = (highpc - lowpc + sizeof (WORD) - 1) / sizeof (WORD);
/*
* scale is a 16 bit fixed point fraction with the decimal
* point at the left
*/
if (bufsize < text) {
/* make sure cast is done first! */
double temp = (double)bufsize;
scale = (uint_t)((temp * (long)0200000L) / text);
} else {
/* scale must be less than 1 */
scale = 0xffff;
}
bufsize *= sizeof (WORD); /* bufsize into # bytes */
profil(buffer, bufsize, (ulong_t)lowpc, scale);
curAnchor = newanchp; /* make latest addition, the cur anchor */
lmutex_unlock(&mon_lock);
}
void OggVorbisEncoder::write(UINT8* samples, UINT32 numSamples)
{
static const UINT32 WRITE_LENGTH = 1024;
UINT32 numFrames = numSamples / mNumChannels;
while (numFrames > 0)
{
UINT32 numFramesToWrite = std::min(numFrames, WRITE_LENGTH);
float** buffer = vorbis_analysis_buffer(&mVorbisState, numFramesToWrite);
if (mBitDepth == 8)
{
for (UINT32 i = 0; i < numFramesToWrite; i++)
{
for (UINT32 j = 0; j < mNumChannels; j++)
{
INT8 sample = *(INT8*)samples;
float encodedSample = sample / 127.0f;
buffer[j][i] = encodedSample;
samples++;
}
}
}
else if (mBitDepth == 16)
{
for (UINT32 i = 0; i < numFramesToWrite; i++)
{
for (UINT32 j = 0; j < mNumChannels; j++)
{
INT16 sample = *(INT16*)samples;
float encodedSample = sample / 32767.0f;
buffer[j][i] = encodedSample;
samples += 2;
}
}
}
else if (mBitDepth == 24)
{
for (UINT32 i = 0; i < numFramesToWrite; i++)
{
for (UINT32 j = 0; j < mNumChannels; j++)
{
INT32 sample = AudioUtility::convert24To32Bits(samples);
float encodedSample = sample / 2147483647.0f;
buffer[j][i] = encodedSample;
samples += 3;
}
}
}
else if (mBitDepth == 32)
{
for (UINT32 i = 0; i < numFramesToWrite; i++)
{
for (UINT32 j = 0; j < mNumChannels; j++)
{
INT32 sample = *(INT32*)samples;
float encodedSample = sample / 2147483647.0f;
buffer[j][i] = encodedSample;
samples += 4;
}
}
}
else
assert(false);
// Signal how many frames were written
vorbis_analysis_wrote(&mVorbisState, numFramesToWrite);
writeBlocks();
numFrames -= numFramesToWrite;
}
}
/*
* infile: input filename
* size: size in blocks of input file
* outfile: output filename
* field: which field will be used for sorting
* buffer: the buffer that is used
* memSize: number of buffer blocks available for use, without counting the last one, which is for output
* nunique: number of unique values
* nios: number of ios
*
* when the input file fits the buffer and there's still a block available for output,
* hashes each record and writes it to the output, if a record of same value is not
* found on the corresponding bucket.
*/
void hashElimination(char *infile, uint size, char *outfile, unsigned char field, block_t *buffer, uint memSize, uint *nunique, uint *nios) {
int out = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);
block_t *bufferOut = buffer + memSize;
emptyBlock(bufferOut);
(*bufferOut).valid = true;
(*bufferOut).blockid = 0;
(*nunique) = 0;
(*nios) += readBlocks(infile, buffer, size);
// creates a hash index. for each value returned from the hash function,
// there is a linkedList of pointers to the records with that specific hash
// value
uint hashSize = size*MAX_RECORDS_PER_BLOCK;
linkedRecordPtr **hashIndex = (linkedRecordPtr**) malloc(hashSize * sizeof (linkedRecordPtr*));
for (uint i = 0; i < hashSize; i++) {
hashIndex[i] = NULL;
}
recordPtr start = newPtr(0);
recordPtr end = newPtr(size * MAX_RECORDS_PER_BLOCK - 1);
for (; start <= end; incr(start)) {
if (!buffer[start.block].valid) {
start.record = MAX_RECORDS_PER_BLOCK - 1;
continue;
}
record_t record = getRecord(buffer, start);
if (record.valid) {
// hashes the record being examined
uint index = hashRecord(infile, record, hashSize, field);
linkedRecordPtr *element = hashIndex[index];
// goes through the linked list for the hash value of the record
// if a record with same value is not found, then a recordPtr is
// added to the linked list and the record itself is written to
// the output. otherwise, it is ignored.
while (element) {
if (compareRecords(record, getRecord(buffer, element->ptr), field) == 0) {
break;
}
element = element->next;
}
if (!element) {
element = (linkedRecordPtr*) malloc(sizeof (linkedRecordPtr));
element->ptr = start;
element->next = hashIndex[index];
hashIndex[index] = element;
(*bufferOut).entries[(*bufferOut).nreserved++] = record;
(*nunique) += 1;
if ((*bufferOut).nreserved == MAX_RECORDS_PER_BLOCK) {
(*nios) += writeBlocks(out, bufferOut, 1);
emptyBlock(bufferOut);
(*bufferOut).blockid += 1;
}
}
}
}
// writes records left in buffer to the outfile
if ((*bufferOut).nreserved != 0) {
(*nios) += writeBlocks(out, bufferOut, 1);
}
destroyHashIndex(hashIndex, size);
close(out);
}
void EliminateDuplicates(char *infile, unsigned char field, block_t *buffer, unsigned int nmem_blocks, char *outfile, unsigned int *nunique, unsigned int *nios) {
if (nmem_blocks < 3) {
printf("At least 3 blocks are required.");
return;
}
// empties the buffer
emptyBuffer(buffer, nmem_blocks);
uint memSize = nmem_blocks - 1;
*nunique = 0;
*nios = 0;
uint fileSize = getSize(infile);
// if the relation fits on the buffer and leaves one block free for output,
// loads it to the buffer and eliminates duplicates using hashing
if (fileSize <= memSize) {
hashElimination(infile, fileSize, outfile, field, buffer, memSize, nunique, nios);
} else if (fileSize == nmem_blocks) {
// if the relation completely fits the buffer, calls useFirstBlock
useFirstBlock(infile, outfile, field, buffer, nmem_blocks, nunique, nios);
} else {
// if the relation is larger than the buffer, then sort it using mergesort,
// BUT during the final merging (during last pass) write to the output
// only one time each value
// the following code is similar to that of MergeSort:
int input, output;
char tmpFile1[] = ".ed1";
char tmpFile2[] = ".ed2";
uint fullSegments = fileSize / nmem_blocks;
uint remainingSegment = fileSize % nmem_blocks;
input = open(infile, O_RDONLY, S_IRWXU);
output = open(tmpFile1, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);
uint nSortedSegs = 0;
uint segmentSize = nmem_blocks;
for (uint i = 0; i <= fullSegments; i++) {
if (fullSegments == i) {
if (remainingSegment != 0) {
segmentSize = remainingSegment;
} else {
break;
}
}
(*nios) += readBlocks(input, buffer, segmentSize);
if (sortBuffer(buffer, segmentSize, field)) {
(*nios) += writeBlocks(output, buffer, segmentSize);
nSortedSegs += 1;
}
}
close(input);
close(output);
segmentSize = nmem_blocks;
uint lastSegmentSize;
if (remainingSegment == 0) {
lastSegmentSize = nmem_blocks;
} else {
lastSegmentSize = remainingSegment;
}
buffer[memSize].valid = true;
while (nSortedSegs != 1) {
input = open(tmpFile1, O_RDONLY, S_IRWXU);
output = open(tmpFile2, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);
uint newSortedSegs = 0;
uint fullMerges = nSortedSegs / memSize;
uint lastMergeSegs = nSortedSegs % memSize;
uint *blocksLeft = (uint*) malloc(memSize * sizeof (uint));
uint segsToMerge = memSize;
bool lastMerge = false;
for (uint mergeCounter = 0; mergeCounter <= fullMerges; mergeCounter++) {
uint firstSegOffset = mergeCounter * memSize * segmentSize;
if (mergeCounter == fullMerges - 1 && lastMergeSegs == 0) {
lastMerge = true;
} else if (mergeCounter == fullMerges) {
if (lastMergeSegs != 0) {
segsToMerge = lastMergeSegs;
lastMerge = true;
} else {
break;
}
}
for (uint i = 0; i < segsToMerge; i++) {
(*nios) += preadBlocks(input, buffer + i, (firstSegOffset + i * segmentSize), 1);
blocksLeft[i] = segmentSize - 1;
}
if (lastMerge) {
blocksLeft[segsToMerge - 1] = lastSegmentSize - 1;
}
(*nios) += mergeElimination(input, output, buffer, memSize, segsToMerge, blocksLeft, segmentSize, firstSegOffset, field, nSortedSegs <= memSize, lastMerge, nunique);
newSortedSegs += 1;
}
free(blocksLeft);
if (lastMergeSegs == 0) {
lastSegmentSize = (memSize - 1) * segmentSize + lastSegmentSize;
} else {
lastSegmentSize = (lastMergeSegs - 1) * segmentSize + lastSegmentSize;
}
segmentSize *= memSize;
nSortedSegs = newSortedSegs;
close(input);
close(output);
char tmp = tmpFile1[3];
tmpFile1[3] = tmpFile2[3];
tmpFile2[3] = tmp;
}
rename(tmpFile1, outfile);
remove(tmpFile2);
}
}
uint mergeElimination(int &input, int &output, block_t *buffer, uint memSize, uint segsToMerge, uint *blocksLeft, uint segmentSize, uint firstSegOffset, unsigned char field, bool lastPass, bool lastMergeOfPass, uint *nunique) {
uint ios = 0;
block_t *bufferOut = buffer + memSize;
uint blocksWritten = 0;
uint sizeOfLastSeg;
if (lastMergeOfPass) {
sizeOfLastSeg = blocksLeft[segsToMerge - 1] + 1;
}
// holds the last unique value written to the output
record_t *lastRecordAdded = NULL;
recordPtr *nextRecord = (recordPtr*) malloc(segsToMerge * sizeof (recordPtr));
for (uint i = 0; i < segsToMerge; i++) {
nextRecord[i].block = i;
nextRecord[i].record = 0;
}
emptyBlock(bufferOut);
(*bufferOut).blockid = 0;
uint segsToMergeCopy = segsToMerge;
while (segsToMergeCopy != 0) {
uint i;
for (i = 0; i < segsToMerge; i++) {
if (buffer[i].valid) {
break;
}
}
record_t minRec = getRecord(buffer, nextRecord[i]);
uint minBuffIndex = i;
for (uint j = i + 1; j < segsToMerge; j++) {
if (buffer[j].valid && compareRecords(getRecord(buffer, nextRecord[j]), minRec, field) < 0) {
minRec = getRecord(buffer, nextRecord[j]);
minBuffIndex = j;
}
}
if (!lastPass) {
(*bufferOut).entries[(*bufferOut).nreserved++] = minRec;
} else {
if (!lastRecordAdded) {
(*bufferOut).entries[(*bufferOut).nreserved++] = minRec;
(*nunique) += 1;
lastRecordAdded = (record_t*) malloc(sizeof (record_t));
memcpy(lastRecordAdded, &minRec, sizeof (record_t));
} else {
if (compareRecords(*lastRecordAdded, minRec, field) != 0) {
(*bufferOut).entries[(*bufferOut).nreserved++] = minRec;
(*nunique) += 1;
memcpy(lastRecordAdded, &minRec, sizeof (record_t));
}
}
}
if ((*bufferOut).nreserved == MAX_RECORDS_PER_BLOCK) {
ios += writeBlocks(output, bufferOut, 1);
(*bufferOut).blockid += 1;
blocksWritten += 1;
emptyBlock(bufferOut);
}
incr(nextRecord[minBuffIndex]);
if (nextRecord[minBuffIndex].record == 0) {
nextRecord[minBuffIndex].block -= 1;
if (blocksLeft[minBuffIndex] > 0) {
uint blockOffset;
if (lastMergeOfPass && minBuffIndex == segsToMerge - 1) {
blockOffset = firstSegOffset + segmentSize * minBuffIndex + sizeOfLastSeg - blocksLeft[minBuffIndex];
} else {
blockOffset = firstSegOffset + segmentSize * minBuffIndex + segmentSize - blocksLeft[minBuffIndex];
}
ios += preadBlocks(input, buffer + minBuffIndex, blockOffset, 1);
blocksLeft[minBuffIndex] -= 1;
if (!buffer[minBuffIndex].valid) {
segsToMergeCopy -= 1;
}
} else {
buffer[minBuffIndex].valid = false;
segsToMergeCopy -= 1;
}
} else {
if (!getRecord(buffer, nextRecord[minBuffIndex]).valid) {
buffer[minBuffIndex].valid = false;
segsToMergeCopy -= 1;
}
}
}
free(nextRecord);
if (lastRecordAdded) {
free(lastRecordAdded);
}
if ((*bufferOut).nreserved != 0) {
ios += writeBlocks(output, bufferOut, 1);
(*bufferOut).blockid += 1;
blocksWritten += 1;
}
if (!lastPass && !lastMergeOfPass) {
for (uint i = 0; i < segmentSize * segsToMerge - blocksWritten; i++) {
ios += writeBlocks(output, buffer, 1);
}
}
return ios;
}
/*
* infile: filename of the input file
* outfile: filename of the output file
* field: which field will be used for sorting
* buffer: the buffer used
* nmem_blocks: size of buffer
* nunique: number of unique values
* nios: number of ios
*
* when the input file size is equal to buffer, the whole file is loaded and
* sorted. then the first block is used as output where only unique values are
* written
*/
void useFirstBlock(char *infile, char *outfile, unsigned char field, block_t *buffer, uint nmem_blocks, uint *nunique, uint *nios) {
int out = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);
(*nios) += readBlocks(infile, buffer, nmem_blocks);
if (sortBuffer(buffer, nmem_blocks, field)) {
// all the unique values of the first block are shifted to the start
// of it. the rest are marked as invalid
recordPtr i = newPtr(1);
recordPtr j = newPtr(1);
(*nunique) += 1;
buffer[0].nreserved = 1;
for (; j.block < 1; incr(j)) {
record_t record = getRecord(buffer, j);
if (record.valid && compareRecords(record, getRecord(buffer, i - 1), field) != 0) {
setRecord(buffer, record, i);
(*nunique) += 1;
incr(i);
buffer[0].nreserved += 1;
}
}
j = newPtr(i, 0);
for (; j.block < 1; incr(j)) {
buffer[j.block].entries[j.record].valid = false;
}
record_t *lastRecordAdded = (record_t*) malloc(sizeof (record_t));
record_t lastUnique = getRecord(buffer, i - 1);
memcpy(lastRecordAdded, &lastUnique, sizeof (record_t));
// if the first block is full after the shifting (meaning that all its
// values were actually unique), writes it to the outfile and empties it
if (buffer[0].nreserved == MAX_RECORDS_PER_BLOCK) {
i.block -= 1;
(*nios) += writeBlocks(out, buffer, 1);
emptyBlock(buffer);
buffer[0].blockid += 1;
}
// write the unique values of the other blocks to the first one. if it
// becomes full writes it to outfile and empties it. at the end, if it
// has records not writtend yet, writes them to the outfile as well.
j = newPtr(MAX_RECORDS_PER_BLOCK);
while (buffer[j.block].valid && j.block < nmem_blocks) {
record_t record = getRecord(buffer, j);
if (!record.valid) {
break;
}
if (compareRecords(record, (*lastRecordAdded), field) != 0) {
setRecord(buffer, record, i);
memcpy(lastRecordAdded, &record, sizeof (record_t));
(*nunique) += 1;
incr(i);
buffer[0].nreserved += 1;
}
if (buffer[0].nreserved == MAX_RECORDS_PER_BLOCK) {
i.block -= 1;
(*nios) += writeBlocks(out, buffer, 1);
emptyBlock(buffer);
buffer[0].blockid += 1;
}
incr(j);
}
if (buffer[0].nreserved != 0) {
(*nios) += writeBlocks(out, buffer, 1);
}
free(lastRecordAdded);
}
close(out);
}
/**
* Write a block to disk, directly.
*
* There should always enough data in src, or it would assert failed
*
* @param index index of block
* @param src a piece of memory which the driver write to disk
* @see writeBlocks(BlockIndex index, Length length, ConstSlice src)
*/
virtual void writeBlock(BlockIndex index, ConstSlice src)
{ writeBlocks(index, 1, src); }