Esempio n. 1
0
void SyntroStoreBlocksRaw::processQueue()
{
	if (m_stream->needRotation())
		m_stream->doRotation();

	writeBlocks();	
}
Esempio n. 2
0
/*
 * Write out file system.
 * Clears boot block, writes superblock, bitmaps, inodes, and root directory.
 */
void writeFS(void)
{
  /* zero out the boot sector (and associated block) */
  __u8 blk[WUFS_BLOCKSIZE];
  memset(blk,0x00,WUFS_BLOCKSIZE);
  writeBlocks(Disk, 0, blk, 1);

  /* write the superblock */
  writeBlocks(Disk, 1, SB, 1);

  /* write the inode bitmap */
  writeBlocks(Disk, 2, IMap, SB->sb_imap_bcnt);

  /* write the data block bit map */
  int lba = 2 + SB->sb_imap_bcnt;
  writeBlocks(Disk, lba, BMap, SB->sb_bmap_bcnt);
  
  /* write the inodes */
  lba += SB->sb_bmap_bcnt;
  writeBlocks(Disk, lba, Inode, SB->sb_inodes/WUFS_INODES_PER_BLOCK);

  /* write the root directory */
  int rootDirLBA = Inode[0].in_block[0];
  writeBlocks(Disk, rootDirLBA, RootDir, 1);

  /* write the badblock indirect block */
  if (indirect_addr) {
    writeBlocks(Disk, indirect_addr, indirect, 1);
  }

  /* free at last, free at last */
  if (Verbose) {
    fprintf(stderr,"file system written.\n");
  }
}
	void OggVorbisEncoder::close()
	{
		if (mClosed)
			return;

		// Mark end of data and flush any remaining data in the buffers
		vorbis_analysis_wrote(&mVorbisState, 0);
		writeBlocks();
		flush();

		ogg_stream_clear(&mOggState);
		vorbis_block_clear(&mVorbisBlock);
		vorbis_dsp_clear(&mVorbisState);
		vorbis_info_clear(&mVorbisInfo);

		mClosed = true;
	}
Esempio n. 4
0
int QgsDxfExport::writeToFile( QIODevice* d )
{
  if ( !d )
  {
    return 1;
  }

  if ( !d->isOpen() && !d->open( QIODevice::WriteOnly ) )
  {
    return 2;
  }

  mTextStream.setDevice( d );

  writeHeader();
  writeTables();
  writeBlocks();
  writeEntities();
  writeEndFile();
  return 0;
}
Esempio n. 5
0
void LC_MakerCamSVG::write(RS_Graphic* graphic) {

    RS_DEBUG->print("RS_MakerCamSVG::write: Writing root node ...");

    graphic->calculateBorders();

    min = graphic->getMin();
    max = graphic->getMax();

    RS2::Unit raw_unit = graphic->getUnit();

    switch (raw_unit) {
        case RS2::Millimeter:
            unit = "mm";
            break;
        case RS2::Centimeter:
            unit = "cm";
            break;
        case RS2::Inch:
            unit = "in";
            break;

        default:
            unit = "";
            break;
    }

    xmlWriter->createRootElement("svg", NAMESPACE_URI_SVG);

    xmlWriter->addNamespaceDeclaration("lc", NAMESPACE_URI_LC);
    xmlWriter->addNamespaceDeclaration("xlink", NAMESPACE_URI_XLINK);

    xmlWriter->addAttribute("width", numXml(max.x - min.x) + unit);
    xmlWriter->addAttribute("height", numXml(max.y - min.y) + unit);
    xmlWriter->addAttribute("viewBox", "0 0 "+ numXml(max.x - min.x) + " " + numXml(max.y - min.y));

    writeBlocks(graphic);
    writeLayers(graphic);
}
Esempio n. 6
0
void LC_MakerCamSVG::write(RS_Graphic* graphic) {

    RS_DEBUG->print("RS_MakerCamSVG::write: Writing root node ...");

    graphic->calculateBorders();

    min = graphic->getMin();
    max = graphic->getMax();

    RS2::Unit raw_unit = graphic->getUnit();
    lengthFactor=1.;
    switch (raw_unit) {
        case RS2::Centimeter:
            unit = "cm";
            break;
        case RS2::Inch:
            unit = "in";
            break;

        default:
            lengthFactor=RS_Units::convert(1., raw_unit, RS2::Millimeter);
            //do not break here, default will use mm and convert length to mm
        case RS2::Millimeter:
            unit = "mm";
            break;
    }

    xmlWriter->createRootElement("svg", NAMESPACE_URI_SVG);

    xmlWriter->addNamespaceDeclaration("lc", NAMESPACE_URI_LC);
    xmlWriter->addNamespaceDeclaration("xlink", NAMESPACE_URI_XLINK);

    xmlWriter->addAttribute("width", lengthXml(max.x - min.x) + unit);
    xmlWriter->addAttribute("height", lengthXml(max.y - min.y) + unit);
    xmlWriter->addAttribute("viewBox", "0 0 "+ lengthXml(max.x - min.x) + " " + lengthXml(max.y - min.y));

    writeBlocks(graphic);
    writeLayers(graphic);
}
Esempio n. 7
0
void bbFileCreate(
	char *inName, 	  /* Input file in a tabular bed format <chrom><start><end> + whatever. */
	char *chromSizes, /* Two column tab-separated file: <chromosome> <size>. */
	int blockSize,	  /* Number of items to bundle in r-tree.  1024 is good. */
	int itemsPerSlot, /* Number of items in lowest level of tree.  64 is good. */
	char *asText,	  /* Field definitions in a string */
	struct asObject *as,  /* Field definitions parsed out */
	boolean doCompress, /* If TRUE then compress data. */
	struct slName *extraIndexList,	/* List of extra indexes to add */
	char *outName)    /* BigBed output file name. */
/* Convert tab-separated bed file to binary indexed, zoomed bigBed version. */
{
/* Set up timing measures. */
verboseTimeInit();
struct lineFile *lf = lineFileOpen(inName, TRUE);

bits16 fieldCount = slCount(as->columnList);
bits16 extraIndexCount = slCount(extraIndexList);

struct bbExIndexMaker *eim = NULL;
if (extraIndexList != NULL)
    eim = bbExIndexMakerNew(extraIndexList, as);

/* Load in chromosome sizes. */
struct hash *chromSizesHash = NULL;

if (sizesIs2Bit)
    chromSizesHash = twoBitChromHash(chromSizes);
else
    chromSizesHash = bbiChromSizesFromFile(chromSizes);
verbose(2, "Read %d chromosomes and sizes from %s\n",  chromSizesHash->elCount, chromSizes);

/* Do first pass, mostly just scanning file and counting hits per chromosome. */
int minDiff = 0;
double aveSize = 0;
bits64 bedCount = 0;
bits32 uncompressBufSize = 0;
struct bbiChromUsage *usageList = bbiChromUsageFromBedFile(lf, chromSizesHash, eim, 
    &minDiff, &aveSize, &bedCount, tabSep);
verboseTime(1, "pass1 - making usageList (%d chroms)", slCount(usageList));
verbose(2, "%d chroms in %s. Average span of beds %f\n", slCount(usageList), inName, aveSize);

/* Open output file and write dummy header. */
FILE *f = mustOpen(outName, "wb");
bbiWriteDummyHeader(f);
bbiWriteDummyZooms(f);

/* Write out autoSql string */
bits64 asOffset = ftell(f);
mustWrite(f, asText, strlen(asText) + 1);
verbose(2, "as definition has %d columns\n", fieldCount);

/* Write out dummy total summary. */
struct bbiSummaryElement totalSum;
ZeroVar(&totalSum);
bits64 totalSummaryOffset = ftell(f);
bbiSummaryElementWrite(f, &totalSum);

/* Write out dummy header extension */
bits64 extHeaderOffset = ftell(f);
bits16 extHeaderSize = 64;
repeatCharOut(f, 0, extHeaderSize);

/* Write out extra index stuff if need be. */
bits64 extraIndexListOffset = 0;
bits64 extraIndexListEndOffset = 0;
if (extraIndexList != NULL)
    {
    extraIndexListOffset = ftell(f);
    int extraIndexSize = 16 + 4*1;   // Fixed record size 16, plus 1 times field size of 4 
    repeatCharOut(f, 0, extraIndexSize*extraIndexCount);
    extraIndexListEndOffset = ftell(f);
    }

/* Write out chromosome/size database. */
bits64 chromTreeOffset = ftell(f);
bbiWriteChromInfo(usageList, blockSize, f);

/* Set up to keep track of possible initial reduction levels. */
int resScales[bbiMaxZoomLevels], resSizes[bbiMaxZoomLevels];
int resTryCount = bbiCalcResScalesAndSizes(aveSize, resScales, resSizes);

/* Write out primary full resolution data in sections, collect stats to use for reductions. */
bits64 dataOffset = ftell(f);
bits32 blockCount = 0;
bits32 maxBlockSize = 0;
struct bbiBoundsArray *boundsArray = NULL;
writeOne(f, bedCount);
if (bedCount > 0)
    {
    blockCount = bbiCountSectionsNeeded(usageList, itemsPerSlot);
    AllocArray(boundsArray, blockCount);
    lineFileRewind(lf);
    if (eim)
	bbExIndexMakerAllocChunkArrays(eim, bedCount);
    writeBlocks(usageList, lf, as, itemsPerSlot, boundsArray, blockCount, doCompress,
	    f, resTryCount, resScales, resSizes, eim, bedCount, fieldCount, &maxBlockSize);
    }
verboseTime(1, "pass2 - checking and writing primary data (%lld records, %d fields)", 
	(long long)bedCount, fieldCount);

/* Write out primary data index. */
bits64 indexOffset = ftell(f);
cirTreeFileBulkIndexToOpenFile(boundsArray, sizeof(boundsArray[0]), blockCount,
    blockSize, 1, NULL, bbiBoundsArrayFetchKey, bbiBoundsArrayFetchOffset, 
    indexOffset, f);
freez(&boundsArray);
verboseTime(2, "index write");

/* Declare arrays and vars that track the zoom levels we actually output. */
bits32 zoomAmounts[bbiMaxZoomLevels];
bits64 zoomDataOffsets[bbiMaxZoomLevels];
bits64 zoomIndexOffsets[bbiMaxZoomLevels];

/* Call monster zoom maker library function that bedGraphToBigWig also uses. */
int zoomLevels = 0;
if (bedCount > 0)
    {
    zoomLevels = bbiWriteZoomLevels(lf, f, blockSize, itemsPerSlot,
	bedWriteReducedOnceReturnReducedTwice, fieldCount,
	doCompress, indexOffset - dataOffset, 
	usageList, resTryCount, resScales, resSizes, 
	zoomAmounts, zoomDataOffsets, zoomIndexOffsets, &totalSum);
    }

/* Write out extra indexes if need be. */
if (eim)
    {
    int i;
    for (i=0; i < eim->indexCount; ++i)
        {
	eim->fileOffsets[i] = ftell(f);
	maxBedNameSize = eim->maxFieldSize[i];
	qsort(eim->chunkArrayArray[i], bedCount, 
	    sizeof(struct bbNamedFileChunk), bbNamedFileChunkCmpByName);
	assert(sizeof(struct bbNamedFileChunk) == sizeof(eim->chunkArrayArray[i][0]));
	bptFileBulkIndexToOpenFile(eim->chunkArrayArray[i], sizeof(eim->chunkArrayArray[i][0]), 
	    bedCount, blockSize, bbNamedFileChunkKey, maxBedNameSize, bbNamedFileChunkVal, 
	    sizeof(bits64) + sizeof(bits64), f);
	verboseTime(1, "Sorting and writing extra index %d", i);
	}
    }

/* Figure out buffer size needed for uncompression if need be. */
if (doCompress)
    {
    int maxZoomUncompSize = itemsPerSlot * sizeof(struct bbiSummaryOnDisk);
    uncompressBufSize = max(maxBlockSize, maxZoomUncompSize);
    }

/* Go back and rewrite header. */
rewind(f);
bits32 sig = bigBedSig;
bits16 version = bbiCurrentVersion;
bits16 summaryCount = zoomLevels;
bits32 reserved32 = 0;
bits64 reserved64 = 0;

bits16 definedFieldCount = bedN;

/* Write fixed header */
writeOne(f, sig);
writeOne(f, version);
writeOne(f, summaryCount);
writeOne(f, chromTreeOffset);
writeOne(f, dataOffset);
writeOne(f, indexOffset);
writeOne(f, fieldCount);
writeOne(f, definedFieldCount);
writeOne(f, asOffset);
writeOne(f, totalSummaryOffset);
writeOne(f, uncompressBufSize);
writeOne(f, extHeaderOffset);
assert(ftell(f) == 64);

/* Write summary headers with data. */
int i;
verbose(2, "Writing %d levels of zoom\n", zoomLevels);
for (i=0; i<zoomLevels; ++i)
    {
    verbose(3, "zoomAmounts[%d] = %d\n", i, (int)zoomAmounts[i]);
    writeOne(f, zoomAmounts[i]);
    writeOne(f, reserved32);
    writeOne(f, zoomDataOffsets[i]);
    writeOne(f, zoomIndexOffsets[i]);
    }
/* Write rest of summary headers with no data. */
for (i=zoomLevels; i<bbiMaxZoomLevels; ++i)
    {
    writeOne(f, reserved32);
    writeOne(f, reserved32);
    writeOne(f, reserved64);
    writeOne(f, reserved64);
    }

/* Write total summary. */
fseek(f, totalSummaryOffset, SEEK_SET);
bbiSummaryElementWrite(f, &totalSum);

/* Write extended header */
fseek(f, extHeaderOffset, SEEK_SET);
writeOne(f, extHeaderSize);
writeOne(f, extraIndexCount);
writeOne(f, extraIndexListOffset);
repeatCharOut(f, 0, 52);    // reserved
assert(ftell(f) - extHeaderOffset == extHeaderSize);

/* Write extra index offsets if need be. */
if (extraIndexCount != 0)
    {
    fseek(f, extraIndexListOffset, SEEK_SET);
    int i;
    for (i=0; i<extraIndexCount; ++i)
        {
	// Write out fixed part of index info
	bits16 type = 0;    // bPlusTree type
	bits16 indexFieldCount = 1;
	writeOne(f, type);
	writeOne(f, indexFieldCount);
	writeOne(f, eim->fileOffsets[i]);
	repeatCharOut(f, 0, 4);  // reserved

	// Write out field list - easy this time because for now always only one field.
	bits16 fieldId = eim->indexFields[i];
	writeOne(f, fieldId);
	repeatCharOut(f, 0, 2); // reserved
	}
    assert(ftell(f) == extraIndexListEndOffset);
    }

/* Write end signature. */
fseek(f, 0L, SEEK_END);
writeOne(f, sig);


/* Clean up. */
lineFileClose(&lf);
carefulClose(&f);
freeHash(&chromSizesHash);
bbiChromUsageFreeList(&usageList);
asObjectFreeList(&as);
}
Esempio n. 8
0
/*
 * int (*alowpc)(), (*ahighpc)(); boundaries of text to be monitored
 * WORD *buffer;	ptr to space for monitor data(WORDs)
 * size_t bufsize;	size of above space(in WORDs)
 * size_t nfunc;	max no. of functions whose calls are counted
 *			(default nfunc is 300 on PDP11, 600 on others)
 */
void
monitor(int (*alowpc)(void), int (*ahighpc)(void), WORD *buffer,
	size_t bufsize, size_t nfunc)
{
	uint_t scale;
	long text;
	char *s;
	struct hdr *hdrp;
	ANCHOR  *newanchp;
	size_t	ssiz;
	int error;
	char	*lowpc = (char *)alowpc;
	char	*highpc = (char *)ahighpc;

	lmutex_lock(&mon_lock);

	if (lowpc == NULL) {		/* true only at the end */
		error = 0;
		if (curAnchor != NULL) { /* if anything was collected!.. */
			profil(NULL, 0, 0, 0);
			if (writeBlocks() == 0)
				error = errno;
		}
		lmutex_unlock(&mon_lock);
		if (error) {
			errno = error;
			perror(mon_out);
		}
		return;
	}

	/*
	 * Ok - they want to submit a block for immediate use, for
	 *	function call count consumption, and execution profile
	 *	histogram computation.
	 * If the block fails sanity tests, just bag it.
	 * Next thing - get name to use. If PROFDIR is NULL, let's
	 *	get out now - they want No Profiling done.
	 *
	 * Otherwise:
	 * Set the block hdr cells.
	 * Get an anchor for the block, and link the anchor+block onto
	 *	the end of the chain.
	 * Init the grabba-cell externs (countbase/limit) for this block.
	 * Finally, call profil and return.
	 */

	ssiz = ((sizeof (struct hdr) + nfunc * sizeof (struct cnt)) /
	    sizeof (WORD));
	if (ssiz >= bufsize || lowpc >= highpc) {
		lmutex_unlock(&mon_lock);
		return;
	}

	if ((s = getenv(PROFDIR)) == NULL) { /* PROFDIR not in environment */
		mon_out = MON_OUT; /* use default "mon.out" */
	} else if (*s == '\0') { /* value of PROFDIR is NULL */
		lmutex_unlock(&mon_lock);
		return; /* no profiling on this run */
	} else { /* construct "PROFDIR/pid.progname" */
		int n;
		pid_t pid;
		char *name;
		size_t len;

		len = strlen(s);
		/* 15 is space for /pid.mon.out\0, if necessary */
		if ((mon_out = libc_malloc(len + strlen(___Argv[0]) + 15))
		    == NULL) {
			lmutex_unlock(&mon_lock);
			perror("");
			return;
		}
		(void) strcpy(mon_out, s);
		name = mon_out + len;
		*name++ = '/'; /* two slashes won't hurt */

		if ((pid = getpid()) <= 0) /* extra test just in case */
			pid = 1; /* getpid returns something inappropriate */

		/* suppress leading zeros */
		for (n = 10000; n > pid; n /= 10)
			;
		for (; ; n /= 10) {
			*name++ = pid/n + '0';
			if (n == 1)
			    break;
			pid %= n;
		}
		*name++ = '.';

		if (___Argv != NULL) {	/* mcrt0.s executed */
			if ((s = strrchr(___Argv[0], '/')) != NULL)
				(void) strcpy(name, s + 1);
			else
				(void) strcpy(name, ___Argv[0]);
		} else {
			(void) strcpy(name, MON_OUT);
		}
	}


	hdrp = (struct hdr *)(uintptr_t)buffer;	/* initialize 1st region */
	hdrp->lpc = lowpc;
	hdrp->hpc = highpc;
	hdrp->nfns = nfunc;

	/* get an anchor for the block */
	newanchp = (curAnchor == NULL) ? &firstAnchor :
	    (ANCHOR *)libc_malloc(sizeof (ANCHOR));

	if (newanchp == NULL) {
		lmutex_unlock(&mon_lock);
		perror("monitor");
		return;
	}

	/* link anchor+block into chain */
	newanchp->monBuffer = hdrp;		/* new, down. */
	newanchp->next  = NULL;			/* new, forward to NULL. */
	newanchp->prior = curAnchor;		/* new, backward. */
	if (curAnchor != NULL)
		curAnchor->next = newanchp;	/* old, forward to new. */
	newanchp->flags = HAS_HISTOGRAM;	/* note it has a histgm area */

	/* got it - enable use by mcount() */
	countbase  = (char *)buffer + sizeof (struct hdr);
	_countlimit = countbase + (nfunc * sizeof (struct cnt));

	/* (set size of region 3) */
	newanchp->histSize = (int)
	    (bufsize * sizeof (WORD) - (_countlimit - (char *)buffer));


	/* done w/regions 1 + 2: setup 3  to activate profil processing. */
	buffer += ssiz;			/* move ptr past 2'nd region */
	bufsize -= ssiz;		/* no. WORDs in third region */
					/* no. WORDs of text */
	text = (highpc - lowpc + sizeof (WORD) - 1) / sizeof (WORD);

	/*
	 * scale is a 16 bit fixed point fraction with the decimal
	 * point at the left
	 */
	if (bufsize < text) {
		/* make sure cast is done first! */
		double temp = (double)bufsize;
		scale = (uint_t)((temp * (long)0200000L) / text);
	} else {
		/* scale must be less than 1 */
		scale = 0xffff;
	}
	bufsize *= sizeof (WORD);	/* bufsize into # bytes */
	profil(buffer, bufsize, (ulong_t)lowpc, scale);


	curAnchor = newanchp;	/* make latest addition, the cur anchor */
	lmutex_unlock(&mon_lock);
}
	void OggVorbisEncoder::write(UINT8* samples, UINT32 numSamples)
	{
		static const UINT32 WRITE_LENGTH = 1024;

		UINT32 numFrames = numSamples / mNumChannels;
		while (numFrames > 0)
		{
			UINT32 numFramesToWrite = std::min(numFrames, WRITE_LENGTH);
			float** buffer = vorbis_analysis_buffer(&mVorbisState, numFramesToWrite);

			if (mBitDepth == 8)
			{
				for (UINT32 i = 0; i < numFramesToWrite; i++)
				{
					for (UINT32 j = 0; j < mNumChannels; j++)
					{
						INT8 sample = *(INT8*)samples;
						float encodedSample = sample / 127.0f;
						buffer[j][i] = encodedSample;

						samples++;
					}
				}
			}
			else if (mBitDepth == 16)
			{
				for (UINT32 i = 0; i < numFramesToWrite; i++)
				{
					for (UINT32 j = 0; j < mNumChannels; j++)
					{
						INT16 sample = *(INT16*)samples;
						float encodedSample = sample / 32767.0f;
						buffer[j][i] = encodedSample;

						samples += 2;
					}
				}
			}
			else if (mBitDepth == 24)
			{
				for (UINT32 i = 0; i < numFramesToWrite; i++)
				{
					for (UINT32 j = 0; j < mNumChannels; j++)
					{
						INT32 sample = AudioUtility::convert24To32Bits(samples);
						float encodedSample = sample / 2147483647.0f;
						buffer[j][i] = encodedSample;

						samples += 3;
					}
				}
			}
			else if (mBitDepth == 32)
			{
				for (UINT32 i = 0; i < numFramesToWrite; i++)
				{
					for (UINT32 j = 0; j < mNumChannels; j++)
					{
						INT32 sample = *(INT32*)samples;
						float encodedSample = sample / 2147483647.0f;
						buffer[j][i] = encodedSample;

						samples += 4;
					}
				}
			}
			else
				assert(false);

			// Signal how many frames were written
			vorbis_analysis_wrote(&mVorbisState, numFramesToWrite);
			writeBlocks();

			numFrames -= numFramesToWrite;
		}
	}
/*
 * infile: input filename
 * size: size in blocks of input file
 * outfile: output filename
 * field: which field will be used for sorting
 * buffer: the buffer that is used
 * memSize: number of buffer blocks available for use, without counting the last one, which is for output
 * nunique: number of unique values
 * nios: number of ios
 * 
 * when the input file fits the buffer and there's still a block available for output,
 * hashes each record and writes it to the output, if a record of same value is not
 * found on the corresponding bucket.
 */
void hashElimination(char *infile, uint size, char *outfile, unsigned char field, block_t *buffer, uint memSize, uint *nunique, uint *nios) {
    int out = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);
    block_t *bufferOut = buffer + memSize;
    emptyBlock(bufferOut);
    (*bufferOut).valid = true;
    (*bufferOut).blockid = 0;

    (*nunique) = 0;
    (*nios) += readBlocks(infile, buffer, size);

    // creates a hash index. for each value returned from the hash function,
    // there is a linkedList of pointers to the records with that specific hash
    // value
    uint hashSize = size*MAX_RECORDS_PER_BLOCK;
    linkedRecordPtr **hashIndex = (linkedRecordPtr**) malloc(hashSize * sizeof (linkedRecordPtr*));
    for (uint i = 0; i < hashSize; i++) {
        hashIndex[i] = NULL;
    }

    recordPtr start = newPtr(0);
    recordPtr end = newPtr(size * MAX_RECORDS_PER_BLOCK - 1);

    for (; start <= end; incr(start)) {
        if (!buffer[start.block].valid) {
            start.record = MAX_RECORDS_PER_BLOCK - 1;
            continue;
        }
        record_t record = getRecord(buffer, start);
        if (record.valid) {
            // hashes the record being examined
            uint index = hashRecord(infile, record, hashSize, field);
            linkedRecordPtr *element = hashIndex[index];
            // goes through the linked list for the hash value of the record
            // if a record with same value is not found, then a recordPtr is
            // added to the linked list and the record itself is written to
            // the output. otherwise, it is ignored.
            while (element) {
                if (compareRecords(record, getRecord(buffer, element->ptr), field) == 0) {
                    break;
                }
                element = element->next;
            }
            if (!element) {
                element = (linkedRecordPtr*) malloc(sizeof (linkedRecordPtr));
                element->ptr = start;
                element->next = hashIndex[index];
                hashIndex[index] = element;
                (*bufferOut).entries[(*bufferOut).nreserved++] = record;
                (*nunique) += 1;
                if ((*bufferOut).nreserved == MAX_RECORDS_PER_BLOCK) {
                    (*nios) += writeBlocks(out, bufferOut, 1);
                    emptyBlock(bufferOut);
                    (*bufferOut).blockid += 1;
                }
            }
        }
    }
    // writes records left in buffer to the outfile
    if ((*bufferOut).nreserved != 0) {
        (*nios) += writeBlocks(out, bufferOut, 1);
    }
    destroyHashIndex(hashIndex, size);
    close(out);
}
void EliminateDuplicates(char *infile, unsigned char field, block_t *buffer, unsigned int nmem_blocks, char *outfile, unsigned int *nunique, unsigned int *nios) {

    if (nmem_blocks < 3) {
        printf("At least 3 blocks are required.");
        return;
    }

    // empties the buffer
    emptyBuffer(buffer, nmem_blocks);
    uint memSize = nmem_blocks - 1;
    *nunique = 0;
    *nios = 0;

    uint fileSize = getSize(infile);

    // if the relation fits on the buffer and leaves one block free for output,
    // loads it to the buffer and eliminates duplicates using hashing
    if (fileSize <= memSize) {
        hashElimination(infile, fileSize, outfile, field, buffer, memSize, nunique, nios);
    } else if (fileSize == nmem_blocks) {
        // if the relation completely fits the buffer, calls useFirstBlock
        useFirstBlock(infile, outfile, field, buffer, nmem_blocks, nunique, nios);
    } else {
        // if the relation is larger than the buffer, then sort it using mergesort,
        // BUT during the final merging (during last pass) write to the output
        // only one time each value

        // the following code is similar to that of MergeSort:

        int input, output;
        char tmpFile1[] = ".ed1";
        char tmpFile2[] = ".ed2";

        uint fullSegments = fileSize / nmem_blocks;
        uint remainingSegment = fileSize % nmem_blocks;

        input = open(infile, O_RDONLY, S_IRWXU);
        output = open(tmpFile1, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);

        uint nSortedSegs = 0;
        uint segmentSize = nmem_blocks;
        for (uint i = 0; i <= fullSegments; i++) {
            if (fullSegments == i) {
                if (remainingSegment != 0) {
                    segmentSize = remainingSegment;
                } else {
                    break;
                }
            }
            (*nios) += readBlocks(input, buffer, segmentSize);
            if (sortBuffer(buffer, segmentSize, field)) {
                (*nios) += writeBlocks(output, buffer, segmentSize);
                nSortedSegs += 1;
            }
        }
        close(input);
        close(output);

        segmentSize = nmem_blocks;
        uint lastSegmentSize;
        if (remainingSegment == 0) {
            lastSegmentSize = nmem_blocks;
        } else {
            lastSegmentSize = remainingSegment;
        }

        buffer[memSize].valid = true;
        while (nSortedSegs != 1) {
            input = open(tmpFile1, O_RDONLY, S_IRWXU);
            output = open(tmpFile2, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);

            uint newSortedSegs = 0;
            uint fullMerges = nSortedSegs / memSize;
            uint lastMergeSegs = nSortedSegs % memSize;
            uint *blocksLeft = (uint*) malloc(memSize * sizeof (uint));
            uint segsToMerge = memSize;
            bool lastMerge = false;

            for (uint mergeCounter = 0; mergeCounter <= fullMerges; mergeCounter++) {
                uint firstSegOffset = mergeCounter * memSize * segmentSize;

                if (mergeCounter == fullMerges - 1 && lastMergeSegs == 0) {
                    lastMerge = true;
                } else if (mergeCounter == fullMerges) {
                    if (lastMergeSegs != 0) {
                        segsToMerge = lastMergeSegs;
                        lastMerge = true;
                    } else {
                        break;
                    }
                }

                for (uint i = 0; i < segsToMerge; i++) {
                    (*nios) += preadBlocks(input, buffer + i, (firstSegOffset + i * segmentSize), 1);
                    blocksLeft[i] = segmentSize - 1;
                }

                if (lastMerge) {
                    blocksLeft[segsToMerge - 1] = lastSegmentSize - 1;
                }

                (*nios) += mergeElimination(input, output, buffer, memSize, segsToMerge, blocksLeft, segmentSize, firstSegOffset, field, nSortedSegs <= memSize, lastMerge, nunique);
                newSortedSegs += 1;
            }
            free(blocksLeft);

            if (lastMergeSegs == 0) {
                lastSegmentSize = (memSize - 1) * segmentSize + lastSegmentSize;
            } else {
                lastSegmentSize = (lastMergeSegs - 1) * segmentSize + lastSegmentSize;
            }
            segmentSize *= memSize;
            nSortedSegs = newSortedSegs;
            close(input);
            close(output);

            char tmp = tmpFile1[3];
            tmpFile1[3] = tmpFile2[3];
            tmpFile2[3] = tmp;
        }
        rename(tmpFile1, outfile);
        remove(tmpFile2);
    }
}
uint mergeElimination(int &input, int &output, block_t *buffer, uint memSize, uint segsToMerge, uint *blocksLeft, uint segmentSize, uint firstSegOffset, unsigned char field, bool lastPass, bool lastMergeOfPass, uint *nunique) {
    uint ios = 0;
    block_t *bufferOut = buffer + memSize;
    uint blocksWritten = 0;
    uint sizeOfLastSeg;
    if (lastMergeOfPass) {
        sizeOfLastSeg = blocksLeft[segsToMerge - 1] + 1;
    }
    // holds the last unique value written to the output
    record_t *lastRecordAdded = NULL;

    recordPtr *nextRecord = (recordPtr*) malloc(segsToMerge * sizeof (recordPtr));
    for (uint i = 0; i < segsToMerge; i++) {
        nextRecord[i].block = i;
        nextRecord[i].record = 0;
    }
    emptyBlock(bufferOut);
    (*bufferOut).blockid = 0;

    uint segsToMergeCopy = segsToMerge;
    while (segsToMergeCopy != 0) {
        uint i;
        for (i = 0; i < segsToMerge; i++) {
            if (buffer[i].valid) {
                break;
            }
        }
        record_t minRec = getRecord(buffer, nextRecord[i]);
        uint minBuffIndex = i;

        for (uint j = i + 1; j < segsToMerge; j++) {
            if (buffer[j].valid && compareRecords(getRecord(buffer, nextRecord[j]), minRec, field) < 0) {
                minRec = getRecord(buffer, nextRecord[j]);
                minBuffIndex = j;
            }
        }

        if (!lastPass) {
            (*bufferOut).entries[(*bufferOut).nreserved++] = minRec;
        } else {
            if (!lastRecordAdded) {
                (*bufferOut).entries[(*bufferOut).nreserved++] = minRec;
                (*nunique) += 1;
                lastRecordAdded = (record_t*) malloc(sizeof (record_t));
                memcpy(lastRecordAdded, &minRec, sizeof (record_t));
            } else {
                if (compareRecords(*lastRecordAdded, minRec, field) != 0) {
                    (*bufferOut).entries[(*bufferOut).nreserved++] = minRec;
                    (*nunique) += 1;
                    memcpy(lastRecordAdded, &minRec, sizeof (record_t));
                }
            }
        }

        if ((*bufferOut).nreserved == MAX_RECORDS_PER_BLOCK) {
            ios += writeBlocks(output, bufferOut, 1);
            (*bufferOut).blockid += 1;
            blocksWritten += 1;
            emptyBlock(bufferOut);
        }

        incr(nextRecord[minBuffIndex]);

        if (nextRecord[minBuffIndex].record == 0) {
            nextRecord[minBuffIndex].block -= 1;
            if (blocksLeft[minBuffIndex] > 0) {
                uint blockOffset;
                if (lastMergeOfPass && minBuffIndex == segsToMerge - 1) {
                    blockOffset = firstSegOffset + segmentSize * minBuffIndex + sizeOfLastSeg - blocksLeft[minBuffIndex];
                } else {
                    blockOffset = firstSegOffset + segmentSize * minBuffIndex + segmentSize - blocksLeft[minBuffIndex];
                }
                ios += preadBlocks(input, buffer + minBuffIndex, blockOffset, 1);
                blocksLeft[minBuffIndex] -= 1;
                if (!buffer[minBuffIndex].valid) {
                    segsToMergeCopy -= 1;
                }
            } else {
                buffer[minBuffIndex].valid = false;
                segsToMergeCopy -= 1;
            }
        } else {
            if (!getRecord(buffer, nextRecord[minBuffIndex]).valid) {
                buffer[minBuffIndex].valid = false;
                segsToMergeCopy -= 1;
            }
        }
    }
    free(nextRecord);
    if (lastRecordAdded) {
        free(lastRecordAdded);
    }

    if ((*bufferOut).nreserved != 0) {
        ios += writeBlocks(output, bufferOut, 1);
        (*bufferOut).blockid += 1;
        blocksWritten += 1;
    }

    if (!lastPass && !lastMergeOfPass) {
        for (uint i = 0; i < segmentSize * segsToMerge - blocksWritten; i++) {
            ios += writeBlocks(output, buffer, 1);
        }
    }
    return ios;
}
/*
 * infile: filename of the input file
 * outfile: filename of the output file
 * field: which field will be used for sorting
 * buffer: the buffer used
 * nmem_blocks: size of buffer
 * nunique: number of unique values
 * nios: number of ios
 * 
 * when the input file size is equal to buffer, the whole file is loaded and
 * sorted. then the first block is used as output where only unique values are
 * written
 */
void useFirstBlock(char *infile, char *outfile, unsigned char field, block_t *buffer, uint nmem_blocks, uint *nunique, uint *nios) {
    int out = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);
    (*nios) += readBlocks(infile, buffer, nmem_blocks);
    if (sortBuffer(buffer, nmem_blocks, field)) {
        // all the unique values of the first block are shifted to the start
        // of it. the rest are marked as invalid
        recordPtr i = newPtr(1);
        recordPtr j = newPtr(1);
        (*nunique) += 1;
        buffer[0].nreserved = 1;
        for (; j.block < 1; incr(j)) {
            record_t record = getRecord(buffer, j);
            if (record.valid && compareRecords(record, getRecord(buffer, i - 1), field) != 0) {
                setRecord(buffer, record, i);
                (*nunique) += 1;
                incr(i);
                buffer[0].nreserved += 1;
            }
        }

        j = newPtr(i, 0);
        for (; j.block < 1; incr(j)) {
            buffer[j.block].entries[j.record].valid = false;
        }

        record_t *lastRecordAdded = (record_t*) malloc(sizeof (record_t));
        record_t lastUnique = getRecord(buffer, i - 1);
        memcpy(lastRecordAdded, &lastUnique, sizeof (record_t));
        // if the first block is full after the shifting (meaning that all its
        // values were actually unique), writes it to the outfile and empties it
        if (buffer[0].nreserved == MAX_RECORDS_PER_BLOCK) {
            i.block -= 1;
            (*nios) += writeBlocks(out, buffer, 1);
            emptyBlock(buffer);
            buffer[0].blockid += 1;
        }

        // write the unique values of the other blocks to the first one. if it
        // becomes full writes it to outfile and empties it. at the end, if it
        // has records not writtend yet, writes them to the outfile as well.
        j = newPtr(MAX_RECORDS_PER_BLOCK);
        while (buffer[j.block].valid && j.block < nmem_blocks) {
            record_t record = getRecord(buffer, j);
            if (!record.valid) {
                break;
            }
            if (compareRecords(record, (*lastRecordAdded), field) != 0) {
                setRecord(buffer, record, i);
                memcpy(lastRecordAdded, &record, sizeof (record_t));
                (*nunique) += 1;
                incr(i);
                buffer[0].nreserved += 1;
            }
            if (buffer[0].nreserved == MAX_RECORDS_PER_BLOCK) {
                i.block -= 1;
                (*nios) += writeBlocks(out, buffer, 1);
                emptyBlock(buffer);
                buffer[0].blockid += 1;
            }
            incr(j);
        }
        if (buffer[0].nreserved != 0) {
            (*nios) += writeBlocks(out, buffer, 1);
        }
        free(lastRecordAdded);
    }
    close(out);
}
Esempio n. 14
0
 /**
  * Write a block to disk, directly.
  *
  * There should always enough data in src, or it would assert failed
  *
  * @param index index of block
  * @param src a piece of memory which the driver write to disk
  * @see writeBlocks(BlockIndex index, Length length, ConstSlice src)
  */
 virtual void writeBlock(BlockIndex index, ConstSlice src)
 { writeBlocks(index, 1, src); }