예제 #1
0
static boolean cursor_load_anim(BYTE *data, int no) {
	BYTE *b = data;
	int riffsize;
	RIFFchunk_t c;
	
	if (!search_chunk(b, "RIFF", "ACON", &c)) {
		WARNING("Not animation icon format");
		return FALSE;
	}
	
	riffsize = c.size;
	// printf("size = %d\n", c.size);
	b = c.data;
	
	while (b < (data + riffsize)) {
		if (search_chunk(b, "LIST", "INFO", &c)) {
			NOTICE("LIST(INFO) ignore size = %d\n", c.size);
			b += c.size + 8;
		} else if (search_chunk(b, "anih", NULL, &c)) {
			BYTE *src = c.data;
			NOTICE("anih size = %d\n", c.size);
			anicurHeader.cbSizeof  = LittleEndian_getW(src, 0);
			anicurHeader.cFrames   = LittleEndian_getW(src, 4);
			anicurHeader.cSteps    = LittleEndian_getW(src, 8);
			anicurHeader.cx        = LittleEndian_getW(src, 12);
			anicurHeader.cy        = LittleEndian_getW(src, 16);
			anicurHeader.cBitCount = LittleEndian_getW(src, 20);
			anicurHeader.cPlanes   = LittleEndian_getW(src, 24);
			anicurHeader.jiffRate  = LittleEndian_getW(src, 28);
			anicurHeader.fl        = LittleEndian_getW(src, 32);
			
			anicurImage.header = &anicurHeader;
			
			b += c.size + 8;
		} else if (search_chunk(b, "rate", NULL, &c)) {
			BYTE *src = c.data;
			NOTICE("rate size = %d\n", c.size);
			if (anicurHeader.fl & 0x01) {
				int i;
				anicurHeader.rate = g_new(int, anicurHeader.cSteps);
				for (i = 0; i < anicurHeader.cSteps; i++) {
					anicurHeader.rate[i] = LittleEndian_getW(src, i*4);
					// printf("rate %d, %d\n", i, anicurHeader.rate[i]);
				}
			}
			b += c.size + 8;
		} else if (search_chunk(b, "icon", NULL, &c)) {
예제 #2
0
/********************************************************************************
 *Function: search_stream
 *Description: Analyze the file by reading 1 chunk (default: 100MB) at a time and 
 *passing it to	search_chunk
 *Return: TRUE/FALSE
 **********************************************************************************/
int search_stream(f_state *s, f_info *i)
{
	u_int64_t		bytesread = 0;
	u_int64_t		f_offset = 0;
	u_int64_t		chunk_size = ((u_int64_t) s->chunk_size) * MEGABYTE;
	unsigned char	*buf = (unsigned char *)malloc(sizeof(char) * chunk_size);

	setup_stream(s, i);

	audit_layout(s);
#ifdef DEBUG
	printf("\n\t READING THE FILE INTO MEMORY\n");
#endif

	while ((bytesread = fread(buf, 1, chunk_size, i->handle)) > 0)
		{
		if (signal_caught == SIGTERM || signal_caught == SIGINT)
			{
			user_interrupt(s, i);
			printf("Cleaning up.\n");
			signal_caught = 0;
			}

#ifdef DEBUG
		printf("\n\tbytes_read:=%llu\n", bytesread);
#endif
		search_chunk(s, buf, i, bytesread, f_offset);
		f_offset += bytesread;
		if (!get_mode(s, mode_quiet))
			{
			fprintf(stderr, "*");

			//displayPosition(s,i,f_offset);
			}

		/*FIX ME***
	* We should jump back and make sure we didn't miss any headers that are 
	* bridged between chunks.  What is the best way to do this?\
  	*/
		}

	if (!get_mode(s, mode_quiet))
		{
		fprintf(stderr, "|\n");
		}

#ifdef DEBUG
	printf("\n\tDONE READING bytes_read:=%llu\n", bytesread);
#endif
	if (signal_caught == SIGTERM || signal_caught == SIGINT)
		{
		user_interrupt(s, i);
		printf("Cleaning up.\n");
		signal_caught = 0;
		}

	free(buf);
	return FALSE;
}
예제 #3
0
파일: dmg.c 프로젝트: Blopeur/qemu-heca
static inline int dmg_read_chunk(BlockDriverState *bs, int sector_num)
{
    BDRVDMGState *s = bs->opaque;

    if(!is_sector_in_chunk(s,s->current_chunk,sector_num)) {
	int ret;
	uint32_t chunk = search_chunk(s,sector_num);

	if(chunk>=s->n_chunks)
	    return -1;

	s->current_chunk = s->n_chunks;
	switch(s->types[chunk]) {
	case 0x80000005: { /* zlib compressed */
	    int i;

	    /* we need to buffer, because only the chunk as whole can be
	     * inflated. */
	    i=0;
	    do {
                ret = bdrv_pread(bs->file, s->offsets[chunk] + i,
                                 s->compressed_chunk+i, s->lengths[chunk]-i);
		if(ret<0 && errno==EINTR)
		    ret=0;
		i+=ret;
	    } while(ret>=0 && ret+i<s->lengths[chunk]);

	    if (ret != s->lengths[chunk])
		return -1;

	    s->zstream.next_in = s->compressed_chunk;
	    s->zstream.avail_in = s->lengths[chunk];
	    s->zstream.next_out = s->uncompressed_chunk;
	    s->zstream.avail_out = 512*s->sectorcounts[chunk];
	    ret = inflateReset(&s->zstream);
	    if(ret != Z_OK)
		return -1;
	    ret = inflate(&s->zstream, Z_FINISH);
	    if(ret != Z_STREAM_END || s->zstream.total_out != 512*s->sectorcounts[chunk])
		return -1;
	    break; }
	case 1: /* copy */
	    ret = bdrv_pread(bs->file, s->offsets[chunk],
                             s->uncompressed_chunk, s->lengths[chunk]);
	    if (ret != s->lengths[chunk])
		return -1;
	    break;
	case 2: /* zero */
	    memset(s->uncompressed_chunk, 0, 512*s->sectorcounts[chunk]);
	    break;
	}
	s->current_chunk = chunk;
    }
    return 0;
}
예제 #4
0
파일: block-dmg.c 프로젝트: CPFL/gxen
static inline int dmg_read_chunk(BDRVDMGState *s,int sector_num)
{
    if(!is_sector_in_chunk(s,s->current_chunk,sector_num)) {
	int ret;
	uint32_t chunk = search_chunk(s,sector_num);

	if(chunk>=s->n_chunks)
	    return -1;

	s->current_chunk = s->n_chunks;
	switch(s->types[chunk]) {
	case 0x80000005: { /* zlib compressed */
	    ret = lseek(s->fd, s->offsets[chunk], SEEK_SET);
	    if(ret<0)
		return -1;

	    /* we need to buffer, because only the chunk as whole can be
	     * inflated. */
	    ret = qemu_read_ok(s->fd, s->compressed_chunk, s->lengths[chunk]);
	    if (ret < 0)
	        return -1;

	    s->zstream.next_in = s->compressed_chunk;
	    s->zstream.avail_in = s->lengths[chunk];
	    s->zstream.next_out = s->uncompressed_chunk;
	    s->zstream.avail_out = 512*s->sectorcounts[chunk];
	    ret = inflateReset(&s->zstream);
	    if(ret != Z_OK)
		return -1;
	    ret = inflate(&s->zstream, Z_FINISH);
	    if(ret != Z_STREAM_END || s->zstream.total_out != 512*s->sectorcounts[chunk])
		return -1;
	    break; }
	case 1: /* copy */
	    ret = qemu_read_ok(s->fd, s->uncompressed_chunk, s->lengths[chunk]);
	    if (ret < 0)
		return -1;
	    break;
	case 2: /* zero */
	    memset(s->uncompressed_chunk, 0, 512*s->sectorcounts[chunk]);
	    break;
	}
	s->current_chunk = chunk;
    }
    return 0;
}
예제 #5
0
파일: dmg.c 프로젝트: heiher/qemu
static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
{
    BDRVDMGState *s = bs->opaque;

    if (!is_sector_in_chunk(s, s->current_chunk, sector_num)) {
        int ret;
        uint32_t chunk = search_chunk(s, sector_num);

        if (chunk >= s->n_chunks) {
            return -1;
        }

        s->current_chunk = s->n_chunks;
        switch (s->types[chunk]) { /* block entry type */
        case 0x80000005: { /* zlib compressed */
            /* we need to buffer, because only the chunk as whole can be
             * inflated. */
            ret = bdrv_pread(bs->file, s->offsets[chunk],
                             s->compressed_chunk, s->lengths[chunk]);
            if (ret != s->lengths[chunk]) {
                return -1;
            }

            s->zstream.next_in = s->compressed_chunk;
            s->zstream.avail_in = s->lengths[chunk];
            s->zstream.next_out = s->uncompressed_chunk;
            s->zstream.avail_out = 512 * s->sectorcounts[chunk];
            ret = inflateReset(&s->zstream);
            if (ret != Z_OK) {
                return -1;
            }
            ret = inflate(&s->zstream, Z_FINISH);
            if (ret != Z_STREAM_END ||
                    s->zstream.total_out != 512 * s->sectorcounts[chunk]) {
                return -1;
            }
            break;
        }
        case 0x80000006: /* bzip2 compressed */
            if (!dmg_uncompress_bz2) {
                break;
            }
            /* we need to buffer, because only the chunk as whole can be
             * inflated. */
            ret = bdrv_pread(bs->file, s->offsets[chunk],
                             s->compressed_chunk, s->lengths[chunk]);
            if (ret != s->lengths[chunk]) {
                return -1;
            }

            ret = dmg_uncompress_bz2((char *)s->compressed_chunk,
                                     (unsigned int) s->lengths[chunk],
                                     (char *)s->uncompressed_chunk,
                                     (unsigned int)
                                     (512 * s->sectorcounts[chunk]));
            if (ret < 0) {
                return ret;
            }
            break;
        case 1: /* copy */
            ret = bdrv_pread(bs->file, s->offsets[chunk],
                             s->uncompressed_chunk, s->lengths[chunk]);
            if (ret != s->lengths[chunk]) {
                return -1;
            }
            break;
        case 2: /* zero */
            /* see dmg_read, it is treated specially. No buffer needs to be
             * pre-filled, the zeroes can be set directly. */
            break;
        }
        s->current_chunk = chunk;
    }
    return 0;
}
예제 #6
0
파일: dmg.c 프로젝트: Jameshzc/qemu
static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
{
    BDRVDMGState *s = bs->opaque;

    if (!is_sector_in_chunk(s, s->current_chunk, sector_num)) {
        int ret;
        uint32_t chunk = search_chunk(s, sector_num);
#ifdef CONFIG_BZIP2
        uint64_t total_out;
#endif

        if (chunk >= s->n_chunks) {
            return -1;
        }

        s->current_chunk = s->n_chunks;
        switch (s->types[chunk]) { /* block entry type */
        case 0x80000005: { /* zlib compressed */
            /* we need to buffer, because only the chunk as whole can be
             * inflated. */
            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                             s->compressed_chunk, s->lengths[chunk]);
            if (ret != s->lengths[chunk]) {
                return -1;
            }

            s->zstream.next_in = s->compressed_chunk;
            s->zstream.avail_in = s->lengths[chunk];
            s->zstream.next_out = s->uncompressed_chunk;
            s->zstream.avail_out = 512 * s->sectorcounts[chunk];
            ret = inflateReset(&s->zstream);
            if (ret != Z_OK) {
                return -1;
            }
            ret = inflate(&s->zstream, Z_FINISH);
            if (ret != Z_STREAM_END ||
                s->zstream.total_out != 512 * s->sectorcounts[chunk]) {
                return -1;
            }
            break; }
#ifdef CONFIG_BZIP2
        case 0x80000006: /* bzip2 compressed */
            /* we need to buffer, because only the chunk as whole can be
             * inflated. */
            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                             s->compressed_chunk, s->lengths[chunk]);
            if (ret != s->lengths[chunk]) {
                return -1;
            }

            ret = BZ2_bzDecompressInit(&s->bzstream, 0, 0);
            if (ret != BZ_OK) {
                return -1;
            }
            s->bzstream.next_in = (char *)s->compressed_chunk;
            s->bzstream.avail_in = (unsigned int) s->lengths[chunk];
            s->bzstream.next_out = (char *)s->uncompressed_chunk;
            s->bzstream.avail_out = (unsigned int) 512 * s->sectorcounts[chunk];
            ret = BZ2_bzDecompress(&s->bzstream);
            total_out = ((uint64_t)s->bzstream.total_out_hi32 << 32) +
                        s->bzstream.total_out_lo32;
            BZ2_bzDecompressEnd(&s->bzstream);
            if (ret != BZ_STREAM_END ||
                total_out != 512 * s->sectorcounts[chunk]) {
                return -1;
            }
            break;
#endif /* CONFIG_BZIP2 */
        case 1: /* copy */
            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                             s->uncompressed_chunk, s->lengths[chunk]);
            if (ret != s->lengths[chunk]) {
                return -1;
            }
            break;
        case 2: /* zero */
            /* see dmg_read, it is treated specially. No buffer needs to be
             * pre-filled, the zeroes can be set directly. */
            break;
        }
        s->current_chunk = chunk;
    }
    return 0;
}
예제 #7
0
static bool parse_smaf_score_track(int fd, struct mp3entry *id3)
{
    /* temporary buffer */
    unsigned char *tmp = (unsigned char*)id3->path;
    unsigned char *p = tmp;
    /* contents stored buffer */
    unsigned char *buf = id3->id3v2buf;
    int bufsize = sizeof(id3->id3v2buf);

    unsigned int chunksize;
    unsigned int datasize;
    int valsize;

    int codepage;

    /* parse Optional Data Chunk */
    read(fd, tmp, 21);
    if (memcmp(tmp + 5, "OPDA", 4) != 0)
    {
        DEBUGF("metadata error: missing Optional Data Chunk\n");
        return false;
    }

    /* Optional Data Chunk size */
    chunksize = get_long_be(tmp + 9);

    /* parse Data Chunk */
    if (memcmp(tmp + 13, "Dch", 3) != 0)
    {
        DEBUGF("metadata error: missing Data Chunk\n");
        return false;
    }

    codepage = convert_smaf_codetype(tmp[16]);
    if (codepage < 0)
    {
        DEBUGF("metadata error: smaf unsupport codetype: %d\n", tmp[16]);
        return false;
    }

    /* Data Chunk size */
    datasize = get_long_be(tmp + 17);
    while ((id3->title == NULL || id3->artist == NULL || id3->composer == NULL)
           && (datasize > 0 && bufsize > 0))
    {
        if (read(fd, tmp, 4) <= 0)
            return false;

        valsize = (tmp[2] << 8) | tmp[3];
        datasize -= (valsize + 4);
        switch ((tmp[0]<<8)|tmp[1])
        {
            case TAG_TITLE:
                id3->title = buf;
                read_score_track_contets(fd, codepage, valsize, &buf, &bufsize);
                break;
            case TAG_ARTIST:
                id3->artist = buf;
                read_score_track_contets(fd, codepage, valsize, &buf, &bufsize);
                break;
            case TAG_COMPOSER:
                id3->composer = buf;
                read_score_track_contets(fd, codepage, valsize, &buf, &bufsize);
                break;
            default:
                lseek(fd, valsize, SEEK_CUR);
                break;
        }
    }

    /* search Score Track Chunk */
    lseek(fd, 29 + chunksize, SEEK_SET);

    if (search_chunk(fd, "MTR", 3) == 0)
    {
        DEBUGF("metadata error: missing Score Track Chunk\n");
        return false;
    }

    /*
     * search next chunk
     * usually, next chunk ('M***') found within 40 bytes.
     */
    chunksize = 40;
    read(fd, tmp, chunksize);

    tmp[chunksize] = 'M'; /* stopper */
    while (*p != 'M')
        p++;

    chunksize -= (p - tmp);
    if (chunksize == 0)
    {
        DEBUGF("metadata error: missing Score Track Stream PCM Data Chunk");
        return false;
    }

    /* search Score Track Stream PCM Data Chunk */
    lseek(fd, -chunksize, SEEK_CUR);
    if (search_chunk(fd, "Mtsp", 4) == 0)
    {
        DEBUGF("metadata error: missing Score Track Stream PCM Data Chunk\n");
        return false;
    }

    /*
     * parse Score Track Stream Wave Data Chunk
     *  tmp
     *    +4-7: chunk size (WaveType(3bytes) + wave data count)
     *    +8:   bit 7 0:mono/1:stereo, bit 4-6 format, bit 0-3: base bit
     *    +9:   frequency (MSB)
     *    +10:  frequency (LSB)
     */
    read(fd, tmp, 11);
    if (memcmp(tmp, "Mwa", 3) != 0)
    {
        DEBUGF("metadata error: missing Score Track Stream Wave Data Chunk\n");
        return false;
    }

    /* set track length and bitrate */
    id3->frequency = (tmp[9] << 8) | tmp[10];
    set_length(id3, tmp[8], tmp[8] & 0x0f, get_long_be(tmp + 4) - 3);
    return true;
}
예제 #8
0
static bool parse_smaf_audio_track(int fd, struct mp3entry *id3, unsigned int datasize)
{
    /* temporary buffer */
    unsigned char *tmp = (unsigned char*)id3->path;
    /* contents stored buffer */
    unsigned char *buf = id3->id3v2buf;
    int bufsize = sizeof(id3->id3v2buf);

    unsigned int chunksize = datasize;
    int valsize;

    int codepage;

    /* parse contents info */
    read(fd, tmp, 5);
    codepage = convert_smaf_codetype(tmp[2]);
    if (codepage < 0)
    {
        DEBUGF("metadata error: smaf unsupport codetype: %d\n", tmp[2]);
        return false;
    }

    datasize -= 5;
    while ((id3->title == NULL || id3->artist == NULL || id3->composer == NULL)
           && (datasize > 0 && bufsize > 0))
    {
        if (read(fd, tmp, 3) <= 0)
            return false;

        if (tmp[2] != ':')
        {
            DEBUGF("metadata error: illegal tag: %c%c%c\n", tmp[0], tmp[1], tmp[2]);
            return false;
        }
        switch ((tmp[0]<<8)|tmp[1])
        {
            case TAG_TITLE:
                id3->title = buf;
                valsize = read_audio_track_contets(fd, codepage, &buf, &bufsize);
                break;
            case TAG_ARTIST:
                id3->artist = buf;
                valsize = read_audio_track_contets(fd, codepage, &buf, &bufsize);
                break;
            case TAG_COMPOSER:
                id3->composer = buf;
                valsize = read_audio_track_contets(fd, codepage, &buf, &bufsize);
                break;
            default:
                valsize = read_audio_track_contets(fd, codepage, NULL, &bufsize);
                break;
        }
        datasize -= (valsize + 3);
    }

    /* search PCM Audio Track Chunk */
    lseek(fd, 16 + chunksize, SEEK_SET);

    chunksize = search_chunk(fd, "ATR", 3);
    if (chunksize == 0)
    {
        DEBUGF("metadata error: missing PCM Audio Track Chunk\n");
        return false;
    }

    /*
     * get format
     *  tmp
     *    +0: Format Type
     *    +1: Sequence Type
     *    +2: bit 7 0:mono/1:stereo, bit 4-6 format, bit 0-3: frequency
     *    +3: bit 4-7: base bit
     *    +4: TimeBase_D
     *    +5: TimeBase_G
     *
     * Note: If PCM Audio Track does not include Sequence Data Chunk,
     *       tmp+6 is the start position of Wave Data Chunk.
     */
    read(fd, tmp, 6);

    /* search Wave Data Chunk */
    chunksize = search_chunk(fd, "Awa", 3);
    if (chunksize == 0)
    {
        DEBUGF("metadata error: missing Wave Data Chunk\n");
        return false;
    }

    /* set track length and bitrate */
    id3->frequency = convert_smaf_audio_frequency(tmp[2] & 0x0f);
    set_length(id3, tmp[2], tmp[3] >> 4, chunksize);
    return true;
}
예제 #9
0
파일: simd_exporter.c 프로젝트: fomy/dan
/*  
 * Chunk level, without file semantics 
 * Dedup
 * (no trace for chunk-level no-dedup model)
 */
void chunk_dedup_simd_trace(char **path, int count, int weighted, char *pophashfile)
{
	if (weighted) {
		fprintf(stderr, "CHUNK:DEDUP:WEIGHTED\n");
		printf("CHUNK:DEDUP:WEIGHTED\n");
	} else {
		fprintf(stderr, "CHUNK:DEDUP:NOT WEIGHTED\n");
		printf("CHUNK:DEDUP:NOT WEIGHTED\n");
	}

	init_iterator("CHUNK");

	struct chunk_rec chunk;
	memset(&chunk, 0, sizeof(chunk));

	int64_t psize = 0;
	int64_t lsize = 0;
	int64_t total_chunks = 0;
	/* USE part */
	int64_t sum4mean = 0;
	int64_t count4mean = 0;
	while (iterate_chunk(&chunk, 0) == 0) {

		int64_t sum = chunk.csize;
		sum *= chunk.rcount;

		lsize += sum;
		psize += chunk.csize;

		total_chunks += chunk.rcount;

		if (weighted) {
			sum4mean += sum * chunk.csize;
			count4mean += chunk.csize;
			print_a_chunk(chunk.csize, sum);
		} else {
			sum4mean += sum; 
			count4mean += chunk.csize;
			print_a_chunk(chunk.csize, chunk.rcount);
		}
	}

	printf("%.6f\n", 1.0*lsize/psize);
	fprintf(stderr, "D/F = %.4f, total_chunks = %"PRId64"\n", 1.0*lsize/psize, 
			total_chunks);
	fprintf(stderr, "mean = %.4f, per DF = %.6f\n", 1.0*sum4mean/count4mean, 
			1.0*sum4mean*psize/count4mean/lsize);

	close_iterator();

	char buf[4096];
	struct hashfile_handle *handle;
	const struct chunk_info *ci;

	int64_t restore_logical_bytes = 0;
	int64_t restore_physical_bytes = 0;
	int64_t restore_chunks = 0;
	GHashTable* chunks = g_hash_table_new_full(g_int_hash, hash20_equal, free, 
			NULL);

	/* RAID Failure part */
	/* 1 - 99 */
	int step = 1;
	/* All chunks lost */
	puts("0");

	if (pophashfile) {
		int popfd = open(pophashfile, O_RDONLY);
		char pophashbuf[20];
		while (read(popfd, pophashbuf, 20) == 20) {
			char *pophash = malloc(20);
			memcpy(pophash, pophashbuf, 20);

			/* restoring a pop chunk */
			memcpy(chunk.hash, pophash, 20);
			assert(search_chunk(&chunk));

			int64_t sum = chunk.csize;
			sum *= chunk.rcount;
			restore_chunks += chunk.rcount;

			restore_physical_bytes += chunk.csize;
			restore_logical_bytes += sum;

			int progress = restore_physical_bytes * 100/psize;
			while (progress >= step && step <= 99) {
				if (weighted) {
					printf("%.6f\n", 1.0*restore_logical_bytes/lsize);
					fprintf(stderr, "%.6f\n", 1.0*restore_logical_bytes/lsize);
				} else {
					printf("%.6f\n", 1.0*restore_chunks/total_chunks);
					fprintf(stderr, "%.6f\n", 1.0*restore_chunks/total_chunks);
				}
				step++;
			}

			assert(!g_hash_table_contains(chunks, pophash));
			g_hash_table_insert(chunks, pophash, NULL);
		}
		close(popfd);
	}

	int pc = 0;
	for (; pc < count; pc++) {
		handle = hashfile_open(path[pc]);

		if (!handle) {
			fprintf(stderr, "Error opening hash file: %d!", errno);
			exit(-1);
		}


		while (1) {
			int ret = hashfile_next_file(handle);
			if (ret < 0) {
				fprintf(stderr,
						"Cannot get next file from a hashfile: %d!\n",
						errno);
				exit(-1);
			}
			if (ret == 0)
				break;

			while (1) {
				ci = hashfile_next_chunk(handle);
				if (!ci) /* exit the loop if it was the last chunk */
					break;

				int hashsize = hashfile_hash_size(handle)/8;
				int chunksize = ci->size;
				memcpy(chunk.hash, ci->hash, hashsize);
				memcpy(&chunk.hash[hashsize], &chunksize, sizeof(chunksize));
				chunk.hashlen = hashfile_hash_size(handle)/8 + sizeof(chunksize);

				if (!g_hash_table_contains(chunks, chunk.hash)) {
					assert(search_chunk(&chunk));
					int64_t sum = chunk.csize;
					sum *= chunk.rcount;
					restore_chunks += chunk.rcount;

					restore_physical_bytes += chunk.csize;
					restore_logical_bytes += sum;

					int progress = restore_physical_bytes * 100/psize;
					while (progress >= step && step <= 99) {
						if (weighted) {
							printf("%.6f\n", 1.0*restore_logical_bytes/lsize);
							fprintf(stderr, "%.6f\n", 1.0*restore_logical_bytes/lsize);
						} else {
							printf("%.6f\n", 1.0*restore_chunks/total_chunks);
							fprintf(stderr, "%.6f\n", 1.0*restore_chunks/total_chunks);
						}
						step++;
					}
					char* hash = malloc(20);
					memcpy(hash, chunk.hash, 20);
					g_hash_table_insert(chunks, hash, NULL);
				}
			}
		}
		hashfile_close(handle);
	}
	g_hash_table_destroy(chunks);

	puts("1.0");
}
예제 #10
0
파일: simd_exporter.c 프로젝트: fomy/dan
void file_dedup_simd_trace(char** path, int count,  int weighted, char *pophashfile)
{
	if (weighted) {
		printf("FILE:DEDUP:WEIGHTED\n");
		fprintf(stderr, "FILE:DEDUP:WEIGHTED\n");
	} else {
		printf("FILE:DEDUP:NOT WEIGHTED\n");
		fprintf(stderr, "FILE:DEDUP:NOT WEIGHTED\n");
	}

	init_iterator("CHUNK");

	struct chunk_rec chunk;
	memset(&chunk, 0, sizeof(chunk));
	struct file_rec fr;
	memset(&fr, 0, sizeof(fr));

	/* USE part */
	int64_t psize = 0;
	int64_t lsize = 0;
	while (iterate_chunk(&chunk, 0) == 0) {

		int64_t sum = chunk.csize;
		sum *= chunk.rcount;
		lsize += sum;
		psize += chunk.csize;
		if (!weighted) {
			printf("%d\n", chunk.fcount);
		} else {
			int i = 0;
			int prev = -1;
			int64_t sum = 0;
			for (; i<chunk.rcount; i++) {
				int fid = chunk.list[chunk.rcount+i];
				if (fid == prev)
					continue;
				fr.fid = fid;
				search_file(&fr);

				sum+=fr.fsize;
				prev = fid;
			}
			printf("%"PRId64"\n", sum);
		}
	}

	printf("%.6f\n", 1.0*lsize/psize);
	fprintf(stderr, "LS = %.4f GB, PS = %.4f GB, D/F = %.4f\n", 
			1.0*lsize/1024/1024/1024,
			1.0*psize/1024/1024/1024, 1.0*lsize/psize);

	close_iterator();

	char buf[4096];
	struct hashfile_handle *handle;
	const struct chunk_info *ci;

	int64_t sys_file_number = get_file_number();

	/* All files lost */
	puts("0");

	int64_t restore_bytes = 0;
	int64_t restore_files = 0;
	int64_t restore_file_bytes = 0;

	/* RAID Failure part */
	/* 1 - 99 */
	int step = 1;

	GHashTable* files = g_hash_table_new_full(g_int_hash, g_int_equal, 
			NULL, free);
	GHashTable* chunks = g_hash_table_new_full(g_int_hash, hash20_equal, 
			free, NULL);

	if (pophashfile) {
		int popfd = open(pophashfile, O_RDONLY);
		char pophashbuf[20];
		while (read(popfd, pophashbuf, 20) == 20) {
			char *pophash = malloc(20);
			memcpy(pophash, pophashbuf, 20);

			/* restoring a pop chunk */
			memcpy(chunk.hash, pophash, 20);
			assert(search_chunk(&chunk));

			int i = 0;
			for (;i < chunk.rcount; i++) {
				int fid = chunk.list[chunk.rcount + i];
				struct restoring_file* rfile = g_hash_table_lookup(files, &fid);
				if (!rfile) {
					fr.fid = fid;
					search_file(&fr);

					rfile = malloc(sizeof(*rfile));

					rfile->id = fid;
					rfile->chunk_num = fr.cnum;
					rfile->size = fr.fsize;

					g_hash_table_insert(files, &rfile->id, rfile);
				}
				rfile->chunk_num--;

				if (rfile->chunk_num == 0) {
					/* a file is restored */
					/*fprintf(stderr, "complete file %d\n", fid);*/
					restore_files++;
					restore_file_bytes += rfile->size;
				}
				assert(rfile->chunk_num >= 0);
			}

			restore_bytes += chunk.csize;
			int progress = restore_bytes * 100 / psize;
			while (progress >= step && step <= 99) {
				if (!weighted)
					printf("%.6f\n", 1.0*restore_files/sys_file_number);
				else
					printf("%.6f\n", 1.0*restore_file_bytes/lsize);
				step++;
			}

			assert(!g_hash_table_contains(chunks, pophash));
			g_hash_table_insert(chunks, pophash, NULL);
		}
		close(popfd);
	}

	int pc = 0;
	for (; pc < count; pc++) {

		handle = hashfile_open(path[pc]);

		if (!handle) {
			fprintf(stderr, "Error opening hash file: %d!", errno);
			exit(-1);
		}

		while (1) {
			int ret = hashfile_next_file(handle);
			if (ret < 0) {
				fprintf(stderr,
						"Cannot get next file from a hashfile: %d!\n",
						errno);
				exit(-1);
			}
			if (ret == 0)
				break;

			while (1) {
				ci = hashfile_next_chunk(handle);
				if (!ci) /* exit the loop if it was the last chunk */
					break;

				int hashsize = hashfile_hash_size(handle)/8;
				int chunksize = ci->size;
				memcpy(chunk.hash, ci->hash, hashsize);
				memcpy(&chunk.hash[hashsize], &chunksize, sizeof(chunksize));
				chunk.hashlen = hashfile_hash_size(handle)/8 + sizeof(chunksize);

				if (!g_hash_table_contains(chunks, chunk.hash)) {
					/* restore a chunk */
					assert(search_chunk(&chunk));
					int i = 0;
					for (; i < chunk.rcount; i++) {
						int fid = chunk.list[chunk.rcount + i];
						struct restoring_file* rfile = 
							g_hash_table_lookup(files, &fid);
						if (!rfile) {
							fr.fid = fid;
							search_file(&fr);

							rfile = malloc(sizeof(*rfile));

							rfile->id = fid;
							rfile->chunk_num = fr.cnum;
							rfile->size = fr.fsize;

							g_hash_table_insert(files, &rfile->id, rfile);
						}
						rfile->chunk_num--;

						if(rfile->chunk_num == 0){
							/* a file is restored */
							/*fprintf(stderr, "complete file %d\n", fid);*/
							restore_files++;
							restore_file_bytes += rfile->size;
						}
						assert(rfile->chunk_num >= 0);
					}

					restore_bytes += chunk.csize;
					int progress = restore_bytes * 100/psize;
					while (progress >= step && step <= 99) {
						if (!weighted)
							printf("%.6f\n", 1.0*restore_files/sys_file_number);
						else
							printf("%.6f\n", 1.0*restore_file_bytes/lsize);
						step++;
					}
					char* hash = malloc(20);
					memcpy(hash, chunk.hash, 20);
					g_hash_table_insert(chunks, hash, hash);
				}
			}
		}

		hashfile_close(handle);
	}
	puts("1.0");

	g_hash_table_destroy(files);
	g_hash_table_destroy(chunks);
	fprintf(stderr, "restore %.4f GB\n", 1.0*restore_file_bytes/1024/1024/1024);

}