Esempio n. 1
0
primitiveUncompressWithOffset(void)
{
	const char*compressed;
	size_t compressedLength;
	sqInt compressedObj;
	size_t inputOffset;
	sqInt num;
	char* ptr;
	snappy_status status;
	char* uncompressed;
	size_t uncompressedLength;

	if (!((interpreterProxy->methodArgumentCount()) == 2)) {
		interpreterProxy->primitiveFail(); return;
	}
	compressedObj = interpreterProxy->stackValue(1);
	/* begin charPointerFor: */
	if (!(interpreterProxy->isBytes(compressedObj))) {
		compressed = ((char*) null);
		goto l1;
	}
	ptr = ((char*) (interpreterProxy->firstIndexableField(compressedObj)));
	compressed = ((char*) ptr);
l1:	/* end charPointerFor: */;
	if (!(compressed)) {
		interpreterProxy->primitiveFail(); return;
	}
	/* begin stackPositiveIntegerValue: */
	num = interpreterProxy->stackValue(0);
	if ((interpreterProxy->isIntegerValue(num))
	 && (num < 0)) {
		inputOffset = ((sqInt) null);
		goto l2;
	}
	inputOffset = ((sqInt) (interpreterProxy->positive32BitValueOf(num)));
l2:	/* end stackPositiveIntegerValue: */;
	compressedLength = interpreterProxy->byteSizeOf(compressedObj);
	status = snappy_uncompressed_length(compressed + inputOffset, compressedLength, &uncompressedLength);
	if (!(status == 0)) {
		interpreterProxy->primitiveFailFor(status); return;
	}
	uncompressed = malloc(uncompressedLength);
	status = snappy_uncompress(compressed + inputOffset, compressedLength, uncompressed, &uncompressedLength);
	if (!(status == 0)) {
		/* begin returnErrorInfoFor: */
		interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
		interpreterProxy->pushInteger(-status);
		return;
	}
	interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
	interpreterProxy->push(oopFromCBytessized(uncompressed, uncompressedLength));
	free(uncompressed);
}
static int snp_decompress(struct crypto_tfm *tfm, const u8 *src,
			      unsigned int slen, u8 *dst, unsigned int *dlen)
{
	size_t ulen;

	if (!snappy_uncompressed_length(src, slen, &ulen))
		return -EIO;
	if (*dlen < ulen)
		return -EINVAL;
	*dlen = ulen;
	return snappy_uncompress(src, slen, dst) ? 0 : -EIO;
}
Esempio n. 3
0
static ZEND_FUNCTION(snappy_uncompress)
{
    zval *data;
    char *output = NULL;
    size_t output_len;

    if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
                              "z", &data) == FAILURE) {
        RETURN_FALSE;
    }

    if (Z_TYPE_P(data) != IS_STRING) {
        zend_error(E_WARNING,
                   "snappy_uncompress : expects parameter to be string.");
        RETURN_FALSE;
    }

    if (snappy_uncompressed_length(Z_STRVAL_P(data), Z_STRLEN_P(data),
                                   &output_len) != SNAPPY_OK) {
        zend_error(E_WARNING, "snappy_uncompress : output length error");
        RETURN_FALSE;
    }

    output = (char *)emalloc(output_len);
    if (!output) {
        zend_error(E_WARNING, "snappy_uncompress : memory error");
        RETURN_FALSE;
    }

    if (snappy_uncompress(Z_STRVAL_P(data), Z_STRLEN_P(data),
                          output, &output_len) == SNAPPY_OK) {
#if ZEND_MODULE_API_NO >= 20141001
        RETVAL_STRINGL(output, output_len);
#else
        RETVAL_STRINGL(output, output_len, 1);
#endif
    } else {
        zend_error(E_WARNING, "snappy_uncompress : data error");
        RETVAL_FALSE;
    }

    efree(output);
}
Esempio n. 4
0
primitiveUncompress(void)
{
	const char*compressed;
	size_t compressedLength;
	sqInt compressedObj;
	char* ptr;
	snappy_status status;
	char* uncompressed;
	size_t uncompressedLength;

	if (!((interpreterProxy->methodArgumentCount()) == 1)) {
		interpreterProxy->primitiveFail(); return;
	}
	compressedObj = interpreterProxy->stackValue(0);
	/* begin charPointerFor: */
	if (!(interpreterProxy->isBytes(compressedObj))) {
		compressed = ((char*) null);
		goto l1;
	}
	ptr = ((char*) (interpreterProxy->firstIndexableField(compressedObj)));
	compressed = ((char*) ptr);
l1:	/* end charPointerFor: */;
	if (!(compressed)) {
		interpreterProxy->primitiveFail(); return;
	}
	compressedLength = interpreterProxy->byteSizeOf(compressedObj);
	status = snappy_uncompressed_length(compressed, compressedLength, &uncompressedLength);
	if (!(status == 0)) {
		interpreterProxy->primitiveFailFor(status); return;
	}
	uncompressed = malloc(uncompressedLength);
	status = snappy_uncompress(compressed, compressedLength, uncompressed, &uncompressedLength);
	if (!(status == 0)) {
		/* begin returnErrorInfoFor: */
		interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
		interpreterProxy->pushInteger(-status);
		return;
	}
	interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
	interpreterProxy->push(oopFromCBytessized(uncompressed, uncompressedLength));
	free(uncompressed);
}
Esempio n. 5
0
static int decode_snappy(avro_codec_t c, void * data, int64_t len)
{
        uint32_t crc;
        size_t outlen;

        if (snappy_uncompressed_length(data, len-4, &outlen) != SNAPPY_OK) {
		avro_set_error("Uncompressed length error in snappy");
		return 1;
        }

	if (!c->block_data) {
		c->block_data = avro_malloc(outlen);
		c->block_size = outlen;
	} else if ( (size_t)c->block_size < outlen) {
		c->block_data = avro_realloc(c->block_data, c->block_size, outlen);
		c->block_size = outlen;
	}

	if (!c->block_data)
	{
		avro_set_error("Cannot allocate memory for snappy");
		return 1;
	}

        if (snappy_uncompress(data, len-4, c->block_data, &outlen) != SNAPPY_OK)
        {
                avro_set_error("Error uncompressing block with Snappy");
		return 1;
	}

        crc = __bswap_32(crc32(0, c->block_data, outlen));
        if (memcmp(&crc, (char*)data+len-4, 4))
        {
                avro_set_error("CRC32 check failure uncompressing block with Snappy");
		return 1;
	}

        c->used_size = outlen;

	return 0;
}
Esempio n. 6
0
primitiveUncompressedLength(void)
{
	const char*compressed;
	size_t compressedLength;
	sqInt compressedObj;
	char* ptr;
	size_t result;
	snappy_status status;

	if (!((interpreterProxy->methodArgumentCount()) == 1)) {
		interpreterProxy->primitiveFail(); return;
	}
	compressedObj = interpreterProxy->stackValue(0);
	/* begin charPointerFor: */
	if (!(interpreterProxy->isBytes(compressedObj))) {
		compressed = ((char*) null);
		goto l1;
	}
	ptr = ((char*) (interpreterProxy->firstIndexableField(compressedObj)));
	compressed = ((char*) ptr);
l1:	/* end charPointerFor: */;
	if (!(compressed)) {
		interpreterProxy->primitiveFail(); return;
	}
	compressedLength = interpreterProxy->byteSizeOf(compressedObj);
	status = snappy_uncompressed_length(compressed, compressedLength, &result);
	if (status == SnOK) {
		/* begin returnIntegerFor: */
		interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
		if (result <= 1073741823) {
			interpreterProxy->pushInteger(result);
		}
		else {
			interpreterProxy->push(interpreterProxy->positive32BitIntegerFor(result));
		}
		return;
	}
	/* begin returnErrorInfoFor: */
	interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
	interpreterProxy->pushInteger(-status);
}
Esempio n. 7
0
int pread_bin(int fd, off_t pos, char **ret_ptr)
{
    char *new_buf;
    int len = pread_bin_int(fd, pos, ret_ptr);
    if(len < 0)
    {
        return len;
    }
    size_t new_len;
    if((*ret_ptr)[0] == 1) //Snappy
    {
        if(snappy_uncompressed_length((*ret_ptr) + 1, len - 1, &new_len)
                != SNAPPY_OK)
        {
            //marked as compressed but snappy doesn't see it as valid.
            free(*ret_ptr);
            return -1;
        }

        new_buf = (char*) malloc(new_len);
        snappy_status ss = (snappy_uncompress((*ret_ptr) + 1, len - 1, new_buf, &new_len));
        if(ss == SNAPPY_OK)
        {
            free(*ret_ptr);
            *ret_ptr = new_buf;
            return new_len;
        }
        else
        {
            free(*ret_ptr);
            return -1;
        }
    }
    else
    {
        return len;
    }
}
Esempio n. 8
0
couchstore_error_t decode_index_header(const char *bytes,
                                       size_t len,
                                       index_header_t **header)
{
    index_header_t *h = NULL;
    char *b = NULL, *uncomp = NULL;
    uint16_t num_seqs, i, j, sz, num_part_versions;
    size_t uncompLen;

    /* First 16 bytes are md5 checksum (group signature). */
    if (len <= 16) {
        return COUCHSTORE_ERROR_CORRUPT;
    }

    if (snappy_uncompressed_length(bytes + 16, len, &uncompLen) != SNAPPY_OK) {
        return COUCHSTORE_ERROR_CORRUPT;
    }

    b = uncomp = (char *) malloc(uncompLen);
    if (b == NULL) {
        return COUCHSTORE_ERROR_ALLOC_FAIL;
    }

    if (snappy_uncompress(bytes + 16, len - 16, b, &uncompLen) != SNAPPY_OK) {
        goto alloc_error;
    }

    h = (index_header_t *) malloc(sizeof(index_header_t));
    if (h == NULL) {
        goto alloc_error;
    }
    h->seqs = NULL;
    h->id_btree_state = NULL;
    h->view_states = NULL;
    h->replicas_on_transfer = NULL;
    h->pending_transition.active = NULL;
    h->pending_transition.passive = NULL;
    h->pending_transition.unindexable = NULL;
    h->unindexable_seqs = NULL;
    memcpy(h->signature, bytes, 16);

    h->version = (uint8_t) b[0];
    b += 1;

    h->num_partitions = dec_uint16(b);
    b += 2;

    memcpy(&h->active_bitmask, b, BITMASK_BYTE_SIZE);
    b += BITMASK_BYTE_SIZE;
    memcpy(&h->passive_bitmask, b, BITMASK_BYTE_SIZE);
    b += BITMASK_BYTE_SIZE;
    memcpy(&h->cleanup_bitmask, b, BITMASK_BYTE_SIZE);
    b += BITMASK_BYTE_SIZE;

    num_seqs = dec_uint16(b);
    b += 2;

    h->seqs = sorted_list_create(part_seq_cmp);
    if (h->seqs == NULL) {
        goto alloc_error;
    }

    for (i = 0; i < num_seqs; ++i) {
        part_seq_t pseq;

        pseq.part_id = dec_uint16(b);
        b += 2;
        pseq.seq = dec_uint48(b);
        b += 6;

        if (sorted_list_add(h->seqs, &pseq, sizeof(pseq)) != 0) {
            goto alloc_error;
        }
    }

    sz = dec_uint16(b);
    b += 2;
    h->id_btree_state = read_root((void *) b, (int) sz);
    b += sz;

    h->num_views = (uint8_t) b[0];
    b += 1;

    h->view_states = (node_pointer **) malloc(sizeof(node_pointer *) * h->num_views);
    if (h->view_states == NULL) {
        goto alloc_error;
    }

    for (i = 0; i < (uint16_t) h->num_views; ++i) {
        sz = dec_uint16(b);
        b += 2;
        h->view_states[i] = read_root((void *) b, (int) sz);
        b += sz;
    }

    h->has_replica = b[0] == 0 ? 0 : 1;
    b += 1;

    sz = dec_uint16(b);
    b += 2;
    h->replicas_on_transfer = sorted_list_create(part_id_cmp);
    if (h->replicas_on_transfer == NULL) {
        goto alloc_error;
    }

    for (i = 0; i < sz; ++i) {
        uint16_t part_id = dec_uint16(b);
        b += 2;

        if (sorted_list_add(h->replicas_on_transfer, &part_id, sizeof(part_id)) != 0) {
            goto alloc_error;
        }
    }

    sz = dec_uint16(b);
    b += 2;

    h->pending_transition.active = sorted_list_create(part_id_cmp);
    if (h->pending_transition.active == NULL) {
        goto alloc_error;
    }

    for (i = 0; i < sz; ++i) {
        uint16_t part_id = dec_uint16(b);
        b += 2;

        if (sorted_list_add(h->pending_transition.active,
                            &part_id, sizeof(part_id)) != 0) {
            goto alloc_error;
        }
    }

    sz = dec_uint16(b);
    b += 2;

    h->pending_transition.passive = sorted_list_create(part_id_cmp);
    if (h->pending_transition.passive == NULL) {
        goto alloc_error;
    }

    for (i = 0; i < sz; ++i) {
        uint16_t part_id = dec_uint16(b);
        b += 2;

        if (sorted_list_add(h->pending_transition.passive,
                            &part_id, sizeof(part_id)) != 0) {
            goto alloc_error;
        }
    }

    sz = dec_uint16(b);
    b += 2;

    h->pending_transition.unindexable = sorted_list_create(part_id_cmp);
    if (h->pending_transition.unindexable == NULL) {
        goto alloc_error;
    }

    for (i = 0; i < sz; ++i) {
        uint16_t part_id = dec_uint16(b);
        b += 2;

        if (sorted_list_add(h->pending_transition.unindexable,
                            &part_id, sizeof(part_id)) != 0) {
            goto alloc_error;
        }
    }

    num_seqs = dec_uint16(b);
    b += 2;

    h->unindexable_seqs = sorted_list_create(part_seq_cmp);
    if (h->unindexable_seqs == NULL) {
        goto alloc_error;
    }

    for (i = 0; i < num_seqs; ++i) {
        part_seq_t pseq;

        pseq.part_id = dec_uint16(b);
        b += 2;
        pseq.seq = dec_uint48(b);
        b += 6;

        if (sorted_list_add(h->unindexable_seqs, &pseq, sizeof(pseq)) != 0) {
            goto alloc_error;
        }
    }

    if (h->version >= 2) {
        num_part_versions = dec_uint16(b);
        b += 2;

        h->part_versions = sorted_list_create(part_versions_cmp);
        if (h->part_versions == NULL) {
            goto alloc_error;
        }

        for (i = 0; i < num_part_versions; ++i) {
            part_version_t pver;

            pver.part_id = dec_uint16(b);
            b += 2;
            pver.num_failover_log = dec_uint16(b);
            b += 2;
            pver.failover_log = (failover_log_t *) malloc(
                sizeof(failover_log_t) * pver.num_failover_log);

            if (pver.failover_log == NULL) {
                goto alloc_error;
            }

            for (j = 0; j < pver.num_failover_log; ++j) {
                memcpy(&pver.failover_log[j].uuid, b, 8);
                b += 8;
                pver.failover_log[j].seq = dec_uint64(b);
                b += 8;
            }
            if (sorted_list_add(h->part_versions, &pver, sizeof(pver)) != 0) {
                free(pver.failover_log);
                goto alloc_error;
            }
        }
    }

    free(uncomp);
    *header = h;

    return COUCHSTORE_SUCCESS;

 alloc_error:
    free_index_header(h);
    free(uncomp);
    return COUCHSTORE_ERROR_ALLOC_FAIL;
}
Esempio n. 9
0
int bp__uncompressed_length(const char* compressed,
                            size_t compressed_length,
                            size_t* result) {
  int ret = snappy_uncompressed_length(compressed, compressed_length, result);
  return ret == SNAPPY_OK ? BP_OK : BP_EDECOMP;
}
Esempio n. 10
0
File: hap.c Progetto: jossgray/hap
unsigned int HapDecode(const void *inputBuffer, unsigned long inputBufferBytes,
                       HapDecodeCallback callback, void *info,
                       void *outputBuffer, unsigned long outputBufferBytes,
                       unsigned long *outputBufferBytesUsed,
                       unsigned int *outputBufferTextureFormat)
{
    int result = HapResult_No_Error;
    uint32_t sectionHeaderLength;
    uint32_t sectionLength;
    unsigned int sectionType;
    unsigned int textureFormat;
    unsigned int compressor;
    const void *sectionStart;
    size_t bytesUsed = 0;

    /*
     Check arguments
     */
    if (inputBuffer == NULL
        || outputBuffer == NULL
        || outputBufferTextureFormat == NULL
        )
    {
        return HapResult_Bad_Arguments;
    }

    /*
     One top-level section type describes texture-format and second-stage compression
     */
    result = hap_read_section_header(inputBuffer, (uint32_t)inputBufferBytes, &sectionHeaderLength, &sectionLength, &sectionType);

    if (result != HapResult_No_Error)
    {
        return result;
    }
    
    /*
     Hap compressor/format constants can be unpacked by reading the top and bottom four bits.
     */
    compressor = hap_top_4_bits(sectionType);
    textureFormat = hap_bottom_4_bits(sectionType);

    if (compressor == kHapCompressorComplex && callback == NULL)
    {
        return HapResult_Bad_Arguments;
    }

    /*
     Pass the texture format out
     */
    *outputBufferTextureFormat = hap_texture_format_constant_for_format_identifier(textureFormat);
    if (*outputBufferTextureFormat == 0)
    {
        return HapResult_Bad_Frame;
    }

    sectionStart = ((uint8_t *)inputBuffer) + sectionHeaderLength;
    
    if (compressor == kHapCompressorComplex)
    {
        /*
         The top-level section should contain a Decode Instructions Container followed by frame data
         */

        const char *frame_data = NULL;
        size_t bytes_remaining = 0;

        int chunk_count = 0;
        const void *compressors = NULL;
        const void *chunk_sizes = NULL;
        const void *chunk_offsets = NULL;

        result = hap_read_section_header(sectionStart, inputBufferBytes - sectionHeaderLength, &sectionHeaderLength, &sectionLength, &sectionType);

        if (result == HapResult_No_Error && sectionType != kHapSectionDecodeInstructionsContainer)
        {
            result = HapResult_Bad_Frame;
        }

        if (result != HapResult_No_Error)
        {
            return result;
        }

        /*
         Frame data follows immediately after the Decode Instructions Container
         */
        frame_data = ((const char *)sectionStart) + sectionHeaderLength + sectionLength;

        /*
         Step through the sections inside the Decode Instructions Container
         */
        sectionStart = ((uint8_t *)sectionStart) + sectionHeaderLength;
        bytes_remaining = sectionLength;

        while (bytes_remaining > 0) {
            unsigned int section_chunk_count = 0;
            result = hap_read_section_header(sectionStart, bytes_remaining, &sectionHeaderLength, &sectionLength, &sectionType);
            if (result != HapResult_No_Error)
            {
                return result;
            }
            sectionStart = ((uint8_t *)sectionStart) + sectionHeaderLength;
            switch (sectionType) {
                case kHapSectionChunkSecondStageCompressorTable:
                    compressors = sectionStart;
                    section_chunk_count = sectionLength;
                    break;
                case kHapSectionChunkSizeTable:
                    chunk_sizes = sectionStart;
                    section_chunk_count = sectionLength / 4;
                    break;
                case kHapSectionChunkOffsetTable:
                    chunk_offsets = sectionStart;
                    section_chunk_count = sectionLength / 4;
                    break;
                default:
                    // Ignore unrecognized sections
                    break;
            }

            /*
             If we calculated a chunk count and already have one, make sure they match
             */
            if (section_chunk_count != 0)
            {
                if (chunk_count != 0 && section_chunk_count != chunk_count)
                {
                    return HapResult_Bad_Frame;
                }
                chunk_count = section_chunk_count;
            }

            sectionStart = ((uint8_t *)sectionStart) + sectionLength;
            bytes_remaining -= sectionHeaderLength + sectionLength;
        }

        /*
         The Chunk Second-Stage Compressor Table and Chunk Size Table are required
         */
        if (compressors == NULL || chunk_sizes == NULL)
        {
            return HapResult_Bad_Frame;
        }

        if (chunk_count > 0)
        {
            /*
             Step through the chunks, storing information for their decompression
             */
            HapChunkDecodeInfo *chunk_info = (HapChunkDecodeInfo *)malloc(sizeof(HapChunkDecodeInfo) * chunk_count);

            size_t running_compressed_chunk_size = 0;
            size_t running_uncompressed_chunk_size = 0;
            int i;

            if (chunk_info == NULL)
            {
                return HapResult_Internal_Error;
            }

            for (i = 0; i < chunk_count; i++) {

                chunk_info[i].compressor = *(((uint8_t *)compressors) + i);

                chunk_info[i].compressed_chunk_size = hap_read_4_byte_uint(((uint8_t *)chunk_sizes) + (i * 4));

                if (chunk_offsets)
                {
                    chunk_info[i].compressed_chunk_data = frame_data + hap_read_4_byte_uint(((uint8_t *)chunk_offsets) + (i * 4));
                }
                else
                {
                    chunk_info[i].compressed_chunk_data = frame_data + running_compressed_chunk_size;
                }

                running_compressed_chunk_size += chunk_info[i].compressed_chunk_size;

                if (chunk_info[i].compressor == kHapCompressorSnappy)
                {
                    snappy_status snappy_result = snappy_uncompressed_length(chunk_info[i].compressed_chunk_data,
                        chunk_info[i].compressed_chunk_size,
                        &(chunk_info[i].uncompressed_chunk_size));

                    if (snappy_result != SNAPPY_OK)
                    {
                        switch (snappy_result)
                        {
                        case SNAPPY_INVALID_INPUT:
                            result = HapResult_Bad_Frame;
                            break;
                        default:
                            result = HapResult_Internal_Error;
                            break;
                        }
                        break;
                    }
                }
                else
                {
                    chunk_info[i].uncompressed_chunk_size = chunk_info[i].compressed_chunk_size;
                }

                chunk_info[i].uncompressed_chunk_data = (char *)(((uint8_t *)outputBuffer) + running_uncompressed_chunk_size);
                running_uncompressed_chunk_size += chunk_info[i].uncompressed_chunk_size;
            }

            if (result == HapResult_No_Error && running_uncompressed_chunk_size > outputBufferBytes)
            {
                result = HapResult_Buffer_Too_Small;
            }

            if (result == HapResult_No_Error)
            {
                /*
                 Perform decompression
                 */
                bytesUsed = running_uncompressed_chunk_size;

                callback((HapDecodeWorkFunction)hap_decode_chunk, chunk_info, chunk_count, info);

                /*
                 Check to see if we encountered any errors and report one of them
                 */
                for (i = 0; i < chunk_count; i++)
                {
                    if (chunk_info[i].result != HapResult_No_Error)
                    {
                        result = chunk_info[i].result;
                        break;
                    }
                }
            }

            free(chunk_info);

            if (result != HapResult_No_Error)
            {
                return result;
            }
        }
    }
    else if (compressor == kHapCompressorSnappy)
    {
        /*
         Only one section is present containing a single block of snappy-compressed S3 data
         */
        snappy_status snappy_result = snappy_uncompressed_length((const char *)sectionStart, sectionLength, &bytesUsed);
        if (snappy_result != SNAPPY_OK)
        {
            return HapResult_Internal_Error;
        }
        if (bytesUsed > outputBufferBytes)
        {
            return HapResult_Buffer_Too_Small;
        }
        snappy_result = snappy_uncompress((const char *)sectionStart, sectionLength, (char *)outputBuffer, &bytesUsed);
        if (snappy_result != SNAPPY_OK)
        {
            return HapResult_Internal_Error;
        }
    }
    else if (compressor == kHapCompressorNone)
    {
        /*
         Only one section is present containing a single block of uncompressed S3 data
         */
        bytesUsed = sectionLength;
        if (sectionLength > outputBufferBytes)
        {
            return HapResult_Buffer_Too_Small;
        }
        memcpy(outputBuffer, sectionStart, sectionLength);
    }
    else
    {
        return HapResult_Bad_Frame;
    }
    /*
     Fill out the remaining return value
     */
    if (outputBufferBytesUsed != NULL)
    {
        *outputBufferBytesUsed = bytesUsed;
    }
    
    return HapResult_No_Error;
}
Esempio n. 11
0
static int snappy_in_java_uncompress(FILE *infp, FILE *outfp, int skip_magic)
{
  snappy_in_java_header_t header;
  work_buffer_t wb;
  int err = 1;
  int outfd;

  wb.c = NULL;
  wb.uc = NULL;

  if (!skip_magic) {
    /* read header */
    if (fread_unlocked(&header, sizeof(header), 1, infp) != 1) {
      print_error("Failed to read a file: %s\n", strerror(errno));
      goto cleanup;
    }

    /* check header */
    if (memcmp(header.magic, SNAPPY_IN_JAVA_MAGIC, SNAPPY_IN_JAVA_MAGIC_LEN) != 0) {
      print_error("This is not a snappy-java file.\n");
      goto cleanup;
    }
  }

  /* Use a file descriptor 'outfd' instead of the stdio file pointer 'outfp'
   * to reduce the number of write system calls.
   */
  fflush(outfp);
  outfd = fileno(outfp);

  /* read body */
  work_buffer_init(&wb, MAX_BLOCK_SIZE);
  for (;;) {
    int compressed_flag;
    size_t length = 0;
    unsigned int crc32c = 0;

    /* read compressed flag */
    compressed_flag = getc_unlocked(infp);
    switch (compressed_flag) {
    case EOF:
      /* read all blocks */
      err = 0;
      goto cleanup;
    case COMPRESSED_FLAG:
    case UNCOMPRESSED_FLAG:
      /* pass */
      break;
    default:
      print_error("Unknown compressed flag 0x%02x\n", compressed_flag);
      goto cleanup;
    }

    /* read data length. */
    length |= (getc_unlocked(infp) << 8);
    length |= (getc_unlocked(infp) << 0);

    /* read crc32c. */
    crc32c |= (getc_unlocked(infp) << 24);
    crc32c |= (getc_unlocked(infp) << 16);
    crc32c |= (getc_unlocked(infp) <<  8);
    crc32c |= (getc_unlocked(infp) <<  0);

    /* check read error */
    if (feof_unlocked(infp)) {
      print_error("Unexpected end of file.\n");
      goto cleanup;
    } else if (ferror_unlocked(infp)) {
      print_error("Failed to read a file: %s\n", strerror(errno));
      goto cleanup;
    }

    /* read data */
    if (fread_unlocked(wb.c, length, 1, infp) != 1) {
      if (feof_unlocked(infp)) {
        print_error("Unexpected end of file\n");
      } else {
        print_error("Failed to read a file: %s\n", strerror(errno));
      }
      goto cleanup;
    }
    trace("read %ld bytes.\n", (long)(length));

    if (compressed_flag == COMPRESSED_FLAG) {
      /* check the uncompressed length */
      size_t uncompressed_length;
      err = snappy_uncompressed_length(wb.c, length, &uncompressed_length);
      if (err != 0) {
        print_error("Invalid data: GetUncompressedLength failed %d\n", err);
        goto cleanup;
      }
      err = 1;
      if (uncompressed_length > wb.uclen) {
        print_error("Invalid data: too long uncompressed length\n");
        goto cleanup;
      }

      /* uncompress and write */
      if (snappy_uncompress(wb.c, length, wb.uc, &uncompressed_length)) {
        print_error("Invalid data: RawUncompress failed\n");
        goto cleanup;
      }
      if (check_and_write_block(outfd, wb.uc, uncompressed_length, 1, crc32c)) {
        goto cleanup;
      }
    } else {
      if (check_and_write_block(outfd, wb.c, length, 1, crc32c)) {
        goto cleanup;
      }
    }
  }
 cleanup:
  work_buffer_free(&wb);
  return err;
}
Esempio n. 12
0
/*
 * Decode raw data of chunk's current page into corresponding r/d/bool reader
 * and values_buffer, uncompress data if needed.
 */
static void
decodeCurrentPage(ParquetColumnReader *columnReader)
{
	ColumnChunkMetadata_4C *chunkmd;
	ParquetDataPage page;
	ParquetPageHeader header;
	uint8_t			*buf;	/* store uncompressed or decompressed page data */
	MemoryContext	oldContext;

	chunkmd	= columnReader->columnMetadata;
	page	= columnReader->currentPage;
	header	= page->header;

	oldContext = MemoryContextSwitchTo(columnReader->memoryContext);

	/*----------------------------------------------------------------
	 * Decompress raw data. After this, buf & page->data points to
	 * uncompressed/decompressed data.
	 *----------------------------------------------------------------*/
	if (chunkmd->codec == UNCOMPRESSED)
	{
		buf = page->data;
	}
	else
	{
		/*
		 * make `buf` points to decompressed buffer,
		 * which should be large enough for uncompressed data.
		 */
		if (chunkmd->r > 0)
		{
			/* repeatable column creates decompression buffer for each page */
			buf = palloc0(header->uncompressed_page_size);
		}
		else
		{	
			/* non-repeatable column reuses pageBuffer for decompression */
			if (columnReader->pageBuffer == NULL)
			{
				columnReader->pageBufferLen = header->uncompressed_page_size * BUFFER_SCALE_FACTOR;
				columnReader->pageBuffer = palloc0(columnReader->pageBufferLen);
			}
			else if (columnReader->pageBufferLen < header->uncompressed_page_size)
			{
				columnReader->pageBufferLen = header->uncompressed_page_size * BUFFER_SCALE_FACTOR;
				columnReader->pageBuffer = repalloc(columnReader->pageBuffer, columnReader->pageBufferLen);
			}
			buf = (uint8_t *) columnReader->pageBuffer;
		}

		/*
		 * call corresponding decompress routine
		 */
		switch (chunkmd->codec)
		{
			case SNAPPY:
			{
				size_t uncompressedLen;
				if (snappy_uncompressed_length((char *) page->data,
											   header->compressed_page_size,
											   &uncompressedLen) != SNAPPY_OK)
				{
					ereport(ERROR,
							(errcode(ERRCODE_GP_INTERNAL_ERROR),
							 errmsg("invalid snappy compressed data for column %s, page number %d",
									chunkmd->colName, columnReader->dataPageProcessed)));
				}

				Insist(uncompressedLen == header->uncompressed_page_size);

				if (snappy_uncompress((char *) page->data,		header->compressed_page_size,
									  (char *) buf,				&uncompressedLen) != SNAPPY_OK)
				{
					ereport(ERROR,
							(errcode(ERRCODE_GP_INTERNAL_ERROR),
							 errmsg("failed to decompress snappy data for column %s, page number %d, "
									"uncompressed size %d, compressed size %d",
									chunkmd->colName, columnReader->dataPageProcessed,
									header->uncompressed_page_size, header->compressed_page_size)));
				}
				
				page->data = buf;
				break;
			}
			case GZIP:
			{
				int ret;
				/* 15(default windowBits for deflate) + 16(ouput GZIP header/tailer) */
				const int windowbits = 31;

				z_stream stream;
				stream.zalloc	= Z_NULL;
				stream.zfree	= Z_NULL;
				stream.opaque	= Z_NULL;
				stream.avail_in	= header->compressed_page_size;
				stream.next_in	= (Bytef *) page->data;
				
				ret = inflateInit2(&stream, windowbits);
				if (ret != Z_OK)
				{
					ereport(ERROR,
							(errcode(ERRCODE_GP_INTERNAL_ERROR),
							 errmsg("zlib inflateInit2 failed: %s", stream.msg)));
				}

				size_t uncompressedLen = header->uncompressed_page_size;

				stream.avail_out = uncompressedLen;
				stream.next_out  = (Bytef *) buf;
				ret = inflate(&stream, Z_FINISH);
				if (ret != Z_STREAM_END)
				{
					ereport(ERROR,
							(errcode(ERRCODE_GP_INTERNAL_ERROR),
							 errmsg("zlib inflate failed: %s", stream.msg)));
				
				}
				/* should fill all uncompressed_page_size bytes */
				Assert(stream.avail_out == 0);

				inflateEnd(&stream);

				page->data = buf;
				break;
			}
			case LZO:
				/* TODO */
				Insist(false);
				break;
			default:
				Insist(false);
				break;
		}
	}

	/*----------------------------------------------------------------
	 * get r/d/value part
	 *----------------------------------------------------------------*/
	if(chunkmd->r != 0)
	{
		int num_repetition_bytes = /*le32toh(*/ *((uint32_t *) buf) /*)*/;
		buf += 4;

		page->repetition_level_reader = (RLEDecoder *) palloc0(sizeof(RLEDecoder));
		RLEDecoder_Init(page->repetition_level_reader,
						widthFromMaxInt(chunkmd->r),
						buf,
						num_repetition_bytes);

		buf += num_repetition_bytes;
	}

	if(chunkmd->d != 0)
	{
		int num_definition_bytes = /*le32toh(*/ *((uint32_t *) buf) /*)*/;
		buf += 4;

		page->definition_level_reader = (RLEDecoder *) palloc0(sizeof(RLEDecoder));
		RLEDecoder_Init(page->definition_level_reader,
						widthFromMaxInt(chunkmd->d),
						buf,
						num_definition_bytes);

		buf += num_definition_bytes;
	}

	if (chunkmd->type == BOOLEAN)
	{
		page->bool_values_reader = (ByteBasedBitPackingDecoder *)
				palloc0(sizeof(ByteBasedBitPackingDecoder));
		BitPack_InitDecoder(page->bool_values_reader, buf, /*bitwidth=*/1);
	}
	else
	{
		page->values_buffer = buf;
	}

	MemoryContextSwitchTo(oldContext);
}
Esempio n. 13
0
static int snappy_java_uncompress(FILE *infp, FILE *outfp, int skip_magic)
{
  snappy_java_header_t header;
  work_buffer_t wb;
  int err = 1;
  int outfd;

  wb.c = NULL;
  wb.uc = NULL;

  if (skip_magic) {
    /* read header except magic */
    if (fread_unlocked(&header.version, sizeof(header) - sizeof(header.magic), 1, infp) != 1) {
      print_error("Failed to read a file: %s\n", strerror(errno));
      goto cleanup;
    }
  } else {
    /* read header */
    if (fread_unlocked(&header, sizeof(header), 1, infp) != 1) {
      print_error("Failed to read a file: %s\n", strerror(errno));
      goto cleanup;
    }

    /* check magic */
    if (memcmp(header.magic, SNAPPY_JAVA_MAGIC, SNAPPY_JAVA_MAGIC_LEN) != 0) {
      print_error("This is not a snappy-java file.\n");
      goto cleanup;
    }
  }

  /* check rest header */
  header.version = ntohl(header.version);
  if (header.version != SNAPPY_JAVA_FILE_VERSION) {
    print_error("Unknown snappy-java version %d\n", header.version);
    goto cleanup;
  }

  header.compatible_version = ntohl(header.compatible_version);
  if (header.compatible_version != SNAPPY_JAVA_FILE_VERSION) {
    print_error("Unknown snappy-java compatible version %d\n", header.compatible_version);
    goto cleanup;
  }

  /* Use a file descriptor 'outfd' instead of the stdio file pointer 'outfp'
   * to reduce the number of write system calls.
   */
  fflush(outfp);
  outfd = fileno(outfp);

  /* read body */
  work_buffer_init(&wb, DEFAULT_BLOCK_SIZE);
  for (;;) {
    /* read the compressed length in a block */
    size_t compressed_length = 0;
    size_t uncompressed_length = wb.uclen;
    int idx;

    for (idx = 3; idx >= 0; idx--) {
      int chr = getc_unlocked(infp);
      if (chr == -1) {
        if (idx == 3) {
          /* read all blocks */
          err = 0;
          goto cleanup;
        }
        print_error("Unexpected end of file.\n");
        goto cleanup;
      }
      compressed_length |= (chr << (idx * 8));
    }

    trace("read 4 bytes (compressed_length = %ld)\n", (long)compressed_length);
    if (compressed_length == 0) {
      print_error("Invalid compressed length %ld\n", (long)compressed_length);
      goto cleanup;
    }
    if (compressed_length > wb.clen) {
      work_buffer_resize(&wb, compressed_length, 0);
    }

    /* read the compressed data */
    if (fread_unlocked(wb.c, compressed_length, 1, infp) != 1) {
      if (feof_unlocked(infp)) {
        print_error("Unexpected end of file\n");
      } else {
        print_error("Failed to read a file: %s\n", strerror(errno));
      }
      goto cleanup;
    }
    trace("read %ld bytes.\n", (long)(compressed_length));

    /* check the uncompressed length */
    err = snappy_uncompressed_length(wb.c, compressed_length, &uncompressed_length);
    if (err != 0) {
      print_error("Invalid data: GetUncompressedLength failed %d\n", err);
      goto cleanup;
    }
    err = 1;
    if (uncompressed_length > wb.uclen) {
      work_buffer_resize(&wb, 0, uncompressed_length);
    }

    /* uncompress and write */
    if (snappy_uncompress(wb.c, compressed_length, wb.uc, &uncompressed_length)) {
      print_error("Invalid data: RawUncompress failed\n");
      goto cleanup;
    }
    if (write_full(outfd, wb.uc, uncompressed_length) != uncompressed_length) {
      print_error("Failed to write a file: %s\n", strerror(errno));
      goto cleanup;
    }
    trace("write %ld bytes\n", (long)uncompressed_length);
  }
 cleanup:
  work_buffer_free(&wb);
  return err;
}
Esempio n. 14
0
File: hap.c Progetto: Vidvox/hap
unsigned int hap_decode_single_texture(const void *texture_section, uint32_t texture_section_length,
                                       unsigned int texture_section_type,
                                       HapDecodeCallback callback, void *info,
                                       void *outputBuffer, unsigned long outputBufferBytes,
                                       unsigned long *outputBufferBytesUsed,
                                       unsigned int *outputBufferTextureFormat)
{
    int result = HapResult_No_Error;
    unsigned int textureFormat;
    unsigned int compressor;
    size_t bytesUsed = 0;

    /*
     One top-level section type describes texture-format and second-stage compression
     Hap compressor/format constants can be unpacked by reading the top and bottom four bits.
     */
    compressor = hap_top_4_bits(texture_section_type);
    textureFormat = hap_bottom_4_bits(texture_section_type);

    /*
     Pass the texture format out
     */
    *outputBufferTextureFormat = hap_texture_format_constant_for_format_identifier(textureFormat);
    if (*outputBufferTextureFormat == 0)
    {
        return HapResult_Bad_Frame;
    }

    if (compressor == kHapCompressorComplex)
    {
        /*
         The top-level section should contain a Decode Instructions Container followed by frame data
         */
        int chunk_count = 0;
        const void *compressors = NULL;
        const void *chunk_sizes = NULL;
        const void *chunk_offsets = NULL;
        const char *frame_data = NULL;

        result = hap_decode_header_complex_instructions(texture_section, texture_section_length, &chunk_count, &compressors, &chunk_sizes, &chunk_offsets, &frame_data);

        if (result != HapResult_No_Error)
        {
            return result;
        }

        if (chunk_count > 0)
        {
            /*
             Step through the chunks, storing information for their decompression
             */
            HapChunkDecodeInfo *chunk_info = (HapChunkDecodeInfo *)malloc(sizeof(HapChunkDecodeInfo) * chunk_count);

            size_t running_compressed_chunk_size = 0;
            size_t running_uncompressed_chunk_size = 0;
            int i;

            if (chunk_info == NULL)
            {
                return HapResult_Internal_Error;
            }

            for (i = 0; i < chunk_count; i++) {

                chunk_info[i].compressor = *(((uint8_t *)compressors) + i);

                chunk_info[i].compressed_chunk_size = hap_read_4_byte_uint(((uint8_t *)chunk_sizes) + (i * 4));

                if (chunk_offsets)
                {
                    chunk_info[i].compressed_chunk_data = frame_data + hap_read_4_byte_uint(((uint8_t *)chunk_offsets) + (i * 4));
                }
                else
                {
                    chunk_info[i].compressed_chunk_data = frame_data + running_compressed_chunk_size;
                }

                running_compressed_chunk_size += chunk_info[i].compressed_chunk_size;

                if (chunk_info[i].compressor == kHapCompressorSnappy)
                {
                    snappy_status snappy_result = snappy_uncompressed_length(chunk_info[i].compressed_chunk_data,
                        chunk_info[i].compressed_chunk_size,
                        &(chunk_info[i].uncompressed_chunk_size));

                    if (snappy_result != SNAPPY_OK)
                    {
                        switch (snappy_result)
                        {
                        case SNAPPY_INVALID_INPUT:
                            result = HapResult_Bad_Frame;
                            break;
                        default:
                            result = HapResult_Internal_Error;
                            break;
                        }
                        break;
                    }
                }
                else
                {
                    chunk_info[i].uncompressed_chunk_size = chunk_info[i].compressed_chunk_size;
                }

                chunk_info[i].uncompressed_chunk_data = (char *)(((uint8_t *)outputBuffer) + running_uncompressed_chunk_size);
                running_uncompressed_chunk_size += chunk_info[i].uncompressed_chunk_size;
            }

            if (result == HapResult_No_Error && running_uncompressed_chunk_size > outputBufferBytes)
            {
                result = HapResult_Buffer_Too_Small;
            }

            if (result == HapResult_No_Error)
            {
                /*
                 Perform decompression
                 */
                bytesUsed = running_uncompressed_chunk_size;

                if (chunk_count == 1)
                {
                    /*
                     We don't invoke the callback for one chunk, just decode it directly
                     */
                    hap_decode_chunk(chunk_info, 0);
                }
                else
                {
                    callback((HapDecodeWorkFunction)hap_decode_chunk, chunk_info, chunk_count, info);
                }

                /*
                 Check to see if we encountered any errors and report one of them
                 */
                for (i = 0; i < chunk_count; i++)
                {
                    if (chunk_info[i].result != HapResult_No_Error)
                    {
                        result = chunk_info[i].result;
                        break;
                    }
                }
            }

            free(chunk_info);

            if (result != HapResult_No_Error)
            {
                return result;
            }
        }
    }
    else if (compressor == kHapCompressorSnappy)
    {
        /*
         Only one section is present containing a single block of snappy-compressed texture data
         */
        snappy_status snappy_result = snappy_uncompressed_length((const char *)texture_section, texture_section_length, &bytesUsed);
        if (snappy_result != SNAPPY_OK)
        {
            return HapResult_Internal_Error;
        }
        if (bytesUsed > outputBufferBytes)
        {
            return HapResult_Buffer_Too_Small;
        }
        snappy_result = snappy_uncompress((const char *)texture_section, texture_section_length, (char *)outputBuffer, &bytesUsed);
        if (snappy_result != SNAPPY_OK)
        {
            return HapResult_Internal_Error;
        }
    }
    else if (compressor == kHapCompressorNone)
    {
        /*
         Only one section is present containing a single block of uncompressed texture data
         */
        bytesUsed = texture_section_length;
        if (texture_section_length > outputBufferBytes)
        {
            return HapResult_Buffer_Too_Small;
        }
        memcpy(outputBuffer, texture_section, texture_section_length);
    }
    else
    {
        return HapResult_Bad_Frame;
    }
    /*
     Fill out the remaining return value
     */
    if (outputBufferBytesUsed != NULL)
    {
        *outputBufferBytesUsed = bytesUsed;
    }
    
    return HapResult_No_Error;
}
Esempio n. 15
0
static int foldprint(Db *db, DocInfo *docinfo, void *ctx)
{
    int *count = (int *) ctx;
    Doc *doc = NULL;
    uint64_t cas;
    uint32_t expiry, flags;
    couchstore_error_t docerr;
    (*count)++;

    if (dumpJson) {
        printf("{\"seq\":%"PRIu64",\"id\":\"", docinfo->db_seq);
        printjquote(&docinfo->id);
        printf("\",");
    } else {
        if (mode == DumpBySequence) {
            printf("Doc seq: %"PRIu64"\n", docinfo->db_seq);
            printf("     id: ");
            printsb(&docinfo->id);
        } else {
            printf("  Doc ID: ");
            printsb(&docinfo->id);
            if (docinfo->db_seq > 0) {
                printf("     seq: %"PRIu64"\n", docinfo->db_seq);
            }
        }
    }
    if (docinfo->bp == 0 && docinfo->deleted == 0 && !dumpJson) {
        printf("         ** This b-tree node is corrupt; raw node value follows:*\n");
        printf("    raw: ");
        printsbhex(&docinfo->rev_meta, 1);
        return 0;
    }
    if (dumpJson) {
        printf("\"rev\":%"PRIu64",\"content_meta\":%d,", docinfo->rev_seq,
                                                         docinfo->content_meta);
        printf("\"physical_size\":%zu,", docinfo->size);
    } else {
        printf("     rev: %"PRIu64"\n", docinfo->rev_seq);
        printf("     content_meta: %d\n", docinfo->content_meta);
        printf("     size (on disk): %zu\n", docinfo->size);
    }
    if (docinfo->rev_meta.size >= sizeof(CouchbaseRevMeta)) {
        const CouchbaseRevMeta* meta = (const CouchbaseRevMeta*)docinfo->rev_meta.buf;
        cas = decode_raw64(meta->cas);
        expiry = decode_raw32(meta->expiry);
        flags = decode_raw32(meta->flags);
        if (dumpJson) {
            printf("\"cas\":\"%"PRIu64"\",\"expiry\":%"PRIu32",\"flags\":%"PRIu32",",
                    cas, expiry, flags);
        } else {
            printf("     cas: %"PRIu64", expiry: %"PRIu32", flags: %"PRIu32"\n", cas,
                    expiry,
                    flags);
        }
    }
    if (docinfo->deleted) {
        if (dumpJson) {
            printf("\"deleted\":true,");
        } else {
            printf("     doc deleted\n");
        }
    }

    if (!noBody) {
        docerr = couchstore_open_doc_with_docinfo(db, docinfo, &doc, 0);
        if (docerr != COUCHSTORE_SUCCESS) {
            if (dumpJson) {
                printf("\"body\":null}\n");
            } else {
                printf("     could not read document body: %s\n", couchstore_strerror(docerr));
            }
        } else if (doc && (docinfo->content_meta & COUCH_DOC_IS_COMPRESSED)) {
            size_t rlen;
            snappy_uncompressed_length(doc->data.buf, doc->data.size, &rlen);
            char *decbuf = (char *) malloc(rlen);
            size_t uncompr_len;
            snappy_uncompress(doc->data.buf, doc->data.size, decbuf, &uncompr_len);
            sized_buf uncompr_body;
            uncompr_body.size = uncompr_len;
            uncompr_body.buf = decbuf;
            if (dumpJson) {
                printf("\"size\":%zu,", uncompr_body.size);
                printf("\"snappy\":true,\"body\":\"");
                if (dumpHex) {
                    printsbhexraw(&uncompr_body);
                } else {
                    printjquote(&uncompr_body);
                }
                printf("\"}\n");
            } else {
                printf("     size: %zu\n", uncompr_body.size);
                printf("     data: (snappy) ");
                if (dumpHex) {
                    printsbhexraw(&uncompr_body);
                    printf("\n");
                } else {
                    printsb(&uncompr_body);
                }
            }
        } else if (doc) {
            if (dumpJson) {
                printf("\"size\":%zu,", doc->data.size);
                printf("\"body\":\"");
                printjquote(&doc->data);
                printf("\"}\n");
            } else {
                printf("     size: %zu\n", doc->data.size);
                printf("     data: ");
                if (dumpHex) {
                    printsbhexraw(&doc->data);
                    printf("\n");
                } else {
                    printsb(&doc->data);
                }
            }
        }
    } else {
        if (dumpJson) {
            printf("\"body\":null}\n");
        } else {
            printf("\n");
        }
    }

    couchstore_free_document(doc);
    return 0;
}
Esempio n. 16
0
int
couchdb_seekread(uint64_t key, char *data, size_t len, off_t offset,
		int whence)
{
	char *db_key = NULL;
	lcb_size_t	nkey = 0;
	lcb_error_t	ret;
	int	lock_result = -1;
	lcb_get_cmd_t item;
	const lcb_get_cmd_t* const get_items[] = { &item };
	char *compressed_data = NULL, *uncompressed_data = NULL;
	size_t	compressed_len = 0, uncompressed_len = 0;
	int	res = 0;

	memset(&item, 0, sizeof(lcb_get_cmd_t));
	db_key = (char *) malloc(MAX_KEY_SIZE);
	if (NULL == db_key) {
		syslog(LOG_ERR, "%s: Failed to allocate memory for db_key\n",
			__FUNCTION__);
		return -ENOMEM;
	}
	sprintf(db_key,"%"PRId64"", key);
	nkey = strlen(db_key);

	item.v.v0.key = db_key;
	item.v.v0.nkey = nkey;


	/* Couchbase doesn't provide internal locking mechanism. We need
	   to handle the race conditions between various threads */
	lock_result = pthread_rwlock_rdlock(&db_rwlock);
	if (lock_result != 0) {
		syslog(LOG_ERR, "%s: Unable to obtain read lock. Error = %d \n",
			__FUNCTION__, errno);
		return -errno;
	}

	ret = lcb_get(couchdb_handle, NULL, 1, get_items);

	if (ret != LCB_SUCCESS) {
		free(data);
		data = NULL;
	}

	/* Block the thread till this operation completes */
	lcb_wait(couchdb_handle);

	lock_result = pthread_rwlock_unlock(&db_rwlock);

	compressed_data = value;
	compressed_len = num_bytes;
	if ((res = snappy_uncompressed_length(compressed_data, compressed_len,
			&uncompressed_len)) != SNAPPY_OK) {
		syslog(LOG_INFO, "%s: Corrupted data\n", __FUNCTION__);
		return (-res);
	}

	if (len > (uncompressed_len - offset)) {
		syslog(LOG_ERR, "%s: partial read error\n",
			__FUNCTION__);
		return (-1);
	}

	uncompressed_data = (char *) malloc(uncompressed_len);
	if (uncompressed_data == NULL) {
		syslog(LOG_ERR, "%s: Unable to allocate memory for "
			"uncompressed_data\n", __FUNCTION__);
		return (-ENOMEM);
	}

	if ((res = snappy_uncompress((const char *)compressed_data,
 			compressed_len, uncompressed_data,
			&uncompressed_len)) == SNAPPY_OK) {
		memcpy((char *)data, (char *)uncompressed_data + offset, len);
		free(uncompressed_data);
		return (len);
	}

	free(uncompressed_data);
	return (-res);
}
Esempio n. 17
0
int main(int ac, char **av)
{
	int opt;
	int to_stdout = 0;

	while ((opt = getopt(ac, av, "dcs")) != -1) {
		switch (opt) { 
		case 'd':
			mode = uncompress;
			break;
		case 'c':
			mode = compress;
			break;
		case 's':
			to_stdout = 1;
			break;
		default:
			usage();
		}
	}

	char *map;
	size_t size;
	if (!av[optind])
		usage();

	if (mode == undef && match_suffix(av[optind], ".snp"))
		mode = uncompress;
	else
		mode = compress;

	map = mapfile(av[optind], O_RDONLY, &size);
	if (!map) { 
		fprintf(stderr, "Cannot open %s: %s\n", av[1], strerror(errno));
		exit(1);
	}
		
	int err;
	char *out;	
	size_t outlen;
	if (mode == uncompress) {
		if (!snappy_uncompressed_length(map, size, &outlen)) {
			fprintf(stderr, "Cannot read length in %s\n", 
				av[optind]);
			exit(1);
		}
	} else {	
		outlen = snappy_max_compressed_length(size);
	}
	
	out = xmalloc(outlen);

	if (mode == compress) {
		struct snappy_env env;
		snappy_init_env(&env);
		err = snappy_compress(&env, map, size, out, &outlen);
	} else
		err = snappy_uncompress(map, size, out);

	if (err) {
		fprintf(stderr, "Cannot process %s: %s\n", av[optind], 
			strerror(-err));
		exit(1);
	}

	char *file;
	int fd;
	if (to_stdout) {
		if(av[optind + 1])
			usage();
		fd = 1;
		file = "<stdout>";
	} else {
		if (av[optind + 1] && av[optind + 2])
			usage();
		fd = open_output(av[optind], av[optind + 1], &file);
	}

	err = 0;
	if (write(fd, out, outlen) != outlen) {
		fprintf(stderr, "Cannot write to %s: %s\n", 
			file,
			strerror(errno));
		err = 1;
	}

	return err;
}
Esempio n. 18
0
/*
 * ./test -u uri -r request_size -a total_size
 * uri: /tmp/testenv/testfs
 * request_size: 4096bytes
 * total_size: 409600bytes
 */
int main(int argc, char *argv[]){
    if (log4c_init()) {
        g_message("log4c_init error!");
    }
    GError *error = NULL;
    GOptionContext *context;
    context = g_option_context_new("- hlfs test -");
    g_option_context_add_main_entries(context, entries, NULL);
    g_option_context_set_help_enabled(context, TRUE);
    g_option_group_set_error_hook(g_option_context_get_main_group(context),
            (GOptionErrorFunc)error_func);
    if (!g_option_context_parse(context, &argc, &argv, &error)) {
        g_message("option parsing failed: %s", error->message);
        exit(EXIT_FAILURE);
    }
	g_option_context_free(context);
    g_print("TEST: uri is %s, request size is %d, total size is %d\n", uri, request_size, total_size);
    char *content = (char*)g_malloc0(request_size);
    HLFS_CTRL * ctrl = init_hlfs(uri);
    g_assert(ctrl != NULL);
    uint64_t ret = 0;
    ret = hlfs_open(ctrl,1);
    g_assert(ret == 0);
    g_print("TEST  hlfs open over \n");
	g_print("test hlfs write\n");
	sleep(2);
    int offset = 0;
    while(offset < total_size){
        ret = hlfs_write(ctrl,content,request_size,offset);
        g_assert(ret==request_size);
        offset +=request_size;
        printf("offset:%d\n",offset);
    }
    g_print("TEST  hlfs write over \n");
	g_print("test hlfs read\n");
	sleep(2);
    offset = 0;
    while(offset < total_size){
        ret = hlfs_read(ctrl,content,request_size,offset);
        g_assert(ret==request_size);
        offset +=request_size;
        printf("offset:%d\n",offset);
    }

	g_print("again ------------------------\n");
    offset = 0;
/******************************************************************************************************************************************/



	int fdd;
	int len;
	int rett;
	int i;
	//char read_buf[409600];
	char *contentt = (char*)g_malloc0(40960);

	if((fdd = open("test.c", O_RDONLY )) == -1){
	my_err("open", __LINE__);
	} else {
		g_print("Open file success\n");
	}

	//if(write(fd, write_buf, strlen(write_buf)) != strlen(write_buf)){
	//	my_err("write", __LINE__);
	//}


	if(lseek(fdd, 0, SEEK_END) == -1){
	my_err("lseek", __LINE__);
	}
	if((len = lseek(fdd, 0, SEEK_CUR)) == -1){
		my_err("lseek", __LINE__);
	}
	if((lseek(fdd, 0, SEEK_SET)) == -1){
		my_err("lseek", __LINE__);
	}

	printf("len: %d\n", len);
	if((rett = read(fdd, contentt, len)) < 0){
		my_err("lseek",__LINE__);
	}
	strcpy(content, contentt);
	//for(i = 0; i< len; i ++){
	g_print("%s ", contentt);
	//}
	g_print("\n");


	g_print("/************************************************/");
	if(lseek(fdd, 10, SEEK_END) == -1){
		my_err("lseek", __LINE__);
	}
	
	
	//close (fdd);
/****************************************************************************************************/
 //FILE* srcFile = NULL;

	int srcLength =  lseek(fdd, 0, SEEK_CUR);
	int SNAPPY_OK = 0;
	char* pSrcBuffer =  (char*)g_malloc0(srcLength);
	read(fdd, pSrcBuffer, srcLength);
 	size_t cbOfCompressed = 32+ srcLength + srcLength/6;
  	char* pCompressedBuffer =  (char*)g_malloc0(cbOfCompressed); 
  	if (snappy_compress(pSrcBuffer, srcLength, pCompressedBuffer, &cbOfCompressed) == SNAPPY_OK)
  {
  	 free(pSrcBuffer);
   	pSrcBuffer = NULL;

   	if (snappy_validate_compressed_buffer(pCompressedBuffer, cbOfCompressed) == SNAPPY_OK)
   {
    	size_t cbDecompress = 0;
    	snappy_uncompressed_length(pCompressedBuffer, cbOfCompressed, &cbDecompress);

    	assert(cbDecompress == srcLength);

    	char* pDecompressedBuffer = (char*)g_malloc0(cbDecompress);
    	snappy_uncompress(pCompressedBuffer, cbOfCompressed, pDecompressedBuffer, (size_t*)&cbDecompress);

    	int file;
    	file = open("test1.txt", O_CREAT | O_RDWR );
    //_wfopen_s(&file, _T("123.pdf"), _T("ab"));
	write(file, pDecompressedBuffer, cbDecompress);
    	close(file);

    	free(pDecompressedBuffer);
    	pDecompressedBuffer = NULL;
   }
  }
 

	close (fdd);


/******************************************************************************************************************************************/
 //   while(offset < total_size){
 //       ret = hlfs_write(ctrl,content,request_size,offset);
  //      g_assert(ret==request_size);
  //      offset +=request_size;
  //      printf("offset:%d\n",offset);
  //  }
  //  g_print("TEST  hlfs write over \n");
//	g_print("test hlfs read\n");
//	sleep(2);
  // offset = 0;
   while(offset < len+1){
        ret = hlfs_read(ctrl,content,len,offset);
        g_assert(ret==len);
        offset +=len;
        printf("offset:%d\n",offset);
    }

	g_free(content);
	g_free(contentt);
	ret = hlfs_close(ctrl);
	deinit_hlfs(ctrl);
	g_print("TEST  hlfs test over \n");
	return 0;
}
Esempio n. 19
0
primitiveRawUncompressedLength(void)
{
	const char*compressed;
	size_t compressedLength;
	sqInt compressedObj;
	sqInt num;
	sqInt num1;
	size_t offset;
	char* ptr;
	size_t result;
	snappy_status status;

	if (!((interpreterProxy->methodArgumentCount()) == 3)) {
		interpreterProxy->primitiveFail(); return;
	}
	compressedObj = interpreterProxy->stackValue(2);
	/* begin charPointerFor: */
	if (!(interpreterProxy->isBytes(compressedObj))) {
		compressed = ((char*) null);
		goto l1;
	}
	ptr = ((char*) (interpreterProxy->firstIndexableField(compressedObj)));
	compressed = ((char*) ptr);
l1:	/* end charPointerFor: */;
	if (!(compressed)) {
		interpreterProxy->primitiveFail(); return;
	}
	/* begin stackPositiveIntegerValue: */
	num = interpreterProxy->stackValue(1);
	if ((interpreterProxy->isIntegerValue(num))
	 && (num < 0)) {
		offset = ((sqInt) null);
		goto l2;
	}
	offset = ((sqInt) (interpreterProxy->positive32BitValueOf(num)));
l2:	/* end stackPositiveIntegerValue: */;
	/* begin stackPositiveIntegerValue: */
	num1 = interpreterProxy->stackValue(0);
	if ((interpreterProxy->isIntegerValue(num1))
	 && (num1 < 0)) {
		compressedLength = ((sqInt) null);
		goto l3;
	}
	compressedLength = ((sqInt) (interpreterProxy->positive32BitValueOf(num1)));
l3:	/* end stackPositiveIntegerValue: */;
	status = snappy_uncompressed_length(compressed + offset, compressedLength, &result);
	if (status == SnOK) {
		/* begin returnIntegerFor: */
		interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
		if (result <= 1073741823) {
			interpreterProxy->pushInteger(result);
		}
		else {
			interpreterProxy->push(interpreterProxy->positive32BitIntegerFor(result));
		}
		return;
	}
	/* begin returnErrorInfoFor: */
	interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
	interpreterProxy->pushInteger(-status);
}