static void vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark) { ssize_t l, lcb; assert(mark == SKIP || mark == VERBATIM); /* The NO-OP case, no data, no pending data & no change of mode */ if (vep->last_mark == mark && p == vep->ver_p && vep->o_pending == 0) return; /* * If we changed mode, emit whatever the opposite mode * assembled before the pending bytes. */ if (vep->last_mark != mark && (vep->o_wait > 0 || vep->startup)) { lcb = vep->cb(vep->vc, vep->cb_priv, 0, mark == VERBATIM ? VGZ_RESET : VGZ_ALIGN); if (lcb - vep->o_last > 0) vep_emit_common(vep, lcb - vep->o_last, vep->last_mark); vep->o_last = lcb; vep->o_wait = 0; } /* Transfer pending bytes CRC into active mode CRC */ if (vep->o_pending) { (void)vep->cb(vep->vc, vep->cb_priv, vep->o_pending, VGZ_NORMAL); if (vep->o_crc == 0) { vep->crc = vep->crcp; vep->o_crc = vep->o_pending; } else { vep->crc = crc32_combine(vep->crc, vep->crcp, vep->o_pending); vep->o_crc += vep->o_pending; } vep->crcp = crc32(0L, Z_NULL, 0); vep->o_wait += vep->o_pending; vep->o_pending = 0; } /* * Process this bit of input */ AN(vep->ver_p); l = p - vep->ver_p; assert(l >= 0); vep->crc = crc32(vep->crc, (const void*)vep->ver_p, l); vep->o_crc += l; vep->ver_p = p; vep->o_wait += l; vep->last_mark = mark; (void)vep->cb(vep->vc, vep->cb_priv, l, VGZ_NORMAL); }
static PyObject * zlibextras_crc32_combine(PyObject *self, PyObject *args) { uint32_t crc1, crc2; uint64_t len2; int32_t crc3_signed; //if (!PyArg_ParseTuple(args, "s#|I:crc32_combine", &crc1, &crc2, &len2)) if (!PyArg_ParseTuple(args, "IIk", &crc1, &crc2, &len2)) return NULL; /* In Python 2.x we return a signed integer regardless of native platform * * long size (the 32bit unsigned long is treated as 32-bit signed and sign * * extended into a 64-bit long inside the integer object). 3.0 does the * * right thing and returns unsigned. http://bugs.python.org/issue1202 */ crc3_signed = (int32_t)crc32_combine(crc1, crc2, len2); return PyInt_FromLong(crc3_signed); }
static void ved_stripgzip(struct req *req, struct busyobj *bo) { ssize_t start, last, stop, lpad; ssize_t l; char *p; uint32_t icrc; uint32_t ilen; uint64_t olen; uint8_t *dbits; uint8_t *pp; uint8_t tailbuf[8]; enum objiter_status ois; void *oi; void *sp; ssize_t sl, ll, dl; struct ecx *ecx; struct req *preq; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC); preq = ecx->preq; if (bo != NULL) VBO_waitstate(bo, BOS_FINISHED); AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED)); /* * This is the interesting case: Deliver all the deflate * blocks, stripping the "LAST" bit of the last one and * padding it, as necessary, to a byte boundary. */ p = ObjGetattr(req->wrk, req->objcore, OA_GZIPBITS, &l); AN(p); assert(l == 32); start = vbe64dec(p); last = vbe64dec(p + 8); stop = vbe64dec(p + 16); olen = ObjGetLen(req->wrk, req->objcore); assert(start > 0 && start < olen * 8); assert(last > 0 && last < olen * 8); assert(stop > 0 && stop < olen * 8); assert(last >= start); assert(last < stop); /* The start bit must be byte aligned. */ AZ(start & 7); /* * XXX: optimize for the case where the 'last' * XXX: bit is in a empty copy block */ memset(tailbuf, 0xdd, sizeof tailbuf); dbits = WS_Alloc(req->ws, 8); AN(dbits); ll = 0; oi = ObjIterBegin(req->wrk, req->objcore); do { ois = ObjIter(req->objcore, oi, &sp, &sl); pp = sp; if (sl > 0) { /* Skip over the GZIP header */ dl = start / 8 - ll; if (dl > 0) { /* Before start, skip */ if (dl > sl) dl = sl; ll += dl; sl -= dl; pp += dl; } } if (sl > 0) { /* The main body of the object */ dl = last / 8 - ll; if (dl > 0) { if (dl > sl) dl = sl; if (ved_bytes(req, preq, VDP_NULL, pp, dl)) break; ll += dl; sl -= dl; pp += dl; } } if (sl > 0 && ll == last / 8) { /* Remove the "LAST" bit */ dbits[0] = *pp; dbits[0] &= ~(1U << (last & 7)); if (ved_bytes(req, preq, VDP_NULL, dbits, 1)) break; ll++; sl--; pp++; } if (sl > 0) { /* Last block */ dl = stop / 8 - ll; if (dl > 0) { if (dl > sl) dl = sl; if (ved_bytes(req, preq, VDP_NULL, pp, dl)) break; ll += dl; sl -= dl; pp += dl; } } if (sl > 0 && (stop & 7) && ll == stop / 8) { /* Add alignment to byte boundary */ dbits[1] = *pp; ll++; sl--; pp++; switch((int)(stop & 7)) { case 1: /* * x000.... * 00000000 00000000 11111111 11111111 */ case 3: /* * xxx000.. * 00000000 00000000 11111111 11111111 */ case 5: /* * xxxxx000 * 00000000 00000000 11111111 11111111 */ dbits[2] = 0x00; dbits[3] = 0x00; dbits[4] = 0xff; dbits[5] = 0xff; lpad = 5; break; case 2: /* xx010000 00000100 00000001 00000000 */ dbits[1] |= 0x08; dbits[2] = 0x20; dbits[3] = 0x80; dbits[4] = 0x00; lpad = 4; break; case 4: /* xxxx0100 00000001 00000000 */ dbits[1] |= 0x20; dbits[2] = 0x80; dbits[3] = 0x00; lpad = 3; break; case 6: /* xxxxxx01 00000000 */ dbits[1] |= 0x80; dbits[2] = 0x00; lpad = 2; break; case 7: /* * xxxxxxx0 * 00...... * 00000000 00000000 11111111 11111111 */ dbits[2] = 0x00; dbits[3] = 0x00; dbits[4] = 0x00; dbits[5] = 0xff; dbits[6] = 0xff; lpad = 6; break; case 0: /* xxxxxxxx */ default: WRONG("compiler must be broken"); } if (ved_bytes(req, preq, VDP_NULL, dbits + 1, lpad)) break; } if (sl > 0) { /* Recover GZIP tail */ dl = olen - ll; assert(dl >= 0); if (dl > sl) dl = sl; if (dl > 0) { assert(dl <= 8); l = ll - (olen - 8); assert(l >= 0); assert(l <= 8); assert(l + dl <= 8); memcpy(tailbuf + l, pp, dl); ll += dl; sl -= dl; pp += dl; } } } while (ois == OIS_DATA || ois == OIS_STREAM); ObjIterEnd(req->objcore, &oi); (void)ved_bytes(req, preq, VDP_FLUSH, NULL, 0); icrc = vle32dec(tailbuf); ilen = vle32dec(tailbuf + 4); ecx->crc = crc32_combine(ecx->crc, icrc, ilen); ecx->l_crc += ilen; }
VDP_ESI(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { uint8_t *q, *r; ssize_t l = 0; uint32_t icrc = 0; uint8_t tailbuf[8 + 5]; const uint8_t *pp; struct ecx *ecx, *pecx; int retval = 0; if (act == VDP_INIT) { AZ(*priv); ALLOC_OBJ(ecx, ECX_MAGIC); AN(ecx); ecx->preq = req; *priv = ecx; return (0); } CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); if (act == VDP_FINI) { FREE_OBJ(ecx); *priv = NULL; return (0); } pp = ptr; while (1) { switch (ecx->state) { case 0: ecx->p = ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, &l); AN(ecx->p); assert(l > 0); ecx->e = ecx->p + l; if (*ecx->p == VEC_GZ) { ecx->isgzip = 1; ecx->p++; } if (req->esi_level == 0) { /* * Only the top level document gets to * decide this. */ if (ecx->isgzip) { assert(sizeof gzip_hdr == 10); /* Send out the gzip header */ retval = VDP_bytes(req, VDP_NULL, gzip_hdr, 10); ecx->l_crc = 0; ecx->crc = crc32(0L, Z_NULL, 0); } } ecx->state = 1; break; case 1: if (ecx->p >= ecx->e) { ecx->state = 2; break; } switch (*ecx->p) { case VEC_V1: case VEC_V2: case VEC_V8: ecx->l = ved_decode_len(req, &ecx->p); if (ecx->l < 0) return (-1); if (ecx->isgzip) { assert(*ecx->p == VEC_C1 || *ecx->p == VEC_C2 || *ecx->p == VEC_C8); l = ved_decode_len(req, &ecx->p); if (l < 0) return (-1); icrc = vbe32dec(ecx->p); ecx->p += 4; if (ecx->isgzip) { ecx->crc = crc32_combine( ecx->crc, icrc, l); ecx->l_crc += l; } } ecx->state = 3; break; case VEC_S1: case VEC_S2: case VEC_S8: ecx->l = ved_decode_len(req, &ecx->p); if (ecx->l < 0) return (-1); Debug("SKIP1(%d)\n", (int)ecx->l); ecx->state = 4; break; case VEC_INCL: ecx->p++; q = (void*)strchr((const char*)ecx->p, '\0'); AN(q); q++; r = (void*)strchr((const char*)q, '\0'); AN(r); if (VDP_bytes(req, VDP_FLUSH, NULL, 0)) { ecx->p = ecx->e; break; } Debug("INCL [%s][%s] BEGIN\n", q, ecx->p); ved_include(req, (const char*)q, (const char*)ecx->p, ecx); Debug("INCL [%s][%s] END\n", q, ecx->p); ecx->p = r + 1; break; default: VSLb(req->vsl, SLT_Error, "ESI corruption line %d 0x%02x [%s]\n", __LINE__, *ecx->p, ecx->p); WRONG("ESI-codes: Illegal code"); } break; case 2: if (ecx->isgzip && req->esi_level == 0) { /* * We are bytealigned here, so simply emit * a gzip literal block with finish bit set. */ tailbuf[0] = 0x01; tailbuf[1] = 0x00; tailbuf[2] = 0x00; tailbuf[3] = 0xff; tailbuf[4] = 0xff; /* Emit CRC32 */ vle32enc(tailbuf + 5, ecx->crc); /* MOD(2^32) length */ vle32enc(tailbuf + 9, ecx->l_crc); (void)VDP_bytes(req, VDP_NULL, tailbuf, 13); } if (req->transport->deliver == VED_Deliver) { CAST_OBJ_NOTNULL(pecx, req->transport_priv, ECX_MAGIC); pecx->crc = crc32_combine(pecx->crc, ecx->crc, ecx->l_crc); pecx->l_crc += ecx->l_crc; } retval = VDP_bytes(req, VDP_FLUSH, NULL, 0); ecx->state = 99; return (retval); case 3: case 4: /* * There is no guarantee that the 'l' bytes are all * in the same storage segment, so loop over storage * until we have processed them all. */ if (ecx->l <= len) { if (ecx->state == 3) retval = VDP_bytes(req, act, pp, ecx->l); len -= ecx->l; pp += ecx->l; ecx->state = 1; break; } if (ecx->state == 3 && len > 0) retval = VDP_bytes(req, act, pp, len); ecx->l -= len; return (retval); case 99: /* * VEP does not account for the PAD+CRC+LEN * so we can see up to approx 15 bytes here. */ return (retval); default: WRONG("FOO"); break; } if (retval) return (retval); } }
static void ved_stripgzip(struct req *req, const struct boc *boc) { ssize_t l; char *p; uint32_t icrc; uint32_t ilen; uint8_t *dbits; struct ecx *ecx; struct ved_foo foo; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC); INIT_OBJ(&foo, VED_FOO_MAGIC); foo.req = req; foo.preq = ecx->preq; memset(foo.tailbuf, 0xdd, sizeof foo.tailbuf); /* OA_GZIPBITS is not valid until BOS_FINISHED */ if (boc != NULL) ObjWaitState(req->objcore, BOS_FINISHED); AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED)); /* * This is the interesting case: Deliver all the deflate * blocks, stripping the "LAST" bit of the last one and * padding it, as necessary, to a byte boundary. */ p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &l); AN(p); assert(l == 32); foo.start = vbe64dec(p); foo.last = vbe64dec(p + 8); foo.stop = vbe64dec(p + 16); foo.olen = ObjGetLen(req->wrk, req->objcore); assert(foo.start > 0 && foo.start < foo.olen * 8); assert(foo.last > 0 && foo.last < foo.olen * 8); assert(foo.stop > 0 && foo.stop < foo.olen * 8); assert(foo.last >= foo.start); assert(foo.last < foo.stop); /* The start bit must be byte aligned. */ AZ(foo.start & 7); dbits = WS_Alloc(req->ws, 8); AN(dbits); foo.dbits = dbits; (void)ObjIterate(req->wrk, req->objcore, &foo, ved_objiterate); /* XXX: error check ?? */ (void)ved_bytes(req, foo.preq, VDP_FLUSH, NULL, 0); icrc = vle32dec(foo.tailbuf); ilen = vle32dec(foo.tailbuf + 4); ecx->crc = crc32_combine(ecx->crc, icrc, ilen); ecx->l_crc += ilen; }
static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) { ZLibData* d = (ZLibData*)drv_data; int res; switch(command) { case DEFLATE_INIT: if (len != 4) goto badarg; if (d->state != ST_NONE) goto badarg; res = deflateInit(&d->s, i32(buf)); if (res == Z_OK) { d->state = ST_DEFLATE; d->want_crc = 0; d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); case DEFLATE_INIT2: { int wbits; if (len != 20) goto badarg; if (d->state != ST_NONE) goto badarg; wbits = i32(buf+8); res = deflateInit2(&d->s, i32(buf), i32(buf+4), wbits, i32(buf+12), i32(buf+16)); if (res == Z_OK) { d->state = ST_DEFLATE; d->want_crc = (wbits < 0); d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); } case DEFLATE_SETDICT: if (d->state != ST_DEFLATE) goto badarg; res = deflateSetDictionary(&d->s, (unsigned char*)buf, len); if (res == Z_OK) { return zlib_value(d->s.adler, rbuf, rlen); } else { return zlib_return(res, rbuf, rlen); } case DEFLATE_RESET: if (len != 0) goto badarg; if (d->state != ST_DEFLATE) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = deflateReset(&d->s); return zlib_return(res, rbuf, rlen); case DEFLATE_END: if (len != 0) goto badarg; if (d->state != ST_DEFLATE) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = deflateEnd(&d->s); d->state = ST_NONE; return zlib_return(res, rbuf, rlen); case DEFLATE_PARAMS: if (len != 8) goto badarg; if (d->state != ST_DEFLATE) goto badarg; res = deflateParams(&d->s, i32(buf), i32(buf+4)); return zlib_return(res, rbuf, rlen); case DEFLATE: if (d->state != ST_DEFLATE) goto badarg; if (len != 4) goto badarg; res = zlib_deflate(d, i32(buf)); return zlib_return(res, rbuf, rlen); case INFLATE_INIT: if (len != 0) goto badarg; if (d->state != ST_NONE) goto badarg; res = inflateInit(&d->s); if (res == Z_OK) { d->state = ST_INFLATE; d->inflate_eos_seen = 0; d->want_crc = 0; d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); case INFLATE_INIT2: { int wbits; if (len != 4) goto badarg; if (d->state != ST_NONE) goto badarg; wbits = i32(buf); res = inflateInit2(&d->s, wbits); if (res == Z_OK) { d->state = ST_INFLATE; d->inflate_eos_seen = 0; d->want_crc = (wbits < 0); d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); } case INFLATE_SETDICT: if (d->state != ST_INFLATE) goto badarg; res = inflateSetDictionary(&d->s, (unsigned char*)buf, len); return zlib_return(res, rbuf, rlen); case INFLATE_SYNC: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; if (driver_sizeq(d->port) == 0) { res = Z_BUF_ERROR; } else { int vlen; SysIOVec* iov = driver_peekq(d->port, &vlen); d->s.next_in = iov[0].iov_base; d->s.avail_in = iov[0].iov_len; res = inflateSync(&d->s); } return zlib_return(res, rbuf, rlen); case INFLATE_RESET: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = inflateReset(&d->s); d->inflate_eos_seen = 0; return zlib_return(res, rbuf, rlen); case INFLATE_END: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = inflateEnd(&d->s); if (res == Z_OK && d->inflate_eos_seen == 0) { res = Z_DATA_ERROR; } d->state = ST_NONE; return zlib_return(res, rbuf, rlen); case INFLATE: if (d->state != ST_INFLATE) goto badarg; if (len != 4) goto badarg; res = zlib_inflate(d, i32(buf)); if (res == Z_NEED_DICT) { return zlib_value2(3, d->s.adler, rbuf, rlen); } else { return zlib_return(res, rbuf, rlen); } case GET_QSIZE: return zlib_value(driver_sizeq(d->port), rbuf, rlen); case GET_BUFSZ: return zlib_value(d->binsz_need, rbuf, rlen); case SET_BUFSZ: { int need; if (len != 4) goto badarg; need = i32(buf); if ((need < 16) || (need > 0x00ffffff)) goto badarg; if (d->binsz_need != need) { d->binsz_need = need; if (d->bin != NULL) { if (d->s.avail_out == d->binsz) { driver_free_binary(d->bin); d->bin = NULL; d->binsz = 0; } else zlib_output(d); } } return zlib_return(Z_OK, rbuf, rlen); } case CRC32_0: return zlib_value(d->crc, rbuf, rlen); case CRC32_1: { uLong crc = crc32(0L, Z_NULL, 0); crc = crc32(crc, (unsigned char*) buf, len); return zlib_value(crc, rbuf, rlen); } case CRC32_2: { uLong crc; if (len < 4) goto badarg; crc = (unsigned int) i32(buf); crc = crc32(crc, (unsigned char*) buf+4, len-4); return zlib_value(crc, rbuf, rlen); } case ADLER32_1: { uLong adler = adler32(0L, Z_NULL, 0); adler = adler32(adler, (unsigned char*) buf, len); return zlib_value(adler, rbuf, rlen); } case ADLER32_2: { uLong adler; if (len < 4) goto badarg; adler = (unsigned int) i32(buf); adler = adler32(adler, (unsigned char*) buf+4, len-4); return zlib_value(adler, rbuf, rlen); } case CRC32_COMBINE: { uLong crc, crc1, crc2, len2; if (len != 12) goto badarg; crc1 = (unsigned int) i32(buf); crc2 = (unsigned int) i32(buf+4); len2 = (unsigned int) i32(buf+8); crc = crc32_combine(crc1, crc2, len2); return zlib_value(crc, rbuf, rlen); } case ADLER32_COMBINE: { uLong adler, adler1, adler2, len2; if (len != 12) goto badarg; adler1 = (unsigned int) i32(buf); adler2 = (unsigned int) i32(buf+4); len2 = (unsigned int) i32(buf+8); adler = adler32_combine(adler1, adler2, len2); return zlib_value(adler, rbuf, rlen); } } badarg: errno = EINVAL; return zlib_return(Z_ERRNO, rbuf, rlen); }
void main(void) { char* input_data = malloc(DATA_SIZE); FILE* input_file; input_file = fopen(INPUT_FILE, "r"); if(input_file == NULL){ fprintf(stderr, "Failed to open %s\n", INPUT_FILE); exit(1); } int i = 0; char buffer[BLOCK_SIZE]; while(fgets(buffer, BLOCK_SIZE, input_file)){ strcpy(input_data+i,buffer); i = i + BLOCK_SIZE - 1; } fclose(input_file); /*Replace the line field ascii with \0*/ input_data[strlen(input_data) - 1] = '\0'; struct stopwatch_t* sw = stopwatch_create(); /*--------------------------------------------------------------------------------------*/ stopwatch_init(); stopwatch_start(sw); printf("crcSlow() 0x%X ", crcSlow(input_data, strlen(input_data))); stopwatch_stop(sw); printf(" Time: %Lg\n", stopwatch_elapsed(sw)); /*--------------------------------------------------------------------------------------*/ stopwatch_start(sw); size_t input_data_len = strlen(input_data); int input_blocks = input_data_len / BLOCK_SIZE; int extra_blocks = 0; if(input_data_len % BLOCK_SIZE != 0) extra_blocks = 1; int total_blocks = input_blocks + extra_blocks; int *result = malloc(total_blocks * sizeof(int)); omp_set_num_threads(16); unsigned int ans = 0; char* block_data = malloc(input_blocks * (BLOCK_SIZE + 1)); char* block_addr; i = 0; #pragma omp parallel for default(none) shared(input_blocks, input_data, result, block_data) private (i, block_addr) for(i = 0; i < input_blocks; ++i){ block_addr = block_data + (BLOCK_SIZE + 1) * i; strncpy(block_addr, input_data + BLOCK_SIZE * i, BLOCK_SIZE); *(block_addr + BLOCK_SIZE) = '\0'; result[i] = CrcHash(block_addr, BLOCK_SIZE); } int rem = input_data_len % BLOCK_SIZE; char* last_block_data = malloc(rem + 1); if(extra_blocks == 1){ strncpy(last_block_data, input_data + BLOCK_SIZE * input_blocks, rem); *(last_block_data + rem) = '\0'; result[input_blocks] = CrcHash(last_block_data, rem); } i=0; for(i = 0; i < input_blocks; ++i){ ans = crc32_combine(ans, result[i], BLOCK_SIZE); } if(extra_blocks == 1) ans = crc32_combine(ans, result[i], rem); stopwatch_stop(sw); printf("Parallel() 0x%X Time: %Lg \n",ans, stopwatch_elapsed(sw)); /*--------------------------------------------------------------------------------------*/ crcInit(); stopwatch_start(sw); printf("crcFast() 0x%X ", crcFast(input_data, strlen(input_data))); stopwatch_stop(sw); printf(" Time: %Lg\n", stopwatch_elapsed(sw)); stopwatch_destroy(sw); /*--------------------------------------------------------------------------------------*/ stopwatch_start(sw); printf("crc_intel() 0x%X ", CrcHash((const void*)input_data, strlen(input_data))); stopwatch_stop(sw); printf(" Time: %Lg\n", stopwatch_elapsed(sw)); /*--------------------------------------------------------------------------------------*/ /*Cleanup*/ free(last_block_data); free(block_data); free(input_data); }