int32_t rstzip3::compress(rstf_unionT * buf, int32_t nrec) { // if interface buffer count is non-zero, copy in records to fill up buffer int32_t done = 0; if (interface_buffer_count) { int32_t n = (rz3_bufsize - interface_buffer_count); if (n > nrec) n = nrec; memcpy(interface_buffer+interface_buffer_count, buf, n*sizeof(rstf_unionT)); interface_buffer_count += n; if (interface_buffer_count == rz3_bufsize) { compress_buffer(interface_buffer, rz3_bufsize); interface_buffer_count = 0; } done += n; if (done == nrec) { return done; } } // at this point, there are no records waiting to be compressed in the buffer while((nrec-done) >= rz3_bufsize) { compress_buffer(buf+done, rz3_bufsize); done += rz3_bufsize; } // at this point, the buffer is empty; there may be some records left to be // compressed, but not enough to fill the buffer if (done < nrec) { memcpy(interface_buffer, buf+done, (nrec-done)*sizeof(rstf_unionT)); interface_buffer_count += (nrec-done); } return nrec; } // int32_t rstzip3::compress(rstf_unionT * buf, int32_t nrec)
static int enc_mac_write(int fd, uint8_t *enc_key, uint8_t *mac_key, uint8_t *salt, uint8_t *buf, int64_t data_size) { uint8_t nonce[NONCE_SIZE]; uint8_t *new_mac = NULL; uint8_t *enc_buf = NULL; uint8_t *comp_buf = NULL; unsigned long comp_size = 0; int ret; comp_buf = compress_buffer(buf, data_size, &comp_size); if (!comp_buf) goto fail; data_size = comp_size; buf = comp_buf; ret = create_salt(nonce, NONCE_SIZE); if (ret != 0) goto fail; enc_buf = calloc(NONCE_SIZE + data_size, 1); if (!enc_buf) goto fail; memcpy(enc_buf, nonce, NONCE_SIZE); encrypt_data(nonce, enc_key, data_size, buf, enc_buf + NONCE_SIZE); new_mac = create_mac(mac_key, NONCE_SIZE + data_size, enc_buf); if (!new_mac) goto fail; ret = write_file(fd, salt, new_mac, nonce, enc_buf + NONCE_SIZE, data_size); return CAPSTONE_EXIT_SUCCESS; fail: return CAPSTONE_EXIT_FAILURE; }
/** * Inits and returns a newly hash_data_t structure. * @returns a newly hash_data_t structure. */ hash_data_t *new_hash_data_t(guchar *data, gssize size_read, guint8 *hash, gshort cmptype) { hash_data_t *hash_data = NULL; compress_t *compress = NULL; hash_data = (hash_data_t *) g_malloc(sizeof(hash_data_t)); g_assert_nonnull(hash_data); if (cmptype != COMPRESS_NONE_TYPE && data != NULL) { compress = compress_buffer(data, size_read, (gint) cmptype); hash_data->data = compress->text; hash_data->read = compress->len; hash_data->uncmplen = size_read; free_variable(compress); /* do not free compress->text as its reference is now used in hash_data->data. */ /* data variable is not used anymore (it has been replaced by compress->text) it has to be freed by the caller */ } else { hash_data->data = data; hash_data->read = size_read; hash_data->uncmplen = size_read; } hash_data->hash = hash; hash_data->cmptype = cmptype; return hash_data; }
static void write_file(Bytef *source, uLongf sourceLen) { Bytef *d_ptr, *dest; uLongf d_len, destLen; struct timeval timer_start, timer_stop; /* destination buffer needs to be at least 0.1% larger than sourceLen * plus 12 bytes */ destLen = (uLongf)((double)sourceLen + ((double)sourceLen * 0.1)) + 12; dest = (Bytef *)HDmalloc(destLen); if (!dest) error("out of memory"); HDgettimeofday(&timer_start, NULL); compress_buffer(dest, &destLen, source, sourceLen); HDgettimeofday(&timer_stop, NULL); compression_time += ((double)timer_stop.tv_sec + ((double)timer_stop.tv_usec) / MICROSECOND) - ((double)timer_start.tv_sec + ((double)timer_start.tv_usec) / MICROSECOND); if (report_once_flag) { HDfprintf(stdout, "\tCompression Ratio: %g\n", ((double)destLen) / (double)sourceLen); report_once_flag = 0; } d_ptr = dest; d_len = destLen; /* loop to make sure we write everything out that we want to write */ for (;;) { int rc = (int)HDwrite(output, d_ptr, (size_t)d_len); if (rc == -1) error(HDstrerror(errno)); if (rc == (int)d_len) break; d_len -= rc; d_ptr += rc; } HDfree(dest); }
rstzip3::~rstzip3() { if (c_nd) { if (interface_buffer_count != 0) { compress_buffer(interface_buffer, interface_buffer_count); interface_buffer_count = 0; } } else { // caller did not wait to read all decompressed records. ignore } if (sdata != NULL) { if (verbose) sdata->print_totals(); if (verbose) { int i; for (i=0; i<rz3_max_ncpus; i++) { if (tdata[i] != NULL) { tdata[i]->valuecache->Report(stderr); } } } if (stats) print_stat_totals(); } if (gzf != NULL) { gzclose(gzf); gzf = NULL; } delete shdr; shdr = NULL; delete sdata; sdata = NULL; int i; for (i=0; i<(rz3_max_ncpus); i++) { if (tdata[i] != NULL) { delete tdata[i]; tdata[i] = NULL; } } delete [] tdata; tdata = NULL; if (interface_buffer != NULL) { delete [] interface_buffer; } } // rstzip3::~rstzip3()
/* * Create the compression buffer, and fill it with the CTF and string * table data. We flush the compression state between the two so the * dictionary used for the string tables won't be polluted with values * that made sense for the CTF data. */ static caddr_t write_compressed_buffer(ctf_header_t *h, ctf_buf_t *buf, size_t *resszp) { resbuf_t resbuf; resbuf.rb_size = RES_BUF_CHUNK_SIZE; resbuf.rb_base = xmalloc(resbuf.rb_size); bcopy(h, resbuf.rb_base, sizeof (ctf_header_t)); resbuf.rb_ptr = resbuf.rb_base + sizeof (ctf_header_t); compress_start(&resbuf); (void) compress_buffer(buf->ctb_base, buf->ctb_ptr - buf->ctb_base, &resbuf); compress_flush(&resbuf, Z_FULL_FLUSH); (void) strtab_write(&buf->ctb_strtab, compress_buffer, &resbuf); compress_end(&resbuf); *resszp = (resbuf.rb_ptr - resbuf.rb_base); return (resbuf.rb_base); }
void wwd_write(const wap_wwd *wwd, std::vector<char> &out_wwd_buffer) { wwd_offsets offsets; std::vector<wwd_plane_offsets> planes_offsets; calculate_offsets(offsets, planes_offsets, *wwd); std::vector<char> wwd_buffer; const wap_wwd_properties &wwdp = wwd->properties; if (wwdp.flags & WAP_WWD_FLAG_COMPRESS) { std::vector<char> main_block(offsets.eof_offset - offsets.main_block_offset); wap::OutputStream main_block_stream(main_block.data(), main_block.size()); write_main_block(main_block_stream, wwd->planes, planes_offsets, wwd->tile_descriptions); std::vector<char> compressed_main_block = compress_buffer(main_block.data(), main_block.size()); wwd_buffer.resize(offsets.main_block_offset + compressed_main_block.size()); wap::OutputStream stream(wwd_buffer.data(), wwd_buffer.size()); uint32_t checksum = wwd_checksum(main_block.data(), main_block.size()); write_header(stream, *wwd, offsets, main_block.size(), checksum); stream.write_buffer(compressed_main_block.data(), compressed_main_block.size()); } else { wwd_buffer.resize(offsets.eof_offset); wap::OutputStream stream(wwd_buffer.data(), wwd_buffer.size()); stream.seek(offsets.main_block_offset); write_main_block(stream, wwd->planes, planes_offsets, wwd->tile_descriptions); char *main_block = wwd_buffer.data() + offsets.main_block_offset; size_t main_block_size = wwd_buffer.size() - offsets.main_block_offset; uint32_t checksum = wwd_checksum(main_block, main_block_size); stream.seek(0); write_header(stream, *wwd, offsets, 0, checksum); } std::swap(wwd_buffer, out_wwd_buffer); }
static int queue_envelope_dump_buffer(struct envelope *ep, char *evpbuf, size_t evpbufsize) { char evpbufcom[sizeof(struct envelope)]; char *evp; size_t evplen; evp = evpbuf; evplen = envelope_dump_buffer(ep, evpbuf, evpbufsize); if (evplen == 0) return (0); if (env->sc_queue_flags & QUEUE_COMPRESS) { evplen = compress_buffer(evp, evplen, evpbufcom, sizeof evpbufcom); if (evplen == 0) return (0); evp = evpbufcom; } memmove(evpbuf, evp, evplen); return (evplen); }
static int packet_send2(ssh_session session) { unsigned int blocksize = (session->current_crypto ? session->current_crypto->out_cipher->blocksize : 8); uint32_t currentlen = buffer_get_rest_len(session->out_buffer); unsigned char *hmac = NULL; char padstring[32] = {0}; int rc = SSH_ERROR; uint32_t finallen,payloadsize,compsize; uint8_t padding; payloadsize = currentlen; #ifdef WITH_ZLIB if (session->current_crypto && session->current_crypto->do_compress_out && buffer_get_rest_len(session->out_buffer)) { if (compress_buffer(session,session->out_buffer) < 0) { goto error; } currentlen = buffer_get_rest_len(session->out_buffer); } #endif /* WITH_ZLIB */ compsize = currentlen; padding = (blocksize - ((currentlen +5) % blocksize)); if(padding < 4) { padding += blocksize; } if (session->current_crypto) { ssh_get_random(padstring, padding, 0); } else { memset(padstring,0,padding); } finallen = htonl(currentlen + padding + 1); if (buffer_prepend_data(session->out_buffer, &padding, sizeof(uint8_t)) < 0) { goto error; } if (buffer_prepend_data(session->out_buffer, &finallen, sizeof(uint32_t)) < 0) { goto error; } if (buffer_add_data(session->out_buffer, padstring, padding) < 0) { goto error; } #ifdef WITH_PCAP if(session->pcap_ctx){ ssh_pcap_context_write(session->pcap_ctx,SSH_PCAP_DIR_OUT, buffer_get_rest(session->out_buffer),buffer_get_rest_len(session->out_buffer) ,buffer_get_rest_len(session->out_buffer)); } #endif hmac = packet_encrypt(session, buffer_get_rest(session->out_buffer), buffer_get_rest_len(session->out_buffer)); if (hmac) { if (buffer_add_data(session->out_buffer, hmac, 20) < 0) { goto error; } } rc = ssh_packet_write(session); session->send_seq++; SSH_LOG(SSH_LOG_PACKET, "packet: wrote [len=%d,padding=%hhd,comp=%d,payload=%d]", ntohl(finallen), padding, compsize, payloadsize); if (buffer_reinit(session->out_buffer) < 0) { rc = SSH_ERROR; } error: return rc; /* SSH_OK, AGAIN or ERROR */ }
static int packet_send2(ssh_session session) { unsigned int blocksize = (session->current_crypto ? session->current_crypto->out_cipher->blocksize : 8); uint32_t currentlen = buffer_get_rest_len(session->out_buffer); unsigned char *hmac = NULL; char padstring[32] = {0}; int rc = SSH_ERROR; uint32_t finallen; uint8_t padding; enter_function(); ssh_log(session, SSH_LOG_PACKET, "Writing on the wire a packet having %u bytes before", currentlen); #if defined(HAVE_LIBZ) && defined(WITH_LIBZ) if (session->current_crypto && session->current_crypto->do_compress_out && buffer_get_rest_len(session->out_buffer)) { ssh_log(session, SSH_LOG_PACKET, "Compressing out_buffer ..."); if (compress_buffer(session,session->out_buffer) < 0) { goto error; } currentlen = buffer_get_rest_len(session->out_buffer); } #endif padding = (blocksize - ((currentlen +5) % blocksize)); if(padding < 4) { padding += blocksize; } if (session->current_crypto) { ssh_get_random(padstring, padding, 0); } else { memset(padstring,0,padding); } finallen = htonl(currentlen + padding + 1); ssh_log(session, SSH_LOG_PACKET, "%d bytes after comp + %d padding bytes = %lu bytes packet", currentlen, padding, (long unsigned int) ntohl(finallen)); if (buffer_prepend_data(session->out_buffer, &padding, sizeof(uint8_t)) < 0) { goto error; } if (buffer_prepend_data(session->out_buffer, &finallen, sizeof(uint32_t)) < 0) { goto error; } if (buffer_add_data(session->out_buffer, padstring, padding) < 0) { goto error; } #ifdef WITH_PCAP if(session->pcap_ctx){ ssh_pcap_context_write(session->pcap_ctx,SSH_PCAP_DIR_OUT, buffer_get_rest(session->out_buffer),buffer_get_rest_len(session->out_buffer) ,buffer_get_rest_len(session->out_buffer)); } #endif hmac = packet_encrypt(session, buffer_get_rest(session->out_buffer), buffer_get_rest_len(session->out_buffer)); if (hmac) { if (buffer_add_data(session->out_buffer, hmac, 20) < 0) { goto error; } } rc = ssh_packet_write(session); session->send_seq++; if (buffer_reinit(session->out_buffer) < 0) { rc = SSH_ERROR; } error: leave_function(); return rc; /* SSH_OK, AGAIN or ERROR */ }
static int packet_send2(ssh_session session) { unsigned int blocksize = (session->current_crypto ? session->current_crypto->out_cipher->blocksize : 8); unsigned int lenfield_blocksize = (session->current_crypto ? session->current_crypto->out_cipher->lenfield_blocksize : 0); enum ssh_hmac_e hmac_type = (session->current_crypto ? session->current_crypto->out_hmac : session->next_crypto->out_hmac); uint32_t currentlen = ssh_buffer_get_len(session->out_buffer); unsigned char *hmac = NULL; char padstring[32] = { 0 }; int rc = SSH_ERROR; uint32_t finallen,payloadsize,compsize; uint8_t padding; ssh_buffer header_buffer = ssh_buffer_new(); payloadsize = currentlen; #ifdef WITH_ZLIB if (session->current_crypto && session->current_crypto->do_compress_out && ssh_buffer_get_len(session->out_buffer)) { if (compress_buffer(session,session->out_buffer) < 0) { goto error; } currentlen = ssh_buffer_get_len(session->out_buffer); } #endif /* WITH_ZLIB */ compsize = currentlen; /* compressed payload + packet len (4) + padding len (1) */ /* totallen - lenfield_blocksize must be equal to 0 (mod blocksize) */ padding = (blocksize - ((blocksize - lenfield_blocksize + currentlen + 5) % blocksize)); if(padding < 4) { padding += blocksize; } if (session->current_crypto != NULL) { int ok; ok = ssh_get_random(padstring, padding, 0); if (!ok) { ssh_set_error(session, SSH_FATAL, "PRNG error"); goto error; } } if (header_buffer == NULL){ ssh_set_error_oom(session); goto error; } finallen = currentlen + padding + 1; rc = ssh_buffer_pack(header_buffer, "db", finallen, padding); if (rc == SSH_ERROR){ goto error; } rc = ssh_buffer_prepend_data(session->out_buffer, ssh_buffer_get(header_buffer), ssh_buffer_get_len(header_buffer)); if (rc < 0) { goto error; } rc = ssh_buffer_add_data(session->out_buffer, padstring, padding); if (rc < 0) { goto error; } #ifdef WITH_PCAP if (session->pcap_ctx) { ssh_pcap_context_write(session->pcap_ctx, SSH_PCAP_DIR_OUT, ssh_buffer_get(session->out_buffer), ssh_buffer_get_len(session->out_buffer), ssh_buffer_get_len(session->out_buffer)); } #endif hmac = ssh_packet_encrypt(session, ssh_buffer_get(session->out_buffer), ssh_buffer_get_len(session->out_buffer)); if (hmac) { rc = ssh_buffer_add_data(session->out_buffer, hmac, hmac_digest_len(hmac_type)); if (rc < 0) { goto error; } } rc = ssh_packet_write(session); session->send_seq++; if (session->raw_counter != NULL) { session->raw_counter->out_bytes += payloadsize; session->raw_counter->out_packets++; } SSH_LOG(SSH_LOG_PACKET, "packet: wrote [len=%d,padding=%hhd,comp=%d,payload=%d]", finallen, padding, compsize, payloadsize); if (ssh_buffer_reinit(session->out_buffer) < 0) { rc = SSH_ERROR; } error: if (header_buffer != NULL) { ssh_buffer_free(header_buffer); } return rc; /* SSH_OK, AGAIN or ERROR */ }
static int packet_send2(ssh_session session) { unsigned int blocksize = (session->current_crypto ? session->current_crypto->out_cipher->blocksize : 8); enum ssh_hmac_e hmac_type = (session->current_crypto ? session->current_crypto->out_hmac : session->next_crypto->out_hmac); uint32_t currentlen = ssh_buffer_get_len(session->out_buffer); unsigned char *hmac = NULL; char padstring[32] = { 0 }; int rc = SSH_ERROR; uint32_t finallen,payloadsize,compsize; uint8_t padding; uint8_t header[sizeof(padding) + sizeof(finallen)] = { 0 }; payloadsize = currentlen; #ifdef WITH_ZLIB if (session->current_crypto && session->current_crypto->do_compress_out && ssh_buffer_get_len(session->out_buffer)) { if (compress_buffer(session,session->out_buffer) < 0) { goto error; } currentlen = ssh_buffer_get_len(session->out_buffer); } #endif /* WITH_ZLIB */ compsize = currentlen; padding = (blocksize - ((currentlen +5) % blocksize)); if(padding < 4) { padding += blocksize; } if (session->current_crypto) { ssh_get_random(padstring, padding, 0); } finallen = htonl(currentlen + padding + 1); memcpy(&header[0], &finallen, sizeof(finallen)); header[sizeof(finallen)] = padding; rc = ssh_buffer_prepend_data(session->out_buffer, &header, sizeof(header)); if (rc < 0) { goto error; } rc = ssh_buffer_add_data(session->out_buffer, padstring, padding); if (rc < 0) { goto error; } #ifdef WITH_PCAP if(session->pcap_ctx){ ssh_pcap_context_write(session->pcap_ctx,SSH_PCAP_DIR_OUT, ssh_buffer_get(session->out_buffer),ssh_buffer_get_len(session->out_buffer) ,ssh_buffer_get_len(session->out_buffer)); } #endif hmac = ssh_packet_encrypt(session, ssh_buffer_get(session->out_buffer), ssh_buffer_get_len(session->out_buffer)); if (hmac) { rc = ssh_buffer_add_data(session->out_buffer, hmac, hmac_digest_len(hmac_type)); if (rc < 0) { goto error; } } rc = ssh_packet_write(session); session->send_seq++; if (session->raw_counter != NULL) { session->raw_counter->out_bytes += payloadsize; session->raw_counter->out_packets++; } SSH_LOG(SSH_LOG_PACKET, "packet: wrote [len=%d,padding=%hhd,comp=%d,payload=%d]", ntohl(finallen), padding, compsize, payloadsize); if (ssh_buffer_reinit(session->out_buffer) < 0) { rc = SSH_ERROR; } error: return rc; /* SSH_OK, AGAIN or ERROR */ }
//========================================================================== void readfile(std::string filename, std::string outfile) { int num_events(0); int num_badevents(0); int num_baduncompress(0); int num_badchksum(0); int num_goodevents(0); int num_duplevents(0); uint32 hltcount(0); std::vector<uint32> hltStats(0); std::vector<unsigned char> compress_buffer(7000000); std::map<uint32, uint32> seenEventMap; bool output(false); if(outfile != "/dev/null") { output = true; } StreamerOutputFile stream_output(outfile); try{ // ----------- init edm::StreamerInputFile stream_reader(filename); //if(output) StreamerOutputFile stream_output(outfile); std::cout << "Trying to Read The Init message from Streamer File: " << std::endl << filename << std::endl; InitMsgView const* init = stream_reader.startMessage(); std::cout << "\n\n-------------INIT Message---------------------" << std::endl; std::cout << "Dump the Init Message from Streamer:-" << std::endl; dumpInitView(init); if(output) { stream_output.write(*init); hltcount = init->get_hlt_bit_cnt(); //Initialize the HLT Stat vector with all ZEROs for(uint32 i = 0; i != hltcount; ++i) hltStats.push_back(0); } // ------- event std::cout << "\n\n-------------EVENT Messages-------------------" << std::endl; bool first_event(true); std::auto_ptr<EventMsgView> firstEvtView(0); std::vector<unsigned char> savebuf(0); EventMsgView const* eview(0); seenEventMap.clear(); while(stream_reader.next()) { eview = stream_reader.currentRecord(); ++num_events; bool good_event(true); if(seenEventMap.find(eview->event()) == seenEventMap.end()) { seenEventMap.insert(std::make_pair(eview->event(), 1)); } else { ++seenEventMap[eview->event()]; ++num_duplevents; std::cout << "??????? duplicate event Id for count " << num_events << " event number " << eview->event() << " seen " << seenEventMap[eview->event()] << " times" << std::endl; } if(first_event) { std::cout << "----------dumping first EVENT-----------" << std::endl; dumpEventView(eview); first_event = false; unsigned char* src = (unsigned char*)eview->startAddress(); unsigned int srcSize = eview->size(); savebuf.resize(srcSize); std::copy(src, src+srcSize, &(savebuf)[0]); firstEvtView.reset(new EventMsgView(&(savebuf)[0])); //firstEvtView, reset(new EventMsgView((void*)eview->startAddress())); if(!test_chksum(firstEvtView.get())) { std::cout << "checksum error for count " << num_events << " event number " << eview->event() << " from host name " << eview->hostName() << std::endl; ++num_badchksum; std::cout << "----------dumping bad checksum EVENT-----------" << std::endl; dumpEventView(eview); good_event = false; } if(!test_uncompress(firstEvtView.get(), compress_buffer)) { std::cout << "uncompress error for count " << num_events << " event number " << firstEvtView->event() << std::endl; ++num_baduncompress; std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl; dumpEventView(firstEvtView.get()); good_event = false; } } else { if(compares_bad(firstEvtView.get(), eview)) { std::cout << "Bad event at count " << num_events << " dumping event " << std::endl << "----------dumping bad EVENT-----------" << std::endl; dumpEventView(eview); ++num_badevents; good_event = false; } if(!test_chksum(eview)) { std::cout << "checksum error for count " << num_events << " event number " << eview->event() << " from host name " << eview->hostName() << std::endl; ++num_badchksum; std::cout << "----------dumping bad checksum EVENT-----------" << std::endl; dumpEventView(eview); good_event = false; } if(!test_uncompress(eview, compress_buffer)) { std::cout << "uncompress error for count " << num_events << " event number " << eview->event() << std::endl; ++num_baduncompress; std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl; dumpEventView(eview); good_event = false; } } if(output && good_event) { ++num_goodevents; stream_output.write(*eview); //get the HLT Packed bytes std::vector<uint8> packedHlt; uint32 hlt_sz = 0; if(hltcount != 0) hlt_sz = 1 + ((hltcount - 1)/4); packedHlt.resize(hlt_sz); firstEvtView->hltTriggerBits(&packedHlt[0]); updateHLTStats(packedHlt, hltcount, hltStats); } if((num_events % 50) == 0) { std::cout << "Read " << num_events << " events, and " << num_badevents << " events with bad headers, and " << num_badchksum << " events with bad check sum, and " << num_baduncompress << " events with bad uncompress" << std::endl; if(output) std::cout << "Wrote " << num_goodevents << " good events " << std::endl; } } std::cout << std::endl << "------------END--------------" << std::endl << "read " << num_events << " events" << std::endl << "and " << num_badevents << " events with bad headers" << std::endl << "and " << num_badchksum << " events with bad check sum" << std::endl << "and " << num_baduncompress << " events with bad uncompress" << std::endl << "and " << num_duplevents << " duplicated event Id" << std::endl; if(output) { uint32 dummyStatusCode = 1234; stream_output.writeEOF(dummyStatusCode, hltStats); std::cout << "Wrote " << num_goodevents << " good events " << std::endl; } }catch(cms::Exception& e){ std::cerr << "Exception caught: " << e.what() << std::endl << "After reading " << num_events << " events, and " << num_badevents << " events with bad headers" << std::endl << "and " << num_badchksum << " events with bad check sum" << std::endl << "and " << num_baduncompress << " events with bad uncompress" << std::endl << "and " << num_duplevents << " duplicated event Id" << std::endl; } }