//--------------------------------------------------------------------- void DeflateStream::compressFinal() { // Close temp stream mTmpWriteStream->close(); // Copy & compress // We do this rather than compress directly because some code seeks // around while writing (e.g. to update size blocks) which is not // possible when compressing on the fly int ret, flush; char in[OGRE_DEFLATE_TMP_SIZE]; char out[OGRE_DEFLATE_TMP_SIZE]; if (deflateInit(mZStream, Z_DEFAULT_COMPRESSION) != Z_OK) { destroy(); OGRE_EXCEPT(Exception::ERR_INVALID_STATE, "Error initialising deflate compressed stream!", "DeflateStream::init"); } std::ifstream inFile; inFile.open(mTempFileName.c_str(), std::ios::in | std::ios::binary); do { inFile.read(in, OGRE_DEFLATE_TMP_SIZE); mZStream->avail_in = (uInt)inFile.gcount(); if (inFile.bad()) { deflateEnd(mZStream); OGRE_EXCEPT(Exception::ERR_INVALID_STATE, "Error reading temp uncompressed stream!", "DeflateStream::init"); } flush = inFile.eof() ? Z_FINISH : Z_NO_FLUSH; mZStream->next_in = (Bytef*)in; /* run deflate() on input until output buffer not full, finish compression if all of source has been read in */ do { mZStream->avail_out = OGRE_DEFLATE_TMP_SIZE; mZStream->next_out = (Bytef*)out; ret = deflate(mZStream, flush); /* no bad return value */ assert(ret != Z_STREAM_ERROR); /* state not clobbered */ size_t compressed = OGRE_DEFLATE_TMP_SIZE - mZStream->avail_out; mCompressedStream->write(out, compressed); } while (mZStream->avail_out == 0); assert(mZStream->avail_in == 0); /* all input will be used */ /* done when last data in file processed */ } while (flush != Z_FINISH); assert(ret == Z_STREAM_END); /* stream will be complete */ (void)ret; deflateEnd(mZStream); inFile.close(); remove(mTempFileName.c_str()); }
int lws_extension_callback_deflate_stream( struct libwebsocket_context *context, struct libwebsocket_extension *ext, struct libwebsocket *wsi, enum libwebsocket_extension_callback_reasons reason, void *user, void *in, size_t len) { struct lws_ext_deflate_stream_conn *conn = (struct lws_ext_deflate_stream_conn *)user; int n; struct lws_tokens *eff_buf = (struct lws_tokens *)in; switch (reason) { /* * for deflate-stream, both client and server sides act the same */ case LWS_EXT_CALLBACK_CLIENT_CONSTRUCT: case LWS_EXT_CALLBACK_CONSTRUCT: conn->zs_in.zalloc = conn->zs_out.zalloc = Z_NULL; conn->zs_in.zfree = conn->zs_out.zfree = Z_NULL; conn->zs_in.opaque = conn->zs_out.opaque = Z_NULL; n = inflateInit2(&conn->zs_in, -LWS_ZLIB_WINDOW_BITS); if (n != Z_OK) { lws_log(LWS_LOG_WARNING, "deflateInit returned %d", n); return 1; } n = deflateInit2(&conn->zs_out, DEFLATE_STREAM_COMPRESSION_LEVEL, Z_DEFLATED, -LWS_ZLIB_WINDOW_BITS, LWS_ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); if (n != Z_OK) { lws_log(LWS_LOG_WARNING, "deflateInit returned %d", n); return 1; } conn->remaining_in = 0; lws_log(LWS_LOG_DEBUG, "zlibs constructed"); break; case LWS_EXT_CALLBACK_DESTROY: (void)inflateEnd(&conn->zs_in); (void)deflateEnd(&conn->zs_out); lws_log(LWS_LOG_DEBUG, "zlibs destructed"); break; case LWS_EXT_CALLBACK_PACKET_RX_PREPARSE: /* * inflate the incoming compressed data * Notice, length may be 0 and pointer NULL * in the case we are flushing with nothing new coming in */ if (conn->remaining_in) { conn->zs_in.next_in = conn->buf_in; conn->zs_in.avail_in = conn->remaining_in; conn->remaining_in = 0; } else { conn->zs_in.next_in = (unsigned char *)eff_buf->token; conn->zs_in.avail_in = eff_buf->token_len; } conn->zs_in.next_out = conn->buf_out; conn->zs_in.avail_out = sizeof(conn->buf_out); n = inflate(&conn->zs_in, Z_SYNC_FLUSH); switch (n) { case Z_NEED_DICT: case Z_DATA_ERROR: case Z_MEM_ERROR: /* * screwed.. close the connection... we will get a * destroy callback to take care of closing nicely */ lws_log(LWS_LOG_ERROR, "zlib error inflate %d", n); return -1; } /* rewrite the buffer pointers and length */ eff_buf->token = (char *)conn->buf_out; eff_buf->token_len = sizeof(conn->buf_out) - conn->zs_in.avail_out; /* copy avail data if not consumed */ if (conn->zs_in.avail_in > 0) { conn->remaining_in = conn->zs_in.avail_in; memcpy(conn->buf_in, conn->zs_in.next_in, conn->zs_in.avail_in); return 1; } /* * if we filled the output buffer, signal that we likely have * more and need to be called again */ if (eff_buf->token_len == sizeof(conn->buf_out)) return 1; /* we don't need calling again until new input data comes */ return 0; case LWS_EXT_CALLBACK_FLUSH_PENDING_TX: case LWS_EXT_CALLBACK_PACKET_TX_PRESEND: /* * deflate the outgoing compressed data */ conn->zs_out.next_in = (unsigned char *)eff_buf->token; conn->zs_out.avail_in = eff_buf->token_len; conn->zs_out.next_out = conn->buf_out; conn->zs_out.avail_out = sizeof(conn->buf_out); n = Z_PARTIAL_FLUSH; if (reason == LWS_EXT_CALLBACK_FLUSH_PENDING_TX) n = Z_FULL_FLUSH; n = deflate(&conn->zs_out, n); if (n == Z_STREAM_ERROR) { /* * screwed.. close the connection... we will get a * destroy callback to take care of closing nicely */ lws_log(LWS_LOG_ERROR, "zlib error deflate"); return -1; } /* rewrite the buffer pointers and length */ eff_buf->token = (char *)conn->buf_out; eff_buf->token_len = sizeof(conn->buf_out) - conn->zs_out.avail_out; /* * if we filled the output buffer, signal that we likely have * more and need to be called again... even in deflate case * we might sometimes need to spill more than came in */ if (eff_buf->token_len == sizeof(conn->buf_out)) return 1; /* we don't need calling again until new input data comes */ return 0; default: break; } return 0; }
/*! */ bool gzfilterstreambuf::writeData( int flush_type ) { // size of data to write int size = ( pptr() - pbase() ) * sizeof( char_type ); if ( size == 0 ) { return true; } if ( M_output_stream == NULL ) { //std::cerr << "create stream" << std::endl; M_output_stream = new std::ostream( &M_strmbuf ); } #ifdef HAVE_LIBZ if ( M_level <= NO_COMPRESSION || BEST_COMPRESSION < M_level ) { M_output_stream->write( M_output_buf, size ); } else { if ( M_impl->comp_stream_ == NULL ) { M_impl->comp_stream_ = new z_stream; M_impl->comp_stream_->zalloc = Z_NULL; M_impl->comp_stream_->zfree = Z_NULL; M_impl->comp_stream_->opaque = NULL; M_impl->comp_stream_->avail_in = 0; M_impl->comp_stream_->next_in = 0; M_impl->comp_stream_->next_out = 0; M_impl->comp_stream_->avail_out = 0; if ( deflateInit( M_impl->comp_stream_, M_level ) != Z_OK ) { return false; } if ( M_write_buf == NULL ) { //std::cerr << "create write buf" << std::endl; M_write_buf = new char[ M_buf_size ]; } M_impl->comp_stream_->next_out = (Bytef*)M_write_buf; M_impl->comp_stream_->avail_out = M_buf_size; } M_impl->comp_stream_->next_in = (Bytef*)M_output_buf; M_impl->comp_stream_->avail_in = size; do { int bytes_out = - M_impl->comp_stream_->total_out; int err = deflate( M_impl->comp_stream_, flush_type ); if ( err != Z_OK && err != Z_STREAM_END ) { //std::cerr << "error deflating [" // << M_impl->comp_stream_->msg << "]" // << std::endl; return false; } bytes_out += M_impl->comp_stream_->total_out; //std::cerr << "compress loop size = " << bytes_out << std::endl; M_output_stream->write( M_write_buf, bytes_out ); M_impl->comp_stream_->next_out = (Bytef*)M_write_buf; M_impl->comp_stream_->avail_out = M_buf_size; } while ( M_impl->comp_stream_->avail_in != 0 ); // we want to keep writing until all the data has been // consumed by the compression stream deflateReset( M_impl->comp_stream_ ); } #else M_output_stream->write( M_output_buf, size ); #endif if ( flush_type != NO_FLUSH ) // == Z_NO_FLUSH { // flush the underlying stream if a flush has been used M_output_stream->flush(); } return true; }
int zipfile_tool(int argc, const char *argv[]) { const char *pMode = NULL; FILE *pInfile = NULL, *pOutfile = NULL; uint infile_size = 0; int level = Z_BEST_COMPRESSION; z_stream stream; int p = 1; const char *pSrc_filename = NULL; const char *pDst_filename = NULL; long file_loc = 0; if(SystemFlags::VERBOSE_MODE_ENABLED) printf("miniz.c version: %s\n", MZ_VERSION); if (argc < 4) { if(SystemFlags::VERBOSE_MODE_ENABLED) { printf("Usage: example3 [options] [mode:c or d] infile outfile\n"); printf("\nModes:\n"); printf("c - Compresses file infile to a zlib stream in file outfile\n"); printf("d - Decompress zlib stream in file infile to file outfile\n"); printf("\nOptions:\n"); printf("-l[0-10] - Compression level, higher values are slower.\n"); } return EXIT_FAILURE; } while ((p < argc) && (argv[p][0] == '-')) { switch (argv[p][1]) { case 'l': { level = atoi(&argv[1][2]); if ((level < 0) || (level > 10)) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Invalid level!\n"); return EXIT_FAILURE; } break; } default: { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Invalid option: %s\n", argv[p]); return EXIT_FAILURE; } } p++; } if ((argc - p) < 3) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Must specify mode, input filename, and output filename after options!\n"); return EXIT_FAILURE; } else if ((argc - p) > 3) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Too many filenames!\n"); return EXIT_FAILURE; } pMode = argv[p++]; if (!strchr("cCdD", pMode[0])) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Invalid mode!\n"); if(SystemFlags::VERBOSE_MODE_ENABLED) return EXIT_FAILURE; } pSrc_filename = argv[p++]; pDst_filename = argv[p++]; if(SystemFlags::VERBOSE_MODE_ENABLED)printf("Mode: %c, Level: %d\nInput File: \"%s\"\nOutput File: \"%s\"\n", pMode[0], level, pSrc_filename, pDst_filename); // Open input file. pInfile = fopen(pSrc_filename, "rb"); if (!pInfile) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Failed opening input file!\n"); return EXIT_FAILURE; } // Determine input file's size. fseek(pInfile, 0, SEEK_END); file_loc = ftell(pInfile); fseek(pInfile, 0, SEEK_SET); if ((file_loc < 0) || (file_loc > INT_MAX)) { // This is not a limitation of miniz or tinfl, but this example. printf("File is too large to be processed by this example.\n"); fclose(pInfile); return EXIT_FAILURE; } infile_size = (uint)file_loc; // Open output file. pOutfile = fopen(pDst_filename, "wb"); if (!pOutfile) { printf("Failed opening output file!\n"); fclose(pInfile); return EXIT_FAILURE; } if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Input file size: %u\n", infile_size); // Init the z_stream memset(&stream, 0, sizeof(stream)); stream.next_in = s_inbuf; stream.avail_in = 0; stream.next_out = s_outbuf; stream.avail_out = BUF_SIZE; if ((pMode[0] == 'c') || (pMode[0] == 'C')) { // Compression. uint infile_remaining = infile_size; if (deflateInit(&stream, level) != Z_OK) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("deflateInit() failed!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } for ( ; ; ) { int status; if (!stream.avail_in) { // Input buffer is empty, so read more bytes from input file. uint n = my_min(BUF_SIZE, infile_remaining); if (fread(s_inbuf, 1, n, pInfile) != n) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Failed reading from input file!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } stream.next_in = s_inbuf; stream.avail_in = n; infile_remaining -= n; //printf("Input bytes remaining: %u\n", infile_remaining); } status = deflate(&stream, infile_remaining ? Z_NO_FLUSH : Z_FINISH); if ((status == Z_STREAM_END) || (!stream.avail_out)) { // Output buffer is full, or compression is done, so write buffer to output file. uint n = BUF_SIZE - stream.avail_out; if (fwrite(s_outbuf, 1, n, pOutfile) != n) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Failed writing to output file!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } stream.next_out = s_outbuf; stream.avail_out = BUF_SIZE; } if (status == Z_STREAM_END) { break; } else if (status != Z_OK) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("deflate() failed with status %i!\n", status); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } } if (deflateEnd(&stream) != Z_OK) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("deflateEnd() failed!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } } else if ((pMode[0] == 'd') || (pMode[0] == 'D')) { // Decompression. uint infile_remaining = infile_size; if (inflateInit(&stream)) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("inflateInit() failed!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } for ( ; ; ) { int status; if (!stream.avail_in) { // Input buffer is empty, so read more bytes from input file. uint n = my_min(BUF_SIZE, infile_remaining); if (fread(s_inbuf, 1, n, pInfile) != n) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Failed reading from input file!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } stream.next_in = s_inbuf; stream.avail_in = n; infile_remaining -= n; } status = inflate(&stream, Z_SYNC_FLUSH); if ((status == Z_STREAM_END) || (!stream.avail_out)) { // Output buffer is full, or decompression is done, so write buffer to output file. uint n = BUF_SIZE - stream.avail_out; if (fwrite(s_outbuf, 1, n, pOutfile) != n) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Failed writing to output file!\n"); return EXIT_FAILURE; } stream.next_out = s_outbuf; stream.avail_out = BUF_SIZE; } if (status == Z_STREAM_END) { break; } else if (status != Z_OK) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("inflate() failed with status %i!\n", status); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } } if (inflateEnd(&stream) != Z_OK) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("inflateEnd() failed!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } } else { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Invalid mode!\n"); if(pInfile) fclose(pInfile); if(pOutfile) fclose(pOutfile); return EXIT_FAILURE; } fclose(pInfile); pInfile = 0; if (EOF == fclose(pOutfile)) { if(SystemFlags::VERBOSE_MODE_ENABLED) printf("Failed writing to output file!\n"); return EXIT_FAILURE; } if(SystemFlags::VERBOSE_MODE_ENABLED) { printf("Total input bytes: %u\n", (mz_uint32)stream.total_in); printf("Total output bytes: %u\n", (mz_uint32)stream.total_out); printf("Success.\n"); } return EXIT_SUCCESS; }
ATCResult ATCLocker_impl::writeFileData(ostream *dst, istream *src, size_t length) { int rest_length = length; while (1) { if (z_.avail_in == 0) { size_t read_length = (rest_length < ATC_BUF_SIZE) ? rest_length : ATC_BUF_SIZE; z_.next_in = reinterpret_cast<Bytef*>(input_buffer_); z_.avail_in = static_cast<uInt>(src->read(input_buffer_, read_length).gcount()); rest_length -= z_.avail_in; total_write_length_ += z_.avail_in; if (total_write_length_ >= total_length_) { z_flush_ = Z_FINISH; } } // TODO: コメントアウトするとなぜか正常に出力しない // if(z_.avail_in == 0) break; z_status_ = deflate(&z_, z_flush_); if (z_status_ == Z_STREAM_END) { break; } if (z_status_ != Z_OK) { if (z_status_ == Z_BUF_ERROR) { return ATC_OK; } else { return ATC_ERR_ZLIB_ERROR; } } if (z_.avail_out == 0) { encryptBuffer(output_buffer_, chain_buffer_); dst->write(output_buffer_, ATC_BUF_SIZE); z_.next_out = reinterpret_cast<Bytef*>(output_buffer_); z_.avail_out = ATC_BUF_SIZE; } } if (z_status_ == Z_STREAM_END) { int32_t count; if ((count = ATC_BUF_SIZE - z_.avail_out) != 0) { char padding_num = (char)z_.avail_out; for(int i = count; i < ATC_BUF_SIZE; i++) { output_buffer_[i] = padding_num; } encryptBuffer(output_buffer_, chain_buffer_); dst->write(output_buffer_, ATC_BUF_SIZE); } return finish(); } return ATC_OK; }
extern int ZEXPORT zipWriteInFileInZip ( zipFile file, const void* buf, unsigned len) { zip_internal* zi; int err=ZIP_OK; if (file == NULL) return ZIP_PARAMERROR; zi = (zip_internal*)file; if (zi->in_opened_file_inzip == 0) return ZIP_PARAMERROR; zi->ci.stream.next_in = (Bytef *)buf; zi->ci.stream.avail_in = len; zi->ci.crc32 = crc32(zi->ci.crc32,(const Bytef *)buf,len); while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) { if (zi->ci.stream.avail_out == 0) { if (zipFlushWriteBuffer(zi) == ZIP_ERRNO) err = ZIP_ERRNO; zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; zi->ci.stream.next_out = zi->ci.buffered_data; } if(err != ZIP_OK) break; if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) { uLong uTotalOutBefore = zi->ci.stream.total_out; err=deflate(&zi->ci.stream, Z_NO_FLUSH); zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; } else { uInt copy_this,i; if (zi->ci.stream.avail_in < zi->ci.stream.avail_out) copy_this = zi->ci.stream.avail_in; else copy_this = zi->ci.stream.avail_out; for (i=0;i<copy_this;i++) *(((char*)zi->ci.stream.next_out)+i) = *(((const char*)zi->ci.stream.next_in)+i); { zi->ci.stream.avail_in -= copy_this; zi->ci.stream.avail_out-= copy_this; zi->ci.stream.next_in+= copy_this; zi->ci.stream.next_out+= copy_this; zi->ci.stream.total_in+= copy_this; zi->ci.stream.total_out+= copy_this; zi->ci.pos_in_buffered_data += copy_this; } } } return err; }
//--------------------------------------------------------------------------- void FileInformation::Export_XmlGz (const QString &ExportFileName, const activefilters& filters) { stringstream Data; // Header Data<<"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"; Data<<"<!-- Created by QCTools " << Version << " -->\n"; Data<<"<ffprobe:ffprobe xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:ffprobe='http://www.ffmpeg.org/schema/ffprobe' xsi:schemaLocation='http://www.ffmpeg.org/schema/ffprobe ffprobe.xsd'>\n"; Data<<" <program_version version=\"" << FFmpeg_Glue::FFmpeg_Version() << "\" copyright=\"Copyright (c) 2007-" << FFmpeg_Glue::FFmpeg_Year() << " the FFmpeg developers\" build_date=\"" __DATE__ "\" build_time=\"" __TIME__ "\" compiler_ident=\"" << FFmpeg_Glue::FFmpeg_Compiler() << "\" configuration=\"" << FFmpeg_Glue::FFmpeg_Configuration() << "\"/>\n"; Data<<"\n"; Data<<" <library_versions>\n"; Data<<FFmpeg_Glue::FFmpeg_LibsVersion(); Data<<" </library_versions>\n"; Data<<" <frames>\n"; // From stats for (size_t Pos=0; Pos<Stats.size(); Pos++) { if (Stats[Pos]) { if(Stats[Pos]->Type_Get() == Type_Video && Glue) { auto videoStats = static_cast<VideoStats*>(Stats[Pos]); videoStats->setWidth(Glue->Width_Get()); videoStats->setHeight(Glue->Height_Get()); } Data<<Stats[Pos]->StatsToXML(filters); } } // Footer Data<<" </frames>"; QString streamsAndFormats; QXmlStreamWriter writer(&streamsAndFormats); writer.setAutoFormatting(true); writer.setAutoFormattingIndent(4); if(streamsStats) streamsStats->writeToXML(&writer); if(formatStats) formatStats->writeToXML(&writer); // add indentation QStringList splitted = streamsAndFormats.split("\n"); for(size_t i = 0; i < splitted.length(); ++i) splitted[i] = QString(qAbs(writer.autoFormattingIndent()), writer.autoFormattingIndent() > 0 ? ' ' : '\t') + splitted[i]; streamsAndFormats = splitted.join("\n"); Data<<streamsAndFormats.toStdString() << "\n\n"; Data<<"</ffprobe:ffprobe>"; SharedFile file; QString name; if(ExportFileName.isEmpty()) { file = SharedFile(new QTemporaryFile()); QFileInfo info(fileName() + ".qctools.xml.gz"); name = info.fileName(); } else { file = SharedFile(new QFile(ExportFileName)); QFileInfo info(ExportFileName); name = info.fileName(); } string DataS=Data.str(); uLongf Buffer_Size=65536; if(file->open(QIODevice::ReadWrite)) { if(name.endsWith(".qctools.xml")) { auto bytesLeft = Data.str().size(); auto writePtr = DataS.c_str(); auto totalBytesWritten = 0; while(bytesLeft) { auto bytesToWrite = std::min(size_t(Buffer_Size), bytesLeft); auto bytesWritten = file->write(writePtr, bytesToWrite); totalBytesWritten += bytesWritten; Q_EMIT statsFileGenerationProgress(totalBytesWritten, DataS.size()); writePtr += bytesToWrite; bytesLeft -= bytesWritten; if(bytesWritten != bytesToWrite) break; } } else { char* Buffer=new char[Buffer_Size]; z_stream strm; strm.next_in = (Bytef *) DataS.c_str(); strm.avail_in = DataS.size() ; strm.total_out = 0; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; if (deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY)>=0) // 15 + 16 are magic values for gzip { do { strm.next_out = (unsigned char*) Buffer; strm.avail_out = Buffer_Size; if (deflate(&strm, Z_FINISH)<0) break; file->write(Buffer, Buffer_Size-strm.avail_out); Q_EMIT statsFileGenerationProgress((char*) strm.next_in - DataS.c_str(), DataS.size()); } while (strm.avail_out == 0); deflateEnd (&strm); } delete[] Buffer; } file->flush(); file->seek(0); } Q_EMIT statsFileGenerated(file, name); m_commentsUpdated = false; }
int main(int argc, char *argv[]) { const char *pMode; FILE *pInfile, *pOutfile; uint infile_size; int level = Z_BEST_COMPRESSION; z_stream stream; int n = 1; const char *pSrc_filename; const char *pDst_filename; printf("LZHAM example1 (streaming compression/decompression)\nUsing library version %s\n", ZLIB_VERSION); if (argc < 4) { printf("Usage: example1 [options] [mode:c or d] infile outfile\n"); printf("\nModes:\n"); printf("c - Compresses file infile to a LZHAM zlib stream in file outfile\n"); printf("d - Decompress a LZHAM zlib stream in file infile to file outfile\n"); printf("\nOptions:\n"); printf("-l[0-10] - Compression level, higher values are slower.\n"); return EXIT_FAILURE; } while ((n < argc) && (argv[n][0] == '-')) { switch (argv[n][1]) { case 'l': { level = atoi(&argv[1][2]); if ((level < 0) || (level > 10)) { printf("Invalid level!\n"); return EXIT_FAILURE; } break; } default: { printf("Invalid option: %s\n", argv[n]); return EXIT_FAILURE; } } n++; } if ((argc - n) < 3) { printf("Must specify mode, input filename, and output filename after options!\n"); return EXIT_FAILURE; } else if ((argc - n) > 3) { printf("Too many filenames!\n"); return EXIT_FAILURE; } pMode = argv[n++]; if (!strchr("cCdD", pMode[0])) { printf("Invalid mode!\n"); return EXIT_FAILURE; } pSrc_filename = argv[n++]; pDst_filename = argv[n++]; printf("Mode: %c, Level: %u\nInput File: \"%s\"\nOutput File: \"%s\"\n", pMode[0], level, pSrc_filename, pDst_filename); // Open input file. pInfile = fopen(pSrc_filename, "rb"); if (!pInfile) { printf("Failed opening input file!\n"); return EXIT_FAILURE; } // Determine input file's size. fseek(pInfile, 0, SEEK_END); infile_size = ftell(pInfile); fseek(pInfile, 0, SEEK_SET); // Open output file. pOutfile = fopen(pDst_filename, "wb"); if (!pOutfile) { printf("Failed opening output file!\n"); return EXIT_FAILURE; } printf("Input file size: %u\n", infile_size); // Init the z_stream memset(&stream, 0, sizeof(stream)); stream.next_in = s_inbuf; stream.avail_in = 0; stream.next_out = s_outbuf; stream.avail_out = BUF_SIZE; if ((pMode[0] == 'c') || (pMode[0] == 'C')) { // Compression. uint infile_remaining = infile_size; if (deflateInit2(&stream, level, LZHAM_Z_LZHAM, WINDOW_BITS, 9, LZHAM_Z_DEFAULT_STRATEGY) != Z_OK) { printf("deflateInit() failed!\n"); return EXIT_FAILURE; } for ( ; ; ) { int status; if (!stream.avail_in) { // Input buffer is empty, so read more bytes from input file. uint n = my_min(BUF_SIZE, infile_remaining); if (fread(s_inbuf, 1, n, pInfile) != n) { printf("Failed reading from input file!\n"); return EXIT_FAILURE; } stream.next_in = s_inbuf; stream.avail_in = n; infile_remaining -= n; //printf("Input bytes remaining: %u\n", infile_remaining); } status = deflate(&stream, infile_remaining ? Z_NO_FLUSH : Z_FINISH); if ((status == Z_STREAM_END) || (!stream.avail_out)) { // Output buffer is full, or compression is done, so write buffer to output file. uint n = BUF_SIZE - stream.avail_out; if (fwrite(s_outbuf, 1, n, pOutfile) != n) { printf("Failed writing to output file!\n"); return EXIT_FAILURE; } stream.next_out = s_outbuf; stream.avail_out = BUF_SIZE; } if (status == Z_STREAM_END) break; else if (status != Z_OK) { printf("deflate() failed with status %i!\n", status); return EXIT_FAILURE; } } if (deflateEnd(&stream) != Z_OK) { printf("deflateEnd() failed!\n"); return EXIT_FAILURE; } } else if ((pMode[0] == 'd') || (pMode[0] == 'D')) { // Decompression. uint infile_remaining = infile_size; if (inflateInit2(&stream, WINDOW_BITS)) { printf("inflateInit() failed!\n"); return EXIT_FAILURE; } for ( ; ; ) { int status; if (!stream.avail_in) { // Input buffer is empty, so read more bytes from input file. uint n = my_min(BUF_SIZE, infile_remaining); if (fread(s_inbuf, 1, n, pInfile) != n) { printf("Failed reading from input file!\n"); return EXIT_FAILURE; } stream.next_in = s_inbuf; stream.avail_in = n; infile_remaining -= n; } status = inflate(&stream, Z_SYNC_FLUSH); if ((status == Z_STREAM_END) || (!stream.avail_out)) { // Output buffer is full, or decompression is done, so write buffer to output file. uint n = BUF_SIZE - stream.avail_out; if (fwrite(s_outbuf, 1, n, pOutfile) != n) { printf("Failed writing to output file!\n"); return EXIT_FAILURE; } stream.next_out = s_outbuf; stream.avail_out = BUF_SIZE; } if (status == Z_STREAM_END) break; else if (status != Z_OK) { printf("inflate() failed with status %i!\n", status); return EXIT_FAILURE; } } if (inflateEnd(&stream) != Z_OK) { printf("inflateEnd() failed!\n"); return EXIT_FAILURE; } } else { printf("Invalid mode!\n"); return EXIT_FAILURE; } fclose(pInfile); if (EOF == fclose(pOutfile)) { printf("Failed writing to output file!\n"); return EXIT_FAILURE; } printf("Total input bytes: %u\n", stream.total_in); printf("Total output bytes: %u\n", stream.total_out); printf("Success.\n"); return EXIT_SUCCESS; }
static void* HW_CaptureThread(void* arg) { HWCapture* capture = (HWCapture*)arg; z_stream stream; unsigned char* zbuffer = 0; int result; unsigned int nodecount = 0; unsigned int have = 0; int savecapture = 1; capture->done = 0; if (capture->outputFile == 0) { savecapture = 0; } if (savecapture) { zbuffer = malloc(ZCHUNK); if (zbuffer == 0) { fprintf(stderr, "Error allocating zbuffer\n"); exit(1); } stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; result = deflateInit(&stream, ZCOMPLEVEL); if (result != Z_OK) { fprintf(stderr, "Error initializing ZLIB\n"); exit(1); } } while(capture->running) { HWBuffer* node = 0; unsigned int available; unsigned int capacity; unsigned char* buffer; unsigned int buffersize; unsigned int bufferpos; pthread_mutex_lock(&capture->mutex); node = HW_BufferChainGetFirst(&capture->chain); if (node) { available = node->capacity - node->size; buffersize = node->size; bufferpos = node->pos; buffer = node->buffer; capacity = node->capacity; node->pos = node->size; } pthread_mutex_unlock(&capture->mutex); if (node && (buffersize - bufferpos > 0)) HW_Process(&capture->process, buffer + bufferpos, buffersize - bufferpos); if (node == 0 || available != 0) { mssleep(1); continue; } if (savecapture) { stream.avail_in = capacity; stream.next_in = buffer; do { stream.avail_out = ZCHUNK; stream.next_out = zbuffer; result = deflate(&stream, Z_NO_FLUSH); if (result == Z_STREAM_ERROR) { fprintf(stderr, "Error compressing stream\n"); exit(1); } have = ZCHUNK - stream.avail_out; capture->compressedsize += have; if (capture->outputFile) { if (fwrite(zbuffer, have, 1, capture->outputFile) != 1) { deflateEnd(&stream); fprintf(stderr, "Write error\n"); exit(1); } } } while(stream.avail_out == 0); } nodecount = 0; pthread_mutex_lock(&capture->mutex); HW_BufferChainDestroyFirst(&capture->chain); node = HW_BufferChainGetFirst(&capture->chain); while(1) { if (node == 0) break; node = node->next; nodecount++; } pthread_mutex_unlock(&capture->mutex); if (nodecount > 1) fprintf(stdout, "WARNING: %d hwbuffers still remaining\n", nodecount); } if (savecapture) { // Write last remaining data in the buffers while(1) { HWBuffer* node = 0; unsigned int available; unsigned int capacity; unsigned char* buffer; pthread_mutex_lock(&capture->mutex); node = HW_BufferChainGetFirst(&capture->chain); if (node) { available = node->capacity - node->size; buffer = node->buffer; capacity = node->capacity; } pthread_mutex_unlock(&capture->mutex); if (node == 0) break; if (available == capacity) break; stream.avail_in = capacity - available; stream.next_in = buffer; do { stream.avail_out = ZCHUNK; stream.next_out = zbuffer; result = deflate(&stream, Z_NO_FLUSH); if (result == Z_STREAM_ERROR) { fprintf(stderr, "Error compressing stream\n"); exit(1); } have = ZCHUNK - stream.avail_out; capture->compressedsize += have; if (capture->outputFile) { if (fwrite(zbuffer, have, 1, capture->outputFile) != 1) { deflateEnd(&stream); fprintf(stderr, "Write error\n"); exit(1); } } } while(stream.avail_out == 0); pthread_mutex_lock(&capture->mutex); HW_BufferChainDestroyFirst(&capture->chain); pthread_mutex_unlock(&capture->mutex); } stream.avail_in = 0; stream.next_in = NULL; do { stream.avail_out = ZCHUNK; stream.next_out = zbuffer; result = deflate(&stream, Z_FINISH); if (result == Z_STREAM_ERROR) { fprintf(stderr, "Error compressing stream\n"); exit(1); } have = ZCHUNK - stream.avail_out; capture->compressedsize += have; if (capture->outputFile) { if (fwrite(zbuffer, have, 1, capture->outputFile) != 1) { deflateEnd(&stream); fprintf(stderr, "Write error\n"); exit(1); } } } while(stream.avail_out == 0); deflateEnd(&stream); free(zbuffer); } capture->done = 1; return 0; }
static void write_bytes_to_file(FILE * file, char *addr, long bytes, int compression) { if (compression == COMPRESSION_LEVEL_NONE) { while (bytes > 0) { sword_t count = fwrite(addr, 1, bytes, file); if (count > 0) { bytes -= count; addr += count; } else { perror("error writing to core file"); lose("core file is incomplete or corrupt\n"); } } #ifdef LISP_FEATURE_SB_CORE_COMPRESSION } else if ((compression >= -1) && (compression <= 9)) { # define ZLIB_BUFFER_SIZE (1u<<16) z_stream stream; unsigned char buf[ZLIB_BUFFER_SIZE]; unsigned char * written, * end; long total_written = 0; int ret; stream.zalloc = NULL; stream.zfree = NULL; stream.opaque = NULL; stream.avail_in = bytes; stream.next_in = (void*)addr; ret = deflateInit(&stream, compression); if (ret != Z_OK) lose("deflateInit: %i\n", ret); do { stream.avail_out = sizeof(buf); stream.next_out = buf; ret = deflate(&stream, Z_FINISH); if (ret < 0) lose("zlib deflate error: %i... exiting\n", ret); written = buf; end = buf+sizeof(buf)-stream.avail_out; total_written += end - written; while (written < end) { long count = fwrite(written, 1, end-written, file); if (count > 0) { written += count; } else { perror("error writing to core file"); lose("core file is incomplete or corrupt\n"); } } } while (stream.avail_out == 0); deflateEnd(&stream); printf("compressed %lu bytes into %lu at level %i\n", bytes, total_written, compression); # undef ZLIB_BUFFER_SIZE #endif } else { #ifdef LISP_FEATURE_SB_CORE_COMPRESSION lose("Unknown core compression level %i, exiting\n", compression); #else lose("zlib-compressed core support not built in this runtime\n"); #endif } if (fflush(file) != 0) { perror("error writing to core file"); lose("core file is incomplete or corrupt\n"); } };
//we are allowed to trash our input here. int QC_encode(progfuncs_t *progfuncs, int len, int method, char *in, int handle) { int i; if (method == 0) //copy { SafeWrite(handle, in, len); return len; } else if (method == 1) //xor encryption { for (i = 0; i < len; i++) in[i] = in[i] ^ 0xA5; SafeWrite(handle, in, len); return len; } else if (method == 2) //compression (ZLIB) { #ifdef AVAIL_ZLIB char out[8192]; z_stream strm = { in, len, 0, out, sizeof(out), 0, NULL, NULL, NULL, NULL, NULL, Z_BINARY, 0, 0 }; i=0; deflateInit(&strm, Z_BEST_COMPRESSION); while(deflate(&strm, Z_FINISH) == Z_OK) { SafeWrite(handle, out, sizeof(out) - strm.avail_out); //compress in chunks of 8192. Saves having to allocate a huge-mega-big buffer i+=sizeof(out) - strm.avail_out; strm.next_out = out; strm.avail_out = sizeof(out); } SafeWrite(handle, out, sizeof(out) - strm.avail_out); i+=sizeof(out) - strm.avail_out; deflateEnd(&strm); return i; #endif Sys_Error("ZLIB compression not supported in this build"); return 0; } //add your compression/decryption routine here. else { Sys_Error("Wierd method"); return 0; } }
int vncEncodeTight::CompressData(BYTE *dest, int streamId, int dataLen, int zlibLevel, int zlibStrategy) { if (dataLen < TIGHT_MIN_TO_COMPRESS) { memcpy(dest, m_buffer, dataLen); return dataLen; } z_streamp pz = &m_zsStruct[streamId]; // Initialize compression stream if needed. if (!m_zsActive[streamId]) { pz->zalloc = Z_NULL; pz->zfree = Z_NULL; pz->opaque = Z_NULL; vnclog.Print(LL_INTINFO, VNCLOG("calling deflateInit2 with zlib level:%d\n"), zlibLevel); int err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS, MAX_MEM_LEVEL, zlibStrategy); if (err != Z_OK) { vnclog.Print(LL_INTINFO, VNCLOG("deflateInit2 returned error:%d:%s\n"), err, pz->msg); return -1; } m_zsActive[streamId] = true; m_zsLevel[streamId] = zlibLevel; } int outBufferSize = dataLen + dataLen / 100 + 16; // Prepare buffer pointers. pz->next_in = (Bytef *)m_buffer; pz->avail_in = dataLen; pz->next_out = (Bytef *)dest; pz->avail_out = outBufferSize; // Change compression parameters if needed. if (zlibLevel != m_zsLevel[streamId]) { vnclog.Print(LL_INTINFO, VNCLOG("calling deflateParams with zlib level:%d\n"), zlibLevel); int err = deflateParams (pz, zlibLevel, zlibStrategy); if (err != Z_OK) { vnclog.Print(LL_INTINFO, VNCLOG("deflateParams returned error:%d:%s\n"), err, pz->msg); return -1; } m_zsLevel[streamId] = zlibLevel; } // Actual compression. if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK || pz->avail_in != 0 || pz->avail_out == 0 ) { vnclog.Print(LL_INTINFO, VNCLOG("deflate() call failed.\n")); return -1; } return SendCompressedData(outBufferSize - pz->avail_out); }
/*! * zlibCompress() * * Input: datain (byte buffer with input data) * nin (number of bytes of input data) * &nout (<return> number of bytes of output data) * Return: dataout (compressed data), or null on error * * Notes: * (1) We repeatedly read in and fill up an input buffer, * compress the data, and read it back out. zlib * uses two byte buffers internally in the z_stream * data structure. We use the bbuffers to feed data * into the fixed bufferin, and feed it out of bufferout, * in the same way that a pair of streams would normally * be used if the data were being read from one file * and written to another. This is done iteratively, * compressing L_BUF_SIZE bytes of input data at a time. */ l_uint8 * zlibCompress(l_uint8 *datain, size_t nin, size_t *pnout) { l_uint8 *dataout; l_int32 status; size_t nbytes; l_uint8 *bufferin, *bufferout; BBUFFER *bbin, *bbout; z_stream z; PROCNAME("zlibCompress"); if (!datain) return (l_uint8 *)ERROR_PTR("datain not defined", procName, NULL); /* Set up fixed size buffers used in z_stream */ if ((bufferin = (l_uint8 *)CALLOC(L_BUF_SIZE, sizeof(l_uint8))) == NULL) return (l_uint8 *)ERROR_PTR("bufferin not made", procName, NULL); if ((bufferout = (l_uint8 *)CALLOC(L_BUF_SIZE, sizeof(l_uint8))) == NULL) return (l_uint8 *)ERROR_PTR("bufferout not made", procName, NULL); /* Set up bbuffers and load bbin with the data */ if ((bbin = bbufferCreate(datain, nin)) == NULL) return (l_uint8 *)ERROR_PTR("bbin not made", procName, NULL); if ((bbout = bbufferCreate(NULL, 0)) == NULL) return (l_uint8 *)ERROR_PTR("bbout not made", procName, NULL); z.zalloc = (alloc_func)0; z.zfree = (free_func)0; z.opaque = (voidpf)0; z.next_in = bufferin; z.avail_in = 0; z.next_out = bufferout; z.avail_out = L_BUF_SIZE; deflateInit(&z, ZLIB_COMPRESSION_LEVEL); for ( ; ; ) { if (z.avail_in == 0) { z.next_in = bufferin; bbufferWrite(bbin, bufferin, L_BUF_SIZE, &nbytes); /* fprintf(stderr, " wrote %d bytes to bufferin\n", nbytes); */ z.avail_in = nbytes; } if (z.avail_in == 0) break; status = deflate(&z, Z_SYNC_FLUSH); /* fprintf(stderr, " status is %d, bytesleft = %d, totalout = %d\n", status, z.avail_out, z.total_out); */ nbytes = L_BUF_SIZE - z.avail_out; if (nbytes) { bbufferRead(bbout, bufferout, nbytes); /* fprintf(stderr, " read %d bytes from bufferout\n", nbytes); */ } z.next_out = bufferout; z.avail_out = L_BUF_SIZE; } deflateEnd(&z); bbufferDestroy(&bbin); dataout = bbufferDestroyAndSaveData(&bbout, pnout); FREE(bufferin); FREE(bufferout); return dataout; }
int sftp_compress_write_data(struct ssh2_packet *pkt) { struct sftp_compress *comp; z_stream *stream; comp = &(write_compresses[write_comp_idx]); stream = &(write_streams[write_comp_idx]); if (comp->use_zlib && comp->stream_ready) { unsigned char buf[16384], *input; char *payload; uint32_t input_len, payload_len = 0, payload_sz; pool *sub_pool; int zres; if (pkt->payload_len == 0) { return 0; } sub_pool = make_sub_pool(pkt->pool); /* Use a copy of the payload, rather than the actual payload itself, * as zlib may alter the payload contents and then encounter an error. */ input_len = pkt->payload_len; input = palloc(sub_pool, input_len); memcpy(input, pkt->payload, input_len); /* Try to guess at how small the compressed data will be. Optimistic * estimate, for now, will be a factor of 2, with a minimum of 1K. */ payload_sz = 1024; if ((input_len * 2) > payload_sz) { payload_sz = input_len * 2; } payload = palloc(sub_pool, payload_sz); stream->next_in = input; stream->avail_in = input_len; stream->avail_out = 0; while (stream->avail_out == 0) { size_t copy_len = 0; pr_signals_handle(); stream->next_out = buf; stream->avail_out = sizeof(buf); zres = deflate(stream, Z_SYNC_FLUSH); switch (zres) { case Z_OK: copy_len = sizeof(buf) - stream->avail_out; /* Allocate more space for the data if necessary. */ if ((payload_len + copy_len) > payload_sz) { uint32_t new_sz; char *tmp; new_sz = payload_sz; while ((payload_len + copy_len) > new_sz) { pr_signals_handle(); /* Keep doubling the size until it is large enough. */ new_sz *= 2; } pr_trace_msg(trace_channel, 20, "allocating larger payload size (%lu bytes) for " "deflated data (%lu bytes) plus existing payload %lu bytes", (unsigned long) new_sz, (unsigned long) copy_len, (unsigned long) payload_len); tmp = palloc(sub_pool, new_sz); memcpy(tmp, payload, payload_len); payload = tmp; payload_sz = new_sz; } memcpy(payload + payload_len, buf, copy_len); payload_len += copy_len; pr_trace_msg(trace_channel, 20, "deflated %lu bytes to %lu bytes", (unsigned long) input_len, (unsigned long) copy_len); break; default: (void) pr_log_writefile(sftp_logfd, MOD_SFTP_VERSION, "unhandled zlib error (%d) while compressing", zres); destroy_pool(sub_pool); errno = EIO; return -1; } } if (payload_len > 0) { if (pkt->payload_len < payload_len) { pkt->payload = palloc(pkt->pool, payload_len); } memcpy(pkt->payload, payload, payload_len); pkt->payload_len = payload_len; pr_trace_msg(trace_channel, 20, "finished deflating (payload len = %lu bytes)", (unsigned long) payload_len); } destroy_pool(sub_pool); } return 0; }
static int deflate_common(struct mbuf *m, struct mbuf *md, size_t *lenp, int mode) /* 0: compress 1: decompress */ { struct mbuf *mprev; struct mbuf *p; struct mbuf *n = NULL, *n0 = NULL, **np; z_stream zs; int error = 0; int zerror; size_t offset; #define MOREBLOCK() \ do { \ /* keep the reply buffer into our chain */ \ if (n) { \ n->m_len = zs.total_out - offset; \ offset = zs.total_out; \ *np = n; \ np = &n->m_next; \ n = NULL; \ } \ \ /* get a fresh reply buffer */ \ n = m_getcl(M_NOWAIT, MT_DATA, 0); \ if (!n) { \ error = ENOBUFS; \ goto fail; \ } \ n->m_len = 0; \ n->m_len = M_TRAILINGSPACE(n); \ n->m_next = NULL; \ /* \ * if this is the first reply buffer, reserve \ * region for ipcomp header. \ */ \ if (*np == NULL) { \ n->m_len -= sizeof(struct ipcomp); \ n->m_data += sizeof(struct ipcomp); \ } \ \ zs.next_out = mtod(n, u_int8_t *); \ zs.avail_out = n->m_len; \ } while (0) for (mprev = m; mprev && mprev->m_next != md; mprev = mprev->m_next) ; if (!mprev) panic("md is not in m in deflate_common"); bzero(&zs, sizeof(zs)); zs.zalloc = deflate_alloc; zs.zfree = deflate_free; zerror = mode ? inflateInit2(&zs, deflate_window_in) : deflateInit2(&zs, deflate_policy, Z_DEFLATED, deflate_window_out, deflate_memlevel, Z_DEFAULT_STRATEGY); if (zerror != Z_OK) { error = ENOBUFS; goto fail; } n0 = n = NULL; np = &n0; offset = 0; zerror = 0; p = md; while (p && p->m_len == 0) { p = p->m_next; } /* input stream and output stream are available */ while (p && zs.avail_in == 0) { /* get input buffer */ if (p && zs.avail_in == 0) { zs.next_in = mtod(p, u_int8_t *); zs.avail_in = p->m_len; p = p->m_next; while (p && p->m_len == 0) { p = p->m_next; } } /* get output buffer */ if (zs.next_out == NULL || zs.avail_out == 0) { MOREBLOCK(); } zerror = mode ? inflate(&zs, Z_NO_FLUSH) : deflate(&zs, Z_NO_FLUSH); if (zerror == Z_STREAM_END) ; /* once more. */ else if (zerror == Z_OK) { /* inflate: Z_OK can indicate the end of decode */ if (mode && !p && zs.avail_out != 0) goto terminate; else ; /* once more. */ } else { if (zs.msg) { ipseclog((LOG_ERR, "ipcomp_%scompress: " "%sflate(Z_NO_FLUSH): %s\n", mode ? "de" : "", mode ? "in" : "de", zs.msg)); } else { ipseclog((LOG_ERR, "ipcomp_%scompress: " "%sflate(Z_NO_FLUSH): unknown error (%d)\n", mode ? "de" : "", mode ? "in" : "de", zerror)); } mode ? inflateEnd(&zs) : deflateEnd(&zs); error = EINVAL; goto fail; } }
/* {{{ libssh2_comp_method_zlib_comp * zlib, a compression standard for all occasions */ static int libssh2_comp_method_zlib_comp(LIBSSH2_SESSION * session, int compress, unsigned char **dest, unsigned long *dest_len, unsigned long payload_limit, int *free_dest, const unsigned char *src, unsigned long src_len, void **abstract) { z_stream *strm = *abstract; /* A short-term alloc of a full data chunk is better than a series of reallocs */ char *out; int out_maxlen = compress ? (src_len + 4) : (2 * src_len); int limiter = 0; /* In practice they never come smaller than this */ if (out_maxlen < 25) { out_maxlen = 25; } if (out_maxlen > (int) payload_limit) { out_maxlen = payload_limit; } strm->next_in = (unsigned char *) src; strm->avail_in = src_len; strm->next_out = (unsigned char *) LIBSSH2_ALLOC(session, out_maxlen); out = (char *) strm->next_out; strm->avail_out = out_maxlen; if (!strm->next_out) { libssh2_error(session, LIBSSH2_ERROR_ALLOC, "Unable to allocate compression/decompression buffer", 0); return -1; } while (strm->avail_in) { int status; if (compress) { status = deflate(strm, Z_PARTIAL_FLUSH); } else { status = inflate(strm, Z_PARTIAL_FLUSH); } if (status != Z_OK) { libssh2_error(session, LIBSSH2_ERROR_ZLIB, "compress/decompression failure", 0); LIBSSH2_FREE(session, out); return -1; } if (strm->avail_in) { unsigned long out_ofs = out_maxlen - strm->avail_out; char *newout; out_maxlen += compress ? (strm->avail_in + 4) : (2 * strm->avail_in); if ((out_maxlen > (int) payload_limit) && !compress && limiter++) { libssh2_error(session, LIBSSH2_ERROR_ZLIB, "Excessive growth in decompression phase", 0); LIBSSH2_FREE(session, out); return -1; } newout = LIBSSH2_REALLOC(session, out, out_maxlen); if (!newout) { libssh2_error(session, LIBSSH2_ERROR_ALLOC, "Unable to expand compress/decompression buffer", 0); LIBSSH2_FREE(session, out); return -1; } out = newout; strm->next_out = (unsigned char *) out + out_ofs; strm->avail_out += compress ? (strm->avail_in + 4) : (2 * strm->avail_in); } else while (!strm->avail_out) { /* Done with input, might be a byte or two in internal buffer during compress * Or potentially many bytes if it's a decompress */ int grow_size = compress ? 8 : 1024; char *newout; if (out_maxlen >= (int) payload_limit) { libssh2_error(session, LIBSSH2_ERROR_ZLIB, "Excessive growth in decompression phase", 0); LIBSSH2_FREE(session, out); return -1; } if (grow_size > (int) (payload_limit - out_maxlen)) { grow_size = payload_limit - out_maxlen; } out_maxlen += grow_size; strm->avail_out = grow_size; newout = LIBSSH2_REALLOC(session, out, out_maxlen); if (!newout) { libssh2_error(session, LIBSSH2_ERROR_ALLOC, "Unable to expand final compress/decompress buffer", 0); LIBSSH2_FREE(session, out); return -1; } out = newout; strm->next_out = (unsigned char *) out + out_maxlen - grow_size; if (compress) { status = deflate(strm, Z_PARTIAL_FLUSH); } else { status = inflate(strm, Z_PARTIAL_FLUSH); } if (status != Z_OK) { libssh2_error(session, LIBSSH2_ERROR_ZLIB, "compress/decompression failure", 0); LIBSSH2_FREE(session, out); return -1; } } } *dest = (unsigned char *) out; *dest_len = out_maxlen - strm->avail_out; *free_dest = 1; return 0; }
static int lzlib_compress(lua_State *L) { const char *next_in = luaL_checkstring(L, 1); int avail_in = lua_rawlen(L, 1); int level = luaL_optint(L, 2, Z_DEFAULT_COMPRESSION); int method = luaL_optint(L, 3, Z_DEFLATED); int windowBits = luaL_optint(L, 4, 15); int memLevel = luaL_optint(L, 5, 8); int strategy = luaL_optint(L, 6, Z_DEFAULT_STRATEGY); int ret; z_stream zs; luaL_Buffer b; luaL_buffinit(L, &b); zs.zalloc = Z_NULL; zs.zfree = Z_NULL; zs.next_out = Z_NULL; zs.avail_out = 0; zs.next_in = Z_NULL; zs.avail_in = 0; ret = deflateInit2(&zs, level, method, windowBits, memLevel, strategy); if (ret != Z_OK) { lua_pushnil(L); lua_pushnumber(L, ret); return 2; } zs.next_in = (Bytef*)next_in; zs.avail_in = avail_in; for(;;) { zs.next_out = (Bytef*)luaL_prepbuffer(&b); zs.avail_out = LUAL_BUFFERSIZE; /* munch some more */ ret = deflate(&zs, Z_FINISH); /* push gathered data */ luaL_addsize(&b, LUAL_BUFFERSIZE - zs.avail_out); /* done processing? */ if (ret == Z_STREAM_END) break; /* error condition? */ if (ret != Z_OK) break; } /* cleanup */ deflateEnd(&zs); luaL_pushresult(&b); lua_pushnumber(L, ret); return 2; }
unique_ptr<IOBuf> GzipHeaderCodec::encode(vector<Header>& headers) noexcept { // Build a sequence of the header names and values, sorted by name. // The purpose of the sort is to make it easier to combine the // values of multiple headers with the same name. The SPDY spec // prohibits any header name from appearing more than once in the // Name/Value list, so we must combine values when serializing. std::sort(headers.begin(), headers.end()); auto& uncompressed = getHeaderBuf(); // Compute the amount of space needed to hold the uncompressed // representation of the headers. This is an upper bound on the // amount of space we'll actually need, because if we end up // combining any headers with the same name, the combined // representation will be smaller than the original. size_t maxUncompressedSize = versionSettings_.nameValueSize; for (const Header& header : headers) { maxUncompressedSize += versionSettings_.nameValueSize; maxUncompressedSize += header.name->length(); maxUncompressedSize += versionSettings_.nameValueSize; maxUncompressedSize += header.value->length(); } // TODO: give on 'onError()' callback if the space in uncompressed buf // cannot fit the headers and then skip the "reserve" code below. We // have already reserved the maximum legal amount of space for // uncompressed headers. VLOG(4) << "reserving " << maxUncompressedSize << " bytes for uncompressed headers"; uncompressed.reserve(0, maxUncompressedSize); // Serialize the uncompressed representation of the headers. uint8_t* dst = uncompressed.writableData(); dst += versionSettings_.nameValueSize; // Leave space for count of headers. HTTPHeaderCode lastCode = HTTP_HEADER_OTHER; const string* lastName = &empty_string; uint8_t* lastValueLenPtr = nullptr; size_t lastValueLen = 0; unsigned numHeaders = 0; for (const Header& header : headers) { if ((header.code != lastCode) || (*header.name != *lastName)) { // Simple case: this header name is different from the previous // one, so we don't need to combine values. numHeaders++; versionSettings_.appendSizeFun(dst, header.name->length()); // lowercasing the header name inline char* nameBegin = (char *)dst; appendString(dst, *header.name); folly::toLowerAscii((char *)nameBegin, header.name->size()); lastValueLenPtr = dst; lastValueLen = header.value->length(); versionSettings_.appendSizeFun(dst, header.value->length()); appendString(dst, *header.value); lastCode = header.code; lastName = header.name; } else { // More complicated case: we do need to combine values. *dst++ = 0; // SPDY uses a null byte as a separator appendString(dst, *header.value); // Go back and rewrite the length field in front of the value lastValueLen += (1 + header.value->length()); uint8_t* tmp = lastValueLenPtr; versionSettings_.appendSizeFun(tmp, lastValueLen); } } // Compute the uncompressed length; if we combined any header values, // we will have used less space than originally estimated. size_t uncompressedLen = dst - uncompressed.writableData(); // Go back and write the count of unique header names at the start. dst = uncompressed.writableData(); versionSettings_.appendSizeFun(dst, numHeaders); // Allocate a contiguous space big enough to hold the compressed headers, // plus any headroom requested by the caller. size_t maxDeflatedSize = deflateBound(&deflater_, uncompressedLen); unique_ptr<IOBuf> out(IOBuf::create(maxDeflatedSize + encodeHeadroom_)); out->advance(encodeHeadroom_); // Compress deflater_.next_in = uncompressed.writableData(); deflater_.avail_in = uncompressedLen; deflater_.next_out = out->writableData(); deflater_.avail_out = maxDeflatedSize; int r = deflate(&deflater_, Z_SYNC_FLUSH); CHECK(r == Z_OK); CHECK(deflater_.avail_in == 0); out->append(maxDeflatedSize - deflater_.avail_out); VLOG(4) << "header size orig=" << uncompressedLen << ", max deflated=" << maxDeflatedSize << ", actual deflated=" << out->length(); encodedSize_.compressed = out->length(); encodedSize_.uncompressed = uncompressedLen; if (stats_) { stats_->recordEncode(Type::GZIP, encodedSize_); } return std::move(out); }
extern int ZEXPORT zipCloseFileInZipRaw ( zipFile file, uLong uncompressed_size, uLong crc32) { zip_internal* zi; uLong compressed_size; int err=ZIP_OK; if (file == NULL) return ZIP_PARAMERROR; zi = (zip_internal*)file; if (zi->in_opened_file_inzip == 0) return ZIP_PARAMERROR; zi->ci.stream.avail_in = 0; if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) while (err==ZIP_OK) { uLong uTotalOutBefore; if (zi->ci.stream.avail_out == 0) { if (zipFlushWriteBuffer(zi) == ZIP_ERRNO) err = ZIP_ERRNO; zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; zi->ci.stream.next_out = zi->ci.buffered_data; } uTotalOutBefore = zi->ci.stream.total_out; err=deflate(&zi->ci.stream, Z_FINISH); zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; } if (err==Z_STREAM_END) err=ZIP_OK; /* this is normal */ if ((zi->ci.pos_in_buffered_data>0) && (err==ZIP_OK)) if (zipFlushWriteBuffer(zi)==ZIP_ERRNO) err = ZIP_ERRNO; if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) { err=deflateEnd(&zi->ci.stream); zi->ci.stream_initialised = 0; } if (!zi->ci.raw) { crc32 = (uLong)zi->ci.crc32; uncompressed_size = (uLong)zi->ci.stream.total_in; } compressed_size = (uLong)zi->ci.stream.total_out; # ifndef NOCRYPT compressed_size += zi->ci.crypt_header_size; # endif ziplocal_putValue_inmemory(zi->ci.central_header+16,crc32,4); /*crc*/ ziplocal_putValue_inmemory(zi->ci.central_header+20, compressed_size,4); /*compr size*/ if (zi->ci.stream.data_type == Z_ASCII) ziplocal_putValue_inmemory(zi->ci.central_header+36,(uLong)Z_ASCII,2); ziplocal_putValue_inmemory(zi->ci.central_header+24, uncompressed_size,4); /*uncompr size*/ if (err==ZIP_OK) err = add_data_in_datablock(&zi->central_dir,zi->ci.central_header, (uLong)zi->ci.size_centralheader); free(zi->ci.central_header); if (err==ZIP_OK) { long cur_pos_inzip = ZTELL(zi->z_filefunc,zi->filestream); if (ZSEEK(zi->z_filefunc,zi->filestream, zi->ci.pos_local_header + 14,ZLIB_FILEFUNC_SEEK_SET)!=0) err = ZIP_ERRNO; if (err==ZIP_OK) err = ziplocal_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ if (err==ZIP_OK) /* compressed size, unknown */ err = ziplocal_putValue(&zi->z_filefunc,zi->filestream,compressed_size,4); if (err==ZIP_OK) /* uncompressed size, unknown */ err = ziplocal_putValue(&zi->z_filefunc,zi->filestream,uncompressed_size,4); if (ZSEEK(zi->z_filefunc,zi->filestream, cur_pos_inzip,ZLIB_FILEFUNC_SEEK_SET)!=0) err = ZIP_ERRNO; } zi->number_entry ++; zi->in_opened_file_inzip = 0; return err; }
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw) { MolochHttpServer_t *server = serverV; if (key_len > 1000) { LOGEXIT("Url too long %.*s", key_len, key); } // Are we overloaded if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding); MOLOCH_THREAD_INCR(server->dropped); if (data) { MOLOCH_SIZE_FREE(buffer, data); } return 1; } MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t); if (headers) { int i; for (i = 0; headers[i]; i++) { request->headerList = curl_slist_append(request->headerList, headers[i]); } } if (server->defaultHeaders) { int i; for (i = 0; server->defaultHeaders[i]; i++) { request->headerList = curl_slist_append(request->headerList, server->defaultHeaders[i]); } } // Do we need to compress item if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; MOLOCH_LOCK(z_strm); z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate"); MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); MOLOCH_UNLOCK(z_strm); } request->server = server; request->func = func; request->uw = uw; request->dataOut = data; request->dataOutLen = data_len; request->easy = curl_easy_init(); if (config.debug >= 2) { curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1); } if (config.insecure) { curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYHOST, 0L); } curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback); curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request); curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback); curl_easy_setopt(request->easy, CURLOPT_ACCEPT_ENCODING, ""); // https://curl.haxx.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html if (request->headerList) { curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList); } if (method[0] != 'G') { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data); } else { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL); curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L); } if (server->headerCb) { curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function); curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request); } curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L); curl_easy_setopt(request->easy, CURLOPT_TIMEOUT, 60L); memcpy(request->key, key, key_len); request->key[key_len] = 0; MOLOCH_LOCK(requests); moloch_http_add_request(server, request, TRUE); MOLOCH_UNLOCK(requests); return 0; }
/* {{{ php_zlib_output_handler() */ static int php_zlib_output_handler(void **handler_context, php_output_context *output_context) { php_zlib_context *ctx = *(php_zlib_context **) handler_context; int flags = Z_SYNC_FLUSH; PHP_OUTPUT_TSRMLS(output_context); if (!php_zlib_output_encoding(TSRMLS_C)) { /* "Vary: Accept-Encoding" header sent along uncompressed content breaks caching in MSIE, so let's just send it with successfully compressed content or unless the complete buffer gets discarded, see http://bugs.php.net/40325; Test as follows: +Vary: $ HTTP_ACCEPT_ENCODING=gzip ./sapi/cgi/php <<<'<?php ob_start("ob_gzhandler"); echo "foo\n";' +Vary: $ HTTP_ACCEPT_ENCODING= ./sapi/cgi/php <<<'<?php ob_start("ob_gzhandler"); echo "foo\n";' -Vary: $ HTTP_ACCEPT_ENCODING=gzip ./sapi/cgi/php <<<'<?php ob_start("ob_gzhandler"); echo "foo\n"; ob_end_clean();' -Vary: $ HTTP_ACCEPT_ENCODING= ./sapi/cgi/php <<<'<?php ob_start("ob_gzhandler"); echo "foo\n"; ob_end_clean();' */ if (output_context->op != (PHP_OUTPUT_HANDLER_START|PHP_OUTPUT_HANDLER_CLEAN|PHP_OUTPUT_HANDLER_FINAL)) { sapi_add_header_ex(ZEND_STRL("Vary: Accept-Encoding"), 1, 1 TSRMLS_CC); } return FAILURE; } if (output_context->op & PHP_OUTPUT_HANDLER_START) { /* start up */ if (Z_OK != deflateInit2(&ctx->Z, ZLIBG(output_compression_level), Z_DEFLATED, ZLIBG(compression_coding), MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) { return FAILURE; } } if (output_context->op & PHP_OUTPUT_HANDLER_CLEAN) { /* free buffers */ deflateEnd(&ctx->Z); if (output_context->op & PHP_OUTPUT_HANDLER_FINAL) { /* discard */ return SUCCESS; } else { /* restart */ if (Z_OK != deflateInit2(&ctx->Z, ZLIBG(output_compression_level), Z_DEFLATED, ZLIBG(compression_coding), MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) { return FAILURE; } ctx->buffer.used = 0; } } else { if (output_context->in.used) { /* append input */ if (ctx->buffer.free < output_context->in.used) { if (!(ctx->buffer.aptr = erealloc_recoverable(ctx->buffer.data, ctx->buffer.used + ctx->buffer.free + output_context->in.used))) { deflateEnd(&ctx->Z); return FAILURE; } ctx->buffer.data = ctx->buffer.aptr; ctx->buffer.free += output_context->in.used; } memcpy(ctx->buffer.data + ctx->buffer.used, output_context->in.data, output_context->in.used); ctx->buffer.free -= output_context->in.used; ctx->buffer.used += output_context->in.used; } output_context->out.size = PHP_ZLIB_BUFFER_SIZE_GUESS(output_context->in.used); output_context->out.data = emalloc(output_context->out.size); output_context->out.free = 1; output_context->out.used = 0; ctx->Z.avail_in = ctx->buffer.used; ctx->Z.next_in = (Bytef *) ctx->buffer.data; ctx->Z.avail_out = output_context->out.size; ctx->Z.next_out = (Bytef *) output_context->out.data; if (output_context->op & PHP_OUTPUT_HANDLER_FINAL) { flags = Z_FINISH; } else if (output_context->op & PHP_OUTPUT_HANDLER_FLUSH) { flags = Z_FULL_FLUSH; } switch (deflate(&ctx->Z, flags)) { case Z_OK: if (flags == Z_FINISH) { deflateEnd(&ctx->Z); return FAILURE; } case Z_STREAM_END: if (ctx->Z.avail_in) { memmove(ctx->buffer.data, ctx->buffer.data + ctx->buffer.used - ctx->Z.avail_in, ctx->Z.avail_in); } ctx->buffer.free += ctx->buffer.used - ctx->Z.avail_in; ctx->buffer.used = ctx->Z.avail_in; output_context->out.used = output_context->out.size - ctx->Z.avail_out; break; default: deflateEnd(&ctx->Z); return FAILURE; } php_output_handler_hook(PHP_OUTPUT_HANDLER_HOOK_GET_FLAGS, &flags TSRMLS_CC); if (!(flags & PHP_OUTPUT_HANDLER_STARTED)) { if (SG(headers_sent) || !ZLIBG(output_compression)) { deflateEnd(&ctx->Z); return FAILURE; } switch (ZLIBG(compression_coding)) { case PHP_ZLIB_ENCODING_GZIP: sapi_add_header_ex(ZEND_STRL("Content-Encoding: gzip"), 1, 1 TSRMLS_CC); break; case PHP_ZLIB_ENCODING_DEFLATE: sapi_add_header_ex(ZEND_STRL("Content-Encoding: deflate"), 1, 1 TSRMLS_CC); break; default: deflateEnd(&ctx->Z); return FAILURE; } sapi_add_header_ex(ZEND_STRL("Vary: Accept-Encoding"), 1, 1 TSRMLS_CC); php_output_handler_hook(PHP_OUTPUT_HANDLER_HOOK_IMMUTABLE, NULL TSRMLS_CC); } if (output_context->op & PHP_OUTPUT_HANDLER_FINAL) { deflateEnd(&ctx->Z); } } return SUCCESS; }
static zip_int64_t compress_read(zip_source_t *src, struct deflate *ctx, void *data, zip_uint64_t len) { int end, ret; zip_int64_t n; zip_uint64_t out_offset; uInt out_len; if (zip_error_code_zip(&ctx->error) != ZIP_ER_OK) return -1; if (len == 0 || ctx->is_stored) { return 0; } out_offset = 0; out_len = (uInt)ZIP_MIN(UINT_MAX, len); ctx->zstr.next_out = (Bytef *)data; ctx->zstr.avail_out = out_len; end = 0; while (!end) { ret = deflate(&ctx->zstr, ctx->eof ? Z_FINISH : 0); switch (ret) { case Z_STREAM_END: if (ctx->can_store && ctx->zstr.total_in <= ctx->zstr.total_out) { ctx->is_stored = true; ctx->size = ctx->zstr.total_in; memcpy(data, ctx->buffer, ctx->size); return (zip_int64_t)ctx->size; } /* fallthrough */ case Z_OK: /* all ok */ if (ctx->zstr.avail_out == 0) { out_offset += out_len; if (out_offset < len) { out_len = (uInt)ZIP_MIN(UINT_MAX, len-out_offset); ctx->zstr.next_out = (Bytef *)data+out_offset; ctx->zstr.avail_out = out_len; } else { ctx->can_store = false; end = 1; } } else if (ctx->eof && ctx->zstr.avail_in == 0) end = 1; break; case Z_BUF_ERROR: if (ctx->zstr.avail_in == 0) { if (ctx->eof) { end = 1; break; } if ((n=zip_source_read(src, ctx->buffer, sizeof(ctx->buffer))) < 0) { _zip_error_set_from_source(&ctx->error, src); end = 1; break; } else if (n == 0) { ctx->eof = true; /* TODO: check against stat of src? */ ctx->size = ctx->zstr.total_in; } else { if (ctx->zstr.total_in > 0) { /* we overwrote a previously filled ctx->buffer */ ctx->can_store = false; } ctx->zstr.next_in = (Bytef *)ctx->buffer; ctx->zstr.avail_in = (uInt)n; } continue; } /* fallthrough */ case Z_NEED_DICT: case Z_DATA_ERROR: case Z_STREAM_ERROR: case Z_MEM_ERROR: zip_error_set(&ctx->error, ZIP_ER_ZLIB, ret); end = 1; break; } } if (ctx->zstr.avail_out < len) { ctx->can_store = false; return (zip_int64_t)(len - ctx->zstr.avail_out); } return (zip_error_code_zip(&ctx->error) == ZIP_ER_OK) ? 0 : -1; }
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw) { MolochHttpServer_t *server = serverV; // Are we overloaded if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding); if (data) { MOLOCH_SIZE_FREE(buffer, data); } return 1; } MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t); if (headers) { int i; for (i = 0; headers[i]; i++) { request->headerList = curl_slist_append(request->headerList, headers[i]); } } // Do we need to compress item if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate"); MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); } request->server = server; request->func = func; request->uw = uw; request->dataOut = data; request->dataOutLen = data_len; request->easy = curl_easy_init(); if (config.debug >= 2) { curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1); } curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback); curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request); curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, server); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, server); if (request->headerList) { curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList); } if (method[0] != 'G') { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data); } else { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL); curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L); } if (server->headerCb) { curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function); curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request); } curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L); char *host = server->names[server->namesPos]; server->namesPos = (server->namesPos + 1) % server->namesCnt; if (strchr(host, ':') == 0) { snprintf(request->url, sizeof(request->url), "%s://%s:%d%.*s", (server->https?"https":"http"), host, server->defaultPort, key_len, key); } else { snprintf(request->url, sizeof(request->url), "%s://%s%.*s", (server->https?"https":"http"), host, key_len, key); } curl_easy_setopt(request->easy, CURLOPT_URL, request->url); MOLOCH_LOCK(requests); #ifdef MOLOCH_HTTP_DEBUG LOG("HTTPDEBUG INCR %s %p %d %s", server->names[0], request, server->outstanding, request->url); #endif server->outstanding++; DLL_PUSH_TAIL(rqt_, &requests, request); if (!requestsTimer) requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL); MOLOCH_UNLOCK(requests); return 0; }
//=========================================================================== // スレッド実行 //=========================================================================== void __fastcall TAttacheCaseFileEncrypt::Execute() { int i, c; int res; float ProgressPercentNumF; //進捗パーセンテージ(浮動小数点) z_stream z; // zlibライブラリとやりとりするための構造体 int flush, status; // zlib //出力する暗号化ファイルのタイムスタンプを元ファイルに合わせる HANDLE hFile; //_WIN32_FIND_DATAW first_fd; ZeroMemory(&first_fd, sizeof(_WIN32_FIND_DATAW)); int len, pos; int FileIndex; String FilePath; int HeaderSize; //ヘッダデータサイズ __int64 CurrentDriveFreeSpaceSize = -1; //保存するドライブの空き容量 //実行可能形式出力ファイルのデータサイズ __int64 ExeAllSize = 0; __int64 ExeSize = 0; //全体のファイルサイズ AllTotalSize = 0; __int64 TotalSize = 0; //バッファ char source_buffer[BUF_SIZE]; char read_buffer[BUF_SIZE]; char out_buffer[BUF_SIZE]; char chain_buffer[BUF_SIZE]; // IVなどを格納するチェインバッファ char margin_buffer[BUF_SIZE]; //ファイルストリーム TFileStream *fsIn; TFileStream *fsOut; TFileStream *fsExe; //オープン中か bool fOpenIn; bool fOpenOut; //メモリストリーム TMemoryStream *pms = new TMemoryStream; // マージンバッファサイズ int MarginBufSize = MARGIN_BUF_SIZE; // PKCS #7 Pading num. unsigned char paddingNum = 0; //--------------------------------------- // 同名ファイルがあるのでダイアログ表示 //--------------------------------------- if ( fConfirmOverwirte == true && fOverwirteYesToAll == false ) { if (FileExists(OutFilePath) == true) { //同名ファイルの上書き確認メッセージダイアログ MsgText = LoadResourceString(&Msgencrypt::_MSG_CONFIRM_OVER_WRITE_SAME_FILE)+"\n"+OutFilePath; Synchronize(&PostConfirmOverwriteMessageForm); if ( MsgReturnVal == mrYes ) { //上書きOKなのでFilePathはそのまま } else if ( MsgReturnVal == mrNo ) { //別名保存でFilePath文字列が書き換えられてきている OutFilePath = MsgReturnPath; } else if ( MsgReturnVal == mrYesToAll ) { //すべて上書き(YesToAll) fOverwirteYesToAll = true; } else if ( MsgReturnVal == mrCancel ) { //キャンセル delete pms; goto LabelStop; } } } //--------------------------------------- // ヘッダ情報の生成&ファイル総サイズ取得 //--------------------------------------- //'暗号化するファイルリストの生成中...' ProgressStatusText = LoadResourceString(&Msgencrypt::_LABEL_STATUS_TITLE_LISTING); if ( CreateHeaderData( pms, InputFileList, FilePathList, AllTotalSize) == false ){ if (Terminated == true) { //ユーザーキャンセルで抜けてきた delete pms; goto LabelStop; } else{ //'暗号化するファイルを開けません。他のアプリケーションが使用中の可能性があります。' MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_FILE_OPEN); MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); delete pms; goto LabelError; } } //----------------------------------- // ディスクの空き容量チェック //----------------------------------- CurrentDriveFreeSpaceSize = GetDiskFreeSpaceNum(OutFilePath); if (CurrentDriveFreeSpaceSize > -1 ) { if ( AllTotalSize > CurrentDriveFreeSpaceSize ) { //"ディスクの空き容量が足りません! 暗号化ファイルを保存できません。\n //暗号化を中止します。;" MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_NO_DISK_FREE_SPACE); MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); delete pms; goto LabelError; } } else{ // -1はネットワークドライブの可能性があるので無視 //(万が一、別のエラーの場合、実際書き込みに移行したときエラーが発生する) } //----------------------------------- // 実行可能形式でかつ // 合計バイト数が4GBを越えたときのエラー //----------------------------------- if ( fExeOutputOption == true && fOver4gbOk == false && AllTotalSize > SIZE_4GB ){ //実行形式ファイルのサイズが4GBを超えてしまう可能性があります!\n //Win32アプリケーションとして実行できなくなるかもしれませんがよろしいですか?'; MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_OVER_4GB_EXE); MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbYes << mbNo; MsgDefaultButton = mbNo; Synchronize(&PostConfirmMessageForm); if ( MsgReturnVal == mbNo) { //キャンセル delete pms; goto LabelStop; } } //----------------------------------- // 暗号化ファイルの生成開始 //----------------------------------- //'暗号化しています...' ProgressStatusText = LoadResourceString(&Msgencrypt::_LABEL_STATUS_TITLE_ENCRYPTING); ProgressMsgText = ExtractFileName(OutFilePath); TotalSize = 0; try{ fsOut = new TFileStream(OutFilePath, fmCreate); fOpenOut = true; } catch(...){ //'保存する先のファイルが開けません。他のアプリケーションが使用中の可能性があります。' MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_OUT_FILE_OPEN) + "\n" + OutFilePath; MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); delete pms; goto LabelError; } //----------------------------------- // 実行可能形式の出力 //----------------------------------- if ( fExeOutputOption == true ){ //----------------------------------- // 自分のお尻から実行データを抽出 //----------------------------------- //自分自身の実行ファイルを開く try{ fsExe = new TFileStream(Application->ExeName, fmOpenRead | fmShareDenyWrite); } catch(...){ //'実行可能形式出力に失敗しました。暗号化処理を中止します。' MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_EXEOUT_FAILED); MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); delete pms; goto LabelError; } //切り出すサイズを取得 fsExe->Seek(-(__int64)sizeof(__int64), TSeekOrigin::soEnd); fsExe->Read(&ExeAllSize, sizeof(__int64)); //処理する合計サイズに実行形式分を加える AllTotalSize += ExeAllSize; //自己実行可能形式データの境界へ fsExe->Seek(-(__int64)ExeAllSize-sizeof(__int64), TSeekOrigin::soEnd); while(fsExe->Read(read_buffer, BUF_SIZE) != 0 ){ ExeSize+=BUF_SIZE; //書き込む if ( ExeSize < ExeAllSize ){ fsOut->Write(read_buffer, BUF_SIZE); TotalSize += BUF_SIZE; } else{ fsOut->Write(read_buffer, ExeSize-ExeAllSize); TotalSize += (ExeSize-ExeAllSize); } //進捗表示 ProgressPercentNumF = (float)TotalSize/AllTotalSize; ProgressPercentNum = (int)(ProgressPercentNumF*100); if (AllTotalSize < 104857600) { // 100MB未満 ProgressPercentNumText = IntToStr(ProgressPercentNum)+"%"; } else{ ProgressPercentNumText = FloatToStrF(ProgressPercentNumF*100, ffNumber, 4, 1)+"%"; } } //自分自身を閉じる delete fsExe; } //----------------------------------- // ヘッダ情報の描き込み //----------------------------------- pms->SaveToStream(fsOut); //fsOutに追記 delete pms; //----------------------------------- // Rijndaelの初期化 //----------------------------------- gentables(); gkey( 8, 8, key); // 初期化ベクトルを生成して先頭に書き込む fillrand(chain_buffer, BUF_SIZE); if ( fsOut->Write(chain_buffer, BUF_SIZE) < BUF_SIZE ){ //''保存先に指定された暗号化ファイルに書き込めません。 MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_OUT_FILE_WRITE) + "\n" + OutFilePath; MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); goto LabelError; } //----------------------------------- // zlib 初期化(圧縮においてすべてのメモリ管理をライブラリに任せる) z.zalloc = Z_NULL; z.zfree = Z_NULL; z.opaque = Z_NULL; //z.next_in = Z_NULL; // 第2引数は圧縮の度合。0~9 の範囲の整数で,0 は無圧縮 // Z_DEFAULT_COMPRESSION (= 6) が標準 if (deflateInit(&z, CompressRateNum) != Z_OK){ //zlibエラー表示はラベル先で goto LabelError; } //出力バッファの初期化 for(i = 0; i < BUF_SIZE; i++){ out_buffer[i] = 0; } // zlibに入出力バッファをセットする z.avail_in = 0; // 入力バッファ中のデータのバイト数 z.next_out = out_buffer; // 出力バッファ残量 z.avail_out = BUF_SIZE; // 出力ポインタ // 通常は deflate() の第2引数は Z_NO_FLUSH にして呼び出す flush = Z_NO_FLUSH; FileIndex = 0; while(!Terminated) { //----------------------------------- //入力 //----------------------------------- if ( z.avail_in == 0 && flush != Z_FINISH){ pos = 0; for(i = 0; i < BUF_SIZE; i++){ source_buffer[i] = 0; read_buffer[i] = 0; } while ( pos < BUF_SIZE ){ //オープン中のファイルがあればそこから読む if ( fOpenIn == true ) { if (pos < BUF_SIZE) { len = fsIn->Read(read_buffer, BUF_SIZE - pos); TotalSize+=len; for (i = 0; i < len; i++) { source_buffer[pos+i] = read_buffer[i]; } if (len < BUF_SIZE - pos) { fOpenIn = false; //ファイルを閉じる delete fsIn; } } pos += len; } //ファイルを開く else{ if (FileIndex < FilePathList->Count) { while(FileIndex < FilePathList->Count){ if (FilePathList->Strings[FileIndex] != "") { try{ fsIn = new TFileStream(FilePathList->Strings[FileIndex], fmOpenRead | fmShareDenyWrite); fOpenIn = true; FileIndex++; break; } catch(...){ //'暗号化するファイルを開けません。他のアプリケーションが使用中の可能性があります。' MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_FILE_OPEN); MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); fOpenIn = false; goto LabelError; } } FileIndex++; } } else{ //読み込むファイルがなくなったので、 //お尻にダミーのマージンデータを挿入する // //【補足】 // 本来はここにあるマージンデータ挿入処理は不要ですが、 // 昔に作った際に復号の際に圧縮データ境界のチェックを // 怠っていたため、このように余分なデータを // 入れておくという力業を使っています(すみません...) fillrand(margin_buffer, BUF_SIZE); for (i = pos; i < BUF_SIZE; i++) { source_buffer[i] = margin_buffer[i]; } pos = BUF_SIZE; MarginBufSize -= BUF_SIZE; }//end if (FileIndex < FilePathList->Count); }//end if ( fOpenIn == true ); }//while ( pos < BUF_SIZE && 0 < MarginBufSize ); if (MarginBufSize < 1) { flush = Z_FINISH; //入力バッファはこれが最後 } z.next_in = source_buffer; z.avail_in = pos; }//end if ( z.avail_in == 0 ); //----------------------------------- //圧縮 //----------------------------------- if ( z.avail_out > 0 ){ status = deflate(&z, flush); } if (status == Z_STREAM_END){ break; } if (status != Z_OK ){ //#define Z_OK 0 //#define Z_STREAM_END 1 //#define Z_NEED_DICT 2 //#define Z_ERRNO (-1) //#define Z_STREAM_ERROR (-2) //#define Z_DATA_ERROR (-3) //#define Z_MEM_ERROR (-4) //#define Z_BUF_ERROR (-5) //#define Z_VERSION_ERROR (-6) goto LabelError; } //----------------------------------- //出力 //----------------------------------- if ( z.avail_out == 0 ){ // CBC - xor the file bytes with the IV bytes for(i = 0; i < BUF_SIZE; i++){ out_buffer[i] ^= chain_buffer[i]; } //encrypt! rijndael_encrypt(out_buffer); len = fsOut->Write(out_buffer, BUF_SIZE); if (len < BUF_SIZE) { //'保存先に指定された暗号化ファイルに書き込めません。 MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_OUT_FILE_WRITE) + "\n" + OutFilePath; MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); goto LabelError; } for(i = 0; i < BUF_SIZE; i++){ chain_buffer[i] = out_buffer[i]; out_buffer[i] = 0; } z.next_out = out_buffer; // 出力バッファ残量を元に戻す z.avail_out = BUF_SIZE; // 出力ポインタを元に戻す } //----------------------------------- //進捗状況表示 if (AllTotalSize == 0) { ProgressPercentNum = 100; ProgressPercentNumText = "100%"; } else if (TotalSize == 0) { ProgressPercentNum = 0; ProgressPercentNumText = "0%"; } else{ ProgressPercentNumF = (float)TotalSize/AllTotalSize; ProgressPercentNum = (int)(ProgressPercentNumF*100); if (AllTotalSize < 104857600) { // 100MB未満 ProgressPercentNumText = IntToStr(ProgressPercentNum)+"%"; } else{ ProgressPercentNumText = FloatToStrF(ProgressPercentNumF*100, ffNumber, 4, 1)+"%"; } } //----------------------------------- if ( fOpenIn == true ){ ProgressMsgText = ExtractFileName(fsIn->FileName); } else{ ProgressMsgText = ExtractFileName(OutFilePath); } }//while(!Terminated); if (Terminated == true) { //ユーザーキャンセルで抜けてきた goto LabelStop; } //残りのバッファ if (z.avail_out > 0) { // PKCS #7 パディング len = BUF_SIZE - z.avail_out; paddingNum = (char)z.avail_out; for(i = len; i < BUF_SIZE; i++){ out_buffer[i] = paddingNum; } // CBC - xor the file bytes with the IV bytes for(i = 0; i < BUF_SIZE; i++){ out_buffer[i] ^= chain_buffer[i]; } //encrypt! rijndael_encrypt(out_buffer); if ((len = fsOut->Write(out_buffer, BUF_SIZE)) != BUF_SIZE){ //'保存先に指定された暗号化ファイルに書き込めません。 MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_OUT_FILE_WRITE) + "\n" + OutFilePath; MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); goto LabelError; } } if (deflateEnd(&z) != Z_OK){ //zlibエラー goto LabelError; } //----------------------------------- // 実行可能形式ファイルは // 末尾へ暗号化データサイズを書き込む //----------------------------------- if ( fExeOutputOption == true ){ ExeSize = fsOut->Seek((__int64)0, TSeekOrigin::soEnd); ExeSize = ExeSize-ExeAllSize; fsOut->Write(&ExeSize, sizeof(__int64)); } //----------------------------------- // 完了 //----------------------------------- ProgressPercentNum = 100; //'完了' ProgressStatusText = LoadResourceString(&Msgencrypt::_LABEL_STATUS_TITLE_COMPLETE); ProgressMsgText = ExtractFileName(OutFilePath); if (fOpenIn == true) { delete fsIn; } if (fOpenOut == true) { delete fsOut; } //出力する暗号化ファイルのタイムスタンプを元ファイルに合わせる if ( fKeepTimeStamp == true && first_fd.cFileName[0] != NULL ) { hFile = CreateFileW(FilePath.c_str(), GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile != INVALID_HANDLE_VALUE) { SetFileTime( hFile, &first_fd.ftCreationTime, &first_fd.ftLastAccessTime, &first_fd.ftLastWriteTime); CloseHandle(hFile); } } StatusNum = 1; return; //----------------------------------- // エラーの後始末 //----------------------------------- LabelError: ProgressPercentNum = 0; if ( status < 0 ){ //'zlibライブラリからエラーを返されました。' //'エラー番号:' MsgText = LoadResourceString(&Msgencrypt::_MSG_ERROR_ZLIB) + IntToStr(status) + "\n" + z.msg; MsgType = mtError; MsgButtons = TMsgDlgButtons() << mbOK; MsgDefaultButton = mbOK; Synchronize(&PostConfirmMessageForm); } //'エラー' ProgressStatusText = LoadResourceString(&Msgencrypt::_LABEL_STATUS_TITLE_ERROR); //'暗号化に失敗しました。' ProgressMsgText = LoadResourceString(&Msgencrypt::_LABEL_STATUS_DETAIL_FAILED); if (fOpenIn == true) { delete fsIn; } if (fOpenOut == true) { delete fsOut; } StatusNum = -1; return; //----------------------------------- // ユーザーキャンセルの後始末 //----------------------------------- LabelStop: ProgressPercentNum = 0; //'キャンセル' ProgressStatusText = LoadResourceString(&Msgencrypt::_LABEL_STATUS_TITLE_USER_CANCEL); //'暗号化が中止されました。' ProgressMsgText = LoadResourceString(&Msgencrypt::_LABEL_STATUS_DETAIL_STOPPED); if (fOpenIn == true) { delete fsIn; } if (fOpenOut == true) { delete fsOut; } StatusNum = -2; return; }
static int bio_zlib_write(BIO *b, const char *in, int inl) { BIO_ZLIB_CTX *ctx; int ret; z_stream *zout; if (!in || !inl) return 0; ctx = (BIO_ZLIB_CTX *)b->ptr; if (ctx->odone) return 0; zout = &ctx->zout; BIO_clear_retry_flags(b); if (!ctx->obuf) { ctx->obuf = malloc(ctx->obufsize); /* Need error here */ if (!ctx->obuf) { COMPerror(ERR_R_MALLOC_FAILURE); return 0; } ctx->optr = ctx->obuf; ctx->ocount = 0; deflateInit(zout, ctx->comp_level); zout->next_out = ctx->obuf; zout->avail_out = ctx->obufsize; } /* Obtain input data directly from supplied buffer */ zout->next_in = (void *)in; zout->avail_in = inl; for (;;) { /* If data in output buffer write it first */ while (ctx->ocount) { ret = BIO_write(b->next_bio, ctx->optr, ctx->ocount); if (ret <= 0) { /* Total data written */ int tot = inl - zout->avail_in; BIO_copy_next_retry(b); if (ret < 0) return (tot > 0) ? tot : ret; return tot; } ctx->optr += ret; ctx->ocount -= ret; } /* Have we consumed all supplied data? */ if (!zout->avail_in) return inl; /* Compress some more */ /* Reset buffer */ ctx->optr = ctx->obuf; zout->next_out = ctx->obuf; zout->avail_out = ctx->obufsize; /* Compress some more */ ret = deflate(zout, 0); if (ret != Z_OK) { COMPerror(COMP_R_ZLIB_DEFLATE_ERROR); ERR_asprintf_error_data("zlib error:%s", zError(ret)); return 0; } ctx->ocount = ctx->obufsize - zout->avail_out; } }
u_int32_t deflate_global(u_int8_t *data, u_int32_t size, int decomp, u_int8_t **out) { z_stream zbuf; u_int8_t *output; u_int32_t count, result; int error, i = 0, j; struct deflate_buf buf[ZBUF]; bzero(&zbuf, sizeof(z_stream)); for (j = 0; j < ZBUF; j++) buf[j].flag = 0; zbuf.next_in = data; /* data that is going to be processed */ zbuf.zalloc = zcalloc; zbuf.zfree = zcfree; zbuf.opaque = Z_NULL; zbuf.avail_in = size; /* Total length of data to be processed */ if (decomp) { /* * Choose a buffer with 4x the size of the input buffer * for the size of the output buffer in the case of * decompression. If it's not sufficient, it will need to be * updated while the decompression is going on */ if (size < 32 * 1024) size *= 4; } buf[i].out = malloc((u_long)size, M_CRYPTO_DATA, M_NOWAIT); if (buf[i].out == NULL) goto bad; buf[i].size = size; buf[i].flag = 1; i++; zbuf.next_out = buf[0].out; zbuf.avail_out = buf[0].size; error = decomp ? inflateInit2(&zbuf, window_inflate) : deflateInit2(&zbuf, Z_DEFAULT_COMPRESSION, Z_METHOD, window_deflate, Z_MEMLEVEL, Z_DEFAULT_STRATEGY); if (error != Z_OK) goto bad; for (;;) { error = decomp ? inflate(&zbuf, Z_PARTIAL_FLUSH) : deflate(&zbuf, Z_FINISH); if (error == Z_STREAM_END) break; if (error != Z_OK) goto bad; if (zbuf.avail_out == 0 && i < (ZBUF - 1)) { /* we need more output space, allocate size */ if (size < 32 * 1024) size *= 2; buf[i].out = malloc((u_long)size, M_CRYPTO_DATA, M_NOWAIT); if (buf[i].out == NULL) goto bad; zbuf.next_out = buf[i].out; buf[i].size = size; buf[i].flag = 1; zbuf.avail_out = buf[i].size; i++; } else goto bad; /* out of buffers */ } result = count = zbuf.total_out; *out = malloc((u_long)result, M_CRYPTO_DATA, M_NOWAIT); if (*out == NULL) goto bad; if (decomp) inflateEnd(&zbuf); else deflateEnd(&zbuf); output = *out; for (j = 0; buf[j].flag != 0; j++) { if (count > buf[j].size) { bcopy(buf[j].out, *out, buf[j].size); *out += buf[j].size; free(buf[j].out, M_CRYPTO_DATA); count -= buf[j].size; } else { /* it should be the last buffer */ bcopy(buf[j].out, *out, count); *out += count; free(buf[j].out, M_CRYPTO_DATA); count = 0; } } *out = output; return result; bad: *out = NULL; for (j = 0; buf[j].flag != 0; j++) free(buf[j].out, M_CRYPTO_DATA); if (decomp) inflateEnd(&zbuf); else deflateEnd(&zbuf); return 0; }
int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile, bool hardnested_mode) { uint8_t *fpga_config; uint32_t i; int32_t ret; uint8_t c; z_stream compressed_fpga_stream; if (hardnested_mode) { fpga_config = malloc(num_infiles * HARDNESTED_TABLE_SIZE); } else { fpga_config = malloc(num_infiles * FPGA_CONFIG_SIZE); } // read the input files. Interleave them into fpga_config[] i = 0; do { if (i >= num_infiles * (hardnested_mode?HARDNESTED_TABLE_SIZE:FPGA_CONFIG_SIZE)) { if (hardnested_mode) { fprintf(stderr, "Input file too big (> %lu bytes). This is probably not a hardnested bitflip state table.\n", HARDNESTED_TABLE_SIZE); } else { fprintf(stderr, "Input files too big (total > %lu bytes). These are probably not PM3 FPGA config files.\n", num_infiles*FPGA_CONFIG_SIZE); } for(uint16_t j = 0; j < num_infiles; j++) { fclose(infile[j]); } free(fpga_config); return(EXIT_FAILURE); } for(uint16_t j = 0; j < num_infiles; j++) { for(uint16_t k = 0; k < FPGA_INTERLEAVE_SIZE; k++) { c = (uint8_t)fgetc(infile[j]); if (!feof(infile[j])) { fpga_config[i++] = c; } else if (num_infiles > 1) { fpga_config[i++] = '\0'; } } } } while (!all_feof(infile, num_infiles)); // initialize zlib structures compressed_fpga_stream.next_in = fpga_config; compressed_fpga_stream.avail_in = i; compressed_fpga_stream.zalloc = fpga_deflate_malloc; compressed_fpga_stream.zfree = fpga_deflate_free; compressed_fpga_stream.opaque = Z_NULL; ret = deflateInit2(&compressed_fpga_stream, COMPRESS_LEVEL, Z_DEFLATED, COMPRESS_WINDOW_BITS, COMPRESS_MEM_LEVEL, COMPRESS_STRATEGY); // estimate the size of the compressed output uint32_t outsize_max = deflateBound(&compressed_fpga_stream, compressed_fpga_stream.avail_in); uint8_t *outbuf = malloc(outsize_max); compressed_fpga_stream.next_out = outbuf; compressed_fpga_stream.avail_out = outsize_max; if (ret == Z_OK) { ret = deflateTune(&compressed_fpga_stream, COMPRESS_GOOD_LENGTH, COMPRESS_MAX_LAZY, COMPRESS_MAX_NICE_LENGTH, COMPRESS_MAX_CHAIN); } if (ret == Z_OK) { ret = deflate(&compressed_fpga_stream, Z_FINISH); } fprintf(stdout, "compressed %u input bytes to %lu output bytes\n", i, compressed_fpga_stream.total_out); if (ret != Z_STREAM_END) { fprintf(stderr, "Error in deflate(): %i %s\n", ret, compressed_fpga_stream.msg); free(outbuf); deflateEnd(&compressed_fpga_stream); for(uint16_t j = 0; j < num_infiles; j++) { fclose(infile[j]); } fclose(outfile); free(infile); free(fpga_config); return(EXIT_FAILURE); } for (i = 0; i < compressed_fpga_stream.total_out; i++) { fputc(outbuf[i], outfile); } free(outbuf); deflateEnd(&compressed_fpga_stream); for(uint16_t j = 0; j < num_infiles; j++) { fclose(infile[j]); } fclose(outfile); free(infile); free(fpga_config); return(EXIT_SUCCESS); }
static fz_image *render_to_pixmap(fz_context *ctx, HBITMAP hbmp, SizeI size) { int w = size.dx, h = size.dy; int stride = ((w * 3 + 3) / 4) * 4; unsigned char *data = (unsigned char *)fz_malloc(ctx, stride * h); BITMAPINFO bmi = { 0 }; bmi.bmiHeader.biSize = sizeof(bmi.bmiHeader); bmi.bmiHeader.biWidth = w; bmi.bmiHeader.biHeight = -h; bmi.bmiHeader.biPlanes = 1; bmi.bmiHeader.biBitCount = 24; bmi.bmiHeader.biCompression = BI_RGB; HDC hDC = GetDC(nullptr); int res = GetDIBits(hDC, hbmp, 0, h, data, &bmi, DIB_RGB_COLORS); ReleaseDC(nullptr, hDC); if (!res) { fz_free(ctx, data); fz_throw(ctx, FZ_ERROR_GENERIC, "GetDIBits failed"); } // convert BGR with padding to RGB without padding unsigned char *out = data; bool is_grayscale = true; for (int y = 0; y < h; y++) { const unsigned char *in = data + y * stride; unsigned char green, blue; for (int x = 0; x < w; x++) { is_grayscale = is_grayscale && in[0] == in[1] && in[0] == in[2]; blue = *in++; green = *in++; *out++ = *in++; *out++ = green; *out++ = blue; } } // convert grayscale RGB to proper grayscale if (is_grayscale) { const unsigned char *in = out = data; for (int i = 0; i < w * h; i++) { *out++ = *in++; in += 2; } } fz_compressed_buffer *buf = nullptr; fz_var(buf); fz_try(ctx) { buf = fz_malloc_struct(ctx, fz_compressed_buffer); buf->buffer = fz_new_buffer(ctx, w * h * 4 + 10); buf->params.type = FZ_IMAGE_FLATE; buf->params.u.flate.predictor = 1; z_stream zstm = { 0 }; zstm.next_in = data; zstm.avail_in = out - data; zstm.next_out = buf->buffer->data; zstm.avail_out = buf->buffer->cap; res = deflateInit(&zstm, 9); if (res != Z_OK) fz_throw(ctx, FZ_ERROR_GENERIC, "deflate failure %d", res); res = deflate(&zstm, Z_FINISH); if (res != Z_STREAM_END) fz_throw(ctx, FZ_ERROR_GENERIC, "deflate failure %d", res); buf->buffer->len = zstm.total_out; res = deflateEnd(&zstm); if (res != Z_OK) fz_throw(ctx, FZ_ERROR_GENERIC, "deflate failure %d", res); } fz_always(ctx) { fz_free(ctx, data); } fz_catch(ctx) { fz_free_compressed_buffer(ctx, buf); fz_rethrow(ctx); } fz_colorspace *cs = is_grayscale ? fz_device_gray(ctx) : fz_device_rgb(ctx); return fz_new_image(ctx, w, h, 8, cs, 96, 96, 0, 0, nullptr, nullptr, buf, nullptr); }
int streaming_commons_call_deflate_flush (z_stream *stream) { return deflate(stream, Z_SYNC_FLUSH); }
int ELF32(int f, Elf32_Ehdr *b, unsigned char **output) /* Caller must free output manually */ { int i,j; unsigned char *tempbuffer=NULL; int tempbufferpos=0; int maxoutputsize=TEMPBUFSIZE; tempbuffer=malloc(TEMPBUFSIZE); //setup zlib z_stream strm; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; deflateInit(&strm, 9); *output=malloc(maxoutputsize); //allocate 256KB. This "should" be enough, but reallocate if more is needed strm.avail_out=maxoutputsize-3*sizeof(uint32_t); //first if it's an exe, followed by the compressed size, followed by the decompressed size strm.next_out=(unsigned char *)&(*output)[sizeof(uint32_t)*3]; *(uint32_t *)(&(*output)[0])=(b->e_type==ET_EXEC); /* printf("e_shoff=%x\n", b->e_shoff); printf("e_shentsize=%d\n", b->e_shentsize); printf("e_shnum=%d\n", b->e_shnum); printf("e_shstrndx=%d\n", b->e_shstrndx);*/ Elf32_Shdr *sectionHeaders=malloc(b->e_shentsize*b->e_shnum); if (pread(f, sectionHeaders, b->e_shentsize*b->e_shnum, b->e_shoff)==-1) { //printf("Failure to read sectionHeaders\n"); deflateEnd(&strm); if (sectionHeaders) free(sectionHeaders); if (output) free(output); if (tempbuffer) free(tempbuffer); return -1; } unsigned char **stringTable=calloc(b->e_shnum, sizeof(unsigned char*) ); loadStringTable32(f, sectionHeaders, stringTable, b->e_shstrndx); for (i=0; i<b->e_shnum; i++) { //printf("Section %d (%x): name=%s\n", i, sectionHeaders[i].sh_addr, &stringTable[b->e_shstrndx][sectionHeaders[i].sh_name]); if ((sectionHeaders[i].sh_type==SHT_SYMTAB) || (sectionHeaders[i].sh_type==SHT_DYNSYM)) { // printf("Symbol data:\n", i); // printf("sh_addr=%x\n", sectionHeaders[i].sh_addr); //printf("sh_offset=%x\n", sectionHeaders[i].sh_offset); //printf("sh_size=%x\n", sectionHeaders[i].sh_size); //printf("sh_link=%d (string table)\n", sectionHeaders[i].sh_link); //printf("sh_info=%d\n", sectionHeaders[i].sh_info); Elf32_Sym *symbolTable=malloc(sectionHeaders[i].sh_size); if (pread(f, symbolTable, sectionHeaders[i].sh_size, sectionHeaders[i].sh_offset)==-1) { // printf("Failure reading symbol table\n"); return -1; } int maxindex=sectionHeaders[i].sh_size / sizeof(Elf32_Sym); loadStringTable32(f, sectionHeaders, stringTable, sectionHeaders[i].sh_link); //printf("maxindex=%d\n", maxindex); for (j=0; j<maxindex; j++) { //printf("symbolTable[%d]:\n", i); //printf("st_name=%s\n", &stringTable[sectionHeaders[i].sh_link][symbolTable[j].st_name] ); // printf("st_value=%x\n", symbolTable[j].st_value); //printf("st_size=%d\n", symbolTable[j].st_size); // printf("st_info=%d\n", symbolTable[j].st_info); //printf(" Bind=%d\n", ELF32_ST_BIND(symbolTable[j].st_info)); //printf(" Type=%d\n", ELF32_ST_TYPE(symbolTable[j].st_info)); // printf("st_other=%d\n", symbolTable[j].st_other); // printf("st_shndx=%d\n", symbolTable[j].st_shndx); if (symbolTable[j].st_value) { //add it to the tempbuffer char *symbolname=(char *)&stringTable[sectionHeaders[i].sh_link][symbolTable[j].st_name]; size_t namelength=strlen(symbolname); int entrysize=sizeof(symbolinfo)+namelength; if (tempbufferpos+entrysize>=TEMPBUFSIZE) { //compress the current temp buffer //printf("compressing\n"); strm.avail_in=tempbufferpos; strm.next_in=tempbuffer; while (strm.avail_in) { if (deflate(&strm, Z_NO_FLUSH)!=Z_OK) { //printf("FAILURE TO COMPRESS!\n"); return -1; } //printf("strm.avail_out=%d\n", strm.avail_out); if (strm.avail_out==0) { //printf("Out buffer full. Reallocating\n"); *output=realloc(*output, maxoutputsize*2); strm.next_out=(unsigned char *)&(*output)[maxoutputsize]; strm.avail_out=maxoutputsize; maxoutputsize=maxoutputsize*2; } } tempbufferpos=0; } psymbolinfo si=(psymbolinfo)&tempbuffer[tempbufferpos]; si->address=symbolTable[j].st_value; si->size=symbolTable[j].st_size; si->type=symbolTable[j].st_info; si->namelength=namelength; memcpy(&si->name, symbolname, namelength); tempbufferpos+=entrysize; } } free(symbolTable); } } for (i=0; i<b->e_shnum; i++) { if (stringTable[i]) free(stringTable[i]); } free(stringTable); free(sectionHeaders); printf("end:\n"); strm.avail_in=tempbufferpos; strm.next_in=tempbuffer; while (1) { i=deflate(&strm, Z_FINISH); printf("i=%d\n", i); if (i==Z_STREAM_END) //done break; if (i!=Z_OK) { printf("Failure to compress: %i\n", i); return -1; } if (strm.avail_out==0) { printf("Out buffer full. Reallocating :%d\n", maxoutputsize*2); *output=realloc(*output, maxoutputsize*2); strm.next_out=(unsigned char *)&(*output)[maxoutputsize]; strm.avail_out=maxoutputsize; maxoutputsize=maxoutputsize*2; } else break; }; /*printf("strm.avail_out=%d\n", strm.avail_out); printf("total_in = %lu\n", strm.total_in); printf("total_out = %lu\n", strm.total_out);*/ deflateEnd(&strm); //update the size *(uint32_t *)(&(*output)[4])=strm.total_out+3*sizeof(uint32_t); *(uint32_t *)(&(*output)[8])=strm.total_in; free(tempbuffer); return 0; }