void go() override
{
const std::string in_file = get_arg("file");
std::string out_file, suffix;
parse_extension(in_file, out_file, suffix);
std::ifstream in(in_file);
if(!in.good())
throw CLI_IO_Error("reading", in_file);
std::unique_ptr<Botan::Transform> decompress;
#if defined(BOTAN_HAS_COMPRESSION)
decompress.reset(Botan::make_decompressor(suffix));
#endif
if(!decompress)
throw CLI_Error_Unsupported("Decompression", suffix);
std::ofstream out(out_file);
if(!out.good())
throw CLI_IO_Error("writing", out_file);
do_compress(*decompress, in, out, get_arg_sz("buf-size"));
}
void go() override
{
const std::string comp_type = get_arg("type");
std::unique_ptr<Botan::Transform> compress;
#if defined(BOTAN_HAS_COMPRESSION)
compress.reset(Botan::make_compressor(comp_type, get_arg_sz("level")));
#endif
if(!compress)
{
throw CLI_Error_Unsupported("Compression", comp_type);
}
const std::string in_file = get_arg("file");
std::ifstream in(in_file);
if(!in.good())
{
throw CLI_IO_Error("reading", in_file);
}
const std::string out_file = output_filename(in_file, comp_type);
std::ofstream out(out_file);
if(!in.good())
{
throw CLI_IO_Error("writing", out_file);
}
do_compress(*compress, in, out, get_arg_sz("buf-size"));
}
void send_block_list(Signature *sig)
{
IOSBuf *ios, *ios1, *ios2;
ProtoPutBlock pb;
guint32 length;
void *data;
if(int_option(kOption_verbose) & VERBOSE_FLOW)
g_message("sending block list");
g_assert(sig);
ios = block_list_to_stream(sig);
g_assert(ios);
SCsum_buffer(ios_buffer(ios), ios_buffer_size(ios),
sig->summary->block_list_sha1);
memset(&pb, 0, sizeof(pb));
pb.file_id = sig->summary->file_id;
pb.block_id = 0;
length = ios_buffer_size(ios);
data = ios_buffer(ios);
sig->summary->block_list_length = length;
ios1 = ios_new();
ios_set_buffer(ios1, data, length);
ios2 = do_compress(ios1);
pb.ios = encrypt(ios2);
ios_free(ios1);
ios_free(ios2);
send_put_block(&pb);
ios_free(ios);
return;
}
int exomizer(unsigned char *srcbuf, int len, int load, int start, unsigned char *destbuf)
{
int destlen;
int max_offset = 65536;
int max_passes = 65536;
static match_ctx ctx;
encode_match_data emd;
encode_match_priv optimal_priv;
search_nodep snp;
match_ctx_init(ctx, srcbuf, len, max_offset);
emd->out = NULL;
emd->priv = optimal_priv;
optimal_init(emd);
snp = do_compress(ctx, emd, max_passes);
destlen = generate_output(ctx, snp, sfx_c64ne, optimal_encode, emd,
load, len, start, destbuf);
optimal_free(emd);
#if 0 /* RH */
search_node_free(snp);
#endif /* RH */
match_ctx_free(ctx);
return destlen;
}
void crunch_backwards(struct membuf *inbuf,
struct membuf *outbuf,
struct crunch_options *options, /* IN */
struct crunch_info *info) /* OUT */
{
static match_ctx ctx;
encode_match_data emd;
search_nodep snp;
int outlen;
int safety;
int copy_used;
if(options == NULL)
{
options = default_options;
}
outlen = membuf_memlen(outbuf);
emd->out = NULL;
optimal_init(emd);
LOG(LOG_NORMAL,
("\nPhase 1: Instrumenting file"
"\n-----------------------------\n"));
LOG(LOG_NORMAL, (" Length of indata: %d bytes.\n", membuf_memlen(inbuf)));
match_ctx_init(ctx, inbuf, options->max_len, options->max_offset,
options->use_imprecise_rle);
LOG(LOG_NORMAL, (" Instrumenting file, done.\n"));
emd->out = NULL;
optimal_init(emd);
LOG(LOG_NORMAL,
("\nPhase 2: Calculating encoding"
"\n-----------------------------\n"));
snp = do_compress(ctx, emd, options->exported_encoding,
options->max_passes, options->use_literal_sequences);
LOG(LOG_NORMAL, (" Calculating encoding, done.\n"));
LOG(LOG_NORMAL,
("\nPhase 3: Generating output file"
"\n------------------------------\n"));
LOG(LOG_NORMAL, (" Encoding: %s\n", optimal_encoding_export(emd)));
safety = do_output(ctx, snp, emd, optimal_encode, outbuf, ©_used);
LOG(LOG_NORMAL, (" Length of crunched data: %d bytes.\n",
membuf_memlen(outbuf) - outlen));
optimal_free(emd);
search_node_free(snp);
match_ctx_free(ctx);
if(info != NULL)
{
info->literal_sequences_used = copy_used;
info->needed_safety_offset = safety;
}
}
bool compress_custom(void* input_data, InputFunc input_func,
void* output_data, OutputFunc output_func, int sample_size)
{
GifFileType* input_gif = NULL;
GifFileType* output_gif = NULL;
int result = GIF_ERROR;
// Check sample
if (sample_size <= 0) {
return false;
}
input_gif = DGifOpen(input_data, input_func, &error_code);
if (input_gif == NULL) {
LOGE(EMSG("Can't open input gif"));
return false;
}
DGifSlurp(input_gif);
if (input_gif->ImageCount == 0) {
LOGE(EMSG("Gif frame count is 0"));
DGifCloseFile(input_gif, &error_code);
return false;
}
// Save gif
output_gif = EGifOpen(output_data, output_func, &error_code);
if (output_gif == NULL) {
LOGE(EMSG("Can't open output gif"));
DGifCloseFile(input_gif, &error_code);
return false;
}
if (do_compress(input_gif, output_gif, sample_size)) {
result = EGifSpew(output_gif);
}
// Free
GifFreeExtensions(&output_gif->ExtensionBlockCount, &output_gif->ExtensionBlocks);
if (output_gif->SavedImages) {
GifFreeSavedImages(output_gif);
output_gif->SavedImages = NULL;
}
// Close gif
DGifCloseFile(input_gif, &error_code);
return result == GIF_OK;
}
/*
* Traverse the entry tree, writing data for every item that has
* non-null entry->compressed (i.e. every symlink and non-empty
* regfile).
*/
static unsigned int write_data(struct entry *entry, char *base, unsigned int offset)
{
do {
if (entry->uncompressed) {
if(entry->same) {
set_data_offset(entry, base, entry->same->offset);
entry->offset=entry->same->offset;
} else {
set_data_offset(entry, base, offset);
entry->offset=offset;
offset = do_compress(base, offset, entry->name, entry->uncompressed, entry->size);
}
}
else if (entry->child)
offset = write_data(entry->child, base, offset);
entry=entry->next;
} while (entry);
return offset;
}
gint main (gint argc, gchar **argv)
{
if (argc < 2)
do_usage (argv[0]);
camel_init (NULL, 0);
if (!strcmp(argv[1], "compress"))
return do_compress (argc, argv);
else if (!strcmp(argv[1], "dump"))
return do_dump (argc, argv);
else if (!strcmp(argv[1], "info"))
return do_info (argc, argv);
else if (!strcmp(argv[1], "check"))
return do_check (argc, argv);
else if (!strcmp(argv[1], "perf"))
return do_perf (argc, argv);
do_usage (argv[0]);
return 1;
}
/* Send blocks until the output buffer is sufficiently full.
*/
static void wt_send_blocks(WorkTicket *wt, IOSBuf *ios)
{
DBBlock *dbb;
ProtoPutBlock pb;
IOSBuf *ios1, *ios2;
g_assert(wt);
g_assert(ios);
/* g_assert(wt->action == wt_Archive); */
g_assert(wt->dbf);
while((ios_buffer_size(ios) < desired_queue_size)
&& (wt->next_block_to_queue < wt->sig.blockList->len)) {
dbb = g_ptr_array_index(wt->sig.blockList,
wt->next_block_to_queue++);
g_assert(dbb);
if(dbb->in_new_cover && !dbb->remote_id) {
/* If dbb->remote_id, then it's already in the
remote block store, so don't send. */
cover_fill_block(dbb, wt->fp);
memset(&pb, 0, sizeof(pb));
pb.file_id = wt->dbf->file_id;
pb.block_id = dbb->local_id;
ios1 = ios_new();
ios_set_buffer(ios1, dbb->data, dbb->length);
ios2 = do_compress(ios1);
pb.ios = encrypt(ios2);
ios_free(ios1);
ios_free(ios2);
send_put_block(&pb);
ios_free(pb.ios);
wt->num_blocks_moved++;
g_free(dbb->data);
}
}
if(wt->next_block_to_queue >= wt->sig.blockList->len)
wt->status = wt_AwaitingBlockConfirms;
return;
}
// not threadsafe, make sure there is only
// one thread calling this
void stat_helper::reset()
{
if (compressed_data != NULL && sent == false) return;
tair_stat *new_stat = (tair_stat *) malloc(STAT_TOTAL_SIZE);
assert (new_stat != NULL);
memset(new_stat, 0, STAT_TOTAL_SIZE);
if(curr_stat != NULL)
free(curr_stat);
curr_stat = stat;
stat = new_stat;
// fill persistent info
if (storage_mgr != NULL) {
storage_mgr->get_stats(curr_stat);
}
uint64_t now = tbsys::CTimeUtil::getTime();
uint64_t interval = now - last_send_time;
last_send_time = now;
interval /= 1000000; // conver to second
if (interval == 0) interval = 1;
log_debug("start calculate ratio, interval: %d", interval);
for (int i=0; i<STAT_LENGTH; i++) {
tair_stat *cs = curr_stat + i;
cs->set_get_count(cs->get_count() / interval);
cs->set_put_count(cs->put_count() / interval);
cs->set_remove_count(cs->remove_count() / interval);
cs->set_hit_count(cs->hit_count() / interval);
}
// compress
do_compress();
sent = false;
}
int main(int argc, char * const argv[])
{
int c;
int decompress = 0, files = 1;
int selftest_compression = 0, selftest_decompression = 0;
const char *ifile_name, *ofile_name;
FILE *ifile, *ofile;
while((c = getopt(argc, argv, "S:dc")) != -1) {
switch (c) {
case 'S':
switch (optarg[0]) {
case 'c':
selftest_compression = 1;
break;
case 'd':
selftest_decompression = 1;
break;
default:
goto usage;
}
break;
case 'd':
decompress = 1;
break;
case 'c':
files = 0;
break;
default:
goto usage;
}
}
if (selftest_compression)
return do_selftest_compression();
if (selftest_decompression)
return do_selftest_decompression();
ifile = stdin;
ofile = stdout;
if (files) {
if (optind > argc - 2)
goto usage;
ifile_name = argv[optind];
ofile_name = argv[optind + 1];
if (!(ifile = fopen(ifile_name, "rb"))) {
perror("fopen of ifile_name");
return 2;
}
if (!(ofile = fopen(ofile_name, "wb"))) {
perror("fopen of ofile_name");
return 3;
}
}
if (decompress)
return do_decompress(ifile, ofile);
else
return do_compress(ifile, ofile);
usage:
fprintf(stderr,
"Usage:\n"
"cl_tester [-d] infile outfile\t-\t[de]compress infile to outfile.\n"
"cl_tester [-d] -c\t\t-\t[de]compress stdin to stdout.\n"
"cl_tester -S c\t\t\t-\tSelf-test compression.\n"
"cl_tester -S d\t\t\t-\tSelf-test decompression.\n");
return 1;
}
/* Generate a seekable compressed stream. */
int SzipCompress::run(const char *name, Buffer &origBuf,
const char *outName, Buffer &outBuf)
{
size_t origSize = origBuf.GetLength();
if (origSize == 0) {
log("Won't compress %s: it's empty", name);
return 1;
}
if (SeekableZStreamHeader::validate(origBuf)) {
log("Skipping %s: it's already a szip", name);
return 0;
}
bool compressed = false;
log("Size = %" PRIuSize, origSize);
/* Allocate a buffer the size of the uncompressed data: we don't want
* a compressed file larger than that anyways. */
if (!outBuf.Resize(origSize)) {
log("Couldn't allocate output buffer: %s", strerror(errno));
return 1;
}
/* Find the most appropriate filter */
SeekableZStream::FilterId firstFilter, lastFilter;
bool scanFilters;
if (filter == SeekableZStream::FILTER_MAX) {
firstFilter = SeekableZStream::NONE;
lastFilter = SeekableZStream::FILTER_MAX;
scanFilters = true;
} else {
firstFilter = lastFilter = filter;
++lastFilter;
scanFilters = false;
}
mozilla::ScopedDeletePtr<Buffer> filteredBuf;
Buffer *origData;
for (SeekableZStream::FilterId f = firstFilter; f < lastFilter; ++f) {
FilteredBuffer *filteredTmp = NULL;
Buffer tmpBuf;
if (f != SeekableZStream::NONE) {
debug("Applying filter \"%s\"", filterName[f]);
filteredTmp = new FilteredBuffer();
filteredTmp->Filter(origBuf, f, chunkSize);
origData = filteredTmp;
} else {
origData = &origBuf;
}
if (dictSize && !scanFilters) {
filteredBuf = filteredTmp;
break;
}
debug("Compressing with no dictionary");
if (do_compress(*origData, tmpBuf, NULL, 0, f) == 0) {
if (tmpBuf.GetLength() < outBuf.GetLength()) {
outBuf.Fill(tmpBuf);
compressed = true;
filter = f;
filteredBuf = filteredTmp;
continue;
}
}
delete filteredTmp;
}
origData = filteredBuf ? filteredBuf : &origBuf;
if (dictSize) {
Dictionary<uint64_t> dict(*origData, dictSize ? SzipCompress::winSize : 0);
/* Find the most appropriate dictionary size */
size_t firstDictSize, lastDictSize;
if (dictSize == (size_t) -1) {
/* If we scanned for filters, we effectively already tried dictSize=0 */
firstDictSize = scanFilters ? 4096 : 0;
lastDictSize = SzipCompress::winSize;
} else {
firstDictSize = lastDictSize = dictSize;
}
Buffer tmpBuf;
for (size_t d = firstDictSize; d <= lastDictSize; d += 4096) {
debug("Compressing with dictionary of size %" PRIuSize, d);
if (do_compress(*origData, tmpBuf, static_cast<unsigned char *>(dict)
+ SzipCompress::winSize - d, d, filter))
continue;
if (!compressed || tmpBuf.GetLength() < outBuf.GetLength()) {
outBuf.Fill(tmpBuf);
compressed = true;
dictSize = d;
}
}
}
if (!compressed) {
outBuf.Fill(origBuf);
log("Not compressed");
return 0;
}
if (dictSize == (size_t) -1)
dictSize = 0;
debug("Used filter \"%s\" and dictionary size of %" PRIuSize,
filterName[filter], dictSize);
log("Compressed size is %" PRIuSize, outBuf.GetLength());
/* Sanity check */
Buffer tmpBuf;
SzipDecompress decompress;
if (decompress.run("buffer", outBuf, "buffer", tmpBuf))
return 1;
size_t size = tmpBuf.GetLength();
if (size != origSize) {
log("Compression error: %" PRIuSize " != %" PRIuSize, size, origSize);
return 1;
}
if (memcmp(static_cast<void *>(origBuf), static_cast<void *>(tmpBuf), size)) {
log("Compression error: content mismatch");
return 1;
}
return 0;
}
static
int process_file ( const compress_t *c, lzo_decompress_t decompress,
const char *method_name,
const char *file_name, lzo_uint l,
int t_loops, int c_loops, int d_loops )
{
int i;
unsigned blocks = 0;
unsigned long compressed_len = 0;
my_clock_t t_time = 0, c_time = 0, d_time = 0;
my_clock_t t_start, c_start, d_start;
#ifdef USE_DUMP
FILE *dump = NULL;
if (opt_dump_compressed_data)
dump = fopen(opt_dump_compressed_data,"wb");
#endif
/* process the file */
t_start = my_clock();
for (i = 0; i < t_loops; i++)
{
lzo_uint len, c_len, c_len_max, d_len = 0;
lzo_byte *d = data;
len = l;
c_len = 0;
blocks = 0;
/* process blocks */
if (len > 0 || opt_try_to_compress_0_bytes) do
{
int j;
int r;
const lzo_uint bl = len > opt_block_size ? opt_block_size : len;
lzo_uint bl_overwrite = bl;
#if defined(__LZO_CHECKER)
lzo_byte *dd = NULL;
lzo_byte *b1 = NULL;
lzo_byte *b2 = NULL;
const lzo_uint b1_len = bl + bl / 64 + 16 + 3;
#else
lzo_byte * const dd = d;
lzo_byte * const b1 = block1;
lzo_byte * const b2 = block2;
const lzo_uint b1_len = sizeof(_block1) - 256;
unsigned char random_byte;
random_byte = (unsigned char) my_clock();
#endif
blocks++;
/* may overwrite 3 bytes past the end of the decompressed block */
if (opt_use_asm_fast_decompressor)
bl_overwrite += (lzo_uint) sizeof(int) - 1;
#if defined(__LZO_CHECKER)
/* malloc a block of the exact size to detect any overrun */
dd = malloc(bl_overwrite > 0 ? bl_overwrite : 1);
b1 = malloc(b1_len);
b2 = dd;
if (dd == NULL || b1 == NULL)
{
perror("malloc");
return EXIT_MEM;
}
if (bl > 0)
memcpy(dd,d,bl);
#endif
/* compress the block */
c_len = c_len_max = 0;
c_start = my_clock();
for (j = r = 0; r == 0 && j < c_loops; j++)
{
c_len = b1_len;
r = do_compress(c,dd,bl,b1,&c_len);
if (r == 0 && c_len > c_len_max)
c_len_max = c_len;
}
c_time += my_clock() - c_start;
if (r != 0)
{
printf(" compression failed in block %d (%d) (%lu %lu)\n",
blocks, r, (long)c_len, (long)bl);
return EXIT_LZO_ERROR;
}
/* optimize the block */
if (opt_optimize_compressed_data)
{
d_len = bl;
r = do_optimize(c,b1,c_len,b2,&d_len);
if (r != 0 || d_len != bl)
{
printf(" optimization failed in block %d (%d) "
"(%lu %lu %lu)\n", blocks, r,
(long)c_len, (long)d_len, (long)bl);
return EXIT_LZO_ERROR;
}
}
#ifdef USE_DUMP
/* dump compressed data to disk */
if (dump)
{
lzo_fwrite(dump,b1,c_len);
fflush(dump);
}
#endif
/* decompress the block and verify */
#if defined(__LZO_CHECKER)
lzo_memset(b2,0,bl_overwrite);
#else
init_mem_checker(b2,_block2,bl_overwrite,random_byte);
#endif
d_start = my_clock();
for (j = r = 0; r == 0 && j < d_loops; j++)
{
d_len = bl;
r = do_decompress(c,decompress,b1,c_len,b2,&d_len);
if (d_len != bl)
break;
}
d_time += my_clock() - d_start;
if (r != 0)
{
printf(" decompression failed in block %d (%d) "
"(%lu %lu %lu)\n", blocks, r,
(long)c_len, (long)d_len, (long)bl);
return EXIT_LZO_ERROR;
}
if (d_len != bl)
{
printf(" decompression size error in block %d (%lu %lu %lu)\n",
blocks, (long)c_len, (long)d_len, (long)bl);
return EXIT_LZO_ERROR;
}
if (is_compressor(c))
{
if (lzo_memcmp(d,b2,bl) != 0)
{
lzo_uint x = 0;
while (x < bl && b2[x] == d[x])
x++;
printf(" decompression data error in block %d at offset "
"%lu (%lu %lu)\n", blocks, (long)x,
(long)c_len, (long)d_len);
if (opt_compute_adler32)
printf(" checksum: 0x%08lx 0x%08lx\n",
(long)adler_in, (long)adler_out);
#if 0
printf("Orig: ");
r = (x >= 10) ? -10 : 0 - (int) x;
for (j = r; j <= 10 && x + j < bl; j++)
printf(" %02x", (int)d[x+j]);
printf("\nDecomp:");
for (j = r; j <= 10 && x + j < bl; j++)
printf(" %02x", (int)b2[x+j]);
printf("\n");
#endif
return EXIT_LZO_ERROR;
}
if ((opt_compute_adler32 && adler_in != adler_out) ||
(opt_compute_crc32 && crc_in != crc_out))
{
printf(" checksum error in block %d (%lu %lu)\n",
blocks, (long)c_len, (long)d_len);
printf(" adler32: 0x%08lx 0x%08lx\n",
(long)adler_in, (long)adler_out);
printf(" crc32: 0x%08lx 0x%08lx\n",
(long)crc_in, (long)crc_out);
return EXIT_LZO_ERROR;
}
}
#if defined(__LZO_CHECKER)
/* free in reverse order of allocations */
free(b1);
free(dd);
#else
if (check_mem(b2,_block2,bl_overwrite,random_byte) != 0)
{
printf(" decompression overwrite error in block %d "
"(%lu %lu %lu)\n",
blocks, (long)c_len, (long)d_len, (long)bl);
return EXIT_LZO_ERROR;
}
#endif
d += bl;
len -= bl;
compressed_len += c_len_max;
}
while (len > 0);
}
t_time += my_clock() - t_start;
#ifdef USE_DUMP
if (dump)
fclose(dump);
opt_dump_compressed_data = NULL; /* only dump the first file */
#endif
print_stats(method_name, file_name,
t_loops, c_loops, d_loops,
t_time, c_time, d_time,
compressed_len, l, blocks);
return EXIT_OK;
}
struct backend *backend_connect(struct backend *ret_backend, const char *server,
struct protocol_t *prot, const char *userid,
sasl_callback_t *cb, const char **auth_status)
{
/* need to (re)establish connection to server or create one */
int sock = -1;
int r;
int err = -1;
int ask = 1; /* should we explicitly ask for capabilities? */
struct addrinfo hints, *res0 = NULL, *res;
struct sockaddr_un sunsock;
char buf[2048];
struct sigaction action;
struct backend *ret;
char rsessionid[MAX_SESSIONID_SIZE];
if (!ret_backend) {
ret = xzmalloc(sizeof(struct backend));
strlcpy(ret->hostname, server, sizeof(ret->hostname));
ret->timeout = NULL;
}
else
ret = ret_backend;
if (server[0] == '/') { /* unix socket */
res0 = &hints;
memset(res0, 0, sizeof(struct addrinfo));
res0->ai_family = PF_UNIX;
res0->ai_socktype = SOCK_STREAM;
res0->ai_addr = (struct sockaddr *) &sunsock;
res0->ai_addrlen = sizeof(sunsock.sun_family) + strlen(server) + 1;
#ifdef SIN6_LEN
res0->ai_addrlen += sizeof(sunsock.sun_len);
sunsock.sun_len = res0->ai_addrlen;
#endif
sunsock.sun_family = AF_UNIX;
strlcpy(sunsock.sun_path, server, sizeof(sunsock.sun_path));
/* XXX set that we are preauthed */
/* change hostname to 'config_servername' */
strlcpy(ret->hostname, config_servername, sizeof(ret->hostname));
}
else { /* inet socket */
memset(&hints, 0, sizeof(hints));
hints.ai_family = PF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
err = getaddrinfo(server, prot->service, &hints, &res0);
if (err) {
syslog(LOG_ERR, "getaddrinfo(%s) failed: %s",
server, gai_strerror(err));
goto error;
}
}
/* Setup timeout */
timedout = 0;
action.sa_flags = 0;
action.sa_handler = timed_out;
sigemptyset(&action.sa_mask);
if(sigaction(SIGALRM, &action, NULL) < 0)
{
syslog(LOG_ERR, "Setting timeout in backend_connect failed: sigaction: %m");
/* continue anyway */
}
for (res = res0; res; res = res->ai_next) {
sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (sock < 0)
continue;
alarm(config_getint(IMAPOPT_CLIENT_TIMEOUT));
if (connect(sock, res->ai_addr, res->ai_addrlen) >= 0)
break;
if(errno == EINTR && timedout == 1)
errno = ETIMEDOUT;
close(sock);
sock = -1;
}
/* Remove timeout code */
alarm(0);
signal(SIGALRM, SIG_IGN);
if (sock < 0) {
if (res0 != &hints)
freeaddrinfo(res0);
syslog(LOG_ERR, "connect(%s) failed: %m", server);
goto error;
}
memcpy(&ret->addr, res->ai_addr, res->ai_addrlen);
if (res0 != &hints)
freeaddrinfo(res0);
ret->in = prot_new(sock, 0);
ret->out = prot_new(sock, 1);
ret->sock = sock;
prot_setflushonread(ret->in, ret->out);
ret->prot = prot;
/* use literal+ to send literals */
prot_setisclient(ret->in, 1);
prot_setisclient(ret->out, 1);
if (prot->banner.auto_capa) {
/* try to get the capabilities from the banner */
r = ask_capability(ret, /*dobanner*/1, AUTO_CAPA_BANNER);
if (r) {
/* found capabilities in banner -> don't ask */
ask = 0;
}
}
else {
do { /* read the initial greeting */
if (!prot_fgets(buf, sizeof(buf), ret->in)) {
syslog(LOG_ERR,
"backend_connect(): couldn't read initial greeting: %s",
ret->in->error ? ret->in->error : "(null)");
goto error;
}
} while (strncasecmp(buf, prot->banner.resp,
strlen(prot->banner.resp)));
strncpy(ret->banner, buf, 2048);
}
if (ask) {
/* get the capabilities */
ask_capability(ret, /*dobanner*/0, AUTO_CAPA_NO);
}
/* now need to authenticate to backend server,
unless we're doing LMTP/CSYNC on a UNIX socket (deliver/sync_client) */
if ((server[0] != '/') ||
(strcmp(prot->sasl_service, "lmtp") &&
strcmp(prot->sasl_service, "csync"))) {
char *old_mechlist = backend_get_cap_params(ret, CAPA_AUTH);
const char *my_status;
if ((r = backend_authenticate(ret, userid, cb, &my_status))) {
syslog(LOG_ERR, "couldn't authenticate to backend server: %s",
sasl_errstring(r, NULL, NULL));
free(old_mechlist);
goto error;
}
else {
const void *ssf;
sasl_getprop(ret->saslconn, SASL_SSF, &ssf);
if (*((sasl_ssf_t *) ssf)) {
/* if we have a SASL security layer, compare SASL mech lists
before/after AUTH to check for a MITM attack */
char *new_mechlist;
int auto_capa = (prot->sasl_cmd.auto_capa == AUTO_CAPA_AUTH_SSF);
if (!strcmp(prot->service, "sieve")) {
/* XXX Hack to handle ManageSieve servers.
* No way to tell from protocol if server will
* automatically send capabilities, so we treat it
* as optional.
*/
char ch;
/* wait and probe for possible auto-capability response */
usleep(250000);
prot_NONBLOCK(ret->in);
if ((ch = prot_getc(ret->in)) != EOF) {
prot_ungetc(ch, ret->in);
} else {
auto_capa = AUTO_CAPA_AUTH_NO;
}
prot_BLOCK(ret->in);
}
ask_capability(ret, /*dobanner*/0, auto_capa);
new_mechlist = backend_get_cap_params(ret, CAPA_AUTH);
if (new_mechlist &&
old_mechlist &&
strcmp(new_mechlist, old_mechlist)) {
syslog(LOG_ERR, "possible MITM attack:"
"list of available SASL mechanisms changed");
free(new_mechlist);
free(old_mechlist);
goto error;
}
free(new_mechlist);
}
else if (prot->sasl_cmd.auto_capa == AUTO_CAPA_AUTH_OK) {
/* try to get the capabilities from the AUTH success response */
forget_capabilities(ret);
parse_capability(ret, my_status);
post_parse_capability(ret);
}
if (!(strcmp(prot->service, "imap") &&
(strcmp(prot->service, "pop3")))) {
parse_sessionid(my_status, rsessionid);
syslog(LOG_NOTICE, "proxy %s sessionid=<%s> remote=<%s>", userid, session_id(), rsessionid);
}
}
if (auth_status) *auth_status = my_status;
free(old_mechlist);
}
/* start compression if requested and both client/server support it */
if (config_getswitch(IMAPOPT_PROXY_COMPRESS) && ret &&
CAPA(ret, CAPA_COMPRESS) &&
prot->compress_cmd.cmd &&
do_compress(ret, &prot->compress_cmd)) {
syslog(LOG_ERR, "couldn't enable compression on backend server");
goto error;
}
return ret;
error:
forget_capabilities(ret);
if (ret->in) {
prot_free(ret->in);
ret->in = NULL;
}
if (ret->out) {
prot_free(ret->out);
ret->out = NULL;
}
if (sock >= 0)
close(sock);
if (ret->saslconn) {
sasl_dispose(&ret->saslconn);
ret->saslconn = NULL;
}
if (!ret_backend)
free(ret);
return NULL;
}
static int backend_login(struct backend *ret, const char *userid,
sasl_callback_t *cb, const char **auth_status,
int noauth)
{
int r = 0;
int ask = 1; /* should we explicitly ask for capabilities? */
char buf[2048];
struct protocol_t *prot = ret->prot;
if (prot->type != TYPE_STD) return -1;
if (prot->u.std.banner.auto_capa) {
/* try to get the capabilities from the banner */
r = ask_capability(ret, /*dobanner*/1, AUTO_CAPA_BANNER);
if (r) {
/* found capabilities in banner -> don't ask */
ask = 0;
}
}
else {
do { /* read the initial greeting */
if (!prot_fgets(buf, sizeof(buf), ret->in)) {
syslog(LOG_ERR,
"backend_login(): couldn't read initial greeting: %s",
ret->in->error ? ret->in->error : "(null)");
return -1;
}
} while (strncasecmp(buf, prot->u.std.banner.resp,
strlen(prot->u.std.banner.resp)));
xstrncpy(ret->banner, buf, 2048);
}
if (ask) {
/* get the capabilities */
ask_capability(ret, /*dobanner*/0, AUTO_CAPA_NO);
}
/* now need to authenticate to backend server,
unless we're doing LMTP/CSYNC on a UNIX socket (deliver/sync_client) */
if (!noauth) {
char *old_mechlist = backend_get_cap_params(ret, CAPA_AUTH);
const char *my_status;
if ((r = backend_authenticate(ret, userid, cb, &my_status))) {
syslog(LOG_ERR, "couldn't authenticate to backend server: %s",
sasl_errstring(r, NULL, NULL));
free(old_mechlist);
return -1;
}
else {
const void *ssf;
sasl_getprop(ret->saslconn, SASL_SSF, &ssf);
if (*((sasl_ssf_t *) ssf)) {
/* if we have a SASL security layer, compare SASL mech lists
before/after AUTH to check for a MITM attack */
char *new_mechlist;
int auto_capa = (prot->u.std.sasl_cmd.auto_capa == AUTO_CAPA_AUTH_SSF);
if (!strcmp(prot->service, "sieve")) {
/* XXX Hack to handle ManageSieve servers.
* No way to tell from protocol if server will
* automatically send capabilities, so we treat it
* as optional.
*/
char ch;
/* wait and probe for possible auto-capability response */
usleep(250000);
prot_NONBLOCK(ret->in);
if ((ch = prot_getc(ret->in)) != EOF) {
prot_ungetc(ch, ret->in);
} else {
auto_capa = AUTO_CAPA_AUTH_NO;
}
prot_BLOCK(ret->in);
}
ask_capability(ret, /*dobanner*/0, auto_capa);
new_mechlist = backend_get_cap_params(ret, CAPA_AUTH);
if (new_mechlist &&
old_mechlist &&
strcmp(new_mechlist, old_mechlist)) {
syslog(LOG_ERR, "possible MITM attack:"
"list of available SASL mechanisms changed");
if (new_mechlist) free(new_mechlist);
if (old_mechlist) free(old_mechlist);
return -1;
}
free(new_mechlist);
}
else if (prot->u.std.sasl_cmd.auto_capa == AUTO_CAPA_AUTH_OK) {
/* try to get the capabilities from the AUTH success response */
forget_capabilities(ret);
parse_capability(ret, my_status);
post_parse_capability(ret);
}
if (!(strcmp(prot->service, "imap") &&
(strcmp(prot->service, "pop3")))) {
char rsessionid[MAX_SESSIONID_SIZE];
parse_sessionid(my_status, rsessionid);
syslog(LOG_NOTICE, "auditlog: proxy %s sessionid=<%s> remote=<%s>", userid, session_id(), rsessionid);
}
}
if (auth_status) *auth_status = my_status;
free(old_mechlist);
}
/* start compression if requested and both client/server support it */
if (config_getswitch(IMAPOPT_PROXY_COMPRESS) &&
CAPA(ret, CAPA_COMPRESS) &&
prot->u.std.compress_cmd.cmd) {
r = do_compress(ret, &prot->u.std.compress_cmd);
if (r) {
syslog(LOG_NOTICE, "couldn't enable compression on backend server: %s", error_message(r));
r = 0; /* not a fail-level error */
}
}
return 0;
}
static int
compress_file(struct deflate_compressor *compressor, const tchar *path,
const struct options *options)
{
tchar *newpath = NULL;
struct file_stream in;
struct file_stream out;
struct stat stbuf;
int ret;
int ret2;
if (path != NULL && !options->to_stdout) {
size_t path_nchars, suffix_nchars;
if (!options->force && has_suffix(path, options->suffix)) {
msg("%"TS": already has .%"TS" suffix -- skipping",
path, options->suffix);
ret = -2;
goto out;
}
path_nchars = tstrlen(path);
suffix_nchars = tstrlen(options->suffix);
newpath = xmalloc((path_nchars + 1 + suffix_nchars + 1) *
sizeof(tchar));
tmemcpy(newpath, path, path_nchars);
newpath[path_nchars] = '.';
tmemcpy(&newpath[path_nchars + 1], options->suffix,
suffix_nchars + 1);
}
ret = xopen_for_read(path, &in);
if (ret != 0)
goto out_free_newpath;
ret = stat_file(&in, &stbuf, options->force || newpath == NULL);
if (ret != 0)
goto out_close_in;
ret = xopen_for_write(newpath, options->force, &out);
if (ret != 0)
goto out_close_in;
if (!options->force && isatty(out.fd)) {
msg("Refusing to write compressed data to terminal. "
"Use -f to override.\nFor help, use -h.");
ret = -1;
goto out_close_out;
}
ret = map_file_contents(&in, stbuf.st_size);
if (ret)
goto out_close_out;
ret = do_compress(compressor, &in, &out);
if (ret != 0)
goto out_close_out;
if (path != NULL && newpath != NULL)
restore_metadata(&out, newpath, &stbuf);
ret = 0;
out_close_out:
ret2 = xclose(&out);
if (ret == 0)
ret = ret2;
if (ret != 0 && newpath != NULL)
tunlink(newpath);
out_close_in:
xclose(&in);
if (ret == 0 && path != NULL && newpath != NULL && !options->keep)
tunlink(path);
out_free_newpath:
free(newpath);
out:
return ret;
}
int
compress_filter( void *opaque, int control,
IOBUF a, byte *buf, size_t *ret_len)
{
size_t size = *ret_len;
compress_filter_context_t *zfx = opaque;
z_stream *zs = zfx->opaque;
int rc=0;
if( control == IOBUFCTRL_UNDERFLOW ) {
if( !zfx->status ) {
zs = zfx->opaque = m_alloc_clear( sizeof *zs );
init_uncompress( zfx, zs );
zfx->status = 1;
}
#ifndef __riscos__
zs->next_out = buf;
#else /* __riscos__ */
zs->next_out = (Bytef *) buf;
#endif /* __riscos__ */
zs->avail_out = size;
zfx->outbufsize = size; /* needed only for calculation */
rc = do_uncompress( zfx, zs, a, ret_len );
}
else if( control == IOBUFCTRL_FLUSH ) {
if( !zfx->status ) {
PACKET pkt;
PKT_compressed cd;
if( !zfx->algo )
zfx->algo = DEFAULT_COMPRESS_ALGO;
if( zfx->algo != 1 && zfx->algo != 2 )
BUG();
memset( &cd, 0, sizeof cd );
cd.len = 0;
cd.algorithm = zfx->algo;
init_packet( &pkt );
pkt.pkttype = PKT_COMPRESSED;
pkt.pkt.compressed = &cd;
if( build_packet( a, &pkt ))
log_bug("build_packet(PKT_COMPRESSED) failed\n");
zs = zfx->opaque = m_alloc_clear( sizeof *zs );
init_compress( zfx, zs );
zfx->status = 2;
}
#ifndef __riscos__
zs->next_in = buf;
#else /* __riscos__ */
zs->next_in = (Bytef *) buf;
#endif /* __riscos__ */
zs->avail_in = size;
rc = do_compress( zfx, zs, Z_NO_FLUSH, a );
}
else if( control == IOBUFCTRL_FREE ) {
if( zfx->status == 1 ) {
inflateEnd(zs);
m_free(zs);
zfx->opaque = NULL;
m_free(zfx->outbuf); zfx->outbuf = NULL;
}
else if( zfx->status == 2 ) {
#ifndef __riscos__
zs->next_in = buf;
#else /* __riscos__ */
zs->next_in = (Bytef *) buf;
#endif /* __riscos__ */
zs->avail_in = 0;
do_compress( zfx, zs, Z_FINISH, a );
deflateEnd(zs);
m_free(zs);
zfx->opaque = NULL;
m_free(zfx->outbuf); zfx->outbuf = NULL;
}
if (zfx->release)
zfx->release (zfx);
}
else if( control == IOBUFCTRL_DESC )
*(char**)buf = "compress_filter";
return rc;
}
int
compress_filter_bz2( void *opaque, int control,
IOBUF a, byte *buf, size_t *ret_len)
{
size_t size = *ret_len;
compress_filter_context_t *zfx = opaque;
bz_stream *bzs = zfx->opaque;
int rc=0;
if( control == IOBUFCTRL_UNDERFLOW )
{
if( !zfx->status )
{
bzs = zfx->opaque = xmalloc_clear( sizeof *bzs );
init_uncompress( zfx, bzs );
zfx->status = 1;
}
bzs->next_out = buf;
bzs->avail_out = size;
zfx->outbufsize = size; /* needed only for calculation */
rc = do_uncompress( zfx, bzs, a, ret_len );
}
else if( control == IOBUFCTRL_FLUSH )
{
if( !zfx->status )
{
PACKET pkt;
PKT_compressed cd;
if( zfx->algo != COMPRESS_ALGO_BZIP2 )
BUG();
memset( &cd, 0, sizeof cd );
cd.len = 0;
cd.algorithm = zfx->algo;
init_packet( &pkt );
pkt.pkttype = PKT_COMPRESSED;
pkt.pkt.compressed = &cd;
/* if( build_packet( a, &pkt ))
log_bug("build_packet(PKT_COMPRESSED) failed\n"); */ //FIXME
bzs = zfx->opaque = xmalloc_clear( sizeof *bzs );
init_compress( zfx, bzs );
zfx->status = 2;
}
bzs->next_in = buf;
bzs->avail_in = size;
rc = do_compress( zfx, bzs, BZ_RUN, a );
}
else if( control == IOBUFCTRL_FREE )
{
if( zfx->status == 1 )
{
BZ2_bzDecompressEnd(bzs);
xfree(bzs);
zfx->opaque = NULL;
xfree(zfx->outbuf); zfx->outbuf = NULL;
}
else if( zfx->status == 2 )
{
bzs->next_in = buf;
bzs->avail_in = 0;
do_compress( zfx, bzs, BZ_FINISH, a );
BZ2_bzCompressEnd(bzs);
xfree(bzs);
zfx->opaque = NULL;
xfree(zfx->outbuf); zfx->outbuf = NULL;
}
if (zfx->release)
zfx->release (zfx);
}
else if( control == IOBUFCTRL_DESC )
*(char**)buf = "compress_filter";
return rc;
}