static void shutdown_thread(iow_t *iow, struct lzothread_t *thread)
{
pthread_mutex_lock(&thread->mutex);
/* If this buffer is empty it shouldn't have any data in it, we should have taken
* care of that before.
*/
/* thread->state == EMPTY implies thread->inbuf.offset == 0 */
assert(!(thread->state == EMPTY) || thread->inbuf.offset == 0);
while (thread->state == WAITING) {
pthread_cond_wait(
&thread->out_ready,
&thread->mutex);
}
if (thread->state == FULL) {
wandio_wwrite(DATA(iow)->child,
thread->outbuf.buffer,
thread->outbuf.offset);
thread->state = EMPTY;
thread->inbuf.offset = 0;
}
/* Now the thread should be empty, so ask it to shut down */
assert(thread->state == EMPTY && thread->inbuf.offset == 0);
thread->closing = true;
pthread_cond_signal(&thread->in_ready);
pthread_mutex_unlock(&thread->mutex);
/* And wait for it to die */
pthread_join(thread->thread,NULL);
}
static void lzo_wclose(iow_t *iow)
{
const uint32_t zero = 0;
int i;
/* Flush the last buffer */
pthread_mutex_lock(&get_next_thread(iow)->mutex);
if (get_next_thread(iow)->state == EMPTY && get_next_thread(iow)->inbuf.offset != 0) {
get_next_thread(iow)->state = WAITING;
pthread_cond_signal(&get_next_thread(iow)->in_ready);
}
pthread_mutex_unlock(&get_next_thread(iow)->mutex);
DATA(iow)->next_thread =
(DATA(iow)->next_thread+1) % DATA(iow)->threads;
/* Right, now we have to shutdown all our threads -- in order */
for(i=DATA(iow)->next_thread; i<DATA(iow)->threads; ++i) {
shutdown_thread(iow,&DATA(iow)->thread[i]);
}
for(i=0; i<DATA(iow)->next_thread; ++i) {
shutdown_thread(iow,&DATA(iow)->thread[i]);
}
/* Write out an end of file marker */
wandio_wwrite(DATA(iow)->child,
&zero,
sizeof(zero));
/* And clean everything up */
wandio_wdestroy(DATA(iow)->child);
free(DATA(iow)->thread);
free(iow->data);
free(iow);
}
static void bz_wclose(iow_t *iow)
{
while (BZ2_bzCompress(&DATA(iow)->strm, BZ_FINISH) == BZ_OK) {
/* Need to flush the output buffer */
wandio_wwrite(DATA(iow)->child,
DATA(iow)->outbuff,
sizeof(DATA(iow)->outbuff)-DATA(iow)->strm.avail_out);
DATA(iow)->strm.next_out = DATA(iow)->outbuff;
DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff);
}
BZ2_bzCompressEnd(&DATA(iow)->strm);
wandio_wwrite(DATA(iow)->child,
DATA(iow)->outbuff,
sizeof(DATA(iow)->outbuff)-DATA(iow)->strm.avail_out);
wandio_wdestroy(DATA(iow)->child);
free(iow->data);
free(iow);
}
off_t corsaro_file_write(corsaro_t *corsaro,
corsaro_file_t *file, const void *buffer, off_t len)
{
/* let's not try and write raw bytes to a libtrace file... */
assert(file->mode == CORSARO_FILE_MODE_ASCII ||
file->mode == CORSARO_FILE_MODE_BINARY ||
file->mode == CORSARO_FILE_MODE_UNKNOWN);
assert(file->wand_io != NULL);
return wandio_wwrite(file->wand_io, buffer, len);
}
static off_t bz_wwrite(iow_t *iow, const char *buffer, off_t len)
{
if (DATA(iow)->err == ERR_EOF) {
return 0; /* EOF */
}
if (DATA(iow)->err == ERR_ERROR) {
return -1; /* ERROR! */
}
DATA(iow)->strm.next_in = (char*)buffer;
DATA(iow)->strm.avail_in = len;
while (DATA(iow)->err == ERR_OK && DATA(iow)->strm.avail_in > 0) {
while (DATA(iow)->strm.avail_out <= 0) {
int bytes_written = wandio_wwrite(DATA(iow)->child,
DATA(iow)->outbuff,
sizeof(DATA(iow)->outbuff));
if (bytes_written <= 0) { /* Error */
DATA(iow)->err = ERR_ERROR;
/* Return how much data we managed to write ok */
if (DATA(iow)->strm.avail_in != (uint32_t)len) {
return len-DATA(iow)->strm.avail_in;
}
/* Now return error */
return -1;
}
DATA(iow)->strm.next_out = DATA(iow)->outbuff;
DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff);
}
/* Decompress some data into the output buffer */
int err=BZ2_bzCompress(&DATA(iow)->strm, 0);
switch(err) {
case BZ_RUN_OK:
case BZ_OK:
DATA(iow)->err = ERR_OK;
break;
default:
DATA(iow)->err = ERR_ERROR;
break;
}
}
/* Return the number of bytes compressed */
return len-DATA(iow)->strm.avail_in;
}
/* The writing thread */
static void *thread_consumer(void *userdata)
{
int buffer=0;
bool running = true;
iow_t *state = (iow_t *) userdata;
#ifdef PR_SET_NAME
char namebuf[17];
if (prctl(PR_GET_NAME, namebuf, 0,0,0) == 0) {
namebuf[16] = '\0'; /* Make sure it's NUL terminated */
/* If the filename is too long, overwrite the last few bytes */
if (strlen(namebuf)>9) {
strcpy(namebuf+10,"[iow]");
}
else {
strncat(namebuf," [iow]",16);
}
prctl(PR_SET_NAME, namebuf, 0,0,0);
}
#endif
pthread_mutex_lock(&DATA(state)->mutex);
do {
/* Wait for data that we can write */
while (DATA(state)->buffer[buffer].state == EMPTY) {
/* Unless, of course, the program is over! */
if (DATA(state)->closing)
break;
pthread_cond_wait(&DATA(state)->data_ready,
&DATA(state)->mutex);
}
/* Empty the buffer using the child writer */
pthread_mutex_unlock(&DATA(state)->mutex);
wandio_wwrite(
DATA(state)->iow,
DATA(state)->buffer[buffer].buffer,
DATA(state)->buffer[buffer].len);
pthread_mutex_lock(&DATA(state)->mutex);
/* If we've not reached the end of the file keep going */
running = ( DATA(state)->buffer[buffer].len > 0 );
DATA(state)->buffer[buffer].len = 0;
DATA(state)->buffer[buffer].state = EMPTY;
/* Signal that we've freed up another buffer for the main
* thread to copy data into */
pthread_cond_signal(&DATA(state)->space_avail);
/* Move on to the next buffer */
buffer=(buffer+1) % BUFFERS;
} while(running);
/* If we reach here, it's all over so start tidying up */
wandio_wdestroy(DATA(state)->iow);
pthread_mutex_unlock(&DATA(state)->mutex);
return NULL;
}
static off_t lzo_wwrite(iow_t *iow, const char *buffer, off_t len)
{
off_t ret = 0;
while (len>0) {
off_t size = len;
off_t err;
struct buffer_t outbuf;
if (!DATA(iow)->threads) {
size = min(len, MAX_BLOCK_SIZE);
err=lzo_wwrite_block(buffer, size, &outbuf);
/* Flush the data out */
wandio_wwrite(DATA(iow)->child,
outbuf.buffer,
outbuf.offset);
if (err < 0) {/* Error */
if (ret == 0)
return err;
/* If we've written some data, return that fact now, let them call back
* and try and write more data, fail again then.
*/
return ret;
}
else {
assert(err == size);
buffer += size;
len -= size;
}
}
else {
off_t space;
pthread_mutex_lock(&get_next_thread(iow)->mutex);
/* If this thread is still compressing, wait for it to finish */
while (get_next_thread(iow)->state == WAITING) {
pthread_cond_wait(
&get_next_thread(iow)->out_ready,
&get_next_thread(iow)->mutex);
}
/* Flush any data out thats there */
if (get_next_thread(iow)->state == FULL) {
assert(get_next_thread(iow)->outbuf.offset
< sizeof(get_next_thread(iow)->outbuf.buffer));
wandio_wwrite(DATA(iow)->child,
get_next_thread(iow)->outbuf.buffer,
get_next_thread(iow)->outbuf.offset);
get_next_thread(iow)->state = EMPTY;
get_next_thread(iow)->inbuf.offset = 0;
}
assert(get_next_thread(iow)->state == EMPTY);
/* Figure out how much space we can copy into this buffer */
assert(MAX_BLOCK_SIZE <= sizeof(get_next_thread(iow)->inbuf.buffer));
space = MAX_BLOCK_SIZE-get_next_thread(iow)->inbuf.offset;
size = min(space, size);
assert(size>0);
assert(size <= MAX_BLOCK_SIZE);
assert(get_next_thread(iow)->inbuf.offset + size <= MAX_BLOCK_SIZE);
/* Move our data in */
memcpy(&get_next_thread(iow)->inbuf.buffer[get_next_thread(iow)->inbuf.offset],
buffer,
size);
get_next_thread(iow)->inbuf.offset += size;
/* If the buffer is now full Trigger the thread to start compressing this block,
* and move onto the next block.
*/
if (get_next_thread(iow)->inbuf.offset >= sizeof(get_next_thread(iow)->inbuf.buffer)
||get_next_thread(iow)->inbuf.offset >= MAX_BLOCK_SIZE) {
assert(get_next_thread(iow)->state == EMPTY);
get_next_thread(iow)->state = WAITING;
pthread_cond_signal(&get_next_thread(iow)->in_ready);
pthread_mutex_unlock(&get_next_thread(iow)->mutex);
DATA(iow)->next_thread =
(DATA(iow)->next_thread+1) % DATA(iow)->threads;
}
else
pthread_mutex_unlock(&get_next_thread(iow)->mutex);
/* Update the lengths */
buffer += size;
len -= size;
}
}
return len;
}
iow_t *lzo_wopen(iow_t *child, int compress_level)
{
const int opt_filter = 0;
int flags;
iow_t *iow;
struct buffer_t buffer;
buffer.offset=0;
int i;
if (!child)
return NULL;
if (lzo_init() != LZO_E_OK) {
/* Fail */
return NULL;
}
/* Compress level is useless for LZO, but getting UNUSED into here
* is more trouble than it is worth so this check will at least
* stop us from getting warnings about it.
*/
if (compress_level < 0)
return NULL;
iow = malloc(sizeof(iow_t));
iow->source = &lzo_wsource;
iow->data = malloc(sizeof(struct lzow_t));
DATA(iow)->child = child;
DATA(iow)->err = ERR_OK;
flags = 0;
flags |= F_OS_UNIX & F_OS_MASK; /* Operating System */
flags |= F_CS_NATIVE & F_CS_MASK; /* Character Set */
flags |= F_ADLER32_D; /* We adler32 the uncompressed data */
/* flags |= F_STDIN; */
/* flags |= F_STDOUT */
/* flags |= F_MULTIPART; */
/* flags |= F_H_CRC32; */
write_buf(&buffer, lzop_magic, sizeof(lzop_magic));
write16(&buffer, 0x1010 &0xFFFF); /* version: pretend to be LZOP version 0x1010 from lzop's version.h */
write16(&buffer, lzo_version() & 0xFFFF); /* libversion */
write16(&buffer, opt_filter ? 0x0950 : 0x0940); /* version needed to extract */
write8(&buffer, M_LZO1X_1); /* method */
write8(&buffer, 5); /* level */
write32(&buffer, flags); /* flags */
/* if (flags & F_H_FILTER)
write32(iow, opt_filter);
*/
write32(&buffer, 0x600); /* mode: We assume traces may be sensitive */
write32(&buffer, time(NULL)); /* mtime */
write32(&buffer, 0); /* GMTdiff */
/* Length, filename */
write8(&buffer, strlen("compresseddata"));
write_buf(&buffer, "compresseddata",strlen("compresseddata"));
if (flags & F_H_CRC32) {
write32(&buffer,
lzo_crc32(CRC32_INIT_VALUE,
(const void*)buffer.buffer+sizeof(lzop_magic),
buffer.offset-sizeof(lzop_magic)));
}
else {
uint32_t chksum=lzo_adler32(
ADLER32_INIT_VALUE,
(const void *)buffer.buffer+sizeof(lzop_magic),
buffer.offset-sizeof(lzop_magic));
write32(&buffer, chksum);
}
wandio_wwrite(DATA(iow)->child,
buffer.buffer,
buffer.offset);
/* Set up the thread pool -- one thread per core */
DATA(iow)->threads = min((uint32_t)sysconf(_SC_NPROCESSORS_ONLN),
use_threads);
DATA(iow)->thread = malloc(
sizeof(struct lzothread_t) * DATA(iow)->threads);
DATA(iow)->next_thread = 0;
for(i=0; i<DATA(iow)->threads; ++i) {
pthread_cond_init(&DATA(iow)->thread[i].in_ready, NULL);
pthread_cond_init(&DATA(iow)->thread[i].out_ready, NULL);
pthread_mutex_init(&DATA(iow)->thread[i].mutex, NULL);
DATA(iow)->thread[i].closing = false;
DATA(iow)->thread[i].num = i;
DATA(iow)->thread[i].state = EMPTY;
DATA(iow)->thread[i].inbuf.offset = 0;
pthread_create(&DATA(iow)->thread[i].thread,
NULL,
lzo_compress_thread,
(void*)&DATA(iow)->thread[i]);
}
return iow;
}
