static void shutdown_thread(iow_t *iow, struct lzothread_t *thread) { pthread_mutex_lock(&thread->mutex); /* If this buffer is empty it shouldn't have any data in it, we should have taken * care of that before. */ /* thread->state == EMPTY implies thread->inbuf.offset == 0 */ assert(!(thread->state == EMPTY) || thread->inbuf.offset == 0); while (thread->state == WAITING) { pthread_cond_wait( &thread->out_ready, &thread->mutex); } if (thread->state == FULL) { wandio_wwrite(DATA(iow)->child, thread->outbuf.buffer, thread->outbuf.offset); thread->state = EMPTY; thread->inbuf.offset = 0; } /* Now the thread should be empty, so ask it to shut down */ assert(thread->state == EMPTY && thread->inbuf.offset == 0); thread->closing = true; pthread_cond_signal(&thread->in_ready); pthread_mutex_unlock(&thread->mutex); /* And wait for it to die */ pthread_join(thread->thread,NULL); }
static void lzo_wclose(iow_t *iow) { const uint32_t zero = 0; int i; /* Flush the last buffer */ pthread_mutex_lock(&get_next_thread(iow)->mutex); if (get_next_thread(iow)->state == EMPTY && get_next_thread(iow)->inbuf.offset != 0) { get_next_thread(iow)->state = WAITING; pthread_cond_signal(&get_next_thread(iow)->in_ready); } pthread_mutex_unlock(&get_next_thread(iow)->mutex); DATA(iow)->next_thread = (DATA(iow)->next_thread+1) % DATA(iow)->threads; /* Right, now we have to shutdown all our threads -- in order */ for(i=DATA(iow)->next_thread; i<DATA(iow)->threads; ++i) { shutdown_thread(iow,&DATA(iow)->thread[i]); } for(i=0; i<DATA(iow)->next_thread; ++i) { shutdown_thread(iow,&DATA(iow)->thread[i]); } /* Write out an end of file marker */ wandio_wwrite(DATA(iow)->child, &zero, sizeof(zero)); /* And clean everything up */ wandio_wdestroy(DATA(iow)->child); free(DATA(iow)->thread); free(iow->data); free(iow); }
static void bz_wclose(iow_t *iow) { while (BZ2_bzCompress(&DATA(iow)->strm, BZ_FINISH) == BZ_OK) { /* Need to flush the output buffer */ wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, sizeof(DATA(iow)->outbuff)-DATA(iow)->strm.avail_out); DATA(iow)->strm.next_out = DATA(iow)->outbuff; DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); } BZ2_bzCompressEnd(&DATA(iow)->strm); wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, sizeof(DATA(iow)->outbuff)-DATA(iow)->strm.avail_out); wandio_wdestroy(DATA(iow)->child); free(iow->data); free(iow); }
off_t corsaro_file_write(corsaro_t *corsaro, corsaro_file_t *file, const void *buffer, off_t len) { /* let's not try and write raw bytes to a libtrace file... */ assert(file->mode == CORSARO_FILE_MODE_ASCII || file->mode == CORSARO_FILE_MODE_BINARY || file->mode == CORSARO_FILE_MODE_UNKNOWN); assert(file->wand_io != NULL); return wandio_wwrite(file->wand_io, buffer, len); }
static off_t bz_wwrite(iow_t *iow, const char *buffer, off_t len) { if (DATA(iow)->err == ERR_EOF) { return 0; /* EOF */ } if (DATA(iow)->err == ERR_ERROR) { return -1; /* ERROR! */ } DATA(iow)->strm.next_in = (char*)buffer; DATA(iow)->strm.avail_in = len; while (DATA(iow)->err == ERR_OK && DATA(iow)->strm.avail_in > 0) { while (DATA(iow)->strm.avail_out <= 0) { int bytes_written = wandio_wwrite(DATA(iow)->child, DATA(iow)->outbuff, sizeof(DATA(iow)->outbuff)); if (bytes_written <= 0) { /* Error */ DATA(iow)->err = ERR_ERROR; /* Return how much data we managed to write ok */ if (DATA(iow)->strm.avail_in != (uint32_t)len) { return len-DATA(iow)->strm.avail_in; } /* Now return error */ return -1; } DATA(iow)->strm.next_out = DATA(iow)->outbuff; DATA(iow)->strm.avail_out = sizeof(DATA(iow)->outbuff); } /* Decompress some data into the output buffer */ int err=BZ2_bzCompress(&DATA(iow)->strm, 0); switch(err) { case BZ_RUN_OK: case BZ_OK: DATA(iow)->err = ERR_OK; break; default: DATA(iow)->err = ERR_ERROR; break; } } /* Return the number of bytes compressed */ return len-DATA(iow)->strm.avail_in; }
/* The writing thread */ static void *thread_consumer(void *userdata) { int buffer=0; bool running = true; iow_t *state = (iow_t *) userdata; #ifdef PR_SET_NAME char namebuf[17]; if (prctl(PR_GET_NAME, namebuf, 0,0,0) == 0) { namebuf[16] = '\0'; /* Make sure it's NUL terminated */ /* If the filename is too long, overwrite the last few bytes */ if (strlen(namebuf)>9) { strcpy(namebuf+10,"[iow]"); } else { strncat(namebuf," [iow]",16); } prctl(PR_SET_NAME, namebuf, 0,0,0); } #endif pthread_mutex_lock(&DATA(state)->mutex); do { /* Wait for data that we can write */ while (DATA(state)->buffer[buffer].state == EMPTY) { /* Unless, of course, the program is over! */ if (DATA(state)->closing) break; pthread_cond_wait(&DATA(state)->data_ready, &DATA(state)->mutex); } /* Empty the buffer using the child writer */ pthread_mutex_unlock(&DATA(state)->mutex); wandio_wwrite( DATA(state)->iow, DATA(state)->buffer[buffer].buffer, DATA(state)->buffer[buffer].len); pthread_mutex_lock(&DATA(state)->mutex); /* If we've not reached the end of the file keep going */ running = ( DATA(state)->buffer[buffer].len > 0 ); DATA(state)->buffer[buffer].len = 0; DATA(state)->buffer[buffer].state = EMPTY; /* Signal that we've freed up another buffer for the main * thread to copy data into */ pthread_cond_signal(&DATA(state)->space_avail); /* Move on to the next buffer */ buffer=(buffer+1) % BUFFERS; } while(running); /* If we reach here, it's all over so start tidying up */ wandio_wdestroy(DATA(state)->iow); pthread_mutex_unlock(&DATA(state)->mutex); return NULL; }
static off_t lzo_wwrite(iow_t *iow, const char *buffer, off_t len) { off_t ret = 0; while (len>0) { off_t size = len; off_t err; struct buffer_t outbuf; if (!DATA(iow)->threads) { size = min(len, MAX_BLOCK_SIZE); err=lzo_wwrite_block(buffer, size, &outbuf); /* Flush the data out */ wandio_wwrite(DATA(iow)->child, outbuf.buffer, outbuf.offset); if (err < 0) {/* Error */ if (ret == 0) return err; /* If we've written some data, return that fact now, let them call back * and try and write more data, fail again then. */ return ret; } else { assert(err == size); buffer += size; len -= size; } } else { off_t space; pthread_mutex_lock(&get_next_thread(iow)->mutex); /* If this thread is still compressing, wait for it to finish */ while (get_next_thread(iow)->state == WAITING) { pthread_cond_wait( &get_next_thread(iow)->out_ready, &get_next_thread(iow)->mutex); } /* Flush any data out thats there */ if (get_next_thread(iow)->state == FULL) { assert(get_next_thread(iow)->outbuf.offset < sizeof(get_next_thread(iow)->outbuf.buffer)); wandio_wwrite(DATA(iow)->child, get_next_thread(iow)->outbuf.buffer, get_next_thread(iow)->outbuf.offset); get_next_thread(iow)->state = EMPTY; get_next_thread(iow)->inbuf.offset = 0; } assert(get_next_thread(iow)->state == EMPTY); /* Figure out how much space we can copy into this buffer */ assert(MAX_BLOCK_SIZE <= sizeof(get_next_thread(iow)->inbuf.buffer)); space = MAX_BLOCK_SIZE-get_next_thread(iow)->inbuf.offset; size = min(space, size); assert(size>0); assert(size <= MAX_BLOCK_SIZE); assert(get_next_thread(iow)->inbuf.offset + size <= MAX_BLOCK_SIZE); /* Move our data in */ memcpy(&get_next_thread(iow)->inbuf.buffer[get_next_thread(iow)->inbuf.offset], buffer, size); get_next_thread(iow)->inbuf.offset += size; /* If the buffer is now full Trigger the thread to start compressing this block, * and move onto the next block. */ if (get_next_thread(iow)->inbuf.offset >= sizeof(get_next_thread(iow)->inbuf.buffer) ||get_next_thread(iow)->inbuf.offset >= MAX_BLOCK_SIZE) { assert(get_next_thread(iow)->state == EMPTY); get_next_thread(iow)->state = WAITING; pthread_cond_signal(&get_next_thread(iow)->in_ready); pthread_mutex_unlock(&get_next_thread(iow)->mutex); DATA(iow)->next_thread = (DATA(iow)->next_thread+1) % DATA(iow)->threads; } else pthread_mutex_unlock(&get_next_thread(iow)->mutex); /* Update the lengths */ buffer += size; len -= size; } } return len; }
iow_t *lzo_wopen(iow_t *child, int compress_level) { const int opt_filter = 0; int flags; iow_t *iow; struct buffer_t buffer; buffer.offset=0; int i; if (!child) return NULL; if (lzo_init() != LZO_E_OK) { /* Fail */ return NULL; } /* Compress level is useless for LZO, but getting UNUSED into here * is more trouble than it is worth so this check will at least * stop us from getting warnings about it. */ if (compress_level < 0) return NULL; iow = malloc(sizeof(iow_t)); iow->source = &lzo_wsource; iow->data = malloc(sizeof(struct lzow_t)); DATA(iow)->child = child; DATA(iow)->err = ERR_OK; flags = 0; flags |= F_OS_UNIX & F_OS_MASK; /* Operating System */ flags |= F_CS_NATIVE & F_CS_MASK; /* Character Set */ flags |= F_ADLER32_D; /* We adler32 the uncompressed data */ /* flags |= F_STDIN; */ /* flags |= F_STDOUT */ /* flags |= F_MULTIPART; */ /* flags |= F_H_CRC32; */ write_buf(&buffer, lzop_magic, sizeof(lzop_magic)); write16(&buffer, 0x1010 &0xFFFF); /* version: pretend to be LZOP version 0x1010 from lzop's version.h */ write16(&buffer, lzo_version() & 0xFFFF); /* libversion */ write16(&buffer, opt_filter ? 0x0950 : 0x0940); /* version needed to extract */ write8(&buffer, M_LZO1X_1); /* method */ write8(&buffer, 5); /* level */ write32(&buffer, flags); /* flags */ /* if (flags & F_H_FILTER) write32(iow, opt_filter); */ write32(&buffer, 0x600); /* mode: We assume traces may be sensitive */ write32(&buffer, time(NULL)); /* mtime */ write32(&buffer, 0); /* GMTdiff */ /* Length, filename */ write8(&buffer, strlen("compresseddata")); write_buf(&buffer, "compresseddata",strlen("compresseddata")); if (flags & F_H_CRC32) { write32(&buffer, lzo_crc32(CRC32_INIT_VALUE, (const void*)buffer.buffer+sizeof(lzop_magic), buffer.offset-sizeof(lzop_magic))); } else { uint32_t chksum=lzo_adler32( ADLER32_INIT_VALUE, (const void *)buffer.buffer+sizeof(lzop_magic), buffer.offset-sizeof(lzop_magic)); write32(&buffer, chksum); } wandio_wwrite(DATA(iow)->child, buffer.buffer, buffer.offset); /* Set up the thread pool -- one thread per core */ DATA(iow)->threads = min((uint32_t)sysconf(_SC_NPROCESSORS_ONLN), use_threads); DATA(iow)->thread = malloc( sizeof(struct lzothread_t) * DATA(iow)->threads); DATA(iow)->next_thread = 0; for(i=0; i<DATA(iow)->threads; ++i) { pthread_cond_init(&DATA(iow)->thread[i].in_ready, NULL); pthread_cond_init(&DATA(iow)->thread[i].out_ready, NULL); pthread_mutex_init(&DATA(iow)->thread[i].mutex, NULL); DATA(iow)->thread[i].closing = false; DATA(iow)->thread[i].num = i; DATA(iow)->thread[i].state = EMPTY; DATA(iow)->thread[i].inbuf.offset = 0; pthread_create(&DATA(iow)->thread[i].thread, NULL, lzo_compress_thread, (void*)&DATA(iow)->thread[i]); } return iow; }