static int
lcm_logprov_publish (lcm_logprov_t *lcm, const char *channel, const void *data,
unsigned int datalen)
{
if(!lcm->writer) {
dbg(DBG_LCM, "Called publish(), but lcm file provider is not in write mode\n");
return -1;
}
int channellen = strlen(channel);
int64_t mem_sz = sizeof(lcm_eventlog_event_t) + channellen + 1 + datalen;
lcm_eventlog_event_t *le = (lcm_eventlog_event_t*) malloc(mem_sz);
memset(le, 0, mem_sz);
GTimeVal tv;
g_get_current_time(&tv);
le->timestamp = (int64_t) tv.tv_sec * 1000000 + tv.tv_usec;;
le->channellen = channellen;
le->datalen = datalen;
// log_write_event will handle le.eventnum.
le->channel = ((char*)le) + sizeof(lcm_eventlog_event_t);
strcpy(le->channel, channel);
le->data = le->channel + channellen + 1;
assert((char*)le->data + datalen == (char*)le + mem_sz);
memcpy(le->data, data, datalen);
lcm_eventlog_write_event(lcm->log, le);
free(le);
return 0;
}
int
LogFile::writeEvent(LogEvent* event)
{
lcm_eventlog_event_t evt;
evt.eventnum = event->eventnum;
evt.timestamp = event->timestamp;
evt.channellen = event->channel.size();
evt.datalen = event->datalen;
evt.channel = (char*) event->channel.c_str();
evt.data = event->data;
return lcm_eventlog_write_event(eventlog, &evt);
}
static PyObject *
pylog_write_next_event (PyLogObject *self, PyObject *args)
{
int64_t utime = 0;
char *channel = NULL;
int channellen = 0;
// TODO use a buffer object instead of a string
uint8_t *data = NULL;
int datalen = 0;
if (!PyArg_ParseTuple(args, "Ls#s#",
&utime, &channel, &channellen, &data, &datalen)) {
return NULL;
}
if (!self->eventlog) {
PyErr_SetString (PyExc_ValueError, "event log already closed");
return NULL;
}
if (self->mode != 'w') {
PyErr_SetString (PyExc_RuntimeError,
"writing not allowed in read mode");
return NULL;
}
lcm_eventlog_event_t le; //msvc needs init of all fields seperately
le.eventnum = 0;
le.timestamp = utime;
le.channellen = channellen;
le.datalen = datalen;
le.channel = channel;
le.data = data;
if (0 != lcm_eventlog_write_event (self->eventlog, &le)) {
PyErr_SetFromErrno (PyExc_IOError);
return NULL;
}
Py_INCREF (Py_None);
return Py_None;
}
int
main(int argc, char **argv)
{
int verbose = 0;
int filterChannels =0;
char *pattern = strdup(".*");
char *dest_fname = NULL;
int64_t start_utime = 0;
int64_t end_utime = -1;
int have_end_utime = 0;
int invert_regex = 0;
char *optstring = "hc:vs:e:i";
int c;
while ((c = getopt_long(argc, argv, optstring, NULL, 0)) >= 0) {
switch (c)
{
case 'h':
usage();
break;
case 's':
{
char *eptr = NULL;
double start_time = strtod(optarg, &eptr);
if (*eptr != 0)
usage();
start_utime = (int64_t) (start_time * 1000000);
}
break;
case 'e':
{
char *eptr = NULL;
double end_time = strtod(optarg, &eptr);
if (*eptr != 0)
usage();
end_utime = (int64_t) (end_time * 1000000);
have_end_utime = 1;
}
break;
case 'i':
invert_regex = 1;
filterChannels = 1;
break;
case 'c':
free(pattern);
pattern = strdup(optarg);
filterChannels = 1;
break;
case 'v':
verbose = 1;
break;
default:
usage();
break;
};
}
if (start_utime < 0 || (have_end_utime && end_utime < start_utime))
usage();
if (optind > argc - 3)
usage();
regex_t preg;
if (0 != regcomp(&preg, pattern, REG_NOSUB | REG_EXTENDED)) {
fprintf(stderr, "bad regex\n");
exit(1);
}
int num_src_logs = argc - optind - 1;
fprintf(stderr, "Splicing together %d logs\n", num_src_logs);
lcm_eventlog_t *src_logs[num_src_logs];
for (int i = 0; i < argc - optind - 1; i++) {
char * src_fname = argv[optind + i];
src_logs[i] = lcm_eventlog_create(src_fname, "r");
if (!src_logs[i]) {
perror("Unable to open source logfile");
for (int j = 0; j < i; j++)
lcm_eventlog_destroy(src_logs[j]);
regfree(&preg);
return 1;
}
}
dest_fname = argv[argc - 1];
lcm_eventlog_t *dst_log = lcm_eventlog_create(dest_fname, "w");
if (!dst_log) {
perror("Unable to open destination logfile");
for (int i = 0; i < num_src_logs; i++)
lcm_eventlog_destroy(src_logs[i]);
regfree(&preg);
return 1;
}
GHashTable *counts = g_hash_table_new_full(g_str_hash, g_str_equal, free,
free);
int nwritten = 0;
int have_first_event_timestamp = 0;
int64_t first_event_timestamp = -1;
lcm_eventlog_event_t *events[num_src_logs];
for (int i = 0; i < num_src_logs; i++)
events[i] = lcm_eventlog_read_next_event(src_logs[i]);
while (1) {
lcm_eventlog_event_t *event;
int mind = -1;
int64_t mtime = INT64_MAX;
for (int i = 0; i < num_src_logs; i++) {
if (events[i] == NULL)
continue;
else if (events[i]->timestamp < mtime) {
mtime = events[i]->timestamp;
mind = i;
}
}
if (mind < 0) //all are null
break;
event = events[mind];
events[mind] = lcm_eventlog_read_next_event(src_logs[mind]);
if (!have_first_event_timestamp) {
first_event_timestamp = event->timestamp;
have_first_event_timestamp = 1;
}
int64_t elapsed = event->timestamp - first_event_timestamp;
if (elapsed < start_utime) {
lcm_eventlog_free_event(event);
continue;
}
if (have_end_utime && elapsed > end_utime) {
lcm_eventlog_free_event(event);
break;
}
int copy_to_dest = 1;
if (filterChannels) {
int regmatch = regexec(&preg, event->channel, 0, NULL, 0);
copy_to_dest = (regmatch == 0 && !invert_regex) || (regmatch != 0
&& invert_regex);
}
if (copy_to_dest) {
lcm_eventlog_write_event(dst_log, event);
nwritten++;
if (verbose) {
int *count = g_hash_table_lookup(counts, event->channel);
if (!count) {
count = (int*) malloc(sizeof(int));
*count = 1;
g_hash_table_insert(counts, strdup(event->channel), count);
printf("matched channel %s\n", event->channel);
} else {
*count += 1;
}
}
}
lcm_eventlog_free_event(event);
}
if (verbose) {
g_hash_table_foreach(counts, _verbose_entry_summary, NULL);
printf("=====\n");
printf("Events written: %d\n", nwritten);
}
regfree(&preg);
for (int i = 0; i < num_src_logs; i++)
lcm_eventlog_destroy(src_logs[i]);
lcm_eventlog_destroy(dst_log);
g_hash_table_destroy(counts);
return 0;
}
int main(int argc, char **argv)
{
int verbose = 0;
char *pattern = g_strdup(".*");
char *source_fname = NULL;
char *dest_fname = NULL;
int64_t start_utime = 0;
int64_t end_utime = -1;
int have_end_utime = 0;
int invert_regex = 0;
char *optstring = "hc:vs:e:i";
int c;
while ((c = getopt_long(argc, argv, optstring, NULL, 0)) >= 0) {
switch (c) {
case 'h':
usage();
break;
case 's': {
char *eptr = NULL;
double start_time = strtod(optarg, &eptr);
if (*eptr != 0)
usage();
start_utime = (int64_t)(start_time * 1000000);
} break;
case 'e': {
char *eptr = NULL;
double end_time = strtod(optarg, &eptr);
if (*eptr != 0)
usage();
end_utime = (int64_t)(end_time * 1000000);
have_end_utime = 1;
} break;
case 'i':
invert_regex = 1;
break;
case 'c':
g_free(pattern);
pattern = g_strdup(optarg);
break;
case 'v':
verbose = 1;
break;
default:
usage();
break;
};
}
if (start_utime < 0 || (have_end_utime && end_utime < start_utime))
usage();
if (optind != argc - 2)
usage();
GRegex *regex;
GError *rerr = NULL;
regex = g_regex_new(pattern, (GRegexCompileFlags) 0, (GRegexMatchFlags) 0, &rerr);
if (rerr) {
fprintf(stderr, "bad regex\n");
exit(1);
}
source_fname = argv[argc - 2];
dest_fname = argv[argc - 1];
lcm_eventlog_t *src_log = lcm_eventlog_create(source_fname, "r");
if (!src_log) {
perror("Unable to open source logfile");
g_regex_unref(regex);
return 1;
}
lcm_eventlog_t *dst_log = lcm_eventlog_create(dest_fname, "w");
if (!dst_log) {
perror("Unable to open destination logfile");
lcm_eventlog_destroy(src_log);
g_regex_unref(regex);
return 1;
}
GHashTable *counts = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, free);
int nwritten = 0;
int have_first_event_timestamp = 0;
int64_t first_event_timestamp = 0;
for (lcm_eventlog_event_t *event = lcm_eventlog_read_next_event(src_log); event != NULL;
event = lcm_eventlog_read_next_event(src_log)) {
if (!have_first_event_timestamp) {
first_event_timestamp = event->timestamp;
have_first_event_timestamp = 1;
}
int64_t elapsed = event->timestamp - first_event_timestamp;
if (elapsed < start_utime) {
lcm_eventlog_free_event(event);
continue;
}
if (have_end_utime && elapsed > end_utime) {
lcm_eventlog_free_event(event);
break;
}
int regmatch = g_regex_match(regex, event->channel, (GRegexMatchFlags) 0, NULL);
if ((!regmatch && invert_regex) || (regmatch && !invert_regex)) {
lcm_eventlog_write_event(dst_log, event);
nwritten++;
if (verbose) {
int *count = (int *) g_hash_table_lookup(counts, event->channel);
if (!count) {
count = (int *) malloc(sizeof(int));
*count = 1;
g_hash_table_insert(counts, g_strdup(event->channel), count);
printf("matched channel %s\n", event->channel);
} else {
*count += 1;
}
}
}
lcm_eventlog_free_event(event);
}
if (verbose) {
g_hash_table_foreach(counts, _verbose_entry_summary, NULL);
printf("=====\n");
printf("Events written: %d\n", nwritten);
}
g_regex_unref(regex);
lcm_eventlog_destroy(src_log);
lcm_eventlog_destroy(dst_log);
g_hash_table_destroy(counts);
return 0;
}
static PyObject *
pylog_write_next_event (PyLogObject *self, PyObject *args)
{
int64_t utime = 0;
char *channel = NULL;
int channellen = 0;
// TODO use a buffer object instead of a string
uint8_t *data = NULL;
int datalen = 0;
if (!PyArg_ParseTuple(args, "Ls#s#",
&utime, &channel, &channellen, &data, &datalen)) {
return NULL;
}
if (!self->eventlog) {
PyErr_SetString (PyExc_ValueError, "event log already closed");
return NULL;
}
if (self->mode != 'w') {
PyErr_SetString (PyExc_RuntimeError,
"writing not allowed in read mode");
return NULL;
}
lcm_eventlog_event_t le = {
.eventnum = 0,
.timestamp = utime,
.channellen = channellen,
.datalen = datalen,
.channel = channel,
.data = data
};
if (0 != lcm_eventlog_write_event (self->eventlog, &le)) {
PyErr_SetFromErrno (PyExc_IOError);
return NULL;
}
Py_INCREF (Py_None);
return Py_None;
}
static PyObject *
pylog_size (PyLogObject *self)
{
struct stat sbuf;
if (0 != fstat (fileno (self->eventlog->f), &sbuf)) {
PyErr_SetFromErrno (PyExc_IOError);
return NULL;
}
return PyLong_FromLongLong (sbuf.st_size);
}
static PyObject *
pylog_ftell (PyLogObject *self)
{
return PyLong_FromLongLong (ftello(self->eventlog->f));
}
static PyMethodDef pylog_methods[] = {
{ "close", (PyCFunction)pylog_close, METH_NOARGS, "" },
{ "seek", (PyCFunction)pylog_seek, METH_O, "" },
{ "seek_to_timestamp", (PyCFunction)pylog_seek_to_timestamp, METH_O, "" },
{ "read_next_event", (PyCFunction)pylog_read_next_event, METH_NOARGS, "" },
{ "write_event", (PyCFunction)pylog_write_next_event, METH_VARARGS, "" },
{ "size", (PyCFunction)pylog_size, METH_NOARGS, "" },
{ "ftell", (PyCFunction)pylog_ftell, METH_NOARGS, "" },
{ NULL, NULL }
};
// ==================== class administrative methods ====================
static PyObject *
pylog_repr(PyLogObject *s)
{
char buf[512];
PyOS_snprintf(buf, sizeof(buf),
"<Log object ... TODO>");
return PyString_FromString(buf);
}
static void*
write_thread(void *user_data)
{
logger_t *logger = (logger_t*) user_data;
GTimeVal start_time;
g_get_current_time(&start_time);
int num_splits = 0;
int64_t next_split_sec = start_time.tv_sec + logger->auto_split_hours * SECONDS_PER_HOUR;
while(1) {
void *msg = g_async_queue_pop(logger->write_queue);
// Is it time to start a new logfile?
int split_log = 0;
if(logger->auto_split_hours) {
GTimeVal now;
g_get_current_time(&now);
split_log = (now.tv_sec > next_split_sec);
} else if(logger->auto_split_mb) {
double logsize_mb = (double)logger->logsize / (1 << 20);
split_log = (logsize_mb > logger->auto_split_mb);
}
if(_reset_logfile) {
split_log = 1;
_reset_logfile = 0;
}
if(split_log) {
// Yes. open up a new log file
lcm_eventlog_destroy(logger->log);
if(0 != open_logfile(logger))
exit(1);
num_splits++;
int64_t log_duration_sec = logger->auto_split_hours * SECONDS_PER_HOUR;
next_split_sec = start_time.tv_sec + (num_splits + 1) * log_duration_sec;
logger->logsize = 0;
logger->last_report_logsize = 0;
}
// Should the write thread exit?
g_mutex_lock(logger->mutex);
if(logger->write_thread_exit_flag) {
g_mutex_unlock(logger->mutex);
return NULL;
}
// nope. write the event to disk
lcm_eventlog_event_t *le = (lcm_eventlog_event_t*) msg;
int64_t sz = sizeof(lcm_eventlog_event_t) + le->channellen + 1 + le->datalen;
logger->write_queue_size -= sz;
g_mutex_unlock(logger->mutex);
if(0 != lcm_eventlog_write_event(logger->log, le)) {
static int64_t last_spew_utime = 0;
char *reason = strdup(strerror(errno));
int64_t now = timestamp_now();
if(now - last_spew_utime > 500000) {
fprintf(stderr, "lcm_eventlog_write_event: %s\n", reason);
last_spew_utime = now;
}
free(reason);
free(le);
if(errno == ENOSPC) {
exit(1);
} else {
continue;
}
}
if (logger->fflush_interval_ms >= 0 &&
(le->timestamp - logger->last_fflush_time) > logger->fflush_interval_ms*1000) {
fflush(logger->log->f);
logger->last_fflush_time = le->timestamp;
}
// bookkeeping, cleanup
int64_t offset_utime = le->timestamp - logger->time0;
logger->nevents++;
logger->events_since_last_report ++;
logger->logsize += 4 + 8 + 8 + 4 + le->channellen + 4 + le->datalen;
free(le);
if (offset_utime - logger->last_report_time > 1000000) {
double dt = (offset_utime - logger->last_report_time)/1000000.0;
double tps = logger->events_since_last_report / dt;
double kbps = (logger->logsize - logger->last_report_logsize) / dt / 1024.0;
printf("Summary: %s ti:%4"PRIi64"sec Events: %-9"PRIi64" ( %4"PRIi64" MB ) TPS: %8.2f KB/s: %8.2f\n",
logger->fname,
timestamp_seconds(offset_utime),
logger->nevents, logger->logsize/1048576,
tps, kbps);
logger->last_report_time = offset_utime;
logger->events_since_last_report = 0;
logger->last_report_logsize = logger->logsize;
}
}
}
