Пример #1
0
static int
lcm_logprov_publish (lcm_logprov_t *lcm, const char *channel, const void *data,
        unsigned int datalen)
{
    if(!lcm->writer) {
        dbg(DBG_LCM, "Called publish(), but lcm file provider is not in write mode\n");
        return -1;
    }
    int channellen = strlen(channel);

    int64_t mem_sz = sizeof(lcm_eventlog_event_t) + channellen + 1 + datalen;

    lcm_eventlog_event_t *le = (lcm_eventlog_event_t*) malloc(mem_sz);
    memset(le, 0, mem_sz);

    GTimeVal tv;
    g_get_current_time(&tv);
    le->timestamp = (int64_t) tv.tv_sec * 1000000 + tv.tv_usec;;
    le->channellen = channellen;
    le->datalen = datalen;
    // log_write_event will handle le.eventnum.

    le->channel = ((char*)le) + sizeof(lcm_eventlog_event_t);
    strcpy(le->channel, channel);
    le->data = le->channel + channellen + 1;
    assert((char*)le->data + datalen == (char*)le + mem_sz);
    memcpy(le->data, data, datalen);

    lcm_eventlog_write_event(lcm->log, le);
    free(le);

    return 0;
}
Пример #2
0
int 
LogFile::writeEvent(LogEvent* event)
{
    lcm_eventlog_event_t evt;
    evt.eventnum = event->eventnum;
    evt.timestamp = event->timestamp;
    evt.channellen = event->channel.size();
    evt.datalen = event->datalen;
    evt.channel = (char*) event->channel.c_str();
    evt.data = event->data;
    return lcm_eventlog_write_event(eventlog, &evt);
}
Пример #3
0
static PyObject *
pylog_write_next_event (PyLogObject *self, PyObject *args)
{
    int64_t utime = 0;
    char *channel = NULL;
    int channellen = 0;

    // TODO use a buffer object instead of a string
    uint8_t *data = NULL;
    int datalen = 0;

    if (!PyArg_ParseTuple(args, "Ls#s#", 
                &utime, &channel, &channellen, &data, &datalen)) {
        return NULL;
    }

    if (!self->eventlog) {
        PyErr_SetString (PyExc_ValueError, "event log already closed");
        return NULL;
    }

    if (self->mode != 'w') {
        PyErr_SetString (PyExc_RuntimeError, 
                "writing not allowed in read mode");
        return NULL;
    }

    lcm_eventlog_event_t le; //msvc needs init of all fields seperately
    le.eventnum = 0;
    le.timestamp = utime;
    le.channellen = channellen;
    le.datalen = datalen;
    le.channel = channel;
    le.data = data;
    

    if (0 != lcm_eventlog_write_event (self->eventlog, &le)) {
        PyErr_SetFromErrno (PyExc_IOError);
        return NULL;
    }

    Py_INCREF (Py_None);
    return Py_None;
}
Пример #4
0
int
main(int argc, char **argv)
{
    int verbose = 0;
    int filterChannels =0;
    char *pattern = strdup(".*");
    char *dest_fname = NULL;
    int64_t start_utime = 0;
    int64_t end_utime = -1;
    int have_end_utime = 0;
    int invert_regex = 0;

    char *optstring = "hc:vs:e:i";
    int c;

    while ((c = getopt_long(argc, argv, optstring, NULL, 0)) >= 0) {
        switch (c)
            {
        case 'h':
            usage();
            break;
        case 's':
            {
                char *eptr = NULL;
                double start_time = strtod(optarg, &eptr);
                if (*eptr != 0)
                    usage();
                start_utime = (int64_t) (start_time * 1000000);
            }
            break;
        case 'e':
            {
                char *eptr = NULL;
                double end_time = strtod(optarg, &eptr);
                if (*eptr != 0)
                    usage();
                end_utime = (int64_t) (end_time * 1000000);
                have_end_utime = 1;
            }
            break;
        case 'i':
            invert_regex = 1;
            filterChannels = 1;
            break;
        case 'c':
            free(pattern);
            pattern = strdup(optarg);
            filterChannels = 1;
            break;
        case 'v':
            verbose = 1;
            break;
        default:
            usage();
            break;
            };
    }

    if (start_utime < 0 || (have_end_utime && end_utime < start_utime))
        usage();

    if (optind > argc - 3)
        usage();

    regex_t preg;
    if (0 != regcomp(&preg, pattern, REG_NOSUB | REG_EXTENDED)) {
        fprintf(stderr, "bad regex\n");
        exit(1);
    }

    int num_src_logs = argc - optind - 1;
    fprintf(stderr, "Splicing together %d logs\n", num_src_logs);
    lcm_eventlog_t *src_logs[num_src_logs];
    for (int i = 0; i < argc - optind - 1; i++) {
        char * src_fname = argv[optind + i];
        src_logs[i] = lcm_eventlog_create(src_fname, "r");
        if (!src_logs[i]) {
            perror("Unable to open source logfile");
            for (int j = 0; j < i; j++)
                lcm_eventlog_destroy(src_logs[j]);
            regfree(&preg);
            return 1;
        }
    }

    dest_fname = argv[argc - 1];

    lcm_eventlog_t *dst_log = lcm_eventlog_create(dest_fname, "w");
    if (!dst_log) {
        perror("Unable to open destination logfile");
        for (int i = 0; i < num_src_logs; i++)
            lcm_eventlog_destroy(src_logs[i]);
        regfree(&preg);
        return 1;
    }

    GHashTable *counts = g_hash_table_new_full(g_str_hash, g_str_equal, free,
            free);
    int nwritten = 0;
    int have_first_event_timestamp = 0;
    int64_t first_event_timestamp = -1;

    lcm_eventlog_event_t *events[num_src_logs];
    for (int i = 0; i < num_src_logs; i++)
        events[i] = lcm_eventlog_read_next_event(src_logs[i]);
    while (1) {
        lcm_eventlog_event_t *event;
        int mind = -1;
        int64_t mtime = INT64_MAX;
        for (int i = 0; i < num_src_logs; i++) {
            if (events[i] == NULL)
                continue;
            else if (events[i]->timestamp < mtime) {
                mtime = events[i]->timestamp;
                mind = i;
            }
        }
        if (mind < 0) //all are null
            break;

        event = events[mind];
        events[mind] = lcm_eventlog_read_next_event(src_logs[mind]);

        if (!have_first_event_timestamp) {
            first_event_timestamp = event->timestamp;
            have_first_event_timestamp = 1;
        }

        int64_t elapsed = event->timestamp - first_event_timestamp;
        if (elapsed < start_utime) {
            lcm_eventlog_free_event(event);
            continue;
        }
        if (have_end_utime && elapsed > end_utime) {
            lcm_eventlog_free_event(event);
            break;
        }

        int copy_to_dest = 1;
        if (filterChannels) {
            int regmatch = regexec(&preg, event->channel, 0, NULL, 0);
            copy_to_dest = (regmatch == 0 && !invert_regex) || (regmatch != 0
                    && invert_regex);
        }
        if (copy_to_dest) {
            lcm_eventlog_write_event(dst_log, event);
            nwritten++;

            if (verbose) {
                int *count = g_hash_table_lookup(counts, event->channel);
                if (!count) {
                    count = (int*) malloc(sizeof(int));
                    *count = 1;
                    g_hash_table_insert(counts, strdup(event->channel), count);
                    printf("matched channel %s\n", event->channel);
                } else {
                    *count += 1;
                }
            }
        }
        lcm_eventlog_free_event(event);
    }

    if (verbose) {
        g_hash_table_foreach(counts, _verbose_entry_summary, NULL);
        printf("=====\n");
        printf("Events written: %d\n", nwritten);
    }

    regfree(&preg);
    for (int i = 0; i < num_src_logs; i++)
        lcm_eventlog_destroy(src_logs[i]);
    lcm_eventlog_destroy(dst_log);
    g_hash_table_destroy(counts);
    return 0;
}
Пример #5
0
int main(int argc, char **argv)
{
    int verbose = 0;
    char *pattern = g_strdup(".*");
    char *source_fname = NULL;
    char *dest_fname = NULL;
    int64_t start_utime = 0;
    int64_t end_utime = -1;
    int have_end_utime = 0;
    int invert_regex = 0;

    char *optstring = "hc:vs:e:i";
    int c;

    while ((c = getopt_long(argc, argv, optstring, NULL, 0)) >= 0) {
        switch (c) {
        case 'h':
            usage();
            break;
        case 's': {
            char *eptr = NULL;
            double start_time = strtod(optarg, &eptr);
            if (*eptr != 0)
                usage();
            start_utime = (int64_t)(start_time * 1000000);
        } break;
        case 'e': {
            char *eptr = NULL;
            double end_time = strtod(optarg, &eptr);
            if (*eptr != 0)
                usage();
            end_utime = (int64_t)(end_time * 1000000);
            have_end_utime = 1;
        } break;
        case 'i':
            invert_regex = 1;
            break;
        case 'c':
            g_free(pattern);
            pattern = g_strdup(optarg);
            break;
        case 'v':
            verbose = 1;
            break;
        default:
            usage();
            break;
        };
    }

    if (start_utime < 0 || (have_end_utime && end_utime < start_utime))
        usage();

    if (optind != argc - 2)
        usage();

    GRegex *regex;
    GError *rerr = NULL;
    regex = g_regex_new(pattern, (GRegexCompileFlags) 0, (GRegexMatchFlags) 0, &rerr);
    if (rerr) {
        fprintf(stderr, "bad regex\n");
        exit(1);
    }

    source_fname = argv[argc - 2];
    dest_fname = argv[argc - 1];

    lcm_eventlog_t *src_log = lcm_eventlog_create(source_fname, "r");
    if (!src_log) {
        perror("Unable to open source logfile");
        g_regex_unref(regex);
        return 1;
    }
    lcm_eventlog_t *dst_log = lcm_eventlog_create(dest_fname, "w");
    if (!dst_log) {
        perror("Unable to open destination logfile");
        lcm_eventlog_destroy(src_log);
        g_regex_unref(regex);
        return 1;
    }

    GHashTable *counts = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, free);
    int nwritten = 0;
    int have_first_event_timestamp = 0;
    int64_t first_event_timestamp = 0;

    for (lcm_eventlog_event_t *event = lcm_eventlog_read_next_event(src_log); event != NULL;
         event = lcm_eventlog_read_next_event(src_log)) {
        if (!have_first_event_timestamp) {
            first_event_timestamp = event->timestamp;
            have_first_event_timestamp = 1;
        }

        int64_t elapsed = event->timestamp - first_event_timestamp;
        if (elapsed < start_utime) {
            lcm_eventlog_free_event(event);
            continue;
        }
        if (have_end_utime && elapsed > end_utime) {
            lcm_eventlog_free_event(event);
            break;
        }

        int regmatch = g_regex_match(regex, event->channel, (GRegexMatchFlags) 0, NULL);
        if ((!regmatch && invert_regex) || (regmatch && !invert_regex)) {
            lcm_eventlog_write_event(dst_log, event);
            nwritten++;

            if (verbose) {
                int *count = (int *) g_hash_table_lookup(counts, event->channel);
                if (!count) {
                    count = (int *) malloc(sizeof(int));
                    *count = 1;
                    g_hash_table_insert(counts, g_strdup(event->channel), count);
                    printf("matched channel %s\n", event->channel);
                } else {
                    *count += 1;
                }
            }
        }
        lcm_eventlog_free_event(event);
    }

    if (verbose) {
        g_hash_table_foreach(counts, _verbose_entry_summary, NULL);
        printf("=====\n");
        printf("Events written: %d\n", nwritten);
    }

    g_regex_unref(regex);
    lcm_eventlog_destroy(src_log);
    lcm_eventlog_destroy(dst_log);
    g_hash_table_destroy(counts);
    return 0;
}
Пример #6
0
static PyObject *
pylog_write_next_event (PyLogObject *self, PyObject *args)
{
    int64_t utime = 0;
    char *channel = NULL;
    int channellen = 0;

    // TODO use a buffer object instead of a string
    uint8_t *data = NULL;
    int datalen = 0;

    if (!PyArg_ParseTuple(args, "Ls#s#", 
                &utime, &channel, &channellen, &data, &datalen)) {
        return NULL;
    }

    if (!self->eventlog) {
        PyErr_SetString (PyExc_ValueError, "event log already closed");
        return NULL;
    }

    if (self->mode != 'w') {
        PyErr_SetString (PyExc_RuntimeError, 
                "writing not allowed in read mode");
        return NULL;
    }

    lcm_eventlog_event_t le = {
        .eventnum = 0,
        .timestamp = utime,
        .channellen = channellen,
        .datalen = datalen,
        .channel = channel,
        .data = data
    };

    if (0 != lcm_eventlog_write_event (self->eventlog, &le)) {
        PyErr_SetFromErrno (PyExc_IOError);
        return NULL;
    }

    Py_INCREF (Py_None);
    return Py_None;
}

static PyObject *
pylog_size (PyLogObject *self)
{
    struct stat sbuf;
    if (0 != fstat (fileno (self->eventlog->f), &sbuf)) {
        PyErr_SetFromErrno (PyExc_IOError);
        return NULL;
    }
    return PyLong_FromLongLong (sbuf.st_size);
}

static PyObject *
pylog_ftell (PyLogObject *self)
{
    return PyLong_FromLongLong (ftello(self->eventlog->f));
}

static PyMethodDef pylog_methods[] = {
    { "close", (PyCFunction)pylog_close, METH_NOARGS, "" },
    { "seek", (PyCFunction)pylog_seek, METH_O, "" },
    { "seek_to_timestamp", (PyCFunction)pylog_seek_to_timestamp, METH_O, "" },
    { "read_next_event", (PyCFunction)pylog_read_next_event, METH_NOARGS, "" },
    { "write_event", (PyCFunction)pylog_write_next_event, METH_VARARGS, "" },
    { "size", (PyCFunction)pylog_size, METH_NOARGS, "" },
    { "ftell", (PyCFunction)pylog_ftell, METH_NOARGS, "" },
    { NULL, NULL }
};

// ==================== class administrative methods ====================

static PyObject *
pylog_repr(PyLogObject *s)
{
    char buf[512];
    PyOS_snprintf(buf, sizeof(buf),
                "<Log object ... TODO>");
    return PyString_FromString(buf);
}
Пример #7
0
static void*
write_thread(void *user_data)
{
    logger_t *logger = (logger_t*) user_data;

    GTimeVal start_time;
    g_get_current_time(&start_time);
    int num_splits = 0;
    int64_t next_split_sec = start_time.tv_sec + logger->auto_split_hours * SECONDS_PER_HOUR;

    while(1) {
        void *msg = g_async_queue_pop(logger->write_queue);

        // Is it time to start a new logfile?
        int split_log = 0;
        if(logger->auto_split_hours) {
            GTimeVal now;
            g_get_current_time(&now);
            split_log = (now.tv_sec > next_split_sec);
        } else if(logger->auto_split_mb) {
          double logsize_mb = (double)logger->logsize / (1 << 20);
          split_log = (logsize_mb > logger->auto_split_mb);
        }
        if(_reset_logfile) {
            split_log = 1;
            _reset_logfile = 0;
        }

        if(split_log) {
            // Yes.  open up a new log file
            lcm_eventlog_destroy(logger->log);
            if(0 != open_logfile(logger))
              exit(1);
            num_splits++;
            int64_t log_duration_sec = logger->auto_split_hours * SECONDS_PER_HOUR;
            next_split_sec = start_time.tv_sec + (num_splits + 1) * log_duration_sec;
            logger->logsize = 0;
            logger->last_report_logsize = 0;
        }

        // Should the write thread exit?
        g_mutex_lock(logger->mutex);
        if(logger->write_thread_exit_flag) {
            g_mutex_unlock(logger->mutex);
            return NULL;
        }
        // nope.  write the event to disk
        lcm_eventlog_event_t *le = (lcm_eventlog_event_t*) msg;
        int64_t sz = sizeof(lcm_eventlog_event_t) + le->channellen + 1 + le->datalen;
        logger->write_queue_size -= sz;
        g_mutex_unlock(logger->mutex);

        if(0 != lcm_eventlog_write_event(logger->log, le)) {
            static int64_t last_spew_utime = 0;
            char *reason = strdup(strerror(errno));
            int64_t now = timestamp_now();
            if(now - last_spew_utime > 500000) {
                fprintf(stderr, "lcm_eventlog_write_event: %s\n", reason);
                last_spew_utime = now;
            }
            free(reason);
            free(le);
            if(errno == ENOSPC) {
                exit(1);
            } else {
                continue;
            }
        }
        if (logger->fflush_interval_ms >= 0 &&
            (le->timestamp - logger->last_fflush_time) > logger->fflush_interval_ms*1000) {
            fflush(logger->log->f);
            logger->last_fflush_time = le->timestamp;
        }

        // bookkeeping, cleanup
        int64_t offset_utime = le->timestamp - logger->time0;
        logger->nevents++;
        logger->events_since_last_report ++;
        logger->logsize += 4 + 8 + 8 + 4 + le->channellen + 4 + le->datalen;

        free(le);

        if (offset_utime - logger->last_report_time > 1000000) {
            double dt = (offset_utime - logger->last_report_time)/1000000.0;

            double tps =  logger->events_since_last_report / dt;
            double kbps = (logger->logsize - logger->last_report_logsize) / dt / 1024.0;
            printf("Summary: %s ti:%4"PRIi64"sec Events: %-9"PRIi64" ( %4"PRIi64" MB )      TPS: %8.2f       KB/s: %8.2f\n",
                    logger->fname,
                    timestamp_seconds(offset_utime),
                    logger->nevents, logger->logsize/1048576,
                    tps, kbps);
            logger->last_report_time = offset_utime;
            logger->events_since_last_report = 0;
            logger->last_report_logsize = logger->logsize;
        }
    }
}