static bool
read_string_value(const char *p, const char *e, int ai, lsb_read_value *val)
{
int acnt = 0;
int tag = 0;
int wiretype = 0;
while (p && p < e) {
val->type = LSB_READ_NIL;
p = lsb_pb_read_key(p, &tag, &wiretype);
p = read_string(wiretype, p, e, &val->u.s);
if (p) {
if (ai == acnt++) {
val->type = LSB_READ_STRING;
return true;
}
}
}
return false;
}
bool lsb_decode_heka_message(lsb_heka_message *m,
const char *buf,
size_t len,
lsb_logger *logger)
{
if (!m || !buf || len == 0) {
if (logger && logger->cb) {
logger->cb(logger->context, __func__, 4, LSB_ERR_UTIL_NULL);
}
return false;
}
const char *cp = buf; // current position
const char *lp = buf; // last position
const char *ep = buf + len; // end position
int wiretype = 0;
int tag = 0;
long long val = 0;
bool timestamp = false;
lsb_clear_heka_message(m);
do {
cp = lsb_pb_read_key(cp, &tag, &wiretype);
switch (tag) {
case LSB_PB_UUID:
cp = read_string(wiretype, cp, ep, &m->uuid);
if (m->uuid.len != LSB_UUID_SIZE) cp = NULL;
break;
case LSB_PB_TIMESTAMP:
cp = process_varint(wiretype, cp, ep, &m->timestamp);
if (cp) timestamp = true;
break;
case LSB_PB_TYPE:
cp = read_string(wiretype, cp, ep, &m->type);
break;
case LSB_PB_LOGGER:
cp = read_string(wiretype, cp, ep, &m->logger);
break;
case LSB_PB_SEVERITY:
cp = process_varint(wiretype, cp, ep, &val);
if (cp) m->severity = (int)val;
break;
case LSB_PB_PAYLOAD:
cp = read_string(wiretype, cp, ep, &m->payload);
break;
case LSB_PB_ENV_VERSION:
cp = read_string(wiretype, cp, ep, &m->env_version);
break;
case LSB_PB_PID:
cp = process_varint(wiretype, cp, ep, &val);
if (cp) m->pid = (int)val;
break;
case LSB_PB_HOSTNAME:
cp = read_string(wiretype, cp, ep, &m->hostname);
break;
case LSB_PB_FIELDS:
if (wiretype != 2) {
cp = NULL;
break;
}
if (m->fields_len == m->fields_size) {
int step = 8;
m->fields_size += step;
lsb_heka_field *tmp = realloc(m->fields,
m->fields_size * sizeof(lsb_heka_field));
if (!tmp) {
if (logger && logger->cb) {
logger->cb(logger->context, __func__, 0, "fields reallocation failed");
}
return false;
}
m->fields = tmp;
memset(&m->fields[m->fields_len], 0, step * sizeof(lsb_heka_field));
}
cp = process_fields(&m->fields[m->fields_len], cp, ep);
++m->fields_len;
break;
default:
cp = NULL;
break;
}
if (cp) lp = cp;
} while (cp && cp < ep);
if (!cp) {
if (logger && logger->cb) {
logger->cb(logger->context, __func__, 4, "tag:%d wiretype:%d position:%d",
tag, wiretype, lp - buf);
}
return false;
}
if (!m->uuid.s) {
if (logger && logger->cb) {
logger->cb(logger->context, __func__, 4, "missing " LSB_UUID);
}
return false;
}
if (!timestamp) {
if (logger && logger->cb) {
logger->cb(logger->context, __func__, 4, "missing " LSB_TIMESTAMP);
}
return false;
}
m->raw.s = buf;
m->raw.len = len;
return true;
}
static const char*
process_fields(lsb_heka_field *f, const char *p, const char *e)
{
int tag = 0;
int wiretype = 0;
long long vi = 0;
p = lsb_pb_read_varint(p, e, &vi);
if (!p || vi < 0 || p + vi > e) {
return NULL;
}
e = p + vi; // only process to the end of the current field record
do {
p = lsb_pb_read_key(p, &tag, &wiretype);
switch (tag) {
case LSB_PB_NAME:
p = read_string(wiretype, p, e, &f->name);
break;
case LSB_PB_VALUE_TYPE:
p = process_varint(wiretype, p, e, &vi);
if (p) {
f->value_type = (int)vi;
}
break;
case LSB_PB_REPRESENTATION:
p = read_string(wiretype, p, e, &f->representation);
break;
// don't bother with the value(s) until we actually need them
// since this stream is created by Hindsight
// - tags are guaranteed to be properly ordered (values at the end)
// - there won't be repeated tags for packed values
case LSB_PB_VALUE_STRING:
case LSB_PB_VALUE_BYTES:
if (wiretype != 2) {
p = NULL;
break;
}
f->value.s = p - 1;
f->value.len = e - f->value.s;
p = e;
break;
case LSB_PB_VALUE_INTEGER:
case LSB_PB_VALUE_BOOL:
if (wiretype != 0 && wiretype != 2) {
p = NULL;
break;
}
// fall thru
case LSB_PB_VALUE_DOUBLE:
if (tag == 7 && wiretype != 1 && wiretype != 2) {
p = NULL;
break;
}
if (wiretype == 2) {
p = lsb_pb_read_varint(p, e, &vi);
if (!p || vi < 0 || p + vi > e) {
p = NULL;
break;
}
}
f->value.s = p;
f->value.len = e - f->value.s;
p = e;
break;
default:
p = NULL; // don't allow unknown tags
break;
}
} while (p && p < e);
return p && f->name.s ? p : NULL;
}
static void output_text(lsb_heka_message *msg)
{
static char tstr[64];
if (!msg->raw.s) return;
fprintf(stdout, ":Uuid: %02hhx%02hhx%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx"
"-%02hhx%02hhx-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx\n",
msg->uuid.s[0], msg->uuid.s[1], msg->uuid.s[2], msg->uuid.s[3],
msg->uuid.s[4], msg->uuid.s[5], msg->uuid.s[6], msg->uuid.s[7],
msg->uuid.s[8], msg->uuid.s[9], msg->uuid.s[10], msg->uuid.s[11],
msg->uuid.s[12], msg->uuid.s[13], msg->uuid.s[14], msg->uuid.s[15]);
fprintf(stdout, ":Timestamp: ");
time_t t = floor(msg->timestamp / 1e9);
double frac = msg->timestamp - t * 1e9;
struct tm *tms = gmtime(&t);
strftime(tstr, sizeof(tstr) - 1, "%Y-%m-%dT%H:%M:%S", tms);
fprintf(stdout, "%s.%09lldZ\n", tstr, (long long)frac);
output_cs(":Type", &msg->type, true);
output_cs(":Logger", &msg->logger, true);
fprintf(stdout, ":Severity: %d\n", msg->severity);
output_cs(":Payload", &msg->payload, true);
output_cs(":EnvVersion", &msg->env_version, true);
if (msg->pid == INT_MIN) {
fprintf(stdout, ":Pid: <nil>\n");
} else {
fprintf(stdout, ":Pid: %d\n", msg->pid);
}
output_cs(":Hostname", &msg->hostname, true);
fprintf(stdout, ":Fields:\n");
for (int i = 0; i < msg->fields_len; ++i) {
fprintf(stdout, " | name: %.*s type: %d ", (int)msg->fields[i].name.len,
msg->fields[i].name.s, msg->fields[i].value_type);
output_cs("representation", &msg->fields[i].representation, false);
fprintf(stdout, " value: ");
const char *p = msg->fields[i].value.s;
const char *e = msg->fields[i].value.s + msg->fields[i].value.len;
switch (msg->fields[i].value_type) {
case LSB_PB_STRING:
{
lsb_const_string cs;
int tag = 0;
int wiretype = 0;
while (p && p < e) {
p = lsb_pb_read_key(p, &tag, &wiretype);
p = read_string(wiretype, p, e, &cs);
if (p) {
fprintf(stdout, "%.*s", (int)cs.len, cs.s);
}
}
}
break;
case LSB_PB_BYTES:
{
lsb_const_string cs;
int tag = 0;
int wiretype = 0;
while (p && p < e) {
p = lsb_pb_read_key(p, &tag, &wiretype);
p = read_string(wiretype, p, e, &cs);
if (p) {
for (size_t i = 0; i < cs.len; ++i) {
if (isprint(cs.s[i])) {
if (cs.s[i] == '\\') {
fwrite("\\\\", 2, 1, stdout);
} else {
putchar(cs.s[i]);
}
} else {
fprintf(stdout, "\\x%02hhx", (unsigned char)cs.s[i]);
}
}
}
}
}
break;
case LSB_PB_INTEGER:
{
long long ll = 0;
while (p && p < e) {
p = lsb_pb_read_varint(p, e, &ll);
if (p) {
fprintf(stdout, "%lld", ll);
}
}
}
break;
case LSB_PB_DOUBLE:
{
double d;
for (int i = 0; p <= (e - sizeof(double)); p += sizeof(double), ++i) {
memcpy(&d, p, sizeof(double));
if (i > 0) {
fprintf(stdout, ",");
}
fprintf(stdout, "%.17g", d);
}
}
break;
case LSB_PB_BOOL:
{
long long ll = 0;
while (p && p < e) {
p = lsb_pb_read_varint(p, e, &ll);
if (p) {
fprintf(stdout, "%s", ll == 0 ? "false" : "true");
}
}
}
break;
}
fprintf(stdout, "\n");
}
fprintf(stdout, "\n");
return;
}
static const char* process_fields(lua_State *lua, const char *p, const char *e)
{
int tag = 0;
int wiretype = 0;
int has_name = 0;
int value_count = 0;
long long len = 0;
p = lsb_pb_read_varint(p, e, &len);
if (!p || len < 0 || p + len > e) {
return NULL;
}
e = p + len; // only process to the end of the current field record
lua_newtable(lua); // Table to be added to the Fields array index 4
lua_newtable(lua); // Table to hold the value(s) index 5
do {
p = lsb_pb_read_key(p, &tag, &wiretype);
switch (tag) {
case 1:
p = read_string(lua, wiretype, p, e);
if (p) {
lua_setfield(lua, 4, "name");
has_name = 1;
}
break;
case 2:
p = process_varint(lua, "value_type", wiretype, 4, p, e);
break;
case 3:
p = read_string(lua, wiretype, p, e);
if (p) {
lua_setfield(lua, 4, "representation");
}
break;
case 4: // value_string
case 5: // value_bytes
p = read_string(lua, wiretype, p, e);
if (p) {
lua_rawseti(lua, 5, ++value_count);
}
break;
case 6: // value_integer
{
long long val = 0;
switch (wiretype) {
case 0:
p = lsb_pb_read_varint(p, p + len, &val);
if (!p) break;
lua_pushnumber(lua, (lua_Number)val);
lua_rawseti(lua, 5, ++value_count);
break;
case 2:
p = lsb_pb_read_varint(p, e, &len);
if (!p || len < 0 || p + len > e) {
p = NULL;
break;
}
do {
p = lsb_pb_read_varint(p, p + len, &val);
if (!p) break;
lua_pushnumber(lua, (lua_Number)val);
lua_rawseti(lua, 5, ++value_count);
} while (p < e);
break;
default:
p = NULL;
break;
}
}
break;
case 7: // value_double
{
double val = 0;
switch (wiretype) {
case 1:
if (p + sizeof(double) > e) {
p = NULL;
break;
}
memcpy(&val, p, sizeof(double));
p += sizeof(double);
lua_pushnumber(lua, val);
lua_rawseti(lua, 5, ++value_count);
break;
case 2:
p = lsb_pb_read_varint(p, e, &len);
if (!p || len < 0 || p + len > e || len % sizeof(double) != 0) {
p = NULL;
break;
}
do {
memcpy(&val, p, sizeof(double));
p += sizeof(double);
lua_pushnumber(lua, val);
lua_rawseti(lua, 5, ++value_count);
} while (p < e);
break;
default:
p = NULL;
break;
}
}
break;
case 8: // value_bool
{
long long val = 0;
switch (wiretype) {
case 0:
p = lsb_pb_read_varint(p, p + len, &val);
if (!p) break;
lua_pushboolean(lua, (int)val);
lua_rawseti(lua, 5, ++value_count);
break;
case 2:
p = lsb_pb_read_varint(p, e, &len);
if (!p || len < 0 || p + len > e) {
p = NULL;
break;
}
do {
p = lsb_pb_read_varint(p, p + len, &val);
if (!p) break;
lua_pushboolean(lua, (int)val);
lua_rawseti(lua, 5, ++value_count);
} while (p < e);
break;
default:
p = NULL;
break;
}
}
break;
default:
p = NULL; // don't allow unknown tags
break;
}
} while (p && p < e);
lua_setfield(lua, 4, "value");
return has_name ? p : NULL;
}
int heka_decode_message(lua_State *lua)
{
int n = lua_gettop(lua);
if (n != 1 || lua_type(lua, 1) != LUA_TSTRING) {
return luaL_argerror(lua, 0, "must have one string argument");
}
size_t len;
const char *pbstr = lua_tolstring(lua, 1, &len);
if (len < 20) {
return luaL_error(lua, "invalid message, too short");
}
const char *p = pbstr;
const char *lp = p;
const char *e = pbstr + len;
int wiretype = 0;
int tag = 0;
int has_uuid = 0;
int has_timestamp = 0;
int field_count = 0;
lua_newtable(lua); // message table index 2
do {
p = lsb_pb_read_key(p, &tag, &wiretype);
switch (tag) {
case 1:
p = read_string(lua, wiretype, p, e);
if (p && p - lp == 18) {
lua_setfield(lua, 2, "Uuid");
has_uuid = 1;
} else {
p = NULL;
}
break;
case 2:
p = process_varint(lua, "Timestamp", wiretype, 2, p, e);
if (p) {
has_timestamp = 1;
}
break;
case 3:
p = read_string(lua, wiretype, p, e);
if (p) {
lua_setfield(lua, 2, "Type");
}
break;
case 4:
p = read_string(lua, wiretype, p, e);
if (p) {
lua_setfield(lua, 2, "Logger");
}
break;
case 5:
p = process_varint(lua, "Severity", wiretype, 2, p, e);
break;
case 6:
p = read_string(lua, wiretype, p, e);
if (p) {
lua_setfield(lua, 2, "Payload");
}
break;
case 7:
p = read_string(lua, wiretype, p, e);
if (p) {
lua_setfield(lua, 2, "EnvVersion");
}
break;
case 8:
p = process_varint(lua, "Pid", wiretype, 2, p, e);
break;
case 9:
p = read_string(lua, wiretype, p, e);
if (p) {
lua_setfield(lua, 2, "Hostname");
}
break;
case 10:
if (wiretype != 2) {
p = NULL;
break;
}
if (field_count == 0) {
lua_newtable(lua); // Fields table index 3
}
p = process_fields(lua, p, e);
if (p) {
lua_rawseti(lua, 3, ++field_count);
}
break;
default:
p = NULL; // don't allow unknown tags
break;
}
if (p) lp = p;
} while (p && p < e);
if (!p) {
return luaL_error(lua, "error in tag: %d wiretype: %d offset: %d", tag,
wiretype, (const char *)lp - pbstr);
}
if (!has_uuid || !has_timestamp) {
return luaL_error(lua, "missing required field uuid: %s timestamp: %s",
has_uuid ? "found" : "not found",
has_timestamp ? "found" : "not found");
}
if (field_count) {
lua_setfield(lua, 2, "Fields");
}
return 1;
}
