static PyObject *entity_lookup(PyObject *self, PyObject *args)
{
PyObject *idref, *element;
Py_ssize_t i;
if (!PyArg_ParseTuple(args, "O:xml_lookup", &idref))
return NULL;
/* our "document" can have multiple element children */
for (i = 0; i < Container_GET_COUNT(self); i++) {
NodeObject *node = Container_GET_CHILD(self, i);
if (Element_Check(node)) {
element = get_element_by_id(node, idref);
if (element == NULL)
return NULL;
else if (element != Py_None) {
Py_INCREF(element);
return element;
}
}
}
Py_INCREF(Py_None);
return Py_None;
}
/* returns borrowed reference */
Py_LOCAL(PyObject *) /* not inlined as its recursive */
get_element_by_id(NodeObject *node, PyObject *elementId)
{
PyObject *result;
Py_ssize_t i;
for (i = 0; i < Container_GET_COUNT(node); i++) {
NodeObject *child = Container_GET_CHILD(node, i);
if (Element_Check(child)) {
/* Searth the attributes for an ID attr */
PyObject *attributes = Element_ATTRIBUTES(child);
if (attributes != NULL) {
AttrObject *attr;
Py_ssize_t pos = 0;
while ((attr = AttributeMap_Next(attributes, &pos)) != NULL) {
if (Attr_GET_TYPE(attr) == ATTRIBUTE_TYPE_ID) {
switch (PyObject_RichCompareBool(Attr_GET_VALUE(attr),
elementId, Py_EQ)) {
case 1:
/* Found matching element, return it. */
return (PyObject *) child;
case 0:
break;
default:
return NULL;
}
}
}
}
/* Continue on with the children */
result = get_element_by_id(child, elementId);
if (result != Py_None)
/* either NULL (an error) or a node (element found) */
return result;
}
}
return Py_None;
}
int process_startup_xml(char *params, struct fastbit_config *c)
{
struct tm *timeinfo;
char formated_time[17];
std::string path, time_window, record_limit, name_type, name_prefix,
indexes, reorder, create_sp_files, test, template_field_lengths, time_alignment;
pugi::xml_document doc;
doc.load(params);
if (doc) {
pugi::xpath_node ie = doc.select_single_node("fileWriter");
path = ie.node().child_value("path");
/* Make sure path ends with '/' character */
if (path.at(path.size() - 1) != '/') {
c->sys_dir = path + "/";
} else {
c->sys_dir = path;
}
indexes = ie.node().child_value("onTheFlyIndexes");
c->indexes = (indexes == "yes");
create_sp_files = ie.node().child_value("createSpFiles");
c->create_sp_files = (create_sp_files == "yes");
reorder = ie.node().child_value("reorder");
c->reorder = (reorder == "yes");
template_field_lengths = ie.node().child_value("useTemplateFieldLengths");
c->use_template_field_lengths =
(!ie.node().child("useTemplateFieldLengths") || template_field_lengths == "yes");
pugi::xpath_node_set index_e = doc.select_nodes("fileWriter/indexes/element");
for (pugi::xpath_node_set::const_iterator it = index_e.begin(); it != index_e.end(); ++it) {
pugi::xpath_node node = *it;
std::string en = "0";
std::string id = "0";
for (pugi::xml_attribute_iterator ait = node.node().attributes_begin();
ait != node.node().attributes_end(); ++ait) {
if (std::string(ait->name()) == "enterprise") {
en = ait->value();
} else if (std::string(ait->name()) == "id") {
id = ait->value();
}
}
/* Check whether enterprise and field IDs are valid */
int en_int = strtoi(en.c_str(), 10);
int id_int = strtoi(id.c_str(), 10);
if (en_int == INT_MAX || id_int == INT_MAX) {
MSG_ERROR(msg_module, "Invalid enterprise or field ID (enterprise ID: %s, field ID: %s)",
en.c_str(), id.c_str());
return 1;
}
/* Make sure IPv6 elements are indexed */
const ipfix_element_t *element = get_element_by_id(id_int, en_int);
if (element && element->type == ET_IPV6_ADDRESS) {
c->index_en_id->push_back("e" + en + "id" + id + "p0");
c->index_en_id->push_back("e" + en + "id" + id + "p1");
} else {
c->index_en_id->push_back("e" + en + "id" + id);
}
}
if (c->index_en_id->size() > 0 && c->indexes) {
c->indexes = 2; /* Mark elements for indexes */
}
ie = doc.select_single_node("fileWriter/dumpInterval");
time_window = ie.node().child_value("timeWindow");
c->time_window = atoi(time_window.c_str());
record_limit = ie.node().child_value("recordLimit");
c->records_window = atoi(record_limit.c_str());
record_limit = ie.node().child_value("bufferSize");
c->buff_size = atoi(record_limit.c_str());
time_alignment = ie.node().child_value("timeAlignment");
ie = doc.select_single_node("fileWriter/namingStrategy");
name_prefix = ie.node().child_value("prefix");
c->prefix = name_prefix;
time(&(c->last_flush));
name_type = ie.node().child_value("type");
if (name_type == "time") {
c->dump_name = TIME;
if (time_alignment == "yes") {
if (c->time_window > 0) {
/* operators '/' and '*' are used for round down time to time window */
c->last_flush = ((c->last_flush / c->time_window) * c->time_window);
}
}
timeinfo = localtime(&(c->last_flush));
strftime(formated_time, 17, "%Y%m%d%H%M%S", timeinfo);
c->window_dir = c->prefix + std::string(formated_time) + "/";
} else if (name_type == "incremental") {
c->dump_name = INCREMENTAL;
c->window_dir = c->prefix + "000000000001/";
} else if (name_type == "prefix") {
c->dump_name = PREFIX;
if (c->prefix == "") {
c->prefix = "fbitfiles";
}
c->window_dir = c->prefix + "/";
}
} else {
return 1;
}
return 0;
}
/**
* \brief Store data record
*/
void Storage::storeDataRecord(struct metadata *mdata, struct json_conf * config)
{
const char *element_name = NULL;
ELEMENT_TYPE element_type;
offset = 0;
uint16_t trans_len = 0;
const char *trans_str = NULL;
record.clear();
STR_APPEND(record, "{\"@type\": \"ipfix.entry\", ");
struct ipfix_template *templ = mdata->record.templ;
uint8_t *data_record = (uint8_t*) mdata->record.record;
/* get all fields */
uint16_t added = 0;
for (uint16_t count = 0, index = 0; count < templ->field_count; ++count, ++index) {
/* Get Enterprise number and ID */
id = templ->fields[index].ie.id;
length = templ->fields[index].ie.length;
enterprise = 0;
if (id & 0x8000) {
id &= 0x7fff;
enterprise = templ->fields[++index].enterprise_number;
}
/* Get element informations */
const ipfix_element_t * element = get_element_by_id(id, enterprise);
if (element != NULL) {
element_name = element->name;
element_type = element->type;
} else {
// Element not found
if (config->ignoreUnknown) {
offset += realLength(length, data_record, offset);
continue;
}
element_name = rawName(enterprise, id);
element_type = ET_UNASSIGNED;
MSG_DEBUG(msg_module, "Unknown element (%s)", element_name);
}
if (added > 0) {
STR_APPEND(record, ", ");
}
STR_APPEND(record, "\"");
record += config->prefix;
record += element_name;
STR_APPEND(record, "\": ");
switch (element_type) {
case ET_UNSIGNED_8:
case ET_UNSIGNED_16:
case ET_UNSIGNED_32:
case ET_UNSIGNED_64:{
trans_str = translator.toUnsigned(length, &trans_len, data_record, offset,
element, config);
record.append(trans_str, trans_len);
}
break;
case ET_SIGNED_8:
case ET_SIGNED_16:
case ET_SIGNED_32:
case ET_SIGNED_64:
trans_str = translator.toSigned(length, &trans_len, data_record, offset);
record.append(trans_str, trans_len);
break;
case ET_FLOAT_32:
case ET_FLOAT_64:
trans_str = translator.toFloat(length, &trans_len, data_record, offset);
record.append(trans_str, trans_len);
break;
case ET_IPV4_ADDRESS:
record += '"';
trans_str = translator.formatIPv4(read32(data_record + offset), &trans_len);
record.append(trans_str, trans_len);
record += '"';
break;
case ET_IPV6_ADDRESS:
READ_BYTE_ARR(addr6, data_record + offset, IPV6_LEN);
record += '"';
record += translator.formatIPv6(addr6);
record += '"';
break;
case ET_MAC_ADDRESS:
READ_BYTE_ARR(addrMac, data_record + offset, MAC_LEN);
record += '"';
record += translator.formatMac(addrMac);
record += '"';
break;
case ET_DATE_TIME_SECONDS:
record += translator.formatTimestamp(read32(data_record + offset),
t_units::SEC, config);
break;
case ET_DATE_TIME_MILLISECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::MILLISEC, config);
break;
case ET_DATE_TIME_MICROSECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::MICROSEC, config);
break;
case ET_DATE_TIME_NANOSECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::NANOSEC, config);
break;
case ET_STRING:
length = realLength(length, data_record, offset);
record += translator.escapeString(length, data_record + offset,
config);
break;
case ET_BOOLEAN:
case ET_UNASSIGNED:
default:
readRawData(length, data_record, offset);
break;
}
offset += length;
added++;
}
/* Store metadata */
if (processMetadata) {
STR_APPEND(record, ", \"ipfix.metadata\": {");
storeMetadata(mdata);
STR_APPEND(record, "}");
}
STR_APPEND(record, "}\n");
sendData();
}
