/**
* \brief Read string field
*/
void Storage::readString(uint16_t& length, uint8_t *data_record, uint16_t &offset)
{
/* Get string length */
length = realLength(length, data_record, offset);
/* Read string */
record.append((const char *)(data_record + offset), length);
}
/**
* \brief Store data record
*/
void Storage::storeDataRecord(struct metadata *mdata, struct json_conf * config)
{
const char *element_name = NULL;
ELEMENT_TYPE element_type;
offset = 0;
uint16_t trans_len = 0;
const char *trans_str = NULL;
record.clear();
STR_APPEND(record, "{\"@type\": \"ipfix.entry\", ");
struct ipfix_template *templ = mdata->record.templ;
uint8_t *data_record = (uint8_t*) mdata->record.record;
/* get all fields */
uint16_t added = 0;
for (uint16_t count = 0, index = 0; count < templ->field_count; ++count, ++index) {
/* Get Enterprise number and ID */
id = templ->fields[index].ie.id;
length = templ->fields[index].ie.length;
enterprise = 0;
if (id & 0x8000) {
id &= 0x7fff;
enterprise = templ->fields[++index].enterprise_number;
}
/* Get element informations */
const ipfix_element_t * element = get_element_by_id(id, enterprise);
if (element != NULL) {
element_name = element->name;
element_type = element->type;
} else {
// Element not found
if (config->ignoreUnknown) {
offset += realLength(length, data_record, offset);
continue;
}
element_name = rawName(enterprise, id);
element_type = ET_UNASSIGNED;
MSG_DEBUG(msg_module, "Unknown element (%s)", element_name);
}
if (added > 0) {
STR_APPEND(record, ", ");
}
STR_APPEND(record, "\"");
record += config->prefix;
record += element_name;
STR_APPEND(record, "\": ");
switch (element_type) {
case ET_UNSIGNED_8:
case ET_UNSIGNED_16:
case ET_UNSIGNED_32:
case ET_UNSIGNED_64:{
trans_str = translator.toUnsigned(length, &trans_len, data_record, offset,
element, config);
record.append(trans_str, trans_len);
}
break;
case ET_SIGNED_8:
case ET_SIGNED_16:
case ET_SIGNED_32:
case ET_SIGNED_64:
trans_str = translator.toSigned(length, &trans_len, data_record, offset);
record.append(trans_str, trans_len);
break;
case ET_FLOAT_32:
case ET_FLOAT_64:
trans_str = translator.toFloat(length, &trans_len, data_record, offset);
record.append(trans_str, trans_len);
break;
case ET_IPV4_ADDRESS:
record += '"';
trans_str = translator.formatIPv4(read32(data_record + offset), &trans_len);
record.append(trans_str, trans_len);
record += '"';
break;
case ET_IPV6_ADDRESS:
READ_BYTE_ARR(addr6, data_record + offset, IPV6_LEN);
record += '"';
record += translator.formatIPv6(addr6);
record += '"';
break;
case ET_MAC_ADDRESS:
READ_BYTE_ARR(addrMac, data_record + offset, MAC_LEN);
record += '"';
record += translator.formatMac(addrMac);
record += '"';
break;
case ET_DATE_TIME_SECONDS:
record += translator.formatTimestamp(read32(data_record + offset),
t_units::SEC, config);
break;
case ET_DATE_TIME_MILLISECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::MILLISEC, config);
break;
case ET_DATE_TIME_MICROSECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::MICROSEC, config);
break;
case ET_DATE_TIME_NANOSECONDS:
record += translator.formatTimestamp(read64(data_record + offset),
t_units::NANOSEC, config);
break;
case ET_STRING:
length = realLength(length, data_record, offset);
record += translator.escapeString(length, data_record + offset,
config);
break;
case ET_BOOLEAN:
case ET_UNASSIGNED:
default:
readRawData(length, data_record, offset);
break;
}
offset += length;
added++;
}
/* Store metadata */
if (processMetadata) {
STR_APPEND(record, ", \"ipfix.metadata\": {");
storeMetadata(mdata);
STR_APPEND(record, "}");
}
STR_APPEND(record, "}\n");
sendData();
}
void Trace::slowCompressLoop(int window){
static int warned_once = 0;
int distance;
Event *iter;
iteration_t *rtn;
iteration_t *iteration;
vector<iteration_t *> iterations;
vector<iteration_t *>::iterator iteration_it;
int i;
int head_flag; /* Whether target head has been found. */
int match_flag; /* Whether a node matching target head has been found. */
int found_flag; /* */
int target_length=1, merge_length=1;
int target_real_length, merge_real_length;
double ratio;
Event *target_tail = NULL;
Event *target_head = NULL;
Event *merge_tail = NULL;
Event *merge_head = NULL;
#if defined FEATURE_SIG_DIFF && FEATURE_SIG_DIFF > 0
Event* search;
StackSig* target_sig;
StackSig* merge_sig;
#endif
distance = 0;
head_flag = 0;
found_flag = 1;
do {
target_tail = tail;
if(found_flag || !head_flag) {
/* if the found flag is set - start a new search */
target_head = target_tail;
distance = 0;
target_length = merge_length = 1;
} else {
/* otherwise continue a previous search */
target_head = target_head->prev;
target_length++;
merge_length = 1;
}
head_flag = 0;
match_flag = 0;
found_flag = 0;
/* Search backwards in the queue until we find a match with the target tail.
* After this loop is done, either head_flag will be false (and nothing found)
* or the head will be just after the matching tail. */
while(target_head->prev != NULL && (window == -1 || window > distance)) {
if( !target_head->checkLoc(MEMBER)
&& !target_head->checkLoc(PENDING_MEMBER)
&& !target_head->checkLoc(PENDING_LEADER)
&& !target_head->checkLoc(PENDING_TAIL)
&& target_tail->sigMatch(target_head->prev)
&& target_tail->opMatch(target_head->prev)){
head_flag = 1;
break;
}
target_head = target_head->prev;
target_length++;
distance++;
}
/* didn't find head: can skip the rest. */
if (!head_flag) {
/* Warn if the window was exceeded, but only warn once. */
if(!warned_once && window != -1 && window <= distance) {
cerr<<"warning: window exceeded - possible compression missed (window: "<<window<<" distance: "<<distance<<")"<<endl;
warned_once = 1;
}
break;
}
/* If the head flag is true, we've got the target_tail, the target_head, and
* the merge_tail (just before target_head). Now we can try to find a match
* for the target_head somewhere before the merge tail. This will be merge_head.*/
merge_head = target_head->prev;
merge_tail = target_head->prev;
while(merge_head != NULL) {
if( !merge_head->checkLoc(MEMBER)
&& merge_head->sigMatch(target_head)
&& merge_head->opMatch(target_head)){
target_real_length = realLength(target_head, target_tail);
merge_real_length = realLength(merge_head, merge_tail);
assert(target_real_length > 0 && merge_real_length > 0);
if(target_real_length > merge_real_length){
ratio = (double)merge_real_length / (double)target_real_length;
} else {
ratio = (double)target_real_length / (double)merge_real_length;
}
if(1 - ratio < MAX_LENGTH_DIFF){
#if defined FEATURE_SIG_DIFF && FEATURE_SIG_DIFF > 0
/* heuristic that helps to identify the best match */
if( (target_tail->sigEquals(merge_tail) && target_head->sigEquals(merge_head))
|| (merge_length == target_length && merge_length == 1 ) )
#endif
if(merge_head->checkLoc(PENDING_MEMBER) || merge_head->checkLoc(PENDING_LEADER)
|| merge_head->checkLoc(PENDING_TAIL)){
for(iter = merge_head, i = 0; iter; iter = iter->prev, i++){
iterations = pendingIterations.getIterationsByTargetHead(iter);
if(iterations.size() == 1){
if(iterations[0]->target_tail->getId() >= merge_head->getId()){
break;
}
}else{
/* only one pending iteration can have iter as the
* target_head, because it's impossible to choose a
* PENDING_LEADER event as the target_head */
assert(iterations.size()==0);
}
iterations.clear();
}
if(iterations.size() == 1 && (merge_tail == iterations[0]->target_tail)){
match_flag = 1;
break;
}
} else {
match_flag = 1;
break;
}
}
}
merge_head = merge_head->prev;
merge_length++;
}
/* If we didn't find the merge head, we continue to the next iteration to try to
* find a longer iteration. */
if(!match_flag)
continue;
found_flag = 1;
/* mark the loop iteration */
/* 1. check if the current iteration is a trailing iteration, if
* it is a trailing iteration, only the case where merge_tail equals
* to the tail of the pervious iteration is allowed */
if(merge_head->checkLoc(PENDING_MEMBER) || merge_head->checkLoc(PENDING_LEADER)
|| merge_head->checkLoc(PENDING_TAIL)){
for(iter = merge_head, i = 0; iter; iter = iter->prev, i++){
iterations = pendingIterations.getIterationsByTargetHead(iter);
if(iterations.size() == 1){
//if(iterations[0]->target_length - i > 0){
if(iterations[0]->target_tail->getId() >= merge_head->getId()){
break;
}
}else{
/* only one pending iteration can have iter as the
* target_head, because it's impossible to choose a
* PENDING_LEADER event as the target_head */
assert(iterations.size()==0);
}
iterations.clear();
}
if(iterations.size() == 1 && (merge_tail == iterations[0]->target_tail)){
iterations = pendingIterations.getIterationsByTargetTail(merge_tail);
for(iteration_it = iterations.begin();
iteration_it != iterations.end(); iteration_it++)
for(iter = (*iteration_it)->target_tail;
iter != (*iteration_it)->target_head; iter = iter->prev)
if(iter == merge_head){
(*iteration_it)->target_tail->unsetLoc(PENDING_TAIL);
(*iteration_it)->target_tail->setLoc(PENDING_MEMBER);
pendingIterations.updateIterationTargetTail((*iteration_it), target_tail);
(*iteration_it)->target_length += target_length;
break;
}
} else {
found_flag = 0;
continue;
}
}
#if defined FEATURE_SIG_DIFF && FEATURE_SIG_DIFF > 0
if(!target_tail->sigEquals(merge_tail)){
target_sig = target_tail->getSignature();
merge_sig = merge_tail->getSignature();
search = target_tail;
while(search != merge_tail){
search->updateStatsSig(target_sig, merge_sig);
search = search->prev;
}
target_tail->setSignature(*merge_sig);
}
if(target_tail != target_head && !target_head->sigEquals(merge_head)){
target_sig = target_head->getSignature();
merge_sig = merge_head->getSignature();
search = target_tail;
while(search != merge_tail){
search->updateStatsSig(target_sig, merge_sig);
search = search->prev;
}
target_head->setSignature(*merge_sig);
}
#endif
/* 2. if merge_head ... merge_tail is a pending iteration, merge it */
iterations = pendingIterations.getIterationsByTargetHead(merge_head);
rtn = NULL;
for(iteration_it = iterations.begin(); iteration_it != iterations.end(); iteration_it++){
if((*iteration_it)->target_head == merge_head && (*iteration_it)->target_tail == merge_tail){
iteration = *iteration_it;
rtn = compressLoopLCS(iteration);
updateIds(rtn->merge_head);
break;
}
}
/* 3. mark the current iteration as pending */
if(rtn){
pendingIterations.create(rtn->merge_head, rtn->merge_tail, rtn->merge_length,
target_head, target_tail, target_length);
delete rtn;
} else {
pendingIterations.create(merge_head, merge_tail, merge_length, target_head, target_tail, target_length);
for(iter = merge_tail; iter != merge_head && iter != NULL; iter=iter->prev)
iter->setLoc(MEMBER);
assert(iter);
iter->setLoc(LEADER);
}
/* mark the events in the target iteration as PENDING */
target_tail->setLoc(PENDING_TAIL);
if(target_tail != target_head){
for(iter = target_tail->prev; iter != target_head && iter != NULL; iter = iter->prev)
iter->setLoc(PENDING_MEMBER);
}
target_head->setLoc(PENDING_LEADER);
} while(head_flag && !found_flag);
} /* slowCompressLoop */
