static int file_write_block(avro_file_writer_t w) { const avro_encoding_t *enc = &avro_binary_encoding; int rval; if (w->block_count) { int64_t blocklen = avro_writer_tell(w->datum_writer); /* Write the block count */ check(rval, enc->write_long(w->writer, w->block_count)); /* Write the block length */ check(rval, enc->write_long(w->writer, blocklen)); /* Write the block */ check(rval, avro_write(w->writer, w->datum_buffer, blocklen)); /* Write the sync marker */ check(rval, write_sync(w)); /* Reset the datum writer */ avro_writer_reset(w->datum_writer); w->block_count = 0; } return 0; }
/* function call from lmlite with parameters */ void network_devices_status_report(struct networkdevicestatusdata *head, BOOL extender, char* parent_mac) { int i = 0, k = 0; uint8_t* b64buffer = NULL; size_t decodesize = 0; int numElements = 0; struct networkdevicestatusdata* ptr = head; avro_writer_t writer; char * serviceName = "lmlite"; char * dest = "event:raw.kestrel.reports.NetworkDevicesStatus"; char * contentType = "avro/binary"; // contentType "application/json", "avro/binary" uuid_t transaction_id; char trans_id[37]; char CpeMacHoldingBuf[ 20 ] = {0}; unsigned char CpeMacid[ 7 ] = {0}; CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : ENTER \n", __FUNCTION__ )); numElements = NumberofElementsinLinkedList(head); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, numElements = %d\n", numElements )); OneAvroSerializedSize = 0; /* goes thru total number of elements in link list */ writer = prepare_writer_status(); //Reset out writer avro_writer_reset(writer); //Network Device Report avro_value_t adr; avro_generic_value_new(iface, &adr); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, GatewayNetworkDeviceStatusReport\tType: %d\n", avro_value_get_type(&adr))); avro_value_t adrField = {0,0}; avro_value_t array = {0,0}; size_t new_index = 0; //Optional value for unions, mac address is an union avro_value_t optional = {0,0}; // timestamp - long avro_value_get_by_name(&adr, "header", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "timestamp", &adrField, NULL); avro_value_set_branch(&adrField, 1, &optional); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); struct timeval ts; gettimeofday(&ts, NULL); #ifndef UTC_ENABLE int64_t tstamp_av_main = ((int64_t) (ts.tv_sec - getTimeOffsetFromUtc()) * 1000000) + (int64_t) ts.tv_usec; #else int64_t tstamp_av_main = ((int64_t) (ts.tv_sec) * 1000000) + (int64_t) ts.tv_usec; #endif tstamp_av_main = tstamp_av_main/1000; avro_value_set_long(&optional, tstamp_av_main ); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, timestamp = ""%" PRId64 "\n", tstamp_av_main )); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, timestamp\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); // uuid - fixed 16 bytes uuid_generate_random(transaction_id); uuid_unparse(transaction_id, trans_id); avro_value_get_by_name(&adr, "header", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "uuid", &adrField, NULL); avro_value_set_branch(&adrField, 1, &optional); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_fixed(&optional, transaction_id, 16); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, uuid\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //source - string avro_value_get_by_name(&adr, "header", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "source", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&adrField, 1, &optional); avro_value_set_string(&optional, ReportSource); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, source\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); if ( extender == FALSE ) { //cpe_id block /* MAC - Get CPE mac address, do it only pointer is NULL */ memset(CpeMacHoldingBuf, 0, sizeof CpeMacHoldingBuf); memset(CpeMacid, 0, sizeof CpeMacid); if ( macStr == NULL ) { macStr = getDeviceMac(); strncpy( CpemacStr, macStr, sizeof(CpemacStr)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Received DeviceMac from Atom side: %s\n",macStr)); } for (k = 0; k < 6; k++ ) { /* copy 2 bytes */ CpeMacHoldingBuf[ k * 2 ] = CpemacStr[ k * 2 ]; CpeMacHoldingBuf[ k * 2 + 1 ] = CpemacStr[ k * 2 + 1 ]; CpeMacid[ k ] = (unsigned char)strtol(&CpeMacHoldingBuf[ k * 2 ], NULL, 16); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Mac address = %0x\n", CpeMacid[ k ] )); } avro_value_get_by_name(&adr, "cpe_id", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "mac_address", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&adrField, 1, &optional); avro_value_set_fixed(&optional, CpeMacid, 6); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, mac_address\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); // cpe_type - string avro_value_get_by_name(&adr, "cpe_id", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "cpe_type", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&adrField, 1, &optional); avro_value_set_string(&optional, CPE_TYPE_GATEWAY_STRING); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, cpe_type\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); // cpe_parent - Recurrsive CPEIdentifier block avro_value_get_by_name(&adr, "cpe_id", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "cpe_parent", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&adrField, 0, &optional); avro_value_set_null(&optional); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, cpe_parent\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); } else { //cpe_id block avro_value_t parent_optional = {0,0}, parent_adrField = {0,0}; memset(CpeMacHoldingBuf, 0, sizeof CpeMacHoldingBuf); memset(CpeMacid, 0, sizeof CpeMacid); for (k = 0; k < 6; k++ ) { /* copy 2 bytes */ CpeMacHoldingBuf[ k * 2 ] = parent_mac[ k * 3 ]; CpeMacHoldingBuf[ k * 2 + 1 ] = parent_mac[ k * 3 + 1 ]; CpeMacid[ k ] = (unsigned char)strtol(&CpeMacHoldingBuf[ k * 2 ], NULL, 16); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Extender Mac address = %0x\n", CpeMacid[ k ] )); } avro_value_get_by_name(&adr, "cpe_id", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "mac_address", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&adrField, 1, &optional); avro_value_set_fixed(&optional, CpeMacid, 6); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, mac_address\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); // cpe_type - string avro_value_get_by_name(&adr, "cpe_id", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "cpe_type", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&adrField, 1, &optional); avro_value_set_string(&optional, CPE_TYPE_EXTENDER_STRING); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, cpe_type\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); // cpe_parent - Recurrsive CPEIdentifier block avro_value_get_by_name(&adr, "cpe_id", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_get_by_name(&adrField, "cpe_parent", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); /* MAC - Get CPE mac address, do it only pointer is NULL */ if ( macStr == NULL ) { macStr = getDeviceMac(); strncpy( CpemacStr, macStr, sizeof(CpemacStr)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Received DeviceMac from Atom side: %s\n",macStr)); } memset(CpeMacHoldingBuf, 0, sizeof CpeMacHoldingBuf); memset(CpeMacid, 0, sizeof CpeMacid); for (k = 0; k < 6; k++ ) { /* copy 2 bytes */ CpeMacHoldingBuf[ k * 2 ] = CpemacStr[ k * 2 ]; CpeMacHoldingBuf[ k * 2 + 1 ] = CpemacStr[ k * 2 + 1 ]; CpeMacid[ k ] = (unsigned char)strtol(&CpeMacHoldingBuf[ k * 2 ], NULL, 16); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG Parent Mac address = %0x\n", CpeMacid[ k ] )); } // assume 1 parent ONLY // Parent MAC avro_value_set_branch(&adrField, 1, &parent_optional); avro_value_get_by_name(&parent_optional, "mac_address", &parent_adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&parent_adrField, 1, &parent_optional); avro_value_set_fixed(&parent_optional, CpeMacid, 6); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, parent mac_address\tType: %d\n", avro_value_get_type(&parent_optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); // Parent cpe_type avro_value_set_branch(&adrField, 1, &parent_optional); avro_value_get_by_name(&parent_optional, "cpe_type", &parent_adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&parent_adrField, 1, &parent_optional); avro_value_set_string(&parent_optional, CPE_TYPE_GATEWAY_STRING); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, parent cpe_type\tType: %d\n", avro_value_get_type(&parent_optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); // no more parent, set NULL avro_value_set_branch(&adrField, 1, &parent_optional); avro_value_get_by_name(&parent_optional, "cpe_parent", &parent_adrField, NULL); avro_value_set_branch(&parent_adrField, 0, &parent_optional); avro_value_set_null(&parent_optional); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, parent cpe_parent\tType: %d\n", avro_value_get_type(&parent_optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); } //host_table_version block avro_value_get_by_name(&adr, "host_table_version", &adrField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&adrField, 1, &optional); avro_value_set_long(&optional, lmHosts.lastActivity); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, host_table_version\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //Data Field block avro_value_get_by_name(&adr, "data", &adrField, NULL); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, NetworkDeviceStatusReports - data array\tType: %d\n", avro_value_get_type(&adrField))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //adrField now contains a reference to the AssociatedDeviceReportsArray //Device Report avro_value_t dr = {0,0}; //Current Device Report Field avro_value_t drField = {0,0}; while(ptr) { if( (!strcmp(ptr->parent, parent_mac) && (extender == TRUE)) || (extender == FALSE) ) { CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Current Link List Ptr = [0x%lx], numElements = %d\n", (ulong)ptr, numElements )); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \tDevice entry #: %d\n", i + 1)); //Append a DeviceReport item to array avro_value_append(&adrField, &dr, NULL); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \tDevice Status Report\tType: %d\n", avro_value_get_type(&dr))); //data array block memset(CpeMacHoldingBuf, 0, sizeof CpeMacHoldingBuf); memset(CpeMacid, 0, sizeof CpeMacid); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Mac address from node list = %s \n", ptr->device_mac )); for (k = 0; k < 6; k++ ) { /* copy 2 bytes */ CpeMacHoldingBuf[ k * 2 ] = ptr->device_mac[ k * 3 ]; CpeMacHoldingBuf[ k * 2 + 1 ] = ptr->device_mac[ k * 3 + 1 ]; CpeMacid[ k ] = (unsigned char)strtol(&CpeMacHoldingBuf[ k * 2 ], NULL, 16); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Mac address = %0x\n", CpeMacid[ k ] )); } //device_mac - fixed 6 bytes avro_value_get_by_name(&dr, "device_id", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, device_id\tType: %d\n", avro_value_get_type(&drField))); avro_value_get_by_name(&drField, "mac_address", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&drField, 1, &optional); avro_value_set_fixed(&optional, CpeMacid, 6); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \tmac_address\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //device_type - string avro_value_get_by_name(&dr, "device_id", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, device_id\tType: %d\n", avro_value_get_type(&drField))); avro_value_get_by_name(&drField, "device_type", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&drField, 1, &optional); avro_value_set_string(&optional, ptr->device_type); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \tdevice_type\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //timestamp - long avro_value_get_by_name(&dr, "timestamp", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&drField, 1, &optional); int64_t tstamp_av = (int64_t) ptr->timestamp.tv_sec * 1000000 + (int64_t) ptr->timestamp.tv_usec; tstamp_av = tstamp_av/1000; avro_value_set_long(&optional, tstamp_av); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, timestamp = ""%" PRId64 "\n", tstamp_av )); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \ttimestamp\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //interface_name - string avro_value_get_by_name(&dr, "interface_name", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&drField, 1, &optional); //avro_value_set_string(&optional, " aa "); avro_value_set_string(&optional, ptr->interface_name ); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \tinterface_name\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //status - enum avro_value_get_by_name(&dr, "status", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&drField, 1, &optional); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, status\tType: %d\n", avro_value_get_type(&optional))); if ( ptr->is_active ) avro_value_set_enum(&optional, avro_schema_enum_get_by_name(avro_value_get_schema(&optional), "ONLINE")); else avro_value_set_enum(&optional, avro_schema_enum_get_by_name(avro_value_get_schema(&optional), "OFFLINE")); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //hostname - string avro_value_get_by_name(&dr, "hostname", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&drField, 1, &optional); avro_value_set_string(&optional, ptr->hostname); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \thostname\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); //ipaddress - array avro_value_get_by_name(&dr, "ip_addresses", &drField, NULL); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); avro_value_set_branch(&drField, 1, &optional); avro_value_append(&optional, &array, NULL); avro_value_set_string(&array, ptr->ipaddress); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, \tipaddress\tType: %d\n", avro_value_get_type(&optional))); if ( CHK_AVRO_ERR ) CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, %s\n", avro_strerror())); i++; } #if SIMULATION ptr = 0; #else ptr = ptr->next; // next link list #endif /* check for writer size, if buffer is almost full, skip trailing linklist */ avro_value_sizeof(&adr, &AvroSerializedSize); OneAvroSerializedSize = ( OneAvroSerializedSize == 0 ) ? AvroSerializedSize : OneAvroSerializedSize; if ( ( WRITER_BUF_SIZE - AvroSerializedSize ) < OneAvroSerializedSize ) { CcspLMLiteTrace(("RDK_LOG_ERROR, AVRO write buffer is almost full, size = %d func %s, exit!\n", (int)AvroSerializedSize, __FUNCTION__ )); break; } } //Thats the end of that avro_value_write(writer, &adr); avro_value_sizeof(&adr, &AvroSerializedSize); AvroSerializedSize += MAGIC_NUMBER_SIZE + SCHEMA_ID_LENGTH; CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Serialized writer size %d\n", (int)AvroSerializedSize)); //Free up memory avro_value_decref(&adr); avro_writer_free(writer); //free(buffer); /* if ( consoleDebugEnable ) { // b64 encoding decodesize = b64_get_encoded_buffer_size( AvroSerializedSize ); b64buffer = malloc(decodesize * sizeof(uint8_t)); b64_encode( (uint8_t*)AvroSerializedBuf, AvroSerializedSize, b64buffer); fprintf( stderr, "\nAVro serialized data\n"); for (k = 0; k < (int)AvroSerializedSize ; k++) { char buf[30]; if ( ( k % 32 ) == 0 ) fprintf( stderr, "\n"); sprintf(buf, "%02X", (unsigned char)AvroSerializedBuf[k]); fprintf( stderr, "%c%c", buf[0], buf[1] ); } fprintf( stderr, "\n\nB64 data\n"); for (k = 0; k < (int)decodesize; k++) { if ( ( k % 32 ) == 0 ) fprintf( stderr, "\n"); fprintf( stderr, "%c", b64buffer[k]); } fprintf( stderr, "\n\n"); free(b64buffer); }*/ CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Before ND WebPA SEND message call\n")); #ifdef PARODUS_ENABLE CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, serviceName: %s\n", serviceName)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, dest: %s\n", dest)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, trans_id: %s\n", trans_id)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, contentType: %s\n", contentType)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, AvroSerializedBuf: %s\n", AvroSerializedBuf)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, AvroSerializedSize: %d\n", (int)AvroSerializedSize)); #endif // Send data from LMLite to webpa using CCSP bus interface sendWebpaMsg(serviceName, dest, trans_id, contentType, AvroSerializedBuf, AvroSerializedSize); CcspTraceWarning(("NetworkDevicesStatus report sent to Webpa, Destination=%s, Transaction-Id=%s \n",dest,trans_id)); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, After ND WebPA SEND message call\n")); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : EXIT \n", __FUNCTION__ )); #if SIMULATION exit(0); #endif }
void kafka_cache_purge(struct chained_cache *queue[], int index, int safe_action) { struct pkt_primitives *data = NULL; struct pkt_bgp_primitives *pbgp = NULL; struct pkt_nat_primitives *pnat = NULL; struct pkt_mpls_primitives *pmpls = NULL; struct pkt_tunnel_primitives *ptun = NULL; char *pcust = NULL; struct pkt_vlen_hdr_primitives *pvlen = NULL; struct pkt_bgp_primitives empty_pbgp; struct pkt_nat_primitives empty_pnat; struct pkt_mpls_primitives empty_pmpls; struct pkt_tunnel_primitives empty_ptun; char *empty_pcust = NULL; char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN]; char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], dyn_kafka_topic[SRVBUFLEN], *orig_kafka_topic = NULL; int i, j, stop, batch_idx, is_topic_dyn = FALSE, qn = 0, ret, saved_index = index; int mv_num = 0, mv_num_save = 0; time_t start, duration; pid_t writer_pid = getpid(); char *json_buf = NULL; int json_buf_off = 0; #ifdef WITH_AVRO avro_writer_t avro_writer; char *avro_buf = NULL; int avro_buffer_full = FALSE; #endif p_kafka_init_host(&kafkap_kafka_host, config.kafka_config_file); /* setting some defaults */ if (!config.sql_host) config.sql_host = default_kafka_broker_host; if (!config.kafka_broker_port) config.kafka_broker_port = default_kafka_broker_port; if (!config.sql_table) config.sql_table = default_kafka_topic; else { if (strchr(config.sql_table, '$')) { is_topic_dyn = TRUE; orig_kafka_topic = config.sql_table; } } if (config.amqp_routing_key_rr) orig_kafka_topic = config.sql_table; p_kafka_init_topic_rr(&kafkap_kafka_host); p_kafka_set_topic_rr(&kafkap_kafka_host, config.amqp_routing_key_rr); empty_pcust = malloc(config.cpptrs.len); if (!empty_pcust) { Log(LOG_ERR, "ERROR ( %s/%s ): Unable to malloc() empty_pcust. Exiting.\n", config.name, config.type); exit_plugin(1); } memset(&empty_pbgp, 0, sizeof(struct pkt_bgp_primitives)); memset(&empty_pnat, 0, sizeof(struct pkt_nat_primitives)); memset(&empty_pmpls, 0, sizeof(struct pkt_mpls_primitives)); memset(&empty_ptun, 0, sizeof(struct pkt_tunnel_primitives)); memset(empty_pcust, 0, config.cpptrs.len); p_kafka_connect_to_produce(&kafkap_kafka_host); p_kafka_set_broker(&kafkap_kafka_host, config.sql_host, config.kafka_broker_port); if (!is_topic_dyn && !config.amqp_routing_key_rr) p_kafka_set_topic(&kafkap_kafka_host, config.sql_table); p_kafka_set_partition(&kafkap_kafka_host, config.kafka_partition); p_kafka_set_key(&kafkap_kafka_host, config.kafka_partition_key, config.kafka_partition_keylen); if (config.message_broker_output & PRINT_OUTPUT_JSON) p_kafka_set_content_type(&kafkap_kafka_host, PM_KAFKA_CNT_TYPE_STR); else if (config.message_broker_output & PRINT_OUTPUT_AVRO) p_kafka_set_content_type(&kafkap_kafka_host, PM_KAFKA_CNT_TYPE_BIN); else { Log(LOG_ERR, "ERROR ( %s/%s ): Unsupported kafka_output value specified. Exiting.\n", config.name, config.type); exit_plugin(1); } for (j = 0, stop = 0; (!stop) && P_preprocess_funcs[j]; j++) stop = P_preprocess_funcs[j](queue, &index, j); Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - START (PID: %u) ***\n", config.name, config.type, writer_pid); start = time(NULL); if (config.print_markers) { if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) { void *json_obj; char *json_str; json_obj = compose_purge_init_json(config.name, writer_pid); if (json_obj) json_str = compose_json_str(json_obj); if (json_str) { Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str); ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str)); free(json_str); json_str = NULL; } } } if (config.message_broker_output & PRINT_OUTPUT_JSON) { if (config.sql_multi_values) { json_buf = malloc(config.sql_multi_values); if (!json_buf) { Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (json_buf). Exiting ..\n", config.name, config.type); exit_plugin(1); } else memset(json_buf, 0, config.sql_multi_values); } } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO if (!config.avro_buffer_size) config.avro_buffer_size = LARGEBUFLEN; avro_buf = malloc(config.avro_buffer_size); if (!avro_buf) { Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (avro_buf). Exiting ..\n", config.name, config.type); exit_plugin(1); } else memset(avro_buf, 0, config.avro_buffer_size); avro_writer = avro_writer_memory(avro_buf, config.avro_buffer_size); #endif } for (j = 0; j < index; j++) { void *json_obj; char *json_str; if (queue[j]->valid != PRINT_CACHE_COMMITTED) continue; data = &queue[j]->primitives; if (queue[j]->pbgp) pbgp = queue[j]->pbgp; else pbgp = &empty_pbgp; if (queue[j]->pnat) pnat = queue[j]->pnat; else pnat = &empty_pnat; if (queue[j]->pmpls) pmpls = queue[j]->pmpls; else pmpls = &empty_pmpls; if (queue[j]->ptun) ptun = queue[j]->ptun; else ptun = &empty_ptun; if (queue[j]->pcust) pcust = queue[j]->pcust; else pcust = empty_pcust; if (queue[j]->pvlen) pvlen = queue[j]->pvlen; else pvlen = NULL; if (queue[j]->valid == PRINT_CACHE_FREE) continue; if (config.message_broker_output & PRINT_OUTPUT_JSON) { #ifdef WITH_JANSSON json_t *json_obj = json_object(); int idx; for (idx = 0; idx < N_PRIMITIVES && cjhandler[idx]; idx++) cjhandler[idx](json_obj, queue[j]); add_writer_name_and_pid_json(json_obj, config.name, writer_pid); json_str = compose_json_str(json_obj); #endif } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO avro_value_iface_t *avro_iface = avro_generic_class_from_schema(avro_acct_schema); avro_value_t avro_value = compose_avro(config.what_to_count, config.what_to_count_2, queue[j]->flow_type, &queue[j]->primitives, pbgp, pnat, pmpls, ptun, pcust, pvlen, queue[j]->bytes_counter, queue[j]->packet_counter, queue[j]->flow_counter, queue[j]->tcp_flags, &queue[j]->basetime, queue[j]->stitch, avro_iface); size_t avro_value_size; add_writer_name_and_pid_avro(avro_value, config.name, writer_pid); avro_value_sizeof(&avro_value, &avro_value_size); if (avro_value_size > config.avro_buffer_size) { Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: insufficient buffer size (avro_buffer_size=%u)\n", config.name, config.type, config.avro_buffer_size); Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: increase value or look for avro_buffer_size in CONFIG-KEYS document.\n\n", config.name, config.type); exit_plugin(1); } else if (avro_value_size >= (config.avro_buffer_size - avro_writer_tell(avro_writer))) { avro_buffer_full = TRUE; j--; } else if (avro_value_write(avro_writer, &avro_value)) { Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: unable to write value: %s\n", config.name, config.type, avro_strerror()); exit_plugin(1); } else { mv_num++; } avro_value_decref(&avro_value); avro_value_iface_decref(avro_iface); #else if (config.debug) Log(LOG_DEBUG, "DEBUG ( %s/%s ): compose_avro(): AVRO object not created due to missing --enable-avro\n", config.name, config.type); #endif } if (config.message_broker_output & PRINT_OUTPUT_JSON) { char *tmp_str = NULL; if (json_str && config.sql_multi_values) { int json_strlen = (strlen(json_str) ? (strlen(json_str) + 1) : 0); if (json_strlen >= (config.sql_multi_values - json_buf_off)) { if (json_strlen >= config.sql_multi_values) { Log(LOG_ERR, "ERROR ( %s/%s ): kafka_multi_values not large enough to store JSON elements. Exiting ..\n", config.name, config.type); exit(1); } tmp_str = json_str; json_str = json_buf; } else { strcat(json_buf, json_str); mv_num++; string_add_newline(json_buf); json_buf_off = strlen(json_buf); free(json_str); json_str = NULL; } } if (json_str) { if (is_topic_dyn) { P_handle_table_dyn_strings(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, queue[j]); p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic); } if (config.amqp_routing_key_rr) { P_handle_table_dyn_rr(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, &kafkap_kafka_host.topic_rr); p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic); } Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str); ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str)); if (config.sql_multi_values) { json_str = tmp_str; strcpy(json_buf, json_str); mv_num_save = mv_num; mv_num = 1; string_add_newline(json_buf); json_buf_off = strlen(json_buf); } free(json_str); json_str = NULL; if (!ret) { if (!config.sql_multi_values) qn++; else qn += mv_num_save; } else break; } } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO if (!config.sql_multi_values || (mv_num >= config.sql_multi_values) || avro_buffer_full) { if (is_topic_dyn) { P_handle_table_dyn_strings(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, queue[j]); p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic); } if (config.amqp_routing_key_rr) { P_handle_table_dyn_rr(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, &kafkap_kafka_host.topic_rr); p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic); } ret = p_kafka_produce_data(&kafkap_kafka_host, avro_buf, avro_writer_tell(avro_writer)); avro_writer_reset(avro_writer); avro_buffer_full = FALSE; mv_num_save = mv_num; mv_num = 0; if (!ret) qn += mv_num_save; else break; } #endif } } if (config.sql_multi_values) { if (config.message_broker_output & PRINT_OUTPUT_JSON) { if (json_buf && json_buf_off) { /* no handling of dyn routing keys here: not compatible */ Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_buf); ret = p_kafka_produce_data(&kafkap_kafka_host, json_buf, strlen(json_buf)); if (!ret) qn += mv_num; } } else if (config.message_broker_output & PRINT_OUTPUT_AVRO) { #ifdef WITH_AVRO if (avro_writer_tell(avro_writer)) { ret = p_kafka_produce_data(&kafkap_kafka_host, avro_buf, avro_writer_tell(avro_writer)); avro_writer_free(avro_writer); if (!ret) qn += mv_num; } #endif } } duration = time(NULL)-start; if (config.print_markers) { if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) { void *json_obj; char *json_str; json_obj = compose_purge_close_json(config.name, writer_pid, qn, saved_index, duration); if (json_obj) json_str = compose_json_str(json_obj); if (json_str) { sleep(1); /* Let's give a small delay to facilitate purge_close being the last message in batch in case of partitioned topics */ Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str); ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str)); free(json_str); json_str = NULL; } } } p_kafka_close(&kafkap_kafka_host, FALSE); Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - END (PID: %u, QN: %u/%u, ET: %u) ***\n", config.name, config.type, writer_pid, qn, saved_index, duration); if (config.sql_trigger_exec && !safe_action) P_trigger_exec(config.sql_trigger_exec); if (empty_pcust) free(empty_pcust); if (json_buf) free(json_buf); #ifdef WITH_AVRO if (avro_buf) free(avro_buf); #endif }