END_TEST START_TEST (test_text_read) { uint8_t buf [] = "0.123456\t1\t42\tabde\t3.1416\t111\nbleftover text for next line"; char meta [] = "1 mympstrm label:string pi:double fighter:uint32"; MBuffer *mbuf = mbuf_create(); struct oml_message msg; struct schema *schema = schema_from_meta (meta); OmlValue values[3]; oml_value_array_init(values, 3); bzero(&msg, sizeof(msg)); mbuf_write (mbuf, buf, sizeof(buf)); int result = text_read_msg_start (&msg, mbuf); fprintf (stderr, "STRM: %d\n", msg.stream); fprintf (stderr, "SEQN: %u\n", msg.seqno); fprintf (stderr, "TS : %f\n", msg.timestamp); fprintf (stderr, "LEN : %d\n", msg.length); fprintf (stderr, "COUNT: %d\n", msg.count); result = text_read_msg_values (&msg, mbuf, schema, values); result *= 42; oml_value_array_reset(values, 3); }
/** Marshal meta-data for an OML measurement stream's sample * * An OML measurement stream is written as two bytes; the first one is the * counter for the number of elements in the message, and therefore starts at * 0, and the second one is the stream's index. This is followed by a * marshalled int32 value containing the sequence number, and a double value * containing the timestamp. * * A marshalling message should have been prepared in the MBuffer first with * marshal_init(). Actual data can then be marshalled into the message with * marshal_values(). * * * \param mbuf MBuffer to write marshalled data to * \param stream Measurement Stream's index * \param seqno message sequence number * \param now message time * \return 1 if successful, -1 otherwise * \see marshal_init, marshal_values, marshal_finalize */ int marshal_measurements(MBuffer* mbuf, int stream, int seqno, double now) { OmlValueU v; uint8_t s[2] = { 0, (uint8_t)stream }; /* Write num-meas (0, for now), and the stream index */ int result = mbuf_write (mbuf, s, LENGTH (s)); omlc_zero(v); if (result == -1) { logerror("Unable to marshal table number and measurement count (mbuf_write())\n"); mbuf_reset_write (mbuf); return -1; } logdebug("Marshalling sample %d for stream %d\n", seqno, stream); omlc_set_int32(v, seqno); marshal_value(mbuf, OML_INT32_VALUE, &v); omlc_set_double(v, now); marshal_value(mbuf, OML_DOUBLE_VALUE, &v); return 1; }
/** Prepare a long marshalling header into an MBuffer. * * \param mbuf MBuffer to write the mbuf marshalling header to * \return the 0 on success, -1 on failure (\see mbuf_write) */ static int marshal_header_long (MBuffer *mbuf) { uint8_t buf[] = { SYNC_BYTE, SYNC_BYTE, OMB_LDATA_P, 0, 0, 0, 0 }; return mbuf_write (mbuf, buf, LENGTH (buf)); }
/** * \fn bw_push(BufferedWriterHdl* buffSocket, void* chunk, long chunkSize) * \brief Add a chunk to the end of the queue. * \param instance BufferedWriter handle * \param chunk Pointer to chunk to add * \param chunkSize size of chunk * \return 1 if success, 0 otherwise */ int bw_push( BufferedWriterHdl instance, uint8_t* chunk, size_t size ) { BufferedWriter* self = (BufferedWriter*)instance; if (oml_lock(&self->lock, "bw_push")) return 0; if (!self->active) return 0; BufferChain* chain = self->writerChain; if (chain == NULL) return 0; if (chain->mbuf->wr_remaining < size) { chain = self->writerChain = getNextWriteChain(self, chain); } if (mbuf_write(chain->mbuf, chunk, size) < 0) return 0; pthread_cond_signal(&self->semaphore); // oml_unlock(&chain->lock, "bw_push"); oml_unlock(&self->lock, "bw_push"); return 1; }
static bool slshow(void *arg, const char *s) { struct MBuf *mb = arg; if (mbuf_written(mb) > 0) { if (!mbuf_write_byte(mb, ',')) return false; } if (!s) s = "NULL"; //printf("writing: <%s>\n", s); return mbuf_write(mb, s, strlen(s)); }
static bool addln(void *arg, const char *ln, ssize_t len) { struct MBuf *buf = arg; int xlen = len; if (len < 0) return false; if (len > 0 && ln[len - 1] == '\n') xlen--; if (memchr(ln, '\n', xlen)) return false; return mbuf_write(buf, ln, len); }
/** Find the next empty write chunk, sets self->writerChunk to it and returns * it. * * We only use the next one if it is empty. If not, we essentially just filled * up the last chunk and wrapped around to the socket reader. In that case, we * either create a new chunk if the overall buffer can still grow, or we drop * the data from the current one. * * \warning A lock on the current writer chunk should be held prior to calling * this function. It will be released, and the returned writerChunk will be * similarly locked. * * \param self BufferedWriter pointer * \param current locked BufferChunk to use or from which to find the next * \return a locked BufferChunk in which data can be stored */ BufferChunk* getNextWriteChunk(BufferedWriter* self, BufferChunk* current) { int nlost; BufferChunk* nextBuffer; assert(current != NULL); nextBuffer = current->next; oml_unlock(¤t->lock, __FUNCTION__); assert(nextBuffer != NULL); oml_lock(&self->lock, __FUNCTION__); if (nextBuffer == self->nextReaderChunk) { if (self->unallocatedBuffers > 0) { /* The next buffer is the next to be read, but we can allocate more, * allocate a new buffer, insert it after the current writer, and use it */ nextBuffer = createBufferChunk(self); assert(nextBuffer); /** \todo Use existing buffer if allocation fails */ oml_unlock(&self->lock, __FUNCTION__); oml_lock(¤t->lock, __FUNCTION__); nextBuffer->next = current->next; current->next = nextBuffer; /* we have a lock on this one */ oml_unlock(¤t->lock, __FUNCTION__); oml_lock(&self->lock, __FUNCTION__); } else { /* The next buffer is the next to be read, and we cannot allocate more, * use it, dropping unread data, and advance the read pointer */ self->nextReaderChunk = nextBuffer->next; } } self->writerChunk = nextBuffer; nlost = bw_msgcount_reset(self); self->nlost += nlost; oml_unlock(&self->lock, __FUNCTION__); oml_lock(&nextBuffer->lock, __FUNCTION__); if (nlost) { logwarn("%s: Dropping %d samples (%dB)\n", self->outStream->dest, nlost, mbuf_fill(nextBuffer->mbuf)); } mbuf_clear2(nextBuffer->mbuf, 0); // Now we just need to copy the message from current to self->writerChunk int msgSize = mbuf_message_length(current->mbuf); if (msgSize > 0) { mbuf_write(nextBuffer->mbuf, mbuf_message(current->mbuf), msgSize); mbuf_reset_write(current->mbuf); } return nextBuffer; }
END_TEST START_TEST (test_bin_read) { /* DATA_P, count=1, stream=3, { LONG_T 42 } */ uint8_t buf [] = { 0xAA, 0xAA, 0x01, 0x00, 0x00, 0x3, 0x1, // count = 1, stream = 3 0x01, 0x00, 0x00, 0x00, 0x32, // LONG_T 50 0x02, 0x54, 0x00, 0x00, 0x00, 0x05, // DOUBLE_T 42.0 0x01, 0x00, 0x10, 0xF4, 0x47, // LONG_T 1111111 0x02, 0x54, 0x00, 0x00, 0x00, 0x05, // DOUBLE_T 42.0 0x04, 0x03, 'A', 'B', 'C' // STRING_T "ABC" }; char meta [] = "3 mympstrm id:long hitchhiker:double sesame:string"; MBuffer *mbuf = mbuf_create (); struct oml_message msg; struct schema *schema = schema_from_meta (meta); OmlValue values [3]; int result; oml_value_array_init(values, 3); bzero(&msg, sizeof(msg)); int size = sizeof (buf) - 5; uint16_t nv = htons (size); memcpy (buf + 3, &nv, 2); mbuf_write (mbuf, buf, sizeof (buf)); result = bin_read_msg_start (&msg, mbuf); fail_unless(result > 0, "Unable to start reading binary message"); fprintf (stderr, "---\n"); fprintf (stderr, "STRM: %d\n", msg.stream); fprintf (stderr, "SEQN: %u\n", msg.seqno); fprintf (stderr, "TS : %f\n", msg.timestamp); fprintf (stderr, "LEN : %d\n", msg.length); fprintf (stderr, "COUNT: %d\n", msg.count); result = bin_read_msg_values (&msg, mbuf, schema, values); int i = 0; for (i = 0; i < 3; i++) { char s[64]; oml_value_to_s (&values[i], s, 64); fprintf (stderr, "%s\n", s); } oml_value_array_reset(values, 3); }
/** Finalise a marshalled message. * * Depending on the number of values packed, change the type of message, and * write the actual size in the right location of the header, in network byte * order. * * \param mbuf MBuffer where marshalled data is * \return 1 * \see marshal_init, marshal_measurements, marshal_values */ int marshal_finalize(MBuffer* mbuf) { uint8_t* buf = mbuf_message (mbuf); OmlBinMsgType type = marshal_get_msgtype (mbuf); size_t len = mbuf_message_length (mbuf); if (len > UINT32_MAX) { logwarn("Message length %d longer than maximum packet length (%d); " "packet will be truncated\n", len, UINT32_MAX); len = UINT32_MAX; } if (type == OMB_DATA_P && len > UINT16_MAX) { /* * We assumed a short packet, but there is too much data, so we * have to shift the whole buffer down by 2 bytes and convert to a * long packet. */ uint8_t s[2] = {0}; /* Put some padding in the buffer to make sure it has room, and maintains its invariants */ mbuf_write (mbuf, s, sizeof (s)); memmove (&buf[PACKET_HEADER_SIZE+2], &buf[PACKET_HEADER_SIZE], len - PACKET_HEADER_SIZE); len += 2; buf[2] = type = OMB_LDATA_P; } switch (type) { case OMB_DATA_P: len -= PACKET_HEADER_SIZE; // Data length minus header uint16_t nlen16 = htons (len); memcpy (&buf[3], &nlen16, sizeof (nlen16)); break; case OMB_LDATA_P: len -= PACKET_HEADER_SIZE + 2; // Data length minus header uint32_t nlen32 = htonl (len); // pure data length memcpy (&buf[3], &nlen32, sizeof (nlen32)); break; } return 1; }
/** Add some data to the end of the header buffer (lock must be held). * * \warning This function is the same as bw_push_meta except it assumes that * the lock on the header data is already acquired. * * \copydetails bw_push_meta * * \see bw_push_meta * */ int _bw_push_meta(BufferedWriter* instance, uint8_t* data, size_t size) { BufferedWriter* self = (BufferedWriter*)instance; int result = 0; if (!self->active) { return 0; } if (mbuf_write(self->meta_buf, data, size) > 0) { result = 1; /* XXX: There is no point in signalling the semaphore as the * writer will not be able to do anything with the new data. * * Also, it puts everything in a deadlock otherwise */ /* pthread_cond_signal(&self->semaphore); */ } return result; }
/** * \brief write the data after finalysing the data structure * \param writer the net writer that send the measurements * \param ms the stream of measurmenent * \return 1 if successful, 0 otherwise */ int row_end(OmlWriter* writer, OmlMStream* ms) { (void)ms; OmlTextWriter* self = (OmlTextWriter*)writer; MBuffer* mbuf; if ((mbuf = self->mbuf) == NULL) return 0; /* previous use of mbuf failed */ int res; if ((res = mbuf_write(mbuf, (uint8_t*)"\n", 1)) != 0) { mbuf_reset_write(mbuf); } else { // success, lock in message mbuf_begin_write (mbuf); } self->mbuf = NULL; bw_unlock_buf(self->bufferedWriter); return res == 0; }
/** * \fn bw_push_meta(BufferedWriterHdl* buffSocket, void* chunk, long chunkSize) * \brief Add a chunk to the end of the meta description. * * \param instance BufferedWriter handle * \param chunk Pointer to chunk to add * \param chunkSize size of chunk * \return 1 if success, 0 otherwise */ int bw_push_meta( BufferedWriterHdl instance, uint8_t* chunk, size_t size ) { BufferedWriter* self = (BufferedWriter*)instance; int result = 0; if (oml_lock(&self->lock, "bw_push")) return 0; if (!self->active) return 0; if (mbuf_write(self->meta_buf, chunk, size) > 0) { result = 1; pthread_cond_signal(&self->semaphore); } oml_unlock(&self->lock, "bw_push_meta"); return result; }
/** Add some data to the end of the queue (lock must be held). * * \param instance BufferedWriter handle * \param data Pointer to data to add * \param size size of data * \return 1 if success, 0 otherwise * * \warning This function is the same assumes that the * lock on the writer chunk is already acquired. * * \see bw_push, bw_get_write_buf, bw_release_write_buf */ int bw_push(BufferedWriter* instance, uint8_t* data, size_t size) { BufferedWriter* self = (BufferedWriter*)instance; if (!self->active) { return 0; } BufferChunk* chunk = self->writerChunk; if (chunk == NULL) { return 0; } if (mbuf_wr_remaining(chunk->mbuf) < size) { chunk = getNextWriteChunk(self, chunk); } if (mbuf_write(chunk->mbuf, data, size) < 0) { return 0; } pthread_cond_signal(&self->semaphore); return 1; }
// This function finds the next empty write chain, sets +self->writeChain+ to // it and returns. // // We only use the next one if it is empty. If not, we // essentially just filled up the last chain and wrapped // around to the socket reader. In that case, we either create a new chain // if the overall buffer can still grow, or we drop the data from the current one. // // This assumes that the current thread holds the +self->lock+ and the lock on // the +self->writeChain+. // BufferChain* getNextWriteChain( BufferedWriter* self, BufferChain* current ) { assert(current != NULL); BufferChain* nextBuffer = current->next; assert(nextBuffer != NULL); BufferChain* resChain = NULL; if (mbuf_remaining(nextBuffer->mbuf) == 0) { // It's empty, we can use it mbuf_clear2(nextBuffer->mbuf, 0); resChain = nextBuffer; } else if (self->chainsAvailable > 0) { // insert new chain between current and next one. BufferChain* newBuffer = createBufferChain(self); newBuffer->next = nextBuffer; current->next = newBuffer; resChain = newBuffer; } else { // Filled up buffer, time to drop data and reuse current buffer // Current buffer holds most recent added data (we drop from the queue's tail //assert(current->reading == 0); assert(current->reading == 0); o_log (O_LOG_WARN, "Dropping %d bytes of measurement data\n", mbuf_fill(current->mbuf)); mbuf_repack_message2(current->mbuf); return current; } // Now we just need to copy the +message+ from +current+ to +resChain+ int msgSize = mbuf_message_length(current->mbuf); if (msgSize > 0) { mbuf_write(resChain->mbuf, mbuf_message(current->mbuf), msgSize); mbuf_reset_write(current->mbuf); } return resChain; }
/** Marshal a single OmlValueU of type OmlValueT into mbuf. * * Usually called by marshal_values(). On failure, the whole message writing is * reset using mbuf_reset_write(), and marshalling should restart with * marshal_init(), after the MBuffer has been adequately resized or repacked. * * \param mbuf MBuffer to write marshalled data to * \param val_type OmlValueT representing the type of val * \param val pointer to OmlValueU, of type val_type, to marshall * \return 1 on success, or 0 otherwise (marshalling should then restart from marshal_init()) * \see marshal_values, marshal_init, mbuf_reset_write, mbuf_repack_message, mbuf_repack_message2, mbuf_resize */ int marshal_value(MBuffer* mbuf, OmlValueT val_type, OmlValueU* val) { switch (val_type) { case OML_LONG_VALUE: { long v = oml_value_clamp_long (omlc_get_long(*val)); uint32_t uv = (uint32_t)v; uint32_t nv = htonl(uv); uint8_t buf[LONG_T_SIZE+1]; buf[0] = LONG_T; memcpy(&buf[1], &nv, sizeof (nv)); logdebug3("Marshalling long %ld\n", nv); int result = mbuf_write (mbuf, buf, LENGTH (buf)); if (result == -1) { logerror("Failed to marshal OML_LONG_VALUE (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } break; } case OML_INT32_VALUE: case OML_UINT32_VALUE: case OML_INT64_VALUE: case OML_UINT64_VALUE: { uint8_t buf[UINT64_T_SIZE+1]; // Max integer size uint32_t uv32; uint32_t nv32; uint64_t uv64; uint64_t nv64; uint8_t *p_nv; if (oml_size_map[val_type] == 4) { uv32 = omlc_get_uint32(*val); nv32 = htonl(uv32); p_nv = (uint8_t*)&nv32; logdebug3("Marshalling %s %" PRIu32 "\n", oml_type_to_s(val_type), uv32); } else { uv64 = omlc_get_uint64(*val); nv64 = htonll(uv64); p_nv = (uint8_t*)&nv64; logdebug3("Marshalling %s %" PRIu64 "\n", oml_type_to_s(val_type), uv64); } buf[0] = oml_type_map[val_type]; memcpy(&buf[1], p_nv, oml_size_map[val_type]); int result = mbuf_write (mbuf, buf, oml_size_map[val_type] + 1); if (result == -1) { logerror("Failed to marshal %s value (mbuf_write())\n", oml_type_to_s (val_type)); mbuf_reset_write (mbuf); return 0; } break; } case OML_DOUBLE_VALUE: { uint8_t type = DOUBLE_T; double v = omlc_get_double(*val); int exp; double mant = frexp(v, &exp); int8_t nexp = (int8_t)exp; logdebug3("Marshalling double %f\n", v); if (isnan(v)) { type = DOUBLE_NAN; nexp = 0; mant = 0; } else if (nexp != exp) { logerror("Double number '%lf' is out of bounds, sending NaN\n", v); type = DOUBLE_NAN; nexp = 0; mant = 0; } int32_t imant = (int32_t)(mant * (1 << BIG_L)); uint32_t nmant = htonl(imant); uint8_t buf[6] = { type, 0, 0, 0, 0, nexp }; memcpy(&buf[1], &nmant, sizeof (nmant)); int result = mbuf_write (mbuf, buf, LENGTH (buf)); if (result == -1) { logerror("Failed to marshal OML_DOUBLE_VALUE (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } break; } case OML_STRING_VALUE: { char* str = omlc_get_string_ptr(*val); if (str == NULL) { str = ""; logdebug("Attempting to send a NULL string; sending empty string instead\n"); } size_t len = strlen(str); if (len > STRING_T_MAX_SIZE) { logerror("Truncated string '%s'\n", str); len = STRING_T_MAX_SIZE; } logdebug3("Marshalling string '%s' of length %d\n", str, len); uint8_t buf[2] = { STRING_T, (uint8_t)(len & 0xff) }; int result = mbuf_write (mbuf, buf, LENGTH (buf)); if (result == -1) { logerror("Failed to marshal OML_STRING_VALUE type and length (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } result = mbuf_write (mbuf, (uint8_t*)str, len); if (result == -1) { logerror("Failed to marshal OML_STRING_VALUE (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } break; } case OML_BLOB_VALUE: { int result = 0; void *blob = omlc_get_blob_ptr(*val); size_t length = omlc_get_blob_length(*val); if (blob == NULL || length == 0) { logdebug ("Attempting to send NULL or empty blob; blob of length 0 will be sent\n"); length = 0; } uint8_t buf[5] = { BLOB_T, 0, 0, 0, 0 }; size_t n_length = htonl (length); memcpy (&buf[1], &n_length, 4); logdebug3("Marshalling blob of size %d\n", length); result = mbuf_write (mbuf, buf, sizeof (buf)); if (result == -1) { logerror ("Failed to marshall OML_BLOB_VALUE type and length (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } result = mbuf_write (mbuf, blob, length); if (result == -1) { logerror ("Failed to marshall %d bytes of OML_BLOB_VALUE data\n", length); mbuf_reset_write (mbuf); return 0; } break; } case OML_GUID_VALUE: { /* FIXME: Wrap with UINT64 marshalling, just change the type */ uint64_t nv64; uint8_t buf[GUID_T_SIZE+1]; buf[0] = GUID_T; nv64 = htonll(omlc_get_guid(*val)); memcpy(&buf[1], &nv64, sizeof(nv64)); logdebug3("Marshalling GUID %" PRIu64 "\n", nv64); if (-1 == mbuf_write(mbuf, buf, LENGTH(buf))) { logerror("Failed to marshal OML_GUID_VALUE (mbuf_write())\n"); mbuf_reset_write(mbuf); return 0; } break; } case OML_BOOL_VALUE: { uint8_t buf; if (!omlc_get_bool(*val)) { buf = BOOL_FALSE_T; } else { buf = BOOL_TRUE_T; } logdebug3("Marshalling boolean %d\n", BOOL_TRUE_T == buf); if (-1 == mbuf_write(mbuf, &buf, 1)) { logerror("Failed to marshal OML_BOOL_VALUE (mbuf_write())\n"); mbuf_reset_write(mbuf); return 0; } break; } case OML_VECTOR_INT32_VALUE: case OML_VECTOR_UINT32_VALUE: { size_t i; uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 }; uint16_t hn = omlc_get_vector_nof_elts(*val); uint16_t nn = htons(hn); buf[1] = vector_protocol_map[val_type]; memcpy(&buf[2], &nn, sizeof(nn)); if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) { uint32_t elts[hn]; uint32_t *v = omlc_get_vector_ptr(*val); for(i = 0; i < hn; i++) elts[i] = htonl(*((uint32_t*)(v+i))); if(mbuf_write(mbuf, (const uint8_t*)(elts), sizeof(elts)) == -1) { logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn); mbuf_reset_write(mbuf); return 0; } } else { logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type)); mbuf_reset_write(mbuf); return 0; } break; } case OML_VECTOR_INT64_VALUE: case OML_VECTOR_UINT64_VALUE: case OML_VECTOR_DOUBLE_VALUE: { size_t i; uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 }; uint16_t hn = omlc_get_vector_nof_elts(*val); uint16_t nn = htons(hn); buf[1] = vector_protocol_map[val_type]; memcpy(&buf[2], &nn, sizeof(nn)); if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) { uint64_t elts[hn]; uint64_t *v = omlc_get_vector_ptr(*val); for(i = 0; i < hn; i++) elts[i] = htonll(*((uint64_t*)(v+i))); if(mbuf_write(mbuf, (const uint8_t*)(elts), sizeof(elts)) == -1) { logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn); mbuf_reset_write(mbuf); return 0; } } else { logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type)); mbuf_reset_write(mbuf); return 0; } break; } case OML_VECTOR_BOOL_VALUE: { size_t i; uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 }; uint16_t hn = omlc_get_vector_nof_elts(*val); uint16_t nn = htons(hn); buf[1] = vector_protocol_map[val_type]; memcpy(&buf[2], &nn, sizeof(nn)); if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) { uint8_t elts[hn]; bool *v = omlc_get_vector_ptr(*val); for(i = 0; i < hn; i++) elts[i] = v[i] ? BOOL_TRUE_T : BOOL_FALSE_T; if(mbuf_write(mbuf, (const uint8_t*)(elts), hn) == -1) { logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn); mbuf_reset_write(mbuf); return 0; } } else { logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type)); mbuf_reset_write(mbuf); return 0; } break; } default: logerror("%s(): Unsupported value type '%d'\n", __func__, val_type); return 0; } return 1; }