/** Marshal meta-data for an OML measurement stream's sample * * An OML measurement stream is written as two bytes; the first one is the * counter for the number of elements in the message, and therefore starts at * 0, and the second one is the stream's index. This is followed by a * marshalled int32 value containing the sequence number, and a double value * containing the timestamp. * * A marshalling message should have been prepared in the MBuffer first with * marshal_init(). Actual data can then be marshalled into the message with * marshal_values(). * * * \param mbuf MBuffer to write marshalled data to * \param stream Measurement Stream's index * \param seqno message sequence number * \param now message time * \return 1 if successful, -1 otherwise * \see marshal_init, marshal_values, marshal_finalize */ int marshal_measurements(MBuffer* mbuf, int stream, int seqno, double now) { OmlValueU v; uint8_t s[2] = { 0, (uint8_t)stream }; /* Write num-meas (0, for now), and the stream index */ int result = mbuf_write (mbuf, s, LENGTH (s)); omlc_zero(v); if (result == -1) { logerror("Unable to marshal table number and measurement count (mbuf_write())\n"); mbuf_reset_write (mbuf); return -1; } logdebug("Marshalling sample %d for stream %d\n", seqno, stream); omlc_set_int32(v, seqno); marshal_value(mbuf, OML_INT32_VALUE, &v); omlc_set_double(v, now); marshal_value(mbuf, OML_DOUBLE_VALUE, &v); return 1; }
/** Find the next empty write chunk, sets self->writerChunk to it and returns * it. * * We only use the next one if it is empty. If not, we essentially just filled * up the last chunk and wrapped around to the socket reader. In that case, we * either create a new chunk if the overall buffer can still grow, or we drop * the data from the current one. * * \warning A lock on the current writer chunk should be held prior to calling * this function. It will be released, and the returned writerChunk will be * similarly locked. * * \param self BufferedWriter pointer * \param current locked BufferChunk to use or from which to find the next * \return a locked BufferChunk in which data can be stored */ BufferChunk* getNextWriteChunk(BufferedWriter* self, BufferChunk* current) { int nlost; BufferChunk* nextBuffer; assert(current != NULL); nextBuffer = current->next; oml_unlock(¤t->lock, __FUNCTION__); assert(nextBuffer != NULL); oml_lock(&self->lock, __FUNCTION__); if (nextBuffer == self->nextReaderChunk) { if (self->unallocatedBuffers > 0) { /* The next buffer is the next to be read, but we can allocate more, * allocate a new buffer, insert it after the current writer, and use it */ nextBuffer = createBufferChunk(self); assert(nextBuffer); /** \todo Use existing buffer if allocation fails */ oml_unlock(&self->lock, __FUNCTION__); oml_lock(¤t->lock, __FUNCTION__); nextBuffer->next = current->next; current->next = nextBuffer; /* we have a lock on this one */ oml_unlock(¤t->lock, __FUNCTION__); oml_lock(&self->lock, __FUNCTION__); } else { /* The next buffer is the next to be read, and we cannot allocate more, * use it, dropping unread data, and advance the read pointer */ self->nextReaderChunk = nextBuffer->next; } } self->writerChunk = nextBuffer; nlost = bw_msgcount_reset(self); self->nlost += nlost; oml_unlock(&self->lock, __FUNCTION__); oml_lock(&nextBuffer->lock, __FUNCTION__); if (nlost) { logwarn("%s: Dropping %d samples (%dB)\n", self->outStream->dest, nlost, mbuf_fill(nextBuffer->mbuf)); } mbuf_clear2(nextBuffer->mbuf, 0); // Now we just need to copy the message from current to self->writerChunk int msgSize = mbuf_message_length(current->mbuf); if (msgSize > 0) { mbuf_write(nextBuffer->mbuf, mbuf_message(current->mbuf), msgSize); mbuf_reset_write(current->mbuf); } return nextBuffer; }
/** * \brief write the result inside the file * \param writer pointer to writer instance * \param values type of sample * \param value_count size of above array * \return 1 if successful, 0 otherwise */ static int row_cols(OmlWriter* writer, OmlValue* values, int value_count) { OmlTextWriter* self = (OmlTextWriter*)writer; MBuffer* mbuf; if ((mbuf = self->mbuf) == NULL) return 0; /* previous use of mbuf failed */ int i; OmlValue* v = values; for (i = 0; i < value_count; i++, v++) { int res; switch (oml_value_get_type(v)) { case OML_LONG_VALUE: { res = mbuf_print(mbuf, "\t%" PRId32, oml_value_clamp_long (v->value.longValue)); break; } case OML_INT32_VALUE: res = mbuf_print(mbuf, "\t%" PRId32, v->value.int32Value); break; case OML_UINT32_VALUE: res = mbuf_print(mbuf, "\t%" PRIu32, v->value.uint32Value); break; case OML_INT64_VALUE: res = mbuf_print(mbuf, "\t%" PRId64, v->value.int64Value); break; case OML_UINT64_VALUE: res = mbuf_print(mbuf, "\t%" PRIu64, v->value.uint64Value); break; case OML_DOUBLE_VALUE: res = mbuf_print(mbuf, "\t%f", v->value.doubleValue); break; case OML_STRING_VALUE: res = mbuf_print(mbuf, "\t%s", omlc_get_string_ptr(*oml_value_get_value(v))); break; case OML_BLOB_VALUE: { const unsigned int max_bytes = 6; int bytes = v->value.blobValue.length < max_bytes ? v->value.blobValue.length : max_bytes; int i = 0; res = mbuf_print(mbuf, "blob "); for (i = 0; i < bytes; i++) { res = mbuf_print(mbuf, "%02x", ((uint8_t*)v->value.blobValue.ptr)[i]); } res = mbuf_print (mbuf, " ..."); break; } default: res = -1; logerror( "Unsupported value type '%d'\n", oml_value_get_type(v)); return 0; } if (res < 0) { mbuf_reset_write(mbuf); self->mbuf = NULL; return 0; } } return 1; }
/** * \brief before sending datastore information about the time and the stream * \param writer the netwriter to send data * \param ms the stream to store the measruement from * \param now a timestamp that represensent the current time * \return 1 if succesfull, 0 otherwise */ int row_start(OmlWriter* writer, OmlMStream* ms, double now) { OmlTextWriter* self = (OmlTextWriter*)writer; assert(self->bufferedWriter != NULL); MBuffer* mbuf; if ((mbuf = self->mbuf = bw_get_write_buf(self->bufferedWriter, 1)) == NULL) return 0; mbuf_begin_write(mbuf); if (mbuf_print(mbuf, "%f\t%d\t%ld", now, ms->index, ms->seq_no)) { mbuf_reset_write(mbuf); self->mbuf = NULL; return 0; } return 1; }
/** * \brief write the data after finalysing the data structure * \param writer the net writer that send the measurements * \param ms the stream of measurmenent * \return 1 if successful, 0 otherwise */ int row_end(OmlWriter* writer, OmlMStream* ms) { (void)ms; OmlTextWriter* self = (OmlTextWriter*)writer; MBuffer* mbuf; if ((mbuf = self->mbuf) == NULL) return 0; /* previous use of mbuf failed */ int res; if ((res = mbuf_write(mbuf, (uint8_t*)"\n", 1)) != 0) { mbuf_reset_write(mbuf); } else { // success, lock in message mbuf_begin_write (mbuf); } self->mbuf = NULL; bw_unlock_buf(self->bufferedWriter); return res == 0; }
// This function finds the next empty write chain, sets +self->writeChain+ to // it and returns. // // We only use the next one if it is empty. If not, we // essentially just filled up the last chain and wrapped // around to the socket reader. In that case, we either create a new chain // if the overall buffer can still grow, or we drop the data from the current one. // // This assumes that the current thread holds the +self->lock+ and the lock on // the +self->writeChain+. // BufferChain* getNextWriteChain( BufferedWriter* self, BufferChain* current ) { assert(current != NULL); BufferChain* nextBuffer = current->next; assert(nextBuffer != NULL); BufferChain* resChain = NULL; if (mbuf_remaining(nextBuffer->mbuf) == 0) { // It's empty, we can use it mbuf_clear2(nextBuffer->mbuf, 0); resChain = nextBuffer; } else if (self->chainsAvailable > 0) { // insert new chain between current and next one. BufferChain* newBuffer = createBufferChain(self); newBuffer->next = nextBuffer; current->next = newBuffer; resChain = newBuffer; } else { // Filled up buffer, time to drop data and reuse current buffer // Current buffer holds most recent added data (we drop from the queue's tail //assert(current->reading == 0); assert(current->reading == 0); o_log (O_LOG_WARN, "Dropping %d bytes of measurement data\n", mbuf_fill(current->mbuf)); mbuf_repack_message2(current->mbuf); return current; } // Now we just need to copy the +message+ from +current+ to +resChain+ int msgSize = mbuf_message_length(current->mbuf); if (msgSize > 0) { mbuf_write(resChain->mbuf, mbuf_message(current->mbuf), msgSize); mbuf_reset_write(current->mbuf); } return resChain; }
/** Marshal a single OmlValueU of type OmlValueT into mbuf. * * Usually called by marshal_values(). On failure, the whole message writing is * reset using mbuf_reset_write(), and marshalling should restart with * marshal_init(), after the MBuffer has been adequately resized or repacked. * * \param mbuf MBuffer to write marshalled data to * \param val_type OmlValueT representing the type of val * \param val pointer to OmlValueU, of type val_type, to marshall * \return 1 on success, or 0 otherwise (marshalling should then restart from marshal_init()) * \see marshal_values, marshal_init, mbuf_reset_write, mbuf_repack_message, mbuf_repack_message2, mbuf_resize */ int marshal_value(MBuffer* mbuf, OmlValueT val_type, OmlValueU* val) { switch (val_type) { case OML_LONG_VALUE: { long v = oml_value_clamp_long (omlc_get_long(*val)); uint32_t uv = (uint32_t)v; uint32_t nv = htonl(uv); uint8_t buf[LONG_T_SIZE+1]; buf[0] = LONG_T; memcpy(&buf[1], &nv, sizeof (nv)); logdebug3("Marshalling long %ld\n", nv); int result = mbuf_write (mbuf, buf, LENGTH (buf)); if (result == -1) { logerror("Failed to marshal OML_LONG_VALUE (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } break; } case OML_INT32_VALUE: case OML_UINT32_VALUE: case OML_INT64_VALUE: case OML_UINT64_VALUE: { uint8_t buf[UINT64_T_SIZE+1]; // Max integer size uint32_t uv32; uint32_t nv32; uint64_t uv64; uint64_t nv64; uint8_t *p_nv; if (oml_size_map[val_type] == 4) { uv32 = omlc_get_uint32(*val); nv32 = htonl(uv32); p_nv = (uint8_t*)&nv32; logdebug3("Marshalling %s %" PRIu32 "\n", oml_type_to_s(val_type), uv32); } else { uv64 = omlc_get_uint64(*val); nv64 = htonll(uv64); p_nv = (uint8_t*)&nv64; logdebug3("Marshalling %s %" PRIu64 "\n", oml_type_to_s(val_type), uv64); } buf[0] = oml_type_map[val_type]; memcpy(&buf[1], p_nv, oml_size_map[val_type]); int result = mbuf_write (mbuf, buf, oml_size_map[val_type] + 1); if (result == -1) { logerror("Failed to marshal %s value (mbuf_write())\n", oml_type_to_s (val_type)); mbuf_reset_write (mbuf); return 0; } break; } case OML_DOUBLE_VALUE: { uint8_t type = DOUBLE_T; double v = omlc_get_double(*val); int exp; double mant = frexp(v, &exp); int8_t nexp = (int8_t)exp; logdebug3("Marshalling double %f\n", v); if (isnan(v)) { type = DOUBLE_NAN; nexp = 0; mant = 0; } else if (nexp != exp) { logerror("Double number '%lf' is out of bounds, sending NaN\n", v); type = DOUBLE_NAN; nexp = 0; mant = 0; } int32_t imant = (int32_t)(mant * (1 << BIG_L)); uint32_t nmant = htonl(imant); uint8_t buf[6] = { type, 0, 0, 0, 0, nexp }; memcpy(&buf[1], &nmant, sizeof (nmant)); int result = mbuf_write (mbuf, buf, LENGTH (buf)); if (result == -1) { logerror("Failed to marshal OML_DOUBLE_VALUE (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } break; } case OML_STRING_VALUE: { char* str = omlc_get_string_ptr(*val); if (str == NULL) { str = ""; logdebug("Attempting to send a NULL string; sending empty string instead\n"); } size_t len = strlen(str); if (len > STRING_T_MAX_SIZE) { logerror("Truncated string '%s'\n", str); len = STRING_T_MAX_SIZE; } logdebug3("Marshalling string '%s' of length %d\n", str, len); uint8_t buf[2] = { STRING_T, (uint8_t)(len & 0xff) }; int result = mbuf_write (mbuf, buf, LENGTH (buf)); if (result == -1) { logerror("Failed to marshal OML_STRING_VALUE type and length (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } result = mbuf_write (mbuf, (uint8_t*)str, len); if (result == -1) { logerror("Failed to marshal OML_STRING_VALUE (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } break; } case OML_BLOB_VALUE: { int result = 0; void *blob = omlc_get_blob_ptr(*val); size_t length = omlc_get_blob_length(*val); if (blob == NULL || length == 0) { logdebug ("Attempting to send NULL or empty blob; blob of length 0 will be sent\n"); length = 0; } uint8_t buf[5] = { BLOB_T, 0, 0, 0, 0 }; size_t n_length = htonl (length); memcpy (&buf[1], &n_length, 4); logdebug3("Marshalling blob of size %d\n", length); result = mbuf_write (mbuf, buf, sizeof (buf)); if (result == -1) { logerror ("Failed to marshall OML_BLOB_VALUE type and length (mbuf_write())\n"); mbuf_reset_write (mbuf); return 0; } result = mbuf_write (mbuf, blob, length); if (result == -1) { logerror ("Failed to marshall %d bytes of OML_BLOB_VALUE data\n", length); mbuf_reset_write (mbuf); return 0; } break; } case OML_GUID_VALUE: { /* FIXME: Wrap with UINT64 marshalling, just change the type */ uint64_t nv64; uint8_t buf[GUID_T_SIZE+1]; buf[0] = GUID_T; nv64 = htonll(omlc_get_guid(*val)); memcpy(&buf[1], &nv64, sizeof(nv64)); logdebug3("Marshalling GUID %" PRIu64 "\n", nv64); if (-1 == mbuf_write(mbuf, buf, LENGTH(buf))) { logerror("Failed to marshal OML_GUID_VALUE (mbuf_write())\n"); mbuf_reset_write(mbuf); return 0; } break; } case OML_BOOL_VALUE: { uint8_t buf; if (!omlc_get_bool(*val)) { buf = BOOL_FALSE_T; } else { buf = BOOL_TRUE_T; } logdebug3("Marshalling boolean %d\n", BOOL_TRUE_T == buf); if (-1 == mbuf_write(mbuf, &buf, 1)) { logerror("Failed to marshal OML_BOOL_VALUE (mbuf_write())\n"); mbuf_reset_write(mbuf); return 0; } break; } case OML_VECTOR_INT32_VALUE: case OML_VECTOR_UINT32_VALUE: { size_t i; uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 }; uint16_t hn = omlc_get_vector_nof_elts(*val); uint16_t nn = htons(hn); buf[1] = vector_protocol_map[val_type]; memcpy(&buf[2], &nn, sizeof(nn)); if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) { uint32_t elts[hn]; uint32_t *v = omlc_get_vector_ptr(*val); for(i = 0; i < hn; i++) elts[i] = htonl(*((uint32_t*)(v+i))); if(mbuf_write(mbuf, (const uint8_t*)(elts), sizeof(elts)) == -1) { logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn); mbuf_reset_write(mbuf); return 0; } } else { logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type)); mbuf_reset_write(mbuf); return 0; } break; } case OML_VECTOR_INT64_VALUE: case OML_VECTOR_UINT64_VALUE: case OML_VECTOR_DOUBLE_VALUE: { size_t i; uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 }; uint16_t hn = omlc_get_vector_nof_elts(*val); uint16_t nn = htons(hn); buf[1] = vector_protocol_map[val_type]; memcpy(&buf[2], &nn, sizeof(nn)); if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) { uint64_t elts[hn]; uint64_t *v = omlc_get_vector_ptr(*val); for(i = 0; i < hn; i++) elts[i] = htonll(*((uint64_t*)(v+i))); if(mbuf_write(mbuf, (const uint8_t*)(elts), sizeof(elts)) == -1) { logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn); mbuf_reset_write(mbuf); return 0; } } else { logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type)); mbuf_reset_write(mbuf); return 0; } break; } case OML_VECTOR_BOOL_VALUE: { size_t i; uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 }; uint16_t hn = omlc_get_vector_nof_elts(*val); uint16_t nn = htons(hn); buf[1] = vector_protocol_map[val_type]; memcpy(&buf[2], &nn, sizeof(nn)); if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) { uint8_t elts[hn]; bool *v = omlc_get_vector_ptr(*val); for(i = 0; i < hn; i++) elts[i] = v[i] ? BOOL_TRUE_T : BOOL_FALSE_T; if(mbuf_write(mbuf, (const uint8_t*)(elts), hn) == -1) { logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn); mbuf_reset_write(mbuf); return 0; } } else { logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type)); mbuf_reset_write(mbuf); return 0; } break; } default: logerror("%s(): Unsupported value type '%d'\n", __func__, val_type); return 0; } return 1; }