/** Marshal meta-data for an OML measurement stream's sample
*
* An OML measurement stream is written as two bytes; the first one is the
* counter for the number of elements in the message, and therefore starts at
* 0, and the second one is the stream's index. This is followed by a
* marshalled int32 value containing the sequence number, and a double value
* containing the timestamp.
*
* A marshalling message should have been prepared in the MBuffer first with
* marshal_init(). Actual data can then be marshalled into the message with
* marshal_values().
*
*
* \param mbuf MBuffer to write marshalled data to
* \param stream Measurement Stream's index
* \param seqno message sequence number
* \param now message time
* \return 1 if successful, -1 otherwise
* \see marshal_init, marshal_values, marshal_finalize
*/
int
marshal_measurements(MBuffer* mbuf, int stream, int seqno, double now)
{
OmlValueU v;
uint8_t s[2] = { 0, (uint8_t)stream };
/* Write num-meas (0, for now), and the stream index */
int result = mbuf_write (mbuf, s, LENGTH (s));
omlc_zero(v);
if (result == -1) {
logerror("Unable to marshal table number and measurement count (mbuf_write())\n");
mbuf_reset_write (mbuf);
return -1;
}
logdebug("Marshalling sample %d for stream %d\n", seqno, stream);
omlc_set_int32(v, seqno);
marshal_value(mbuf, OML_INT32_VALUE, &v);
omlc_set_double(v, now);
marshal_value(mbuf, OML_DOUBLE_VALUE, &v);
return 1;
}
/** Find the next empty write chunk, sets self->writerChunk to it and returns * it.
*
* We only use the next one if it is empty. If not, we essentially just filled
* up the last chunk and wrapped around to the socket reader. In that case, we
* either create a new chunk if the overall buffer can still grow, or we drop
* the data from the current one.
*
* \warning A lock on the current writer chunk should be held prior to calling
* this function. It will be released, and the returned writerChunk will be
* similarly locked.
*
* \param self BufferedWriter pointer
* \param current locked BufferChunk to use or from which to find the next
* \return a locked BufferChunk in which data can be stored
*/
BufferChunk*
getNextWriteChunk(BufferedWriter* self, BufferChunk* current) {
int nlost;
BufferChunk* nextBuffer;
assert(current != NULL);
nextBuffer = current->next;
oml_unlock(¤t->lock, __FUNCTION__);
assert(nextBuffer != NULL);
oml_lock(&self->lock, __FUNCTION__);
if (nextBuffer == self->nextReaderChunk) {
if (self->unallocatedBuffers > 0) {
/* The next buffer is the next to be read, but we can allocate more,
* allocate a new buffer, insert it after the current writer, and use it */
nextBuffer = createBufferChunk(self);
assert(nextBuffer); /** \todo Use existing buffer if allocation fails */
oml_unlock(&self->lock, __FUNCTION__);
oml_lock(¤t->lock, __FUNCTION__);
nextBuffer->next = current->next;
current->next = nextBuffer; /* we have a lock on this one */
oml_unlock(¤t->lock, __FUNCTION__);
oml_lock(&self->lock, __FUNCTION__);
} else {
/* The next buffer is the next to be read, and we cannot allocate more,
* use it, dropping unread data, and advance the read pointer */
self->nextReaderChunk = nextBuffer->next;
}
}
self->writerChunk = nextBuffer;
nlost = bw_msgcount_reset(self);
self->nlost += nlost;
oml_unlock(&self->lock, __FUNCTION__);
oml_lock(&nextBuffer->lock, __FUNCTION__);
if (nlost) {
logwarn("%s: Dropping %d samples (%dB)\n", self->outStream->dest, nlost, mbuf_fill(nextBuffer->mbuf));
}
mbuf_clear2(nextBuffer->mbuf, 0);
// Now we just need to copy the message from current to self->writerChunk
int msgSize = mbuf_message_length(current->mbuf);
if (msgSize > 0) {
mbuf_write(nextBuffer->mbuf, mbuf_message(current->mbuf), msgSize);
mbuf_reset_write(current->mbuf);
}
return nextBuffer;
}
/**
* \brief write the result inside the file
* \param writer pointer to writer instance
* \param values type of sample
* \param value_count size of above array
* \return 1 if successful, 0 otherwise
*/
static int
row_cols(OmlWriter* writer, OmlValue* values, int value_count)
{
OmlTextWriter* self = (OmlTextWriter*)writer;
MBuffer* mbuf;
if ((mbuf = self->mbuf) == NULL) return 0; /* previous use of mbuf failed */
int i;
OmlValue* v = values;
for (i = 0; i < value_count; i++, v++) {
int res;
switch (oml_value_get_type(v)) {
case OML_LONG_VALUE: {
res = mbuf_print(mbuf, "\t%" PRId32, oml_value_clamp_long (v->value.longValue));
break;
}
case OML_INT32_VALUE: res = mbuf_print(mbuf, "\t%" PRId32, v->value.int32Value); break;
case OML_UINT32_VALUE: res = mbuf_print(mbuf, "\t%" PRIu32, v->value.uint32Value); break;
case OML_INT64_VALUE: res = mbuf_print(mbuf, "\t%" PRId64, v->value.int64Value); break;
case OML_UINT64_VALUE: res = mbuf_print(mbuf, "\t%" PRIu64, v->value.uint64Value); break;
case OML_DOUBLE_VALUE: res = mbuf_print(mbuf, "\t%f", v->value.doubleValue); break;
case OML_STRING_VALUE: res = mbuf_print(mbuf, "\t%s", omlc_get_string_ptr(*oml_value_get_value(v))); break;
case OML_BLOB_VALUE: {
const unsigned int max_bytes = 6;
int bytes = v->value.blobValue.length < max_bytes ? v->value.blobValue.length : max_bytes;
int i = 0;
res = mbuf_print(mbuf, "blob ");
for (i = 0; i < bytes; i++) {
res = mbuf_print(mbuf, "%02x", ((uint8_t*)v->value.blobValue.ptr)[i]);
}
res = mbuf_print (mbuf, " ...");
break;
}
default:
res = -1;
logerror( "Unsupported value type '%d'\n", oml_value_get_type(v));
return 0;
}
if (res < 0) {
mbuf_reset_write(mbuf);
self->mbuf = NULL;
return 0;
}
}
return 1;
}
/**
* \brief before sending datastore information about the time and the stream
* \param writer the netwriter to send data
* \param ms the stream to store the measruement from
* \param now a timestamp that represensent the current time
* \return 1 if succesfull, 0 otherwise
*/
int
row_start(OmlWriter* writer, OmlMStream* ms, double now)
{
OmlTextWriter* self = (OmlTextWriter*)writer;
assert(self->bufferedWriter != NULL);
MBuffer* mbuf;
if ((mbuf = self->mbuf = bw_get_write_buf(self->bufferedWriter, 1)) == NULL)
return 0;
mbuf_begin_write(mbuf);
if (mbuf_print(mbuf, "%f\t%d\t%ld", now, ms->index, ms->seq_no)) {
mbuf_reset_write(mbuf);
self->mbuf = NULL;
return 0;
}
return 1;
}
/**
* \brief write the data after finalysing the data structure
* \param writer the net writer that send the measurements
* \param ms the stream of measurmenent
* \return 1 if successful, 0 otherwise
*/
int
row_end(OmlWriter* writer, OmlMStream* ms)
{
(void)ms;
OmlTextWriter* self = (OmlTextWriter*)writer;
MBuffer* mbuf;
if ((mbuf = self->mbuf) == NULL) return 0; /* previous use of mbuf failed */
int res;
if ((res = mbuf_write(mbuf, (uint8_t*)"\n", 1)) != 0) {
mbuf_reset_write(mbuf);
} else {
// success, lock in message
mbuf_begin_write (mbuf);
}
self->mbuf = NULL;
bw_unlock_buf(self->bufferedWriter);
return res == 0;
}
// This function finds the next empty write chain, sets +self->writeChain+ to
// it and returns.
//
// We only use the next one if it is empty. If not, we
// essentially just filled up the last chain and wrapped
// around to the socket reader. In that case, we either create a new chain
// if the overall buffer can still grow, or we drop the data from the current one.
//
// This assumes that the current thread holds the +self->lock+ and the lock on
// the +self->writeChain+.
//
BufferChain*
getNextWriteChain(
BufferedWriter* self,
BufferChain* current
) {
assert(current != NULL);
BufferChain* nextBuffer = current->next;
assert(nextBuffer != NULL);
BufferChain* resChain = NULL;
if (mbuf_remaining(nextBuffer->mbuf) == 0) {
// It's empty, we can use it
mbuf_clear2(nextBuffer->mbuf, 0);
resChain = nextBuffer;
} else if (self->chainsAvailable > 0) {
// insert new chain between current and next one.
BufferChain* newBuffer = createBufferChain(self);
newBuffer->next = nextBuffer;
current->next = newBuffer;
resChain = newBuffer;
} else {
// Filled up buffer, time to drop data and reuse current buffer
// Current buffer holds most recent added data (we drop from the queue's tail
//assert(current->reading == 0);
assert(current->reading == 0);
o_log (O_LOG_WARN, "Dropping %d bytes of measurement data\n", mbuf_fill(current->mbuf));
mbuf_repack_message2(current->mbuf);
return current;
}
// Now we just need to copy the +message+ from +current+ to +resChain+
int msgSize = mbuf_message_length(current->mbuf);
if (msgSize > 0) {
mbuf_write(resChain->mbuf, mbuf_message(current->mbuf), msgSize);
mbuf_reset_write(current->mbuf);
}
return resChain;
}
/** Marshal a single OmlValueU of type OmlValueT into mbuf.
*
* Usually called by marshal_values(). On failure, the whole message writing is
* reset using mbuf_reset_write(), and marshalling should restart with
* marshal_init(), after the MBuffer has been adequately resized or repacked.
*
* \param mbuf MBuffer to write marshalled data to
* \param val_type OmlValueT representing the type of val
* \param val pointer to OmlValueU, of type val_type, to marshall
* \return 1 on success, or 0 otherwise (marshalling should then restart from marshal_init())
* \see marshal_values, marshal_init, mbuf_reset_write, mbuf_repack_message, mbuf_repack_message2, mbuf_resize
*/
int
marshal_value(MBuffer* mbuf, OmlValueT val_type, OmlValueU* val)
{
switch (val_type) {
case OML_LONG_VALUE: {
long v = oml_value_clamp_long (omlc_get_long(*val));
uint32_t uv = (uint32_t)v;
uint32_t nv = htonl(uv);
uint8_t buf[LONG_T_SIZE+1];
buf[0] = LONG_T;
memcpy(&buf[1], &nv, sizeof (nv));
logdebug3("Marshalling long %ld\n", nv);
int result = mbuf_write (mbuf, buf, LENGTH (buf));
if (result == -1) {
logerror("Failed to marshal OML_LONG_VALUE (mbuf_write())\n");
mbuf_reset_write (mbuf);
return 0;
}
break;
}
case OML_INT32_VALUE:
case OML_UINT32_VALUE:
case OML_INT64_VALUE:
case OML_UINT64_VALUE: {
uint8_t buf[UINT64_T_SIZE+1]; // Max integer size
uint32_t uv32;
uint32_t nv32;
uint64_t uv64;
uint64_t nv64;
uint8_t *p_nv;
if (oml_size_map[val_type] == 4)
{
uv32 = omlc_get_uint32(*val);
nv32 = htonl(uv32);
p_nv = (uint8_t*)&nv32;
logdebug3("Marshalling %s %" PRIu32 "\n", oml_type_to_s(val_type), uv32);
} else {
uv64 = omlc_get_uint64(*val);
nv64 = htonll(uv64);
p_nv = (uint8_t*)&nv64;
logdebug3("Marshalling %s %" PRIu64 "\n", oml_type_to_s(val_type), uv64);
}
buf[0] = oml_type_map[val_type];
memcpy(&buf[1], p_nv, oml_size_map[val_type]);
int result = mbuf_write (mbuf, buf, oml_size_map[val_type] + 1);
if (result == -1)
{
logerror("Failed to marshal %s value (mbuf_write())\n",
oml_type_to_s (val_type));
mbuf_reset_write (mbuf);
return 0;
}
break;
}
case OML_DOUBLE_VALUE: {
uint8_t type = DOUBLE_T;
double v = omlc_get_double(*val);
int exp;
double mant = frexp(v, &exp);
int8_t nexp = (int8_t)exp;
logdebug3("Marshalling double %f\n", v);
if (isnan(v)) {
type = DOUBLE_NAN;
nexp = 0;
mant = 0;
} else if (nexp != exp) {
logerror("Double number '%lf' is out of bounds, sending NaN\n", v);
type = DOUBLE_NAN;
nexp = 0;
mant = 0;
}
int32_t imant = (int32_t)(mant * (1 << BIG_L));
uint32_t nmant = htonl(imant);
uint8_t buf[6] = { type, 0, 0, 0, 0, nexp };
memcpy(&buf[1], &nmant, sizeof (nmant));
int result = mbuf_write (mbuf, buf, LENGTH (buf));
if (result == -1)
{
logerror("Failed to marshal OML_DOUBLE_VALUE (mbuf_write())\n");
mbuf_reset_write (mbuf);
return 0;
}
break;
}
case OML_STRING_VALUE: {
char* str = omlc_get_string_ptr(*val);
if (str == NULL) {
str = "";
logdebug("Attempting to send a NULL string; sending empty string instead\n");
}
size_t len = strlen(str);
if (len > STRING_T_MAX_SIZE) {
logerror("Truncated string '%s'\n", str);
len = STRING_T_MAX_SIZE;
}
logdebug3("Marshalling string '%s' of length %d\n", str, len);
uint8_t buf[2] = { STRING_T, (uint8_t)(len & 0xff) };
int result = mbuf_write (mbuf, buf, LENGTH (buf));
if (result == -1) {
logerror("Failed to marshal OML_STRING_VALUE type and length (mbuf_write())\n");
mbuf_reset_write (mbuf);
return 0;
}
result = mbuf_write (mbuf, (uint8_t*)str, len);
if (result == -1)
{
logerror("Failed to marshal OML_STRING_VALUE (mbuf_write())\n");
mbuf_reset_write (mbuf);
return 0;
}
break;
}
case OML_BLOB_VALUE: {
int result = 0;
void *blob = omlc_get_blob_ptr(*val);
size_t length = omlc_get_blob_length(*val);
if (blob == NULL || length == 0) {
logdebug ("Attempting to send NULL or empty blob; blob of length 0 will be sent\n");
length = 0;
}
uint8_t buf[5] = { BLOB_T, 0, 0, 0, 0 };
size_t n_length = htonl (length);
memcpy (&buf[1], &n_length, 4);
logdebug3("Marshalling blob of size %d\n", length);
result = mbuf_write (mbuf, buf, sizeof (buf));
if (result == -1) {
logerror ("Failed to marshall OML_BLOB_VALUE type and length (mbuf_write())\n");
mbuf_reset_write (mbuf);
return 0;
}
result = mbuf_write (mbuf, blob, length);
if (result == -1) {
logerror ("Failed to marshall %d bytes of OML_BLOB_VALUE data\n", length);
mbuf_reset_write (mbuf);
return 0;
}
break;
}
case OML_GUID_VALUE: {
/* FIXME: Wrap with UINT64 marshalling, just change the type */
uint64_t nv64;
uint8_t buf[GUID_T_SIZE+1];
buf[0] = GUID_T;
nv64 = htonll(omlc_get_guid(*val));
memcpy(&buf[1], &nv64, sizeof(nv64));
logdebug3("Marshalling GUID %" PRIu64 "\n", nv64);
if (-1 == mbuf_write(mbuf, buf, LENGTH(buf))) {
logerror("Failed to marshal OML_GUID_VALUE (mbuf_write())\n");
mbuf_reset_write(mbuf);
return 0;
}
break;
}
case OML_BOOL_VALUE: {
uint8_t buf;
if (!omlc_get_bool(*val)) {
buf = BOOL_FALSE_T;
} else {
buf = BOOL_TRUE_T;
}
logdebug3("Marshalling boolean %d\n", BOOL_TRUE_T == buf);
if (-1 == mbuf_write(mbuf, &buf, 1)) {
logerror("Failed to marshal OML_BOOL_VALUE (mbuf_write())\n");
mbuf_reset_write(mbuf);
return 0;
}
break;
}
case OML_VECTOR_INT32_VALUE:
case OML_VECTOR_UINT32_VALUE: {
size_t i;
uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 };
uint16_t hn = omlc_get_vector_nof_elts(*val);
uint16_t nn = htons(hn);
buf[1] = vector_protocol_map[val_type];
memcpy(&buf[2], &nn, sizeof(nn));
if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) {
uint32_t elts[hn];
uint32_t *v = omlc_get_vector_ptr(*val);
for(i = 0; i < hn; i++)
elts[i] = htonl(*((uint32_t*)(v+i)));
if(mbuf_write(mbuf, (const uint8_t*)(elts), sizeof(elts)) == -1) {
logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn);
mbuf_reset_write(mbuf);
return 0;
}
} else {
logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type));
mbuf_reset_write(mbuf);
return 0;
}
break;
}
case OML_VECTOR_INT64_VALUE:
case OML_VECTOR_UINT64_VALUE:
case OML_VECTOR_DOUBLE_VALUE: {
size_t i;
uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 };
uint16_t hn = omlc_get_vector_nof_elts(*val);
uint16_t nn = htons(hn);
buf[1] = vector_protocol_map[val_type];
memcpy(&buf[2], &nn, sizeof(nn));
if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) {
uint64_t elts[hn];
uint64_t *v = omlc_get_vector_ptr(*val);
for(i = 0; i < hn; i++)
elts[i] = htonll(*((uint64_t*)(v+i)));
if(mbuf_write(mbuf, (const uint8_t*)(elts), sizeof(elts)) == -1) {
logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn);
mbuf_reset_write(mbuf);
return 0;
}
} else {
logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type));
mbuf_reset_write(mbuf);
return 0;
}
break;
}
case OML_VECTOR_BOOL_VALUE: {
size_t i;
uint8_t buf[VECTOR_T_SIZE] = { VECTOR_T, 0, 0, 0 };
uint16_t hn = omlc_get_vector_nof_elts(*val);
uint16_t nn = htons(hn);
buf[1] = vector_protocol_map[val_type];
memcpy(&buf[2], &nn, sizeof(nn));
if(mbuf_write(mbuf, buf, VECTOR_T_SIZE) == 0) {
uint8_t elts[hn];
bool *v = omlc_get_vector_ptr(*val);
for(i = 0; i < hn; i++)
elts[i] = v[i] ? BOOL_TRUE_T : BOOL_FALSE_T;
if(mbuf_write(mbuf, (const uint8_t*)(elts), hn) == -1) {
logerror("%s(): failed to marshal %s of size %" PRIu16 " (mbuf_write())\n", __func__, oml_type_to_s(val_type), hn);
mbuf_reset_write(mbuf);
return 0;
}
} else {
logerror("%s(): failed to marshal %s length (mbuf_write())\n", __func__, oml_type_to_s(val_type));
mbuf_reset_write(mbuf);
return 0;
}
break;
}
default:
logerror("%s(): Unsupported value type '%d'\n", __func__, val_type);
return 0;
}
return 1;
}
