/** * @brief Read an OmlValue value from +mbuf+. * * The mbuf read pointer is assumed to be pointing to the start of a * value. The current line must be completely contained in the mbuf, * including the final newline '\n'. A value is read from the mbuf * assuming that its type matches the one contained in the value * object. * * Values are assumed to be tab-delimited, and so either a tab or a * newline terminates the next field in the buffer. This function * modifies the mbuf contents by null-terminating the field, * replacing the tab or newline with '\0'. * * After this function finishes, if it was successful, the mbuf read * pointer (mbuf_rdptr()) will be modified to point to the first * character following the field. * * @param mbuf The buffer containing the next field value * @param value An OmlValue to store the parse value in. The type of this * value on input determines what type of value the function should * attempt to read. * @param line_length length of the current line in the buffer, measured * from the buffer's current read pointer (mbuf_rdptr()). * * @return -1 if an error occurred; otherwise, the length of the * field in bytes. * */ static int text_read_value (MBuffer *mbuf, OmlValue *value, size_t line_length) { uint8_t *line = mbuf_rdptr (mbuf); size_t field_length = mbuf_find (mbuf, '\t'); int len; int ret = 0; uint8_t save; /* No tab '\t' found on this line --> final field */ if (field_length == (size_t)-1 || field_length > line_length) len = line_length; else len = field_length; save = line[len]; line[len] = '\0'; ret = oml_value_from_s (value, (char*)line); line[len] = save; len++; // Skip the separator if (ret == -1) { return ret; } else { mbuf_read_skip (mbuf, len); return len; } }
/** Unmarshals the content of buffer into an array of values of size * value_count. * * If the returned number is negative, there were more values than could fit in * the array in the buffer, and some were skipped. This number (when * multiplied by -1) indicates by how much the values array should be * extended. If the number is less than * -100, it indicates an error. * * \param mbuf MBuffer to read from * \param header pointer to an OmlBinaryHeader corresponding to this message * \param values array of OmlValue to be filled * \param max_value_count length of the array (XXX: Should be < 100, otherwise confusion may happen with error returns) * \return the number of values found (positive), or the number of values that didn't fit in the array (negative; multiplied by -1), <-100 in case of error * \see unmarshal_init */ int unmarshal_values(MBuffer* mbuf, OmlBinaryHeader* header, OmlValue* values, int max_value_count) { int value_count = header->values; if (0 == value_count) { logwarn("No value to unmarshall\n"); return -102; } if (value_count > max_value_count) { logwarn("Measurement packet contained %d too many values for internal storage (max %d, actual %d); skipping packet\n", (value_count - max_value_count), max_value_count, value_count); logwarn("Message length appears to be %d + 5\n", header->length); mbuf_read_skip (mbuf, header->length + PACKET_HEADER_SIZE); mbuf_begin_read (mbuf); // FIXME: Check for sync return max_value_count - value_count; // value array is too small } int i; OmlValue* val = values; for (i = 0; i < value_count; i++) { if (unmarshal_value(mbuf, &val[i]) == 0) { logwarn("Could not unmarshal values %d of %d\n", i, value_count); return -101; } } return value_count; }
// return 1 if chain has been fully sent, 0 otherwise int processChain( BufferedWriter* self, BufferChain* chain ) { uint8_t* buf = mbuf_rdptr(chain->mbuf); size_t size = mbuf_message_offset(chain->mbuf) - mbuf_read_offset(chain->mbuf); size_t sent = 0; chain->reading = 1; oml_unlock(&self->lock, "processChain"); /* don't keep lock while transmitting */ MBuffer* meta = self->meta_buf; while (size > sent) { long cnt = self->writeFunc(self->writeFuncHdl, (void*)(buf + sent), size - sent, meta->rdptr, meta->fill); if (cnt > 0) { sent += cnt; } else { /* ERROR: Sleep a bit and try again */ /* To be on the safe side, we rewind to the beginning of the * chain and try to resend everything - this is especially important * if the underlying stream needs to reopen and resync. */ mbuf_reset_read(chain->mbuf); size = mbuf_message_offset(chain->mbuf) - mbuf_read_offset(chain->mbuf); sent = 0; sleep(1); } } // get lock back to see what happened while we were busy oml_lock_persistent(self); mbuf_read_skip(chain->mbuf, sent); if (mbuf_write_offset(chain->mbuf) == mbuf_read_offset(chain->mbuf)) { // seem to have sent everything so far, reset chain // mbuf_clear2(chain->mbuf, 0); mbuf_clear2(chain->mbuf, 1); chain->reading = 0; return 1; } return 0; }
/** Send data contained in one chunk. * * \warning This function acquires the lock on the BufferedWriter for the time * it takes to check the double-buffer. * * \warning This function acquires the lock on the chunk being processed for * the time it takes to check it and swap the double buffer. * * \bug The meta buffer should also be protected. * * \param self BufferedWriter to process * \param chunk link of the chunk to process * * \return 1 if chunk has been fully sent, 0 if not, -1 on continuing back-off, -2 otherwise * \see oml_outs_write_f */ static int processChunk(BufferedWriter* self, BufferChunk* chunk) { time_t now; int ret = -2; ssize_t cnt = 0; MBuffer *read_buf = NULL; assert(self); assert(self->meta_buf); assert(self->read_buf); assert(chunk); assert(chunk->mbuf); oml_lock(&self->lock, __FUNCTION__); if (mbuf_message(self->read_buf) > mbuf_rdptr(self->read_buf)) { /* There is unread data in the double buffer */ read_buf = self->read_buf; } oml_unlock(&self->lock, __FUNCTION__); oml_lock(&chunk->lock, __FUNCTION__); if ((NULL == read_buf) && (mbuf_message(chunk->mbuf) >= mbuf_rdptr(chunk->mbuf))) { /* There is unread data in the read buffer, swap MBuffers */ read_buf = chunk->mbuf; chunk->mbuf = self->read_buf; } oml_unlock(&chunk->lock, __FUNCTION__); oml_lock(&self->lock, __FUNCTION__); self->read_buf = read_buf; oml_unlock(&self->lock, __FUNCTION__); if (NULL == read_buf) { /* The current message is not after the read pointer, * we must be on the writer chunk */ ret = 1; goto processChunk_cleanup; } time(&now); if (difftime(now, self->last_failure_time) < self->backoff) { logdebug("%s: Still in back-off period (%ds)\n", self->outStream->dest, self->backoff); ret = -1; goto processChunk_cleanup; } while (mbuf_write_offset(read_buf) > mbuf_read_offset(read_buf)) { oml_lock(&self->meta_lock, __FUNCTION__); cnt = self->outStream->write(self->outStream, mbuf_rdptr(read_buf), mbuf_message_offset(read_buf) - mbuf_read_offset(read_buf), mbuf_rdptr(self->meta_buf), mbuf_fill(self->meta_buf)); oml_unlock(&self->meta_lock, __FUNCTION__); if (cnt > 0) { mbuf_read_skip(read_buf, cnt); if (self->backoff) { self->backoff = 0; loginfo("%s: Connected\n", self->outStream->dest); } } else { self->last_failure_time = now; if (!self->backoff) { self->backoff = 1; } else if (self->backoff < UINT8_MAX) { self->backoff *= 2; } logwarn("%s: Error sending, backing off for %ds\n", self->outStream->dest, self->backoff); goto processChunk_cleanup; } } ret = 1; processChunk_cleanup: return ret; }
/** Unmarshals the next content of an MBuffer into a OmlValue * * \param mbuf MBuffer to read from * \param value pointer to OmlValue to unmarshall the read data into * \return 1 if successful, 0 otherwise */ int unmarshal_value(MBuffer *mbuf, OmlValue *value) { if (mbuf_rd_remaining(mbuf) == 0) { logerror("Tried to unmarshal a value from the buffer, but didn't receive enough data to do that\n"); return 0; } int type = mbuf_read_byte (mbuf); if (type == -1) return 0; switch (type) { case LONG_T: { uint8_t buf [LONG_T_SIZE]; if (mbuf_read (mbuf, buf, LENGTH (buf)) == -1) { logerror("Failed to unmarshal OML_LONG_VALUE; not enough data?\n"); return 0; } uint32_t hv = ntohl(*((uint32_t*)buf)); int32_t v = (int32_t)(hv); /* * The server no longer needs to know about OML_LONG_VALUE, as the * marshalling process now maps OML_LONG_VALUE into OML_INT32_VALUE * (by truncating to [INT_MIN, INT_MAX]. Therefore, unmarshall a * LONG_T value into an OML_INT32_VALUE object. */ oml_value_set_type(value, OML_INT32_VALUE); omlc_set_int32(*oml_value_get_value(value), v); break; } case INT32_T: case UINT32_T: case INT64_T: case UINT64_T: { uint8_t buf [UINT64_T_SIZE]; // Maximum integer size OmlValueT oml_type = protocol_type_map[type]; if (mbuf_read (mbuf, buf, protocol_size_map[type]) == -1) { logerror("Failed to unmarshall %d value; not enough data?\n", type); return 0; } oml_value_set_type(value, oml_type); switch (type) { case INT32_T: omlc_set_int32(*oml_value_get_value(value), ntohl(*((int32_t*)buf))); logdebug3("Unmarshalled %s %" PRId32 "\n", oml_type_to_s(oml_type), omlc_get_int32(*oml_value_get_value(value))); break; case UINT32_T: omlc_set_uint32(*oml_value_get_value(value), ntohl(*((uint32_t*)buf))); logdebug3("Unmarshalled %s %" PRIu32 "\n", oml_type_to_s(oml_type), omlc_get_uint32(*oml_value_get_value(value))); break; case INT64_T: omlc_set_int64(*oml_value_get_value(value), ntohll(*((int64_t*)buf))); logdebug3("Unmarshalled %s %" PRId64 "\n", oml_type_to_s(oml_type), omlc_get_int64(*oml_value_get_value(value))); break; case UINT64_T: omlc_set_uint64(*oml_value_get_value(value), ntohll(*((uint64_t*)buf))); logdebug3("Unmarshalled %s %" PRIu64 "\n", oml_type_to_s(oml_type), omlc_get_uint64(*oml_value_get_value(value))); break; default: logerror("Integer morphed, something magic has just happened\n"); return 0; } break; } case DOUBLE_T: { uint8_t buf [DOUBLE_T_SIZE]; OmlValueT oml_type = protocol_type_map[type]; if (mbuf_read (mbuf, buf, LENGTH (buf)) == -1) { logerror("Failed to unmarshal OML_DOUBLE_VALUE; not enough data?\n"); return 0; } int hmant = (int)ntohl(*((uint32_t*)buf)); double mant = hmant * 1.0 / (1 << BIG_L); int exp = (int8_t) buf[4]; double v = ldexp(mant, exp); oml_value_set_type(value, oml_type); omlc_set_double(*oml_value_get_value(value), v); logdebug3("Unmarshalled double %f\n", omlc_get_double(*oml_value_get_value(value))); break; } case DOUBLE_NAN: { OmlValueT oml_type = protocol_type_map[type]; mbuf_read_skip(mbuf, DOUBLE_T_SIZE); /* The data is irrelevant */ oml_value_set_type(value, oml_type); omlc_set_double(*oml_value_get_value(value), NAN); logdebug("Received NaN\n"); break; } case STRING_T: { int len = 0; uint8_t buf [STRING_T_MAX_SIZE]; len = mbuf_read_byte (mbuf); if (len == -1 || mbuf_read (mbuf, buf, len) == -1) { logerror("Failed to unmarshal OML_STRING_VALUE; not enough data?\n"); return 0; } oml_value_set_type(value, OML_STRING_VALUE); omlc_set_string_copy(*oml_value_get_value(value), buf, len); logdebug3("Unmarshalled string '%s' of length %d\n", omlc_get_string_ptr(*oml_value_get_value(value)), len); break; } case BLOB_T: { uint32_t n_len; if (mbuf_read (mbuf, (uint8_t*)&n_len, 4) == -1) { logerror ("Failed to unmarshal OML_BLOB_VALUE length field; not enough data?\n"); return 0; } size_t len = ntohl (n_len); size_t remaining = mbuf_rd_remaining (mbuf); if (len > remaining) { logerror ("Failed to unmarshal OML_BLOB_VALUE data: not enough data available " "(wanted %d, but only have %d bytes\n", len, remaining); return 0; } void *ptr = mbuf_rdptr (mbuf); oml_value_set_type(value, OML_BLOB_VALUE); omlc_set_blob (*oml_value_get_value(value), ptr, len); /*XXX*/ logdebug3("Unmarshalled blob of size %d\n", len); mbuf_read_skip (mbuf, len); break; } case GUID_T: { uint64_t nv64; uint8_t buf[GUID_T_SIZE]; if(mbuf_read(mbuf, buf, GUID_T_SIZE) == -1) { logerror("Failed to unmarshall OML_GUID_VALUE data; not enough data?\n"); return 0; } memcpy(&nv64, buf, sizeof(nv64)); oml_value_set_type(value, OML_GUID_VALUE); omlc_set_guid(*oml_value_get_value(value), ntohll(nv64)); logdebug3("Unmarshalled GUID %" PRIu64 "\n", omlc_get_guid(*oml_value_get_value(value))); break; } case BOOL_FALSE_T: case BOOL_TRUE_T: oml_value_set_type(value, OML_BOOL_VALUE); omlc_set_bool(*oml_value_get_value(value), (type == BOOL_TRUE_T)?OMLC_BOOL_TRUE:OMLC_BOOL_FALSE); logdebug3("Unmarshalled boolean %d\n", OMLC_BOOL_TRUE == omlc_get_bool(*oml_value_get_value(value))); break; case VECTOR_T: { uint16_t i, nof_elts; int type = mbuf_read_byte(mbuf); if(-1 == type) { logerror("%s(): failed to unmarshall VECTOR_T length\n", __func__); return 0; } if(mbuf_read(mbuf,(uint8_t*)(&nof_elts), sizeof(nof_elts)) == -1) { logerror("%s(): failed to unmarshall VECTOR_T length\n", __func__); return 0; } nof_elts = ntohs(nof_elts); OmlValueT oml_type = vector_type_map[type]; OmlValueU *v = oml_value_get_value(value); switch(type) { case INT32_T: case UINT32_T: { size_t bytes = nof_elts * sizeof(uint32_t); uint32_t *elts = oml_calloc(nof_elts, sizeof(uint32_t)); if(mbuf_read(mbuf, (uint8_t*)(elts), nof_elts * sizeof(uint32_t)) == -1) { logerror("%s(): failed to unmarshall OML_VECTOR_(U)INT32_VALUE\n", __func__); return 0; } for(i = 0; i < nof_elts; i++) elts[i] = ntohl(elts[i]); oml_value_set_type(value, oml_type); omlc_set_vector_ptr(*v, elts); omlc_set_vector_length(*v, bytes); omlc_set_vector_size(*v, bytes); omlc_set_vector_nof_elts(*v, nof_elts); omlc_set_vector_elt_size(*v, sizeof(uint32_t)); break; } case INT64_T: case UINT64_T: case DOUBLE64_T: { size_t bytes = nof_elts * sizeof(uint64_t); uint64_t *elts = oml_calloc(nof_elts, sizeof(uint64_t)); if(mbuf_read(mbuf, (uint8_t*)(elts), nof_elts * sizeof(uint64_t)) == -1) { logerror("%s(): failed to unmarshall OML_VECTOR_(U)INT64_VALUE\n", __func__); return 0; } for(i = 0; i < nof_elts; i++) elts[i] = ntohll(elts[i]); oml_value_set_type(value, oml_type); omlc_set_vector_ptr(*v, elts); omlc_set_vector_length(*v, bytes); omlc_set_vector_size(*v, bytes); omlc_set_vector_nof_elts(*v, nof_elts); omlc_set_vector_elt_size(*v, sizeof(uint64_t)); break; } case BOOL_T: { uint8_t y[nof_elts]; size_t bytes = nof_elts * sizeof(bool); bool *elts = oml_calloc(nof_elts, sizeof(bool)); if(mbuf_read(mbuf, y, nof_elts) == -1) { logerror("%s(): failed to unmarshall OML_VECTOR_BOOL_VALUE\n", __func__); return 0; } for(i = 0; i < nof_elts; i++) elts[i] = ((BOOL_TRUE_T == y[i]) ? true : false); oml_value_set_type(value, oml_type); omlc_set_vector_ptr(*v, elts); omlc_set_vector_length(*v, bytes); omlc_set_vector_size(*v, bytes); omlc_set_vector_nof_elts(*v, nof_elts); omlc_set_vector_elt_size(*v, sizeof(bool)); break; } default: logerror("%s(): bad type for array (t=%d)\n", __func__, type); break; } break; } default: logerror("%s: Unsupported value type '%d'\n", __FUNCTION__, type); return 0; } return 1; }