Пример #1
0
/*!
 * Update data-object table for certain secondary index.
 *
 * @index_data is what client provided
 * @data is what was downloaded from the storage
 */
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request,
	const data_pointer &index_data, const data_pointer &data, update_index_action action)
{
	dnet_indexes indexes;
	if (!data.empty())
		indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");

	// Construct index entry
	index_entry request_index;
	memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
	request_index.data = index_data;

	auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(),
		request_index, dnet_raw_id_less_than<skip_data>());
	if (it != indexes.indexes.end() && it->index == request_index.index) {
		// It's already there
		if (action == insert_data) {
			if (it->data == request_index.data) {
				// All's ok, keep it untouched
				return data;
			} else {
				// Data is not correct, replace it by new one
				it->data = request_index.data;
			}
		} else {
			// Anyway, destroy it
			indexes.indexes.erase(it);
		}
	} else {
		// Index is not created yet
		if (action == insert_data) {
			// Just insert it
			indexes.indexes.insert(it, 1, request_index);
		} else {
			// All's ok, keep it untouched
			return data;
		}
	}

	indexes.shard_id = request->shard_id;
	indexes.shard_count = request->shard_count;

	msgpack::sbuffer buffer;
	msgpack::pack(&buffer, indexes);

	data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
	new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
	new_buffer.write(buffer.data(), buffer.size());

	return std::move(new_buffer);
}
Пример #2
0
/*!
 * Update data-object table for certain secondary index.
 *
 * @index_data is what client provided
 * @data is what was downloaded from the storage
 */
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, const dnet_indexes_request *request,
	const data_pointer &index_data, const data_pointer &data, uint32_t action,
	std::vector<dnet_indexes_reply_entry> * &removed, uint32_t limit)
{
	elliptics_timer timer;

	dnet_indexes indexes;
	if (!data.empty())
		indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");

	const int64_t timer_unpack = timer.restart();

	// Construct index entry
	dnet_index_entry request_index;
	memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
	request_index.data = index_data;
	dnet_current_time(&request_index.time);

	auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index, dnet_raw_id_less_than<skip_data>());

	const int64_t timer_lower_bound = timer.restart();

	if (it != indexes.indexes.end() && it->index == request_index.index) {
		// It's already there
		if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
			// Item exists, update it's data and time if it's capped collection
			if (!removed && it->data == request_index.data) {
				const int64_t timer_compare = timer.restart();
				DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
				typedef long long int lld;
				dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
					 "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
					 id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
					 lld(timer_compare));
				// All's ok, keep it untouched
				return data;
			}
			it->data = request_index.data;
			it->time = request_index.time;
		} else {
			// Anyway, destroy it
			indexes.indexes.erase(it);
		}
	} else {
		// Index is not created yet
		if (action == DNET_INDEXES_FLAGS_INTERNAL_INSERT) {
			// Remove extra elements from capped collection
			if (removed && limit != 0 && indexes.indexes.size() + 1 > limit) {
				dnet_indexes_reply_entry entry;
				memset(&entry, 0, sizeof(entry));

				auto position = it - indexes.indexes.begin();

				while (indexes.indexes.size() + 1 > limit && !indexes.indexes.empty()) {
					auto jt = std::min_element(indexes.indexes.begin(), indexes.indexes.end(), entry_time_less_than);

					// jt will be removed, so it should be modified to be still valid
					if (position > jt - indexes.indexes.begin())
						--position;

					memcpy(entry.id.id, jt->index.id, DNET_ID_SIZE);
					entry.status = DNET_INDEXES_CAPPED_REMOVED;

					indexes.indexes.erase(jt);
					removed->push_back(entry);
				}

				it = indexes.indexes.begin() + position;
			}
			// And just insert new index
			indexes.indexes.insert(it, 1, request_index);
		} else {
			const int64_t timer_compare = timer.restart();
			DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
			typedef long long int lld;
			dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
				 "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
				 id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
				 lld(timer_compare));
			// All's ok, keep it untouched
			return data;
		}
	}

	const int64_t timer_update = timer.restart();

	indexes.shard_id = request->shard_id;
	indexes.shard_count = request->shard_count;

	msgpack::sbuffer buffer;
	msgpack::pack(&buffer, indexes);

	const int64_t timer_pack = timer.restart();

	data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
	new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
	new_buffer.write(buffer.data(), buffer.size());

	const int64_t timer_write = timer.restart();

	DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
	typedef long long int lld;
	dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
		 "unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
		 id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
		 lld(timer_update), lld(timer_pack), lld(timer_write));

	return std::move(new_buffer);
}
Пример #3
0
		static data_pointer copy(const data_pointer &other)
		{
			return copy(other.data(), other.size());
		}
Пример #4
0
MZC_INLINE MStringW::MStringW(data_pointer pData) :
    m_pchData(pData->data())
{
}
Пример #5
0
MZC_INLINE void MStringW::SetData(data_pointer pData)
{
    m_pchData = pData->data();
}
Пример #6
0
/*!
 * Update data-object table for certain secondary index.
 *
 * @index_data is what client provided
 * @data is what was downloaded from the storage
 */
data_pointer convert_index_table(dnet_node *node, dnet_id *cmd_id, dnet_indexes_request *request,
	const data_pointer &index_data, const data_pointer &data, update_index_action action)
{
	elliptics_timer timer;

	raw_dnet_indexes indexes;
	if (!data.empty())
		indexes_unpack(node, cmd_id, data, &indexes, "convert_index_table");

	const int64_t timer_unpack = timer.restart();

	// Construct index entry
	raw_index_entry request_index;
	memcpy(request_index.index.id, request->id.id, sizeof(request_index.index.id));
	request_index.data.data = index_data.data();
	request_index.data.size = index_data.size();

	auto it = std::lower_bound(indexes.indexes.begin(), indexes.indexes.end(), request_index);

	const int64_t timer_lower_bound = timer.restart();

	if (it != indexes.indexes.end() && it->index == request_index.index) {
		// It's already there
		if (action == insert_data) {
			if (it->data == request_index.data) {
				const int64_t timer_compare = timer.restart();
				DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
				typedef long long int lld;
				dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
					 "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
					 id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
					 lld(timer_compare));
				// All's ok, keep it untouched
				return data;
			} else {
				// Data is not correct, replace it by new one
				it->data = request_index.data;
			}
		} else {
			// Anyway, destroy it
			indexes.indexes.erase(it);
		}
	} else {
		// Index is not created yet
		if (action == insert_data) {
			// Just insert it
			indexes.indexes.insert(it, 1, request_index);
		} else {
			const int64_t timer_compare = timer.restart();
			DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
			typedef long long int lld;
			dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
				 "unpack: %lld ms, lower_bound: %lld ms, compare: %lld ms\n",
				 id_str, data.size(), data.size(), lld(timer_unpack), lld(timer_lower_bound),
				 lld(timer_compare));
			// All's ok, keep it untouched
			return data;
		}
	}

	const int64_t timer_update = timer.restart();

	indexes.shard_id = request->shard_id;
	indexes.shard_count = request->shard_count;

	msgpack::sbuffer buffer;
	msgpack::pack(&buffer, indexes);

	const int64_t timer_pack = timer.restart();

	data_buffer new_buffer(DNET_INDEX_TABLE_MAGIC_SIZE + buffer.size());
	new_buffer.write(dnet_bswap64(DNET_INDEX_TABLE_MAGIC));
	new_buffer.write(buffer.data(), buffer.size());

	const int64_t timer_write = timer.restart();

	DNET_DUMP_ID_LEN(id_str, cmd_id, DNET_DUMP_NUM);
	typedef long long int lld;
	dnet_log(node, DNET_LOG_INFO, "INDEXES_INTERNAL: convert: id: %s, data size: %zu, new data size: %zu,"
		 "unpack: %lld ms, lower_bound: %lld ms, update: %lld ms, pack: %lld ms, write: %lld ms\n",
		 id_str, data.size(), new_buffer.size(), lld(timer_unpack), lld(timer_lower_bound),
		 lld(timer_update), lld(timer_pack), lld(timer_write));

	return std::move(new_buffer);
}