void store_data(write_args* args)
{
KineticEntry* entry = &(args->entry);
int32_t objIndex = 0;
while (ByteBuffer_BytesRemaining(args->data) > 0) {
// Configure entry meta-data
ByteBuffer_Reset(&entry->key);
ByteBuffer_AppendCString(&entry->key, args->keyPrefix);
char keySuffix[8];
snprintf(keySuffix, sizeof(keySuffix), "%02d", objIndex);
ByteBuffer_AppendCString(&entry->key, keySuffix);
// Prepare entry with the next object to store
ByteBuffer_Reset(&entry->value);
ByteBuffer_AppendArray(
&entry->value,
ByteBuffer_Consume(
&args->data,
MIN(ByteBuffer_BytesRemaining(args->data), KINETIC_OBJ_SIZE))
);
// Store the object
KineticStatus status = KineticClient_Put(args->session, entry, NULL);
if (status != KINETIC_STATUS_SUCCESS) {
fprintf(stderr, "Kinetic PUT of object %d to host %s failed w/ status: %s\n",
objIndex, args->ip, Kinetic_GetStatusDescription(status));
exit(-1);
}
objIndex++;
}
printf("File stored on Kinetic Device across %d entries\n", objIndex);
}
bool Copy_Com__Seagate__Kinetic__Proto__Command__Range_to_ByteBufferArray(Com__Seagate__Kinetic__Proto__Command__Range* keyRange, ByteBufferArray* keys)
{
bool bufferOverflow = false;
LOGF2("Copying: keyRange=0x%0llX, keys=0x%0llX, max_keys=%lld", keyRange, keys->buffers, keys->count);
if (keyRange != NULL && keys->count > 0 && keys != NULL) {
for (size_t i = 0; i < MIN((size_t)keys->count, (size_t)keyRange->n_keys); i++) {
ByteBuffer_Reset(&keys->buffers[i]);
if (ByteBuffer_Append(&keys->buffers[i], keyRange->keys[i].data, keyRange->keys[i].len) == NULL) {
LOGF2("WANRNING: Buffer overrun for keys[%zd]", i);
bufferOverflow = true;
}
}
keys->used = keyRange->n_keys;
}
return !bufferOverflow;
}
static void build_get_command(KineticOperation* const op,
KineticEntry* const entry,
KineticOperationCallback cb,
Com__Seagate__Kinetic__Proto__Command__MessageType command_id)
{
KineticOperation_ValidateOperation(op);
op->request->message.command.header->messagetype = command_id;
op->request->message.command.header->has_messagetype = true;
op->entry = entry;
KineticMessage_ConfigureKeyValue(&op->request->message, entry);
if (op->entry->value.array.data != NULL) {
ByteBuffer_Reset(&op->entry->value);
op->value.data = op->entry->value.array.data;
op->value.len = op->entry->value.bytesUsed;
}
op->opCallback = cb;
}
KineticStatus KineticBuilder_BuildDelete(KineticOperation* const op,
KineticEntry* const entry)
{
KineticOperation_ValidateOperation(op);
op->request->message.command.header->messagetype = COM__SEAGATE__KINETIC__PROTO__COMMAND__MESSAGE_TYPE__DELETE;
op->request->message.command.header->has_messagetype = true;
op->entry = entry;
KineticMessage_ConfigureKeyValue(&op->request->message, op->entry);
if (op->entry->value.array.data != NULL) {
ByteBuffer_Reset(&op->entry->value);
op->value.data = op->entry->value.array.data;
op->value.len = op->entry->value.bytesUsed;
}
op->opCallback = &KineticCallbacks_Delete;
return KINETIC_STATUS_SUCCESS;
}
void test_put_get_delete_one_entry_by_one_entry(void)
{
uint8_t version_data[DEFAULT_BUFFER_SIZE], tag_data[DEFAULT_BUFFER_SIZE], value_data[DEFAULT_BUFFER_SIZE];
ByteBuffer version_buffer, tag_buffer, value_buffer;
version_buffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0");
ExpectedVersionBuffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0");
tag_buffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue");
ExpectedTagBuffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue");
value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0);
unsigned int i;
for (i=0; i<KV_PAIRS_PER_GROUP; i++)
{
// put object
KineticEntry putEntry = {
.key = generate_entry_key_by_index(i),
.tag = tag_buffer,
.newVersion = version_buffer,
.algorithm = KINETIC_ALGORITHM_SHA1,
.value = generate_entry_value_by_index(i),
.force = true,
.synchronization = KINETIC_SYNCHRONIZATION_WRITETHROUGH,
};
KineticStatus status = KineticClient_Put(Fixture.session, &putEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status);
ByteBuffer_Reset(&value_buffer);
// get object
value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0);
KineticEntry getEntry = {
.key = generate_entry_key_by_index(i),
.dbVersion = version_buffer,
.tag = tag_buffer,
.value = value_buffer,
};
status = KineticClient_Get(Fixture.session, &getEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status);
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, getEntry.dbVersion);
TEST_ASSERT_ByteBuffer_NULL(getEntry.newVersion);
TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_key_by_index(i), getEntry.key);
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedTagBuffer, getEntry.tag);
TEST_ASSERT_EQUAL(KINETIC_ALGORITHM_SHA1, getEntry.algorithm);
TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_value_by_index(i), getEntry.value);
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer);
// delete object
KineticEntry deleteEntry = {
.key = generate_entry_key_by_index(i),
.dbVersion = version_buffer,
};
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer);
status = KineticClient_Delete(Fixture.session, &deleteEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status);
TEST_ASSERT_EQUAL(0, deleteEntry.value.bytesUsed);
// get object again
status = KineticClient_Get(Fixture.session, &getEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_NOT_FOUND, status);
}
}
void test_put_get_delete_one_group_by_one_group(void)
{
uint8_t version_data[DEFAULT_BUFFER_SIZE], tag_data[DEFAULT_BUFFER_SIZE], value_data[DEFAULT_BUFFER_SIZE];
ByteBuffer version_buffer, tag_buffer, value_buffer;
version_buffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0");
ExpectedVersionBuffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0");
tag_buffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue");
ExpectedTagBuffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue");
value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0);
unsigned int i, j;
for (i =0; i<TOTAL_GROUPS; i++)
{
KineticStatus status;
// put a group of entries
for (j=0; j<KV_PAIRS_PER_GROUP; j++)
{
KineticEntry putEntry = {
.key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j),
.tag = tag_buffer,
.newVersion = version_buffer,
.algorithm = KINETIC_ALGORITHM_SHA1,
.value = generate_entry_value_by_index(i*KV_PAIRS_PER_GROUP + j),
.force = true,
.synchronization = KINETIC_SYNCHRONIZATION_WRITETHROUGH,
};
status = KineticClient_Put(Fixture.session, &putEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status);
}
// get key_range
KineticKeyRange range = {
.startKey = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP),
.endKey = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + KV_PAIRS_PER_GROUP -1),
.startKeyInclusive = true,
.endKeyInclusive = true,
.maxReturned = KV_PAIRS_PER_GROUP,
};
ByteBuffer keyBuff[KV_PAIRS_PER_GROUP];
uint8_t keysData[KV_PAIRS_PER_GROUP][DEFAULT_BUFFER_SIZE];
for (j = 0; j < KV_PAIRS_PER_GROUP; j++) {
memset(&keysData[j], 0, DEFAULT_BUFFER_SIZE);
keyBuff[j] = ByteBuffer_Create(&keysData[j], DEFAULT_BUFFER_SIZE, 0);
}
ByteBufferArray keys = {.buffers = keyBuff, .count = KV_PAIRS_PER_GROUP, .used = 0};
status = KineticClient_GetKeyRange(Fixture.session, &range, &keys, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status);
TEST_ASSERT_EQUAL(KV_PAIRS_PER_GROUP, keys.used);
for (j = 0; j < KV_PAIRS_PER_GROUP; j++)
{
TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), keys.buffers[j]);
}
// delete a group of entries
for (j=0; j<KV_PAIRS_PER_GROUP; j++)
{
ByteBuffer_Reset(&value_buffer);
// get object
value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0);
KineticEntry getEntry = {
.key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j),
.dbVersion = version_buffer,
.tag = tag_buffer,
.value = value_buffer,
};
status = KineticClient_Get(Fixture.session, &getEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status);
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, getEntry.dbVersion);
TEST_ASSERT_ByteBuffer_NULL(getEntry.newVersion);
TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), getEntry.key);
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedTagBuffer, getEntry.tag);
TEST_ASSERT_EQUAL(KINETIC_ALGORITHM_SHA1, getEntry.algorithm);
TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_value_by_index(i*KV_PAIRS_PER_GROUP + j), getEntry.value);
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer);
}
// delete a group of entries
for (j=0; j<KV_PAIRS_PER_GROUP; j++)
{
// delete object
KineticEntry deleteEntry = {
.key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j),
.dbVersion = version_buffer,
};
TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer);
status = KineticClient_Delete(Fixture.session, &deleteEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status);
TEST_ASSERT_EQUAL(0, deleteEntry.value.bytesUsed);
// get object again
ByteBuffer_Reset(&value_buffer);
// get object
value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0);
KineticEntry getEntry = {
.key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j),
.dbVersion = version_buffer,
.tag = tag_buffer,
.value = value_buffer,
};
status = KineticClient_Get(Fixture.session, &getEntry, NULL);
TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_NOT_FOUND, status);
}
}
}
KineticStatus KineticPDU_ReceiveMain(KineticPDU* const response)
{
assert(response != NULL);
assert(response->connection != NULL);
const int fd = response->connection->socket;
assert(fd >= 0);
LOGF1("\nReceiving PDU via fd=%d", fd);
KineticStatus status;
KineticMessage* msg = &response->protoData.message;
// Receive the PDU header
ByteBuffer rawHeader =
ByteBuffer_Create(&response->headerNBO, sizeof(KineticPDUHeader), 0);
status = KineticSocket_Read(fd, &rawHeader, rawHeader.array.len);
if (status != KINETIC_STATUS_SUCCESS) {
LOG0("Failed to receive PDU header!");
return status;
}
else {
LOG3("PDU header received successfully");
KineticPDUHeader* headerNBO = &response->headerNBO;
response->header = (KineticPDUHeader) {
.versionPrefix = headerNBO->versionPrefix,
.protobufLength = KineticNBO_ToHostU32(headerNBO->protobufLength),
.valueLength = KineticNBO_ToHostU32(headerNBO->valueLength),
};
KineticLogger_LogHeader(1, &response->header);
}
// Receive the protobuf message
status = KineticSocket_ReadProtobuf(fd, response);
if (status != KINETIC_STATUS_SUCCESS) {
LOG0("Failed to receive PDU protobuf message!");
return status;
}
else {
LOG3("Received PDU protobuf");
KineticLogger_LogProtobuf(2, response->proto);
}
// Validate the HMAC for the recevied protobuf message
if (response->proto->authType == KINETIC_PROTO_MESSAGE_AUTH_TYPE_HMACAUTH) {
if(!KineticHMAC_Validate(
response->proto, response->connection->session.hmacKey)) {
LOG0("Received PDU protobuf message has invalid HMAC!");
msg->has_command = true;
msg->command.status = &msg->status;
msg->status.code = KINETIC_PROTO_COMMAND_STATUS_STATUS_CODE_DATA_ERROR;
return KINETIC_STATUS_DATA_ERROR;
}
else {
LOG3("Received protobuf HMAC validation succeeded");
}
}
else if (response->proto->authType == KINETIC_PROTO_MESSAGE_AUTH_TYPE_PINAUTH) {
LOG0("PIN-based authentication not yet supported!");
return KINETIC_STATUS_DATA_ERROR;
}
else if (response->proto->authType == KINETIC_PROTO_MESSAGE_AUTH_TYPE_UNSOLICITEDSTATUS) {
LOG3("Unsolicited status message is not authenticated");
}
// Extract embedded command, if supplied
KineticProto_Message* pMsg = response->proto;
if (pMsg->has_commandBytes &&
pMsg->commandBytes.data != NULL &&
pMsg->commandBytes.len > 0) {
response->command = KineticProto_command__unpack(
NULL,
pMsg->commandBytes.len,
pMsg->commandBytes.data);
}
status = KineticPDU_GetStatus(response);
if (status == KINETIC_STATUS_SUCCESS) {
LOG2("PDU received successfully!");
}
return status;
}
KineticStatus KineticPDU_ReceiveValue(int socket_desc, ByteBuffer* value, size_t value_length)
{
assert(socket_desc >= 0);
assert(value != NULL);
assert(value->array.data != NULL);
// Receive value payload
LOGF1("Receiving value payload (%lld bytes)...", value_length);
ByteBuffer_Reset(value);
KineticStatus status = KineticSocket_Read(socket_desc, value, value_length);
if (status != KINETIC_STATUS_SUCCESS) {
LOG0("Failed to receive PDU value payload!");
return status;
}
LOG1("Received value payload successfully");
KineticLogger_LogByteBuffer(3, "Value Buffer", *value);
return status;
}
ByteArray ProtobufCBinaryData_to_ByteArray(
ProtobufCBinaryData protoData)
{
return (ByteArray) {
.data = protoData.data, .len = protoData.len
};
}
bool Copy_ProtobufCBinaryData_to_ByteBuffer(ByteBuffer dest, ProtobufCBinaryData src)
{
if (src.data == NULL || src.len == 0) {
return false;
}
if (dest.array.data == NULL || dest.array.len < src.len) {
return false;
}
bool success = (memcpy(dest.array.data, src.data, src.len) == dest.array.data);
if (success) {
dest.bytesUsed = src.len;
}
return success;
}
bool Copy_Com__Seagate__Kinetic__Proto__Command__KeyValue_to_KineticEntry(Com__Seagate__Kinetic__Proto__Command__KeyValue* key_value, KineticEntry* entry)
{
bool bufferOverflow = false;
if (key_value != NULL && entry != NULL) {
ByteBuffer_Reset(&entry->dbVersion);
if (key_value->has_dbversion && key_value->dbversion.len > 0) {
if (entry->dbVersion.array.data == NULL || entry->dbVersion.array.len < key_value->dbversion.len) {
entry->dbVersion.bytesUsed = key_value->dbversion.len;
LOG1(" BUFFER_OVERRUN: dbVersion");
bufferOverflow = true;
}
else {
ByteBuffer_Append(&entry->dbVersion, key_value->dbversion.data, key_value->dbversion.len);
}
}
ByteBuffer_Reset(&entry->key);
if (key_value->has_key && key_value->key.len > 0) {
if (entry->key.array.data == NULL || entry->key.array.len < key_value->key.len) {
entry->key.bytesUsed = key_value->key.len;
LOG1(" BUFFER_OVERRUN: key");
bufferOverflow = true;
}
else {
ByteBuffer_Append(&entry->key, key_value->key.data, key_value->key.len);
}
}
ByteBuffer_Reset(&entry->tag);
if (key_value->has_tag && key_value->tag.len > 0) {
if (entry->tag.array.data == NULL || entry->tag.array.len < key_value->tag.len) {
entry->tag.bytesUsed = key_value->tag.len;
LOG1(" BUFFER_OVERRUN: tag");
bufferOverflow = true;
}
else {
ByteBuffer_Append(&entry->tag, key_value->tag.data, key_value->tag.len);
}
}
if (key_value->has_algorithm) {
entry->algorithm =
KineticAlgorithm_from_Com__Seagate__Kinetic__Proto__Command__Algorithm(
key_value->algorithm);
}
}
return !bufferOverflow;
}
