void store_data(write_args* args) { KineticEntry* entry = &(args->entry); int32_t objIndex = 0; while (ByteBuffer_BytesRemaining(args->data) > 0) { // Configure entry meta-data ByteBuffer_Reset(&entry->key); ByteBuffer_AppendCString(&entry->key, args->keyPrefix); char keySuffix[8]; snprintf(keySuffix, sizeof(keySuffix), "%02d", objIndex); ByteBuffer_AppendCString(&entry->key, keySuffix); // Prepare entry with the next object to store ByteBuffer_Reset(&entry->value); ByteBuffer_AppendArray( &entry->value, ByteBuffer_Consume( &args->data, MIN(ByteBuffer_BytesRemaining(args->data), KINETIC_OBJ_SIZE)) ); // Store the object KineticStatus status = KineticClient_Put(args->session, entry, NULL); if (status != KINETIC_STATUS_SUCCESS) { fprintf(stderr, "Kinetic PUT of object %d to host %s failed w/ status: %s\n", objIndex, args->ip, Kinetic_GetStatusDescription(status)); exit(-1); } objIndex++; } printf("File stored on Kinetic Device across %d entries\n", objIndex); }
bool Copy_Com__Seagate__Kinetic__Proto__Command__Range_to_ByteBufferArray(Com__Seagate__Kinetic__Proto__Command__Range* keyRange, ByteBufferArray* keys) { bool bufferOverflow = false; LOGF2("Copying: keyRange=0x%0llX, keys=0x%0llX, max_keys=%lld", keyRange, keys->buffers, keys->count); if (keyRange != NULL && keys->count > 0 && keys != NULL) { for (size_t i = 0; i < MIN((size_t)keys->count, (size_t)keyRange->n_keys); i++) { ByteBuffer_Reset(&keys->buffers[i]); if (ByteBuffer_Append(&keys->buffers[i], keyRange->keys[i].data, keyRange->keys[i].len) == NULL) { LOGF2("WANRNING: Buffer overrun for keys[%zd]", i); bufferOverflow = true; } } keys->used = keyRange->n_keys; } return !bufferOverflow; }
static void build_get_command(KineticOperation* const op, KineticEntry* const entry, KineticOperationCallback cb, Com__Seagate__Kinetic__Proto__Command__MessageType command_id) { KineticOperation_ValidateOperation(op); op->request->message.command.header->messagetype = command_id; op->request->message.command.header->has_messagetype = true; op->entry = entry; KineticMessage_ConfigureKeyValue(&op->request->message, entry); if (op->entry->value.array.data != NULL) { ByteBuffer_Reset(&op->entry->value); op->value.data = op->entry->value.array.data; op->value.len = op->entry->value.bytesUsed; } op->opCallback = cb; }
KineticStatus KineticBuilder_BuildDelete(KineticOperation* const op, KineticEntry* const entry) { KineticOperation_ValidateOperation(op); op->request->message.command.header->messagetype = COM__SEAGATE__KINETIC__PROTO__COMMAND__MESSAGE_TYPE__DELETE; op->request->message.command.header->has_messagetype = true; op->entry = entry; KineticMessage_ConfigureKeyValue(&op->request->message, op->entry); if (op->entry->value.array.data != NULL) { ByteBuffer_Reset(&op->entry->value); op->value.data = op->entry->value.array.data; op->value.len = op->entry->value.bytesUsed; } op->opCallback = &KineticCallbacks_Delete; return KINETIC_STATUS_SUCCESS; }
void test_put_get_delete_one_entry_by_one_entry(void) { uint8_t version_data[DEFAULT_BUFFER_SIZE], tag_data[DEFAULT_BUFFER_SIZE], value_data[DEFAULT_BUFFER_SIZE]; ByteBuffer version_buffer, tag_buffer, value_buffer; version_buffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0"); ExpectedVersionBuffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0"); tag_buffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue"); ExpectedTagBuffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue"); value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0); unsigned int i; for (i=0; i<KV_PAIRS_PER_GROUP; i++) { // put object KineticEntry putEntry = { .key = generate_entry_key_by_index(i), .tag = tag_buffer, .newVersion = version_buffer, .algorithm = KINETIC_ALGORITHM_SHA1, .value = generate_entry_value_by_index(i), .force = true, .synchronization = KINETIC_SYNCHRONIZATION_WRITETHROUGH, }; KineticStatus status = KineticClient_Put(Fixture.session, &putEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status); ByteBuffer_Reset(&value_buffer); // get object value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0); KineticEntry getEntry = { .key = generate_entry_key_by_index(i), .dbVersion = version_buffer, .tag = tag_buffer, .value = value_buffer, }; status = KineticClient_Get(Fixture.session, &getEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status); TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, getEntry.dbVersion); TEST_ASSERT_ByteBuffer_NULL(getEntry.newVersion); TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_key_by_index(i), getEntry.key); TEST_ASSERT_EQUAL_ByteBuffer(ExpectedTagBuffer, getEntry.tag); TEST_ASSERT_EQUAL(KINETIC_ALGORITHM_SHA1, getEntry.algorithm); TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_value_by_index(i), getEntry.value); TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer); // delete object KineticEntry deleteEntry = { .key = generate_entry_key_by_index(i), .dbVersion = version_buffer, }; TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer); status = KineticClient_Delete(Fixture.session, &deleteEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status); TEST_ASSERT_EQUAL(0, deleteEntry.value.bytesUsed); // get object again status = KineticClient_Get(Fixture.session, &getEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_NOT_FOUND, status); } } void test_put_get_delete_one_group_by_one_group(void) { uint8_t version_data[DEFAULT_BUFFER_SIZE], tag_data[DEFAULT_BUFFER_SIZE], value_data[DEFAULT_BUFFER_SIZE]; ByteBuffer version_buffer, tag_buffer, value_buffer; version_buffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0"); ExpectedVersionBuffer = ByteBuffer_CreateAndAppendCString(version_data, sizeof(version_data), "v1.0"); tag_buffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue"); ExpectedTagBuffer = ByteBuffer_CreateAndAppendCString(tag_data, sizeof(tag_data), "SomeTagValue"); value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0); unsigned int i, j; for (i =0; i<TOTAL_GROUPS; i++) { KineticStatus status; // put a group of entries for (j=0; j<KV_PAIRS_PER_GROUP; j++) { KineticEntry putEntry = { .key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), .tag = tag_buffer, .newVersion = version_buffer, .algorithm = KINETIC_ALGORITHM_SHA1, .value = generate_entry_value_by_index(i*KV_PAIRS_PER_GROUP + j), .force = true, .synchronization = KINETIC_SYNCHRONIZATION_WRITETHROUGH, }; status = KineticClient_Put(Fixture.session, &putEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status); } // get key_range KineticKeyRange range = { .startKey = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP), .endKey = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + KV_PAIRS_PER_GROUP -1), .startKeyInclusive = true, .endKeyInclusive = true, .maxReturned = KV_PAIRS_PER_GROUP, }; ByteBuffer keyBuff[KV_PAIRS_PER_GROUP]; uint8_t keysData[KV_PAIRS_PER_GROUP][DEFAULT_BUFFER_SIZE]; for (j = 0; j < KV_PAIRS_PER_GROUP; j++) { memset(&keysData[j], 0, DEFAULT_BUFFER_SIZE); keyBuff[j] = ByteBuffer_Create(&keysData[j], DEFAULT_BUFFER_SIZE, 0); } ByteBufferArray keys = {.buffers = keyBuff, .count = KV_PAIRS_PER_GROUP, .used = 0}; status = KineticClient_GetKeyRange(Fixture.session, &range, &keys, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status); TEST_ASSERT_EQUAL(KV_PAIRS_PER_GROUP, keys.used); for (j = 0; j < KV_PAIRS_PER_GROUP; j++) { TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), keys.buffers[j]); } // delete a group of entries for (j=0; j<KV_PAIRS_PER_GROUP; j++) { ByteBuffer_Reset(&value_buffer); // get object value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0); KineticEntry getEntry = { .key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), .dbVersion = version_buffer, .tag = tag_buffer, .value = value_buffer, }; status = KineticClient_Get(Fixture.session, &getEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status); TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, getEntry.dbVersion); TEST_ASSERT_ByteBuffer_NULL(getEntry.newVersion); TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), getEntry.key); TEST_ASSERT_EQUAL_ByteBuffer(ExpectedTagBuffer, getEntry.tag); TEST_ASSERT_EQUAL(KINETIC_ALGORITHM_SHA1, getEntry.algorithm); TEST_ASSERT_EQUAL_ByteBuffer(generate_entry_value_by_index(i*KV_PAIRS_PER_GROUP + j), getEntry.value); TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer); } // delete a group of entries for (j=0; j<KV_PAIRS_PER_GROUP; j++) { // delete object KineticEntry deleteEntry = { .key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), .dbVersion = version_buffer, }; TEST_ASSERT_EQUAL_ByteBuffer(ExpectedVersionBuffer, version_buffer); status = KineticClient_Delete(Fixture.session, &deleteEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_SUCCESS, status); TEST_ASSERT_EQUAL(0, deleteEntry.value.bytesUsed); // get object again ByteBuffer_Reset(&value_buffer); // get object value_buffer = ByteBuffer_Create(value_data, DEFAULT_BUFFER_SIZE, 0); KineticEntry getEntry = { .key = generate_entry_key_by_index(i*KV_PAIRS_PER_GROUP + j), .dbVersion = version_buffer, .tag = tag_buffer, .value = value_buffer, }; status = KineticClient_Get(Fixture.session, &getEntry, NULL); TEST_ASSERT_EQUAL_KineticStatus(KINETIC_STATUS_NOT_FOUND, status); } } }
KineticStatus KineticPDU_ReceiveMain(KineticPDU* const response) { assert(response != NULL); assert(response->connection != NULL); const int fd = response->connection->socket; assert(fd >= 0); LOGF1("\nReceiving PDU via fd=%d", fd); KineticStatus status; KineticMessage* msg = &response->protoData.message; // Receive the PDU header ByteBuffer rawHeader = ByteBuffer_Create(&response->headerNBO, sizeof(KineticPDUHeader), 0); status = KineticSocket_Read(fd, &rawHeader, rawHeader.array.len); if (status != KINETIC_STATUS_SUCCESS) { LOG0("Failed to receive PDU header!"); return status; } else { LOG3("PDU header received successfully"); KineticPDUHeader* headerNBO = &response->headerNBO; response->header = (KineticPDUHeader) { .versionPrefix = headerNBO->versionPrefix, .protobufLength = KineticNBO_ToHostU32(headerNBO->protobufLength), .valueLength = KineticNBO_ToHostU32(headerNBO->valueLength), }; KineticLogger_LogHeader(1, &response->header); } // Receive the protobuf message status = KineticSocket_ReadProtobuf(fd, response); if (status != KINETIC_STATUS_SUCCESS) { LOG0("Failed to receive PDU protobuf message!"); return status; } else { LOG3("Received PDU protobuf"); KineticLogger_LogProtobuf(2, response->proto); } // Validate the HMAC for the recevied protobuf message if (response->proto->authType == KINETIC_PROTO_MESSAGE_AUTH_TYPE_HMACAUTH) { if(!KineticHMAC_Validate( response->proto, response->connection->session.hmacKey)) { LOG0("Received PDU protobuf message has invalid HMAC!"); msg->has_command = true; msg->command.status = &msg->status; msg->status.code = KINETIC_PROTO_COMMAND_STATUS_STATUS_CODE_DATA_ERROR; return KINETIC_STATUS_DATA_ERROR; } else { LOG3("Received protobuf HMAC validation succeeded"); } } else if (response->proto->authType == KINETIC_PROTO_MESSAGE_AUTH_TYPE_PINAUTH) { LOG0("PIN-based authentication not yet supported!"); return KINETIC_STATUS_DATA_ERROR; } else if (response->proto->authType == KINETIC_PROTO_MESSAGE_AUTH_TYPE_UNSOLICITEDSTATUS) { LOG3("Unsolicited status message is not authenticated"); } // Extract embedded command, if supplied KineticProto_Message* pMsg = response->proto; if (pMsg->has_commandBytes && pMsg->commandBytes.data != NULL && pMsg->commandBytes.len > 0) { response->command = KineticProto_command__unpack( NULL, pMsg->commandBytes.len, pMsg->commandBytes.data); } status = KineticPDU_GetStatus(response); if (status == KINETIC_STATUS_SUCCESS) { LOG2("PDU received successfully!"); } return status; } KineticStatus KineticPDU_ReceiveValue(int socket_desc, ByteBuffer* value, size_t value_length) { assert(socket_desc >= 0); assert(value != NULL); assert(value->array.data != NULL); // Receive value payload LOGF1("Receiving value payload (%lld bytes)...", value_length); ByteBuffer_Reset(value); KineticStatus status = KineticSocket_Read(socket_desc, value, value_length); if (status != KINETIC_STATUS_SUCCESS) { LOG0("Failed to receive PDU value payload!"); return status; } LOG1("Received value payload successfully"); KineticLogger_LogByteBuffer(3, "Value Buffer", *value); return status; }
ByteArray ProtobufCBinaryData_to_ByteArray( ProtobufCBinaryData protoData) { return (ByteArray) { .data = protoData.data, .len = protoData.len }; } bool Copy_ProtobufCBinaryData_to_ByteBuffer(ByteBuffer dest, ProtobufCBinaryData src) { if (src.data == NULL || src.len == 0) { return false; } if (dest.array.data == NULL || dest.array.len < src.len) { return false; } bool success = (memcpy(dest.array.data, src.data, src.len) == dest.array.data); if (success) { dest.bytesUsed = src.len; } return success; } bool Copy_Com__Seagate__Kinetic__Proto__Command__KeyValue_to_KineticEntry(Com__Seagate__Kinetic__Proto__Command__KeyValue* key_value, KineticEntry* entry) { bool bufferOverflow = false; if (key_value != NULL && entry != NULL) { ByteBuffer_Reset(&entry->dbVersion); if (key_value->has_dbversion && key_value->dbversion.len > 0) { if (entry->dbVersion.array.data == NULL || entry->dbVersion.array.len < key_value->dbversion.len) { entry->dbVersion.bytesUsed = key_value->dbversion.len; LOG1(" BUFFER_OVERRUN: dbVersion"); bufferOverflow = true; } else { ByteBuffer_Append(&entry->dbVersion, key_value->dbversion.data, key_value->dbversion.len); } } ByteBuffer_Reset(&entry->key); if (key_value->has_key && key_value->key.len > 0) { if (entry->key.array.data == NULL || entry->key.array.len < key_value->key.len) { entry->key.bytesUsed = key_value->key.len; LOG1(" BUFFER_OVERRUN: key"); bufferOverflow = true; } else { ByteBuffer_Append(&entry->key, key_value->key.data, key_value->key.len); } } ByteBuffer_Reset(&entry->tag); if (key_value->has_tag && key_value->tag.len > 0) { if (entry->tag.array.data == NULL || entry->tag.array.len < key_value->tag.len) { entry->tag.bytesUsed = key_value->tag.len; LOG1(" BUFFER_OVERRUN: tag"); bufferOverflow = true; } else { ByteBuffer_Append(&entry->tag, key_value->tag.data, key_value->tag.len); } } if (key_value->has_algorithm) { entry->algorithm = KineticAlgorithm_from_Com__Seagate__Kinetic__Proto__Command__Algorithm( key_value->algorithm); } } return !bufferOverflow; }