bool TextRecordReader::NextRecord() {
line_.clear();
int start_offset = file_offset_;
int read_size = 0;
if (file_offset_ < end_offset_) {
read_size = reader_->ReadLine(&line_);
file_offset_ += read_size;
}
if (read_size == 0) {
// No more data to read.
current_key_.reset();
current_value_.reset();
return false;
}
// Serialize offset as an integer.
key_buffer_.clear();
StringOutputStream key_output(&key_buffer_);
SerializationHandler<int>::Serialize(&start_offset, &key_output);
// Serialize line as a string.
value_buffer_.clear();
StringOutputStream value_output(&value_buffer_);
SerializationHandler<std::string>::Serialize(&line_, &value_output);
// Reset key and value input streams.
// FIXME: should use resettable input stream instead.
current_key_.reset(new ArrayInputStream(string_as_array(&key_buffer_),
key_buffer_.size()));
current_value_.reset(new ArrayInputStream(string_as_array(&value_buffer_),
value_buffer_.size()));
return true;
}
bool ReducerContext<KeyIn, ValueIn, KeyOut, ValueOut>::NextPair() {
if (exhausted_) {
// Shouldn't have been called.
return false;
}
// Deserialize current key/value.
SerializationHandler<KeyIn>::Deserialize(input_->current_key(), &key_);
SerializationHandler<ValueIn>::Deserialize(input_->current_value(), &value_);
// Read next pair.
exhausted_ = !input_->NextRecord();
// Check if it has the same as the current key.
if (exhausted_) {
same_as_previous_key_ = false;
} else {
// Read key into buffer.
key_buffer_.clear();
const uint8* key_buffer;
int key_size;
input_->get_key_buffer(&key_buffer, &key_size);
if (static_cast<int>(previous_key_.size()) != key_size) {
same_as_previous_key_ = false;
} else {
same_as_previous_key_ = 0 == memcmp(reinterpret_cast<const void*>(key_buffer),
reinterpret_cast<const void*>(string_as_array(&previous_key_)),
key_size);
}
// Save key.
previous_key_.clear();
previous_key_.append(reinterpret_cast<const char*>(key_buffer), key_size);
}
return true;
}
TEST(ExtensionSetTest, PackedSerializationToArray) {
// Serialize as TestPackedExtensions and parse as TestPackedTypes to insure
// wire compatibility of extensions.
//
// This checks serialization to a flat array by explicitly reserving space in
// the string and calling the generated message's
// SerializeWithCachedSizesToArray.
unittest::TestPackedExtensions source;
unittest::TestPackedTypes destination;
TestUtil::SetPackedExtensions(&source);
int size = source.ByteSize();
string data;
data.resize(size);
uint8* target = reinterpret_cast<uint8*>(string_as_array(&data));
uint8* end = source.SerializeWithCachedSizesToArray(target);
EXPECT_EQ(size, end - target);
EXPECT_TRUE(destination.ParseFromString(data));
TestUtil::ExpectPackedFieldsSet(destination);
}
TEST(ExtensionSetTest, PackedSerializationToStream) {
// Serialize as TestPackedExtensions and parse as TestPackedTypes to insure
// wire compatibility of extensions.
//
// This checks serialization to an output stream by creating an array output
// stream that can only buffer 1 byte at a time - this prevents the message
// from ever jumping to the fast path, ensuring that serialization happens via
// the CodedOutputStream.
unittest::TestPackedExtensions source;
unittest::TestPackedTypes destination;
TestUtil::SetPackedExtensions(&source);
int size = source.ByteSize();
string data;
data.resize(size);
{
io::ArrayOutputStream array_stream(string_as_array(&data), size, 1);
io::CodedOutputStream output_stream(&array_stream);
source.SerializeWithCachedSizes(&output_stream);
ASSERT_FALSE(output_stream.HadError());
}
EXPECT_TRUE(destination.ParseFromString(data));
TestUtil::ExpectPackedFieldsSet(destination);
}
// Get immutable buffer for current key.
virtual void get_key_buffer(const uint8** buffer, int* size) {
// Default implementation.
CopyStreamIntoString(current_key(), default_key_buffer_);
*buffer = reinterpret_cast<const uint8*>(string_as_array(&default_key_buffer_));
*size = default_key_buffer_.size();
}
std::string SHA256HashString(const std::string& str)
{
std::string output(SHA256_LENGTH, 0);
SHA256HashString(str, string_as_array(&output), output.size());
return output;
}