TEST(TestReedSolomonCodes, test_encode_decode)
{
{
kodo::rs_encoder<fifi::binary8>::factory encoder_factory(255, 1600);
kodo::rs_decoder<fifi::binary8>::factory decoder_factory(255, 1600);
uint32_t symbols = rand_symbols(255);
uint32_t symbol_size = rand_symbol_size();
encoder_factory.set_symbols(symbols);
encoder_factory.set_symbol_size(symbol_size);
decoder_factory.set_symbols(symbols);
decoder_factory.set_symbol_size(symbol_size);
auto encoder = encoder_factory.build();
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
encoder->set_symbols(sak::storage(data_in));
uint32_t symbol_count = 0;
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
++symbol_count;
}
// A reed solomon code should be able to decode
// with exactly k symbols
EXPECT_EQ(symbol_count, symbols);
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
}
int req_put(int sockfd, struct message_s *msg)
{
char *filepath = payload_malloc(sockfd, msg, true);
char *repopath = make_path(filepath);
printf("client put %s\n", repopath);
int saveto_fd = open(repopath, O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR);
free(filepath);
free(repopath);
if (saveto_fd == -1) {
perror("cannot open file to write");
close_serving_thread(sockfd);
}
/* always says that you can upload */
write_head(sockfd, TYPE_PUT_REP, 1, 0);
/* receive DATA_FILE */
struct message_s recv_msg;
read_head(sockfd, &recv_msg);
off_t size = payload_size(&recv_msg);
int ret = transfer_file_copy(saveto_fd, sockfd, size, NULL);
close(saveto_fd);
if (ret == -1)
close_serving_thread(sockfd);
return 0;
}
int dir_read(struct object_heap* oh, int dirHandle, struct ByteArray *entNameBuffer) {
struct dirent ent;
struct dirent *result = &ent;
int resultCode, entLen;
if (dirHandle < 0)
return -EINVAL;
#if defined(WIN32) || defined(__CYGWIN__)
result = readdir(oh->dir_index[dirHandle]);
resultCode = errno;
#else
resultCode = readdir_r(oh->dir_index[dirHandle], result, &result);
#endif
if (resultCode != 0) {
/* An error situation. */
assert(resultCode >= 0); /* TODO: read SUSv2. The Darwin manpage is unclear about this */
return -resultCode;
}
if (result == NULL) {
/* End of the directory. */
return 0;
}
entLen = strlen(result->d_name);
if (entLen == 0) {
/* Bizarre! Make up a reasonable response. */
return -ENOENT;
}
assert(payload_size((struct Object *) entNameBuffer) >= entLen);
memcpy(entNameBuffer->elements, result->d_name, entLen);
return entLen;
}
void print_byte_array(struct Object* o) {
word_t i;
char* elements = (char*)inc_ptr(o, object_array_offset(o));
fprintf(stderr, "'");
for (i=0; i < payload_size(o); i++) {
if (elements[i] >= 32 && elements[i] <= 126) {
fprintf(stderr, "%c", elements[i]);
} else {
fprintf(stderr, "\\x%02" PRIxPTR "", (word_t)elements[i]);
}
if (i > 10 && payload_size(o) - i > 100) {
i = payload_size(o) - 20;
fprintf(stderr, "{..snip..}");
}
}
fprintf(stderr, "'");
}
int dir_getcwd(struct ByteArray *pathBuffer) {
#ifdef WIN32
return GetCurrentDirectory((DWORD)payload_size((struct Object *) pathBuffer), pathBuffer->elements);
#else
return ((getcwd((char *)pathBuffer->elements, byte_array_size(pathBuffer)) == NULL)
? -errno : strlen((const char *)pathBuffer->elements));
#endif
}
size_t layer_size(enum Level level, const cap_head* caphead){
/* layer size at physical is not supported, traces does not include necessary data */
if ( level == LEVEL_INVALID || level == LEVEL_PHYSICAL ){
return 0;
}
const enum Level q = (enum Level)((int)level - 1);
return payload_size(q, caphead);
}
/* Executes a new request. A retried request never reach that function (send
* and writes are discarded, and reads and atomics are retried elsewhere.
*/
static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
{
enum resp_states err;
if (pkt->mask & RXE_SEND_MASK) {
if (qp_type(qp) == IB_QPT_UD ||
qp_type(qp) == IB_QPT_SMI ||
qp_type(qp) == IB_QPT_GSI) {
union rdma_network_hdr hdr;
struct sk_buff *skb = PKT_TO_SKB(pkt);
memset(&hdr, 0, sizeof(hdr));
if (skb->protocol == htons(ETH_P_IP))
memcpy(&hdr.roce4grh, ip_hdr(skb), sizeof(hdr.roce4grh));
else if (skb->protocol == htons(ETH_P_IPV6))
memcpy(&hdr.ibgrh, ipv6_hdr(skb), sizeof(hdr.ibgrh));
err = send_data_in(qp, &hdr, sizeof(hdr));
if (err)
return err;
}
err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));
if (err)
return err;
} else if (pkt->mask & RXE_WRITE_MASK) {
err = write_data_in(qp, pkt);
if (err)
return err;
} else if (pkt->mask & RXE_READ_MASK) {
/* For RDMA Read we can increment the msn now. See C9-148. */
qp->resp.msn++;
return RESPST_READ_REPLY;
} else if (pkt->mask & RXE_ATOMIC_MASK) {
err = process_atomic(qp, pkt);
if (err)
return err;
} else
/* Unreachable */
WARN_ON(1);
/* We successfully processed this new request. */
qp->resp.msn++;
/* next expected psn, read handles this separately */
qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
qp->resp.opcode = pkt->opcode;
qp->resp.status = IB_WC_SUCCESS;
if (pkt->mask & RXE_COMP_MASK)
return RESPST_COMPLETE;
else if (qp_type(qp) == IB_QPT_RC)
return RESPST_ACKNOWLEDGE;
else
return RESPST_CLEANUP;
}
word_t byte_array_extract_into(struct ByteArray * fromArray, byte_t* targetBuffer, word_t bufferSize)
{
word_t payloadSize = payload_size((struct Object *) fromArray);
if (bufferSize < payloadSize)
return SLATE_ERROR_RETURN;
copy_bytes_into((byte_t*) fromArray -> elements, bufferSize, targetBuffer);
return payloadSize;
}
static void
handle_pmt(Mpegts *ts, uint8_t *pkt) {
uint8_t *psi = payload(pkt);
uint16_t size = payload_size(psi,pkt);
if(size < 9+4) return;
if(!psi) return;
if(psi[1] != 0) return;
uint32_t length = ((psi[2] & 0xF) << 8) | psi[3];
if(length < 4 || length + 9 > size) return;
psi += 9;
fprintf(stderr, "PMT %d bytes\r\n", length);
}
/* Executes a new request. A retried request never reach that function (send
* and writes are discarded, and reads and atomics are retried elsewhere.
*/
static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
{
enum resp_states err;
if (pkt->mask & RXE_SEND_MASK) {
if (qp_type(qp) == IB_QPT_UD ||
qp_type(qp) == IB_QPT_SMI ||
qp_type(qp) == IB_QPT_GSI) {
union rdma_network_hdr hdr;
build_rdma_network_hdr(&hdr, pkt);
err = send_data_in(qp, &hdr, sizeof(hdr));
if (err)
return err;
}
err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));
if (err)
return err;
} else if (pkt->mask & RXE_WRITE_MASK) {
err = write_data_in(qp, pkt);
if (err)
return err;
} else if (pkt->mask & RXE_READ_MASK) {
/* For RDMA Read we can increment the msn now. See C9-148. */
qp->resp.msn++;
return RESPST_READ_REPLY;
} else if (pkt->mask & RXE_ATOMIC_MASK) {
err = process_atomic(qp, pkt);
if (err)
return err;
} else {
/* Unreachable */
WARN_ON_ONCE(1);
}
/* next expected psn, read handles this separately */
qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
qp->resp.ack_psn = qp->resp.psn;
qp->resp.opcode = pkt->opcode;
qp->resp.status = IB_WC_SUCCESS;
if (pkt->mask & RXE_COMP_MASK) {
/* We successfully processed this new request. */
qp->resp.msn++;
return RESPST_COMPLETE;
} else if (qp_type(qp) == IB_QPT_RC)
return RESPST_ACKNOWLEDGE;
else
return RESPST_CLEANUP;
}
static void
handle_pat(Mpegts *ts, uint8_t *pkt) {
uint8_t *psi = payload(pkt);
uint16_t size = payload_size(psi,pkt);
if(size < 9+4) return;
if(!psi) return;
if(psi[1] != 0) return;
uint32_t length = ((psi[2] & 0xF) << 8) | psi[3];
if(length < 4 || length + 9 > size) return;
psi += 9;
// extract_pat(<<ProgramNum:16, _:3, Pid:13, PAT/binary>>, Descriptors) ->
ts->pmt = ((psi[2] & 0x1F) << 8) | psi[3];
fprintf(stderr, "PMT pid = %d\r\n", ts->pmt);
}
static inline enum comp_state do_read(struct rxe_qp *qp,
struct rxe_pkt_info *pkt,
struct rxe_send_wqe *wqe)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
int ret;
ret = copy_data(rxe, qp->pd, IB_ACCESS_LOCAL_WRITE,
&wqe->dma, payload_addr(pkt),
payload_size(pkt), to_mem_obj, NULL);
if (ret)
return COMPST_ERROR;
if (wqe->dma.resid == 0 && (pkt->mask & RXE_END_MASK))
return COMPST_COMP_ACK;
else
return COMPST_UPDATE_COMP;
}
static enum resp_states write_data_in(struct rxe_qp *qp,
struct rxe_pkt_info *pkt)
{
enum resp_states rc = RESPST_NONE;
int err;
int data_len = payload_size(pkt);
err = rxe_mem_copy(qp->resp.mr, qp->resp.va, payload_addr(pkt),
data_len, to_mem_obj, NULL);
if (err) {
rc = RESPST_ERR_RKEY_VIOLATION;
goto out;
}
qp->resp.va += data_len;
qp->resp.resid -= data_len;
out:
return rc;
}
int main()
{
// Set the number of symbols (i.e. the generation size in RLNC
// terminology) and the size of a symbol in bytes
uint32_t symbols = 42;
uint32_t symbol_size = 100;
typedef kodo::full_rlnc_encoder<fifi::binary8> rlnc_encoder;
typedef kodo::full_rlnc_decoder<fifi::binary8> rlnc_decoder;
// In the following we will make an encoder/decoder factory.
// The factories are used to build actual encoders/decoders
rlnc_encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
rlnc_decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in(encoder->block_size());
// Just for fun - fill the data with random data
for(auto &e: data_in)
e = rand() % 256;
// Assign the data buffer to the encoder so that we may start
// to produce encoded symbols from it
encoder->set_symbols(sak::storage(data_in));
while( !decoder->is_complete() )
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// Pass that packet to the decoder
decoder->decode( &payload[0] );
}
}
int dir_open(struct object_heap* oh, struct ByteArray *dirName) {
int dirHandle = dir_allocate(oh);
if (dirHandle < 0) {
return dirHandle;
} else {
size_t nameLen = payload_size((struct Object *) dirName);
char *name = (char*)malloc(nameLen + 1);
if (name == NULL) {
return -errno;
} else {
memcpy(name, dirName->elements, nameLen);
name[nameLen] = '\0';
oh->dir_index[dirHandle] = opendir(name);
if (oh->dir_index[dirHandle] == NULL) {
int savedErrno = errno;
free(name);
return -savedErrno;
} else {
free(name);
return dirHandle;
}
}
}
}
int main()
{
// Set the number of symbols (i.e. the generation size in RLNC
// terminology) and the size of a symbol in bytes
uint32_t max_symbols = 42;
uint32_t max_symbol_size = 64;
std::string encode_filename = "encode-file.bin";
// Create a test file for encoding.
std::ofstream encode_file;
encode_file.open (encode_filename, std::ios::binary);
uint32_t file_size = 50000;
std::vector<char> encode_data(file_size);
std::vector<char> decode_data;
// Just write some bytes to the file
for(uint32_t i = 0; i < file_size; ++i)
{
encode_data[i] = rand() % 255;
}
encode_file.write(&encode_data[0], file_size);
encode_file.close();
// Select the encoding and decoding algorithms
typedef kodo::full_rlnc_encoder<fifi::binary>
encoder_t;
typedef kodo::full_rlnc_decoder<fifi::binary>
decoder_t;
// Now for the encoder we use a file_encoder with the chosen
// encoding algorithm
typedef kodo::file_encoder<encoder_t>
file_encoder_t;
// For decoding we use an object_decoder with the chosen
// decoding algorithm
typedef kodo::object_decoder<decoder_t>
object_decoder_t;
// Create the encoder factory - builds the individual encoders used
file_encoder_t::factory encoder_factory(max_symbols, max_symbol_size);
// Create the actual file encoder using the encoder factory and
// the filename of the file to be encoded
file_encoder_t file_encoder(encoder_factory, encode_filename);
// Create the decoder factory - build the individual decoders used
object_decoder_t::factory decoder_factory(max_symbols, max_symbol_size);
// Create the object decoder using the decoder factory and the
// size of the file to be decoded
object_decoder_t object_decoder(decoder_factory, file_size);
// Now in the following loop we go through all the encoders
// needed to encode the entire file. We the build the corresponding
// decoder and decode the chunk immediately. In practice where
// encoders and decoders are on different devices e.g. connected
// over a network, we would have to pass also the encoder and decoder
// index between the source and sink to allow the correct data would
// passed from encoder to corresponding decoder.
for(uint32_t i = 0; i < file_encoder.encoders(); ++i)
{
auto encoder = file_encoder.build(i);
auto decoder = object_decoder.build(i);
// Set the encoder non-systematic
if(kodo::has_systematic_encoder<encoder_t>::value)
kodo::set_systematic_off(encoder);
std::vector<uint8_t> payload(encoder->payload_size());
while( !decoder->is_complete() )
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// In practice send the payload over a network, save it to
// a file etc. Then when needed build and pass it to the decoder
// Pass that packet to the decoder
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
data_out.resize(decoder->bytes_used());
decode_data.insert(decode_data.end(),
data_out.begin(),
data_out.end());
}
// Check we properly decoded the data
if (std::equal(decode_data.begin(),
decode_data.end(), encode_data.begin()))
{
std::cout << "Data decoded correctly" << std::endl;
}
else
{
std::cout << "Unexpected failure to decode "
<< "please file a bug report :)" << std::endl;
}
}
/// @example use_cached_symbol_decoder.cpp
///
/// This example shows how to use the cached symbol decoder to "extract"
/// the symbol coding coefficients and the encoded symbol data from an
/// incoming symbol.
int main()
{
// The finite field we will use in the example. You can try
// with other fields by specifying e.g. fifi::binary8 for the
// extension field 2^8
typedef fifi::binary finite_field;
// Set the number of symbols (i.e. the generation size in RLNC
// terminology) and the size of a symbol in bytes
uint32_t symbols = 8;
uint32_t symbol_size = 160;
// Typdefs for the encoder/decoder type we wish to use
typedef kodo::full_rlnc_encoder<finite_field> rlnc_encoder;
typedef kodo::full_rlnc_decoder<finite_field> rlnc_decoder;
typedef kodo::symbol_info_decoder<finite_field> rlnc_info_decoder;
// In the following we will make an encoder/decoder factory.
// The factories are used to build actual encoders/decoders.
// Each stack we use have their own factories.
rlnc_encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
rlnc_decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
rlnc_info_decoder::factory info_decoder_factory(symbols, symbol_size);
auto info_decoder = info_decoder_factory.build();
// Allocate some storage for a "payload" the payload is what we would
// eventually send over a network
std::vector<uint8_t> payload(encoder->payload_size());
// Allocate some data to encode. In this case we make a buffer
// with the same size as the encoder's block size (the max.
// amount a single encoder can encode)
std::vector<uint8_t> data_in(encoder->block_size());
// Just for fun - fill the data with random data
for(auto &e: data_in)
e = rand() % 256;
// Assign the data buffer to the encoder so that we may start
// to produce encoded symbols from it
encoder->set_symbols(sak::storage(data_in));
while( !decoder->is_complete())
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// Here we "simulate" a packet loss of approximately 50%
// by dropping half of the encoded packets.
// When running this example you will notice that the initial
// symbols are received systematically (i.e. uncoded). After
// sending all symbols once uncoded, the encoder will switch
// to full coding, in which case you will see the full encoding
// vectors being sent and received.
if((rand() % 2) == 0)
continue;
// Pass the encoded packet to the info decoder. After this
// information about the coded symbol can be fetched using the
// cached_symbol_decoder API
info_decoder->decode( &payload[0] );
if(!info_decoder->cached_symbol_coded())
{
// The symbol was uncoded so we may ask the cache which of the
// original symbols we have received.
std::cout << "Symbol was uncoded, index = "
<< info_decoder->cached_symbol_index() << std::endl;
// Now we pass the data directly into our actual decoder. This is
// done using the "Codec API" directly, and not through the "Payload
// API" as we would typically do.
decoder->decode_symbol( info_decoder->cached_symbol_data(),
info_decoder->cached_symbol_index());
}
else
{
// The symbol was coded so we may ask the cache to return
// the coding coefficients used to create the encoded symbol.
std::cout << "Symbol was coded, encoding vector = ";
const uint8_t* c = info_decoder->cached_symbol_coefficients();
// We loop through the coefficient buffer and print the coefficients
for(uint32_t i = 0; i < info_decoder->symbols(); ++i)
{
std::cout << (uint32_t) fifi::get_value<finite_field>(c, i)
<< " ";
}
std::cout << std::endl;
// Pass that packet to the decoder, as with the uncoded symbols
// above we pass it directly to the "Codec API"
decoder->decode_symbol(info_decoder->cached_symbol_data(),
info_decoder->cached_symbol_coefficients());
}
}
// The decoder is complete, now copy the symbols from the decoder
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
// Check we properly decoded the data
if (std::equal(data_out.begin(), data_out.end(), data_in.begin()))
{
std::cout << "Data decoded correctly" << std::endl;
}
else
{
std::cout << "Unexpected failure to decode "
<< "please file a bug report :)" << std::endl;
}
}
inline void run_test_on_the_fly_systematic(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Make sure the encoder is systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_on(encoder);
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
// Simulate some loss
if((rand() % 2) == 0)
continue;
uint32_t old_rank = decoder->rank();
decoder->decode( &payload[0] );
if(decoder->rank() > old_rank)
{
// We has a rank increase
if(encoder->rank() == decoder->rank())
{
// The rank of the encoder matches the decoder, if
// this unit test fails you most likely do not have a
// layer which updates the decoding symbol status when
// rank matches. E.g. look at:
// rank_symbol_decoding_status_updater.hpp
EXPECT_TRUE(kodo::is_partial_complete(decoder));
}
}
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
// set symbol 50% of the time ONLY if rank is not full
if(((rand() % 2) == 0) && (encoder->rank() < symbols))
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
EXPECT_TRUE(encoder->rank() >= decoder->rank());
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
void print_object(struct Object* oop) {
if (oop == NULL) {
fprintf(stderr, "<object NULL>\n");
return;
}
if (oop_is_smallint((word_t)oop)) {
fprintf(stderr, "<object int_value: %" PRIdPTR ">\n", object_to_smallint(oop));
} else {
const char* typestr=0;
switch (object_type(oop)) {
case TYPE_OBJECT: typestr = "normal"; break;
case TYPE_OOP_ARRAY: typestr = "oop array"; break;
case TYPE_BYTE_ARRAY: typestr = "byte array"; break;
}
fprintf(stderr, "<object at %p, hash: 0x%" PRIxPTR ", size: %" PRIdPTR ", payload size: %" PRIdPTR ", type: %s>\n", (void*)oop, object_hash(oop), object_size(oop), payload_size(oop), typestr);
}
}
int main()
{
//! [2]
// Set the number of symbols (i.e. the generation size in RLNC
// terminology) and the size of a symbol in bytes
uint32_t max_symbols = 40;
uint32_t max_symbol_size = 64;
uint32_t object_size = 6400;
//! [3]
//! [4]
using storage_encoder = kodo::object::storage_encoder<
kodo::shallow_full_rlnc_encoder<fifi::binary>,
kodo::fixed_partitioning_scheme>;
using storage_decoder = kodo::object::storage_decoder<
kodo::shallow_full_rlnc_decoder<fifi::binary>,
kodo::fixed_partitioning_scheme>;
//! [5]
storage_encoder::factory encoder_factory(max_symbols, max_symbol_size);
storage_decoder::factory decoder_factory(max_symbols, max_symbol_size);
std::vector<uint8_t> data_out(object_size, '\0');
std::vector<uint8_t> data_in(object_size, 'x');
encoder_factory.set_storage(sak::storage(data_in));
decoder_factory.set_storage(sak::storage(data_out));
auto object_encoder = encoder_factory.build();
auto object_decoder = decoder_factory.build();
std::cout << "Object size = " << object_size << std::endl;
std::cout << "Encoder blocks = " << object_encoder->blocks() << std::endl;
std::cout << "Decoder blocks = " << object_decoder->blocks() << std::endl;
//! [6]
for (uint32_t i = 0; i < object_encoder->blocks(); ++i)
{
std::cout << "Block = " << i << " symbols = "
<< object_encoder->symbols(i) << " symbol size = "
<< object_encoder->symbol_size(i) << std::endl;
}
//! [7]
for (uint32_t i = 0; i < object_encoder->blocks(); ++i)
{
auto e = object_encoder->build(i);
auto d = object_decoder->build(i);
std::vector<uint8_t> payload(e->payload_size());
while (!d->is_complete())
{
e->encode( payload.data() );
// Here we would send and receive the payload over a
// network. Lets throw away some packet to simulate.
if (rand() % 2)
{
continue;
}
d->decode( payload.data() );
}
}
// Check we properly decoded the data
if (data_in == data_out)
{
std::cout << "Data decoded correctly" << std::endl;
}
else
{
std::cout << "Unexpected failure to decode "
<< "please file a bug report :)" << std::endl;
}
}
void print_symbol(struct Symbol* name) {
if (fwrite(&name->elements[0], 1, payload_size((struct Object*)name), stderr) != (size_t)payload_size((struct Object*)name)) {
/*handle error*/
}
}
inline void run_test_on_the_fly_systematic_no_errors(uint32_t symbols,
uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
// Encode/decode operations
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Make sure the encoder is systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_on(encoder);
while( !decoder->is_complete() )
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
if(decoder->rank() > 0)
{
EXPECT_TRUE(kodo::is_partial_complete(decoder));
}
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
// set symbol 50% of the time ONLY if rank is not full
if(encoder->rank() < symbols)
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
EXPECT_TRUE(encoder->rank() >= decoder->rank());
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void run_test_on_the_fly(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Set the encoder non-systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_off(encoder);
EXPECT_EQ(encoder->rank(), 0U);
EXPECT_EQ(decoder->rank(), 0U);
while( !decoder->is_complete() )
{
EXPECT_TRUE(encoder->rank() >= decoder->rank());
encoder->encode( &payload[0] );
// Simulate some loss
if((rand() % 2) == 0)
continue;
decoder->decode( &payload[0] );
if(kodo::is_partial_complete(decoder))
{
// Check that we as many pivot elements as expected and that these
// are decoded
uint32_t symbols_uncoded = 0;
for(uint32_t i = 0; i < decoder->symbols(); ++i)
{
if(!decoder->is_symbol_uncoded(i))
continue;
++symbols_uncoded;
auto symbol_storage =
sak::storage(decoder->symbol(i), decoder->symbol_size());
EXPECT_TRUE(sak::is_equal(symbol_storage, symbol_sequence[i]));
}
EXPECT_EQ(symbols_uncoded, decoder->symbols_uncoded());
}
if(((rand() % 2) == 0) && encoder->rank() < symbols)
{
uint32_t i = encoder->rank();
encoder->set_symbol(i, symbol_sequence[i]);
}
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void run_test_systematic_packets_decode()
{
uint32_t symbols = 5;
uint32_t symbol_size = 1400;
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
auto symbol_sequence = sak::split_storage(
sak::storage(data_in), symbol_size);
// Lets start by specifying the first three symbols
encoder->set_symbol(0, symbol_sequence[0]);
encoder->set_symbol(1, symbol_sequence[1]);
encoder->set_symbol(2, symbol_sequence[2]);
// Make sure the encoder is systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_on(encoder);
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
EXPECT_TRUE(decoder->is_partial_complete());
EXPECT_TRUE(decoder->is_symbol_uncoded(0));
encoder->encode( &payload[0] );
// Simulate a packet loss
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
EXPECT_TRUE(decoder->is_partial_complete());
EXPECT_TRUE(decoder->is_symbol_uncoded(0));
EXPECT_TRUE(decoder->is_symbol_uncoded(2));
// We now have to loop since we are producing coded packets
// and we might generate linear dependent packets
uint32_t loop = 0;
while(decoder->symbols_uncoded() != 3)
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
++loop;
// std::cout << "Loop " << loop << std::endl;
}
EXPECT_TRUE(decoder->is_partial_complete());
EXPECT_TRUE(decoder->is_symbol_uncoded(0));
EXPECT_TRUE(decoder->is_symbol_uncoded(1));
EXPECT_TRUE(decoder->is_symbol_uncoded(2));
encoder->set_symbol(3, symbol_sequence[3]);
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
EXPECT_TRUE(decoder->is_partial_complete());
EXPECT_TRUE(decoder->is_symbol_uncoded(3));
// Nothing should happen now since the encoder contains no
// new symbols
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
EXPECT_FALSE(decoder->is_partial_complete());
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
EXPECT_FALSE(decoder->is_partial_complete());
encoder->set_symbol(4, symbol_sequence[4]);
encoder->encode( &payload[0] );
// Packet loss
while(decoder->symbols_uncoded() != 5)
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
++loop;
// std::cout << "Loop " << loop << std::endl;
}
}
inline void
invoke_reuse_incomplete(uint32_t symbols, uint32_t symbol_size)
{
bool do_complete;
typename Encoder::factory encoder_factory(symbols, symbol_size);
typename Decoder::factory decoder_factory(symbols, symbol_size);
// Use factory a lot of times
for (uint32_t i = 0; i < 100; ++i)
{
// Build coders
auto encoder = encoder_factory.build();
auto decoder = decoder_factory.build();
// Prepare buffers
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in(encoder->block_size());
// Fill with random data
for (auto &e: data_in)
e = rand() % 256;
// Put data in encoder
encoder->set_symbols(sak::storage(data_in));
if (rand() % 100 > 90)
{
do_complete = false;
}
else
{
do_complete = true;
}
// Start encoding/decoding
while (!decoder->is_complete())
{
encoder->encode(&payload[0]);
// Loose a packet with probability
if (rand() % 100 > 90)
continue;
decoder->decode(&payload[0]);
// Stop decoding after a while with probability
if (!do_complete && decoder->rank() == symbols - 2)
break;
}
// Check if completed decoders are correct
if (decoder->is_complete())
{
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
ASSERT_TRUE(sak::equal(sak::storage(data_out),
sak::storage(data_in)));
}
}
}
static enum resp_states check_rkey(struct rxe_qp *qp,
struct rxe_pkt_info *pkt)
{
struct rxe_mem *mem = NULL;
u64 va;
u32 rkey;
u32 resid;
u32 pktlen;
int mtu = qp->mtu;
enum resp_states state;
int access;
if (pkt->mask & (RXE_READ_MASK | RXE_WRITE_MASK)) {
if (pkt->mask & RXE_RETH_MASK) {
qp->resp.va = reth_va(pkt);
qp->resp.rkey = reth_rkey(pkt);
qp->resp.resid = reth_len(pkt);
}
access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ
: IB_ACCESS_REMOTE_WRITE;
} else if (pkt->mask & RXE_ATOMIC_MASK) {
qp->resp.va = atmeth_va(pkt);
qp->resp.rkey = atmeth_rkey(pkt);
qp->resp.resid = sizeof(u64);
access = IB_ACCESS_REMOTE_ATOMIC;
} else {
return RESPST_EXECUTE;
}
/* A zero-byte op is not required to set an addr or rkey. */
if ((pkt->mask & (RXE_READ_MASK | RXE_WRITE_OR_SEND)) &&
(pkt->mask & RXE_RETH_MASK) &&
reth_len(pkt) == 0) {
return RESPST_EXECUTE;
}
va = qp->resp.va;
rkey = qp->resp.rkey;
resid = qp->resp.resid;
pktlen = payload_size(pkt);
mem = lookup_mem(qp->pd, access, rkey, lookup_remote);
if (!mem) {
state = RESPST_ERR_RKEY_VIOLATION;
goto err;
}
if (unlikely(mem->state == RXE_MEM_STATE_FREE)) {
state = RESPST_ERR_RKEY_VIOLATION;
goto err;
}
if (mem_check_range(mem, va, resid)) {
state = RESPST_ERR_RKEY_VIOLATION;
goto err;
}
if (pkt->mask & RXE_WRITE_MASK) {
if (resid > mtu) {
if (pktlen != mtu || bth_pad(pkt)) {
state = RESPST_ERR_LENGTH;
goto err;
}
} else {
if (pktlen != resid) {
state = RESPST_ERR_LENGTH;
goto err;
}
if ((bth_pad(pkt) != (0x3 & (-resid)))) {
/* This case may not be exactly that
* but nothing else fits.
*/
state = RESPST_ERR_LENGTH;
goto err;
}
}
}
WARN_ON_ONCE(qp->resp.mr);
qp->resp.mr = mem;
return RESPST_EXECUTE;
err:
if (mem)
rxe_drop_ref(mem);
return state;
}
inline void test_basic_api(uint32_t symbols, uint32_t symbol_size)
{
// Common setting
typename Encoder::factory encoder_factory(symbols, symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(symbols, symbol_size);
auto decoder = decoder_factory.build();
EXPECT_TRUE(symbols == encoder_factory.max_symbols());
EXPECT_TRUE(symbol_size == encoder_factory.max_symbol_size());
EXPECT_TRUE(symbols == encoder->symbols());
EXPECT_TRUE(symbol_size == encoder->symbol_size());
EXPECT_TRUE(symbols == decoder_factory.max_symbols());
EXPECT_TRUE(symbol_size == decoder_factory.max_symbol_size());
EXPECT_TRUE(symbols == decoder->symbols());
EXPECT_TRUE(symbol_size == decoder->symbol_size());
EXPECT_TRUE(encoder->symbol_length() > 0);
EXPECT_TRUE(decoder->symbol_length() > 0);
EXPECT_TRUE(encoder->block_size() == symbols * symbol_size);
EXPECT_TRUE(decoder->block_size() == symbols * symbol_size);
EXPECT_TRUE(encoder_factory.max_payload_size() >=
encoder->payload_size());
EXPECT_TRUE(decoder_factory.max_payload_size() >=
decoder->payload_size());
EXPECT_EQ(encoder_factory.max_payload_size(),
decoder_factory.max_payload_size());
// Encode/decode operations
EXPECT_EQ(encoder->payload_size(), decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
std::vector<uint8_t> data_in_copy(data_in);
sak::mutable_storage storage_in = sak::storage(data_in);
sak::mutable_storage storage_in_copy = sak::storage(data_in_copy);
EXPECT_TRUE(sak::equal(storage_in, storage_in_copy));
// Only used for prime fields, lets reconsider how we implement
// this less intrusive
uint32_t prefix = 0;
if(fifi::is_prime2325<typename Encoder::field_type>::value)
{
// This field only works for multiple of uint32_t
assert((encoder->block_size() % 4) == 0);
uint32_t block_length = encoder->block_size() / 4;
fifi::prime2325_binary_search search(block_length);
prefix = search.find_prefix(storage_in_copy);
// Apply the negated prefix
fifi::apply_prefix(storage_in_copy, ~prefix);
}
encoder->set_symbols(storage_in_copy);
// Set the encoder non-systematic
if(kodo::is_systematic_encoder(encoder))
kodo::set_systematic_off(encoder);
while( !decoder->is_complete() )
{
uint32_t payload_used = encoder->encode( &payload[0] );
EXPECT_TRUE(payload_used <= encoder->payload_size());
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size(), '\0');
decoder->copy_symbols(sak::storage(data_out));
if(fifi::is_prime2325<typename Encoder::field_type>::value)
{
// Now we have to apply the negated prefix to the decoded data
fifi::apply_prefix(sak::storage(data_out), ~prefix);
}
EXPECT_TRUE(std::equal(data_out.begin(),
data_out.end(),
data_in.begin()));
}
inline void invoke_recoding(recoding_parameters param)
{
// Common setting
typename Encoder::factory encoder_factory(
param.m_max_symbols, param.m_max_symbol_size);
encoder_factory.set_symbols(param.m_symbols);
encoder_factory.set_symbol_size(param.m_symbol_size);
auto encoder = encoder_factory.build();
typename Decoder::factory decoder_factory(
param.m_max_symbols, param.m_max_symbol_size);
decoder_factory.set_symbols(param.m_symbols);
decoder_factory.set_symbol_size(param.m_symbol_size);
auto decoder_one = decoder_factory.build();
auto decoder_two = decoder_factory.build();
// If tested with a shallow decoder we have to remember to set the
// buffers to use for the decoding
std::vector<uint8_t> buffer_decoder_one(decoder_one->block_size(), '\0');
std::vector<uint8_t> buffer_decoder_two(decoder_two->block_size(), '\0');
if(kodo::has_shallow_symbol_storage<Decoder>::value)
{
decoder_one->set_symbols(sak::storage(buffer_decoder_one));
decoder_two->set_symbols(sak::storage(buffer_decoder_two));
}
EXPECT_EQ(encoder->payload_size(), decoder_one->payload_size());
EXPECT_EQ(encoder->payload_size(), decoder_two->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
std::vector<uint8_t> data_in = random_vector(encoder->block_size());
encoder->set_symbols(sak::storage(data_in));
// Set the encoder non-systematic
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_off(encoder);
while( !decoder_two->is_complete() )
{
uint32_t encode_size = encoder->encode( &payload[0] );
EXPECT_TRUE(encode_size <= payload.size());
EXPECT_TRUE(encode_size > 0);
decoder_one->decode( &payload[0] );
uint32_t recode_size = decoder_one->recode( &payload[0] );
EXPECT_TRUE(recode_size <= payload.size());
EXPECT_TRUE(recode_size > 0);
decoder_two->decode( &payload[0] );
}
std::vector<uint8_t> data_out_one(decoder_one->block_size(), '\0');
std::vector<uint8_t> data_out_two(decoder_two->block_size(), '\0');
decoder_one->copy_symbols(sak::storage(data_out_one));
decoder_two->copy_symbols(sak::storage(data_out_two));
EXPECT_TRUE(std::equal(data_out_one.begin(),
data_out_one.end(),
data_in.begin()));
EXPECT_TRUE(std::equal(data_out_two.begin(),
data_out_two.end(),
data_in.begin()));
}
void run_test_random_annex_partial(uint32_t max_symbols,
uint32_t max_symbol_size,
uint32_t multiplier)
{
uint32_t object_size = max_symbols * max_symbol_size * multiplier;
object_size -= (rand() % object_size);
uint32_t annex_size = kodo::max_annex_size(
max_symbols, max_symbol_size, object_size);
if(annex_size > 0)
{
// Randomize the actual annex size
annex_size -= (rand() % annex_size);
}
typedef kodo::random_annex_encoder<Encoder, Partitioning>
random_annex_encoder;
typedef kodo::random_annex_decoder<Decoder, Partitioning>
random_annex_decoder;
std::vector<uint8_t> data_in = random_vector(object_size);
std::vector<uint8_t> data_out(object_size, '\0');
typename Encoder::factory encoder_factory(max_symbols, max_symbol_size);
typename Decoder::factory decoder_factory(max_symbols, max_symbol_size);
random_annex_encoder obj_encoder(
annex_size, encoder_factory, sak::storage(data_in));
random_annex_decoder obj_decoder(
annex_size, decoder_factory, obj_encoder.object_size());
EXPECT_TRUE(obj_encoder.encoders() >= 1);
EXPECT_TRUE(obj_decoder.decoders() >= 1);
EXPECT_TRUE(obj_encoder.encoders() == obj_decoder.decoders());
uint32_t bytes_used = 0;
for(uint32_t i = 0; i < obj_encoder.encoders(); ++i)
{
auto encoder = obj_encoder.build(i);
typename random_annex_decoder::pointer_type decoder =
obj_decoder.build(i);
if(kodo::has_systematic_encoder<Encoder>::value)
kodo::set_systematic_off(encoder);
EXPECT_TRUE(encoder->block_size() >= encoder->bytes_used());
EXPECT_TRUE(decoder->block_size() >= decoder->bytes_used());
EXPECT_TRUE(encoder->block_size() == decoder->block_size());
EXPECT_TRUE(encoder->bytes_used() == decoder->bytes_used());
EXPECT_TRUE(encoder->payload_size() == decoder->payload_size());
std::vector<uint8_t> payload(encoder->payload_size());
while(!decoder->is_complete())
{
encoder->encode( &payload[0] );
decoder->decode( &payload[0] );
}
sak::mutable_storage storage = sak::storage(
&data_out[0] + bytes_used, decoder->bytes_used());
decoder.unwrap()->copy_symbols(storage);
bytes_used += decoder->bytes_used();
}
EXPECT_EQ(bytes_used, object_size);
EXPECT_TRUE(std::equal(data_in.begin(),
data_in.end(),
data_out.begin()));
}
// Tests that encoding and decoding a file withe the file encoder
// works.
TEST(TestFileEncoder, test_file_encoder)
{
{
std::string encode_filename = "encode-file";
std::string decode_filename = "decode-file";
// Write a test file
std::ofstream encode_file;
encode_file.open (encode_filename, std::ios::binary);
uint32_t size = 500;
for(uint32_t i = 0; i < size; ++i)
{
char c = rand() % 255;
encode_file.write(&c, 1);
}
encode_file.close();
typedef kodo::full_rlnc_encoder<fifi::binary>
encoder_t;
typedef kodo::full_rlnc_decoder<fifi::binary>
decoder_t;
typedef kodo::file_encoder<encoder_t>
file_encoder_t;
typedef kodo::object_decoder<decoder_t>
object_decoder_t;
uint32_t max_symbols = 10;
uint32_t max_symbol_size = 10;
file_encoder_t::factory encoder_factory(
max_symbols, max_symbol_size);
file_encoder_t file_encoder(encoder_factory, encode_filename);
object_decoder_t::factory decoder_factory(
max_symbols, max_symbol_size);
object_decoder_t object_decoder(decoder_factory, size);
EXPECT_EQ(object_decoder.decoders(), file_encoder.encoders());
// Open the decode file
std::ofstream decode_file;
decode_file.open (decode_filename, std::ios::binary);
for(uint32_t i = 0; i < file_encoder.encoders(); ++i)
{
auto encoder = file_encoder.build(i);
auto decoder = object_decoder.build(i);
EXPECT_EQ(encoder->symbols(), decoder->symbols());
EXPECT_EQ(encoder->symbol_size(), decoder->symbol_size());
EXPECT_EQ(encoder->bytes_used(), decoder->bytes_used());
// Set the encoder non-systematic
if(kodo::is_systematic_encoder(encoder))
kodo::set_systematic_off(encoder);
std::vector<uint8_t> payload(encoder->payload_size());
while( !decoder->is_complete() )
{
// Encode a packet into the payload buffer
encoder->encode( &payload[0] );
// Pass that packet to the decoder
decoder->decode( &payload[0] );
}
std::vector<uint8_t> data_out(decoder->block_size());
decoder->copy_symbols(sak::storage(data_out));
decode_file.write(
reinterpret_cast<char*>(&data_out[0]), decoder->bytes_used());
}
decode_file.close();
}
}
