void RIFFParser::parse()
{
next();
if (chunk_id() != 'RIFF' && chunk_id() != 'RF64')
throw std::runtime_error("Not a RIFF file");
set_format_id(down());
if (chunk_id() == 'RF64')
parse_datasize64();
}
/**
* Is the Id in the set?
*
* @param id The Id to check.
*/
bool get(T id) const noexcept override final {
if (chunk_id(id) >= m_data.size()) {
return false;
}
auto* r = m_data[chunk_id(id)].get();
if (!r) {
return false;
}
return (r[offset(id)] & bitmask(id)) != 0;
}
void RIFFParser::parse_datasize64()
{
next();
if (chunk_id() != 'ds64')
throw std::runtime_error("Invalid RF64 format");
check_eof(stream().seek_forward(8) == 8);
uint32_t x, y;
check_eof(stream().read32le(&x));
check_eof(stream().read32le(&y));
m_chunk_size_map['data'] = ((static_cast<uint64_t>(y) << 32) | x);
check_eof(stream().seek_forward(8) == 8);
uint32_t len;
check_eof(stream().read32le(&len));
for (uint32_t i = 0; i < len; ++i) {
char buff[4];
check_eof(stream().read(buff, 4) == 4);
check_eof(stream().read32le(&x));
check_eof(stream().read32le(&y));
m_chunk_size_map[fourcc(buff)]
= ((static_cast<uint64_t>(y) << 32) | x);
}
done_chunk();
}
bool RIFFParser::get_chunkinfo(uint32_t *fcc, uint64_t *size)
{
char buff[4];
if (stream().read(buff, 4) < 4)
return false;
*fcc = fourcc(buff);
uint32_t x;
check_eof(stream().read32le(&x));
*size = x;
// RF64 support
chunk_map_t::const_iterator it = m_chunk_size_map.find(chunk_id());
if (it != m_chunk_size_map.end())
*size = it->second;
return true;
}
void HandleMaps::issue_command_() {
bool assigned = false; //Describes status of current chunk (file)
while(assigned == false) {
try {
saga::stream::stream worker = service_->serve();
std::string message("Established connection to ");
message += worker.get_url().get_string();
log_->write(message, MR_LOGLEVEL_INFO);
// Ask worker for state.
worker.write(saga::buffer(MASTER_QUESTION_STATE, 6));
if (!SagaStreamUtils::TimedWaitForRead(
worker, job_.get_attribute("protocol.read_timeout", 10))) {
log_->write("Worker didn't respond -- retrying.", MR_LOGLEVEL_DEBUG);
delete service_;
service_ = new saga::stream::server(serverURL_);
continue;
}
char buff[MSG_BUFFER_SIZE];
saga::ssize_t read_bytes = worker.read(saga::buffer(buff));
std::string state(buff, read_bytes);
message.clear();
message = "Worker: " + worker.get_url().get_string() + " has state " + state;
log_->write(message, MR_LOGLEVEL_INFO);
if(state == WORKER_STATE_IDLE)
{
if(finished_.size() == totalChunks_ || unassigned_.size() == 0)
{
//Prevent unneccessary work assignments
worker.write(saga::buffer(MASTER_REQUEST_IDLE, 5));
saga::ssize_t read_bytes = worker.read(saga::buffer(buff));
if(std::string(buff, read_bytes) != WORKER_RESPONSE_ACKNOWLEDGE)
{
log_->write(std::string("Misbehaving worker!"), MR_LOGLEVEL_WARNING);
}
return;
}
std::string chunk_id(getCandidate_(worker));
//Worker is idle
message.clear();
message = "Attempting to issue worker ";
message += worker.get_url().get_string();
message += " to map " + chunk_id;
message += " ...";
log_->write(message, MR_LOGLEVEL_INFO);
//ask where their advert is
worker.write(saga::buffer(MASTER_QUESTION_ADVERT, 7));
memset(buff, 0, MSG_BUFFER_SIZE);
read_bytes = worker.read(saga::buffer(buff));
saga::url advert = saga::url(std::string(buff, read_bytes));
// Store URL->advert mapping.
worker_adverts_[worker.get_url().get_string()] = advert;
message.clear();
message += worker.get_url().get_string();
message += " <==> " + std::string(buff);
message += " ... ";
log_->write(message, MR_LOGLEVEL_INFO);
//Tell worker about data
worker.write(saga::buffer(WORKER_COMMAND_MAP, 3));
memset(buff, 0, MSG_BUFFER_SIZE);
read_bytes = worker.read(saga::buffer(buff));
if(std::string(buff, read_bytes) == WORKER_RESPONSE_ACKNOWLEDGE)
{
worker.write(saga::buffer(WORKER_CHUNK, 5));
memset(buff, 0, MSG_BUFFER_SIZE);
read_bytes = worker.read(saga::buffer(buff));
if(std::string(buff, read_bytes) == WORKER_RESPONSE_ACKNOWLEDGE)
{
// Get chunk pointer for ID.
InputChunk* chunk = chunk_assignments_[chunk_id];
// Give serialized JobDescription, chunk_id and chunk to worker.
// FIXME: shouldn't we store chunk ID with the chunk/jobdesc?
std::string command;
{
StringOutputStream sos(&command);
SerializationHandler<JobDescription>::Serialize(
const_cast<JobDescription*>(&job_), &sos);
SerializationHandler<std::string>::Serialize(&chunk_id, &sos);
input_format_->SerializeInputChunk(chunk, &sos);
}
worker.write(saga::buffer(command, command.size()));
memset(buff, 0, MSG_BUFFER_SIZE);
read_bytes = worker.read(saga::buffer(buff));
if (std::string(buff, read_bytes) == WORKER_RESPONSE_ACKNOWLEDGE)
{
// Add to assigned set.
assigned_.insert(chunk_id);
//Remove from unassigned.
unassigned_.erase(chunk_id);
}
}
}
else
{
message = std::string("Worker did not accept chunk!");
log_->write(message, MR_LOGLEVEL_WARNING);
break;
}
message.clear();
message += "Success: ";
message += advert.get_string() + " is comparing chunk ";
message += chunk_id;
log_->write(message, MR_LOGLEVEL_INFO);
assigned = true;
}
else if(state == WORKER_STATE_DONE_MAP)
{
worker.write(saga::buffer(MASTER_QUESTION_RESULT, 7));
memset(buff, 0, MSG_BUFFER_SIZE);
read_bytes = worker.read(saga::buffer(buff));
std::string result_message(buff, read_bytes);
worker.write(saga::buffer(MASTER_REQUEST_IDLE, 5));
// Parse result message.
size_t split_point = result_message.find_first_of(' ');
std::string chunk_id = result_message.substr(0, split_point);
std::string worker_advert = result_message.substr(split_point+1);
message.clear();
message += "Worker ";
message += worker.get_url().get_string() + " finished chunk ";
message += chunk_id;
log_->write(message, MR_LOGLEVEL_INFO);
// Note which worker completed this chunk.
message.clear();
message += "Noted " + worker_advert + " for " + chunk_id;
committed_chunks_[chunk_id] = worker_advert;
// If in assigned, remove it.
assigned_.erase(chunk_id);
// Put into finished set.
finished_.insert(chunk_id);
}
}
catch(saga::exception const & e) {
std::string message(e.what());
log_->write(message, MR_LOGLEVEL_ERROR);
}
}
}
