Example #1
0
// Upgrade agency, guarded by wakeUp
void Supervision::upgradeAgency() {
  try {
    if (_snapshot(failedServersPrefix).slice().isArray()) {
      Builder builder;
      builder.openArray();
      builder.openObject();
      builder.add(
        _agencyPrefix + failedServersPrefix, VPackValue(VPackValueType::Object));
      for (auto const& failed :
             VPackArrayIterator(_snapshot(failedServersPrefix).slice())) {
        builder.add(failed.copyString(), VPackValue(VPackValueType::Object));
        builder.close();
      }
      builder.close();
      builder.close();
      builder.close();
      transact(_agent, builder);
    }
  } catch (std::exception const&) {
    Builder builder;
    builder.openArray();
    builder.openObject();
    builder.add(
      _agencyPrefix + failedServersPrefix, VPackValue(VPackValueType::Object));
    builder.close();
    builder.close();
    builder.close();
    transact(_agent, builder);    
  }
}
Example #2
0
/// Persist the globally commited truth
bool State::persistReadDB(arangodb::consensus::index_t cind) {
  if (checkCollection("compact")) {
    Builder store;
    store.openObject();
    store.add("readDB", VPackValue(VPackValueType::Array));
    _agent->readDB().dumpToBuilder(store);
    store.close();
    std::stringstream i_str;
    i_str << std::setw(20) << std::setfill('0') << cind;
    store.add("_key", VPackValue(i_str.str()));
    store.close();

    TRI_ASSERT(_vocbase != nullptr);
    auto transactionContext =
        std::make_shared<StandaloneTransactionContext>(_vocbase);
    SingleCollectionTransaction trx(transactionContext, "compact",
                                    TRI_TRANSACTION_WRITE);

    int res = trx.begin();

    if (res != TRI_ERROR_NO_ERROR) {
      THROW_ARANGO_EXCEPTION(res);
    }

    auto result = trx.insert("compact", store.slice(), _options);
    res = trx.finish(result.code);

    return (res == TRI_ERROR_NO_ERROR);
  }

  LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to persist read DB for compaction!";
  return false;
}
int main(int, char* []) {
  // create an object with a few members
  Builder b;

  b(Value(ValueType::Object));
  b.add("foo", Value(42));
  b.add("bar", Value("some string value"));
  b.add("baz", Value(ValueType::Object));
  b.add("qux", Value(true));
  b.add("bart", Value("this is a string"));
  b.close();
  b.add("quux", Value(12345));
  b.close();

  // a Slice is a lightweight accessor for a VPack value
  Slice s(b.start());

  // now fetch the string in the object's "bar" attribute
  if (s.hasKey("bar")) {
    Slice bar(s.get("bar"));
    std::cout << "'bar' attribute value has type: " << bar.type() << std::endl;
  }

  // fetch non-existing attribute "quetzal"
  Slice quetzal(s.get("quetzal"));
  // note: this returns a slice of type None
  std::cout << "'quetzal' attribute value has type: " << quetzal.type()
            << std::endl;
  std::cout << "'quetzal' attribute is None: " << std::boolalpha
            << quetzal.isNone() << std::endl;

  // fetch subattribute "baz.qux"
  Slice qux(s.get(std::vector<std::string>({"baz", "qux"})));
  std::cout << "'baz'.'qux' attribute has type: " << qux.type() << std::endl;
  std::cout << "'baz'.'qux' attribute has bool value: " << std::boolalpha
            << qux.getBoolean() << std::endl;
  std::cout << "Complete value of 'baz' is: " << s.get("baz").toJson()
            << std::endl;

  // fetch non-existing subattribute "bark.foobar"
  Slice foobar(s.get(std::vector<std::string>({"bark", "foobar"})));
  std::cout << "'bark'.'foobar' attribute is None: " << std::boolalpha
            << foobar.isNone() << std::endl;

  // check if subattribute "baz"."bart" does exist
  if (s.hasKey(std::vector<std::string>({"baz", "bart"}))) {
    // access subattribute using operator syntax
    std::cout << "'baz'.'bart' attribute has type: " << s["baz"]["bart"].type()
              << std::endl;
    std::cout << "'baz'.'bart' attribute has value: '"
              << s["baz"]["bart"].copyString() << "'" << std::endl;
  }
}
Example #4
0
HttpHandler::status_t RestAgencyHandler::handleState() {
  Builder body;
  body.add(VPackValue(VPackValueType::Array));
  for (auto const& i : _agent->state().get()) {
    body.add(VPackValue(VPackValueType::Object));
    body.add("index", VPackValue(i.index));
    body.add("term", VPackValue(i.term));
    body.add("leader", VPackValue(i.leaderId));
    body.add("query", VPackSlice(i.entry->data()));
    body.close();
  }
  body.close();
  generateResult(GeneralResponse::ResponseCode::OK, body.slice());
  return HttpHandler::status_t(HANDLER_DONE);
}
Example #5
0
Builder Collection::extract(Slice const& slice, int64_t from, int64_t to) {
  Builder b;
  b.openArray();

  int64_t length = static_cast<int64_t>(slice.length());
  int64_t skip = from;
  int64_t limit = to;

  if (limit < 0) {
    limit = length + limit - skip;
  }
  if (limit > 0) {
    ArrayIterator it(slice);
    while (it.valid()) {
      if (skip > 0) {
        --skip;
      }
      else {
        b.add(it.value());
        if (--limit == 0) {
          break;
        }
      }
      it.next();
    }
  }
  b.close();

  return b;
}
Example #6
0
/// Persist one entry
bool State::persist(arangodb::consensus::index_t index, term_t term,
                    arangodb::velocypack::Slice const& entry) const {
  Builder body;
  body.add(VPackValue(VPackValueType::Object));
  body.add("_key", Value(stringify(index)));
  body.add("term", Value(term));
  body.add("request", entry);
  body.close();
  
  TRI_ASSERT(_vocbase != nullptr);
  auto transactionContext =
    std::make_shared<StandaloneTransactionContext>(_vocbase);
  SingleCollectionTransaction trx(transactionContext, "log",
                                  TRI_TRANSACTION_WRITE);
  
  int res = trx.begin();
  if (res != TRI_ERROR_NO_ERROR) {
    THROW_ARANGO_EXCEPTION(res);
  }
  OperationResult result;
  try {
    result = trx.insert("log", body.slice(), _options);
  } catch (std::exception const& e) {
    LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to persist log entry:"
                                   << e.what();
  }
  res = trx.finish(result.code);

  return (res == TRI_ERROR_NO_ERROR);
}
Example #7
0
static void JS_WriteAgent(v8::FunctionCallbackInfo<v8::Value> const& args) {
  TRI_V8_TRY_CATCH_BEGIN(isolate);
  v8::HandleScope scope(isolate);

  Agent* agent = nullptr;
  try {
    AgencyFeature* feature =
        ApplicationServer::getEnabledFeature<AgencyFeature>("Agency");
    agent = feature->agent();

  } catch (std::exception const& e) {
    TRI_V8_THROW_EXCEPTION_MESSAGE(
        TRI_ERROR_INTERNAL,
        std::string("couldn't access agency feature: ") + e.what());
  }

  query_t query = std::make_shared<Builder>();
  int res = TRI_V8ToVPack(isolate, *query, args[0], false);

  if (res != TRI_ERROR_NO_ERROR) {
    TRI_V8_THROW_EXCEPTION(res);
  }

  write_ret_t ret = agent->write(query);

  if (ret.accepted) {  // Leading

    size_t errors = 0;
    Builder body;
    body.openObject();
    body.add("results", VPackValue(VPackValueType::Array));
    for (auto const& index : ret.indices) {
      body.add(VPackValue(index));
      if (index == 0) {
        errors++;
      }
    }
    body.close();
    body.close();

    // Wait for commit of highest except if it is 0?
    arangodb::consensus::index_t max_index = 0;
    try {
      max_index = *std::max_element(ret.indices.begin(), ret.indices.end());
    } catch (std::exception const& e) {
      LOG_TOPIC(WARN, Logger::AGENCY) << e.what() << " " << __FILE__
                                      << __LINE__;
    }

    if (max_index > 0) {
      agent->waitFor(max_index);
    }

    TRI_V8_RETURN(TRI_VPackToV8(isolate, body.slice()));
  } else {  // Not leading
    TRI_V8_RETURN_FALSE();
  }

  TRI_V8_TRY_CATCH_END
}
Example #8
0
HttpHandler::status_t RestAgencyHandler::handleStores() {
  if (_request->requestType() == GeneralRequest::RequestType::GET) {
    Builder body;
    body.openObject();
    body.add("spearhead", VPackValue(VPackValueType::Array));
    _agent->spearhead().dumpToBuilder(body);
    body.close();
    body.add("read_db", VPackValue(VPackValueType::Array));
    _agent->readDB().dumpToBuilder(body);
    body.close();
    body.close();
    generateResult(GeneralResponse::ResponseCode::OK, body.slice());
  } else {
    generateError(GeneralResponse::ResponseCode::BAD, 400);
  }
  return HttpHandler::status_t(HANDLER_DONE);
}
Example #9
0
Builder Collection::concat(Slice const& slice1, Slice const& slice2) {
  Builder b;
  b.openArray();
  appendArray(b, slice1);
  appendArray(b, slice2);
  b.close();

  return b;
}
Example #10
0
HttpHandler::status_t RestAgencyHandler::handleConfig() {
  Builder body;
  body.add(VPackValue(VPackValueType::Object));
  body.add("term", Value(_agent->term()));
  body.add("leaderId", Value(_agent->leaderID()));
  body.add("configuration", _agent->config().toBuilder()->slice());
  body.close();
  generateResult(GeneralResponse::ResponseCode::OK, body.slice());
  return HttpHandler::status_t(HANDLER_DONE);
}
Example #11
0
std::string Node::toJson() const {
  Builder builder;
  builder.openArray();
  toBuilder(builder);
  builder.close();
  std::string strval = builder.slice()[0].isString()
                           ? builder.slice()[0].copyString()
                           : builder.slice()[0].toJson();
  return strval;
}
Example #12
0
// Get bunch of cluster's unique ids from agency, guarded above
void Supervision::getUniqueIds() {
  uint64_t latestId;
  // Run forever, supervision does not make sense before the agency data
  // is initialized by some other server...
  while (!this->isStopping()) {
    try {
      latestId = std::stoul(_agent->readDB()
                                .get(_agencyPrefix + "/Sync/LatestID")
                                .slice()
                                .toJson());
    } catch (...) {
      std::this_thread::sleep_for(std::chrono::seconds(1));
      continue;
    }

    Builder uniq;
    uniq.openArray();
    uniq.openObject();
    uniq.add(_agencyPrefix + syncLatest, VPackValue(latestId + 100000));  // new
    uniq.close();
    uniq.openObject();
    uniq.add(_agencyPrefix + syncLatest, VPackValue(latestId));  // precond
    uniq.close();
    uniq.close();

    auto result = transact(_agent, uniq);

    if (!result.accepted || result.indices.empty()) {
      LOG_TOPIC(DEBUG, Logger::AGENCY) << "We have lost agency leadership. "
                                          "Stopping any supervision processing "
                                       << __FILE__ << __LINE__;
      return;
    }

    if (result.indices[0]) {
      _agent->waitFor(result.indices[0]);
      _jobId = latestId;
      _jobIdMax = latestId + 100000;
      return;
    }
  }
}
Example #13
0
bool Node::handle<DECREMENT>(VPackSlice const& slice) {
  Builder tmp;
  tmp.openObject();
  try {
    tmp.add("tmp", Value(this->slice().getInt() - 1));
  } catch (std::exception const&) {
    tmp.add("tmp", Value(-1));
  }
  tmp.close();
  *this = tmp.slice().get("tmp");
  return true;
}
Example #14
0
void Node::rebuildVecBuf() const {
  if (_vecBufDirty) {              // Dirty vector buffer
    Builder tmp;
    tmp.openArray();
    for (auto const& i : _value) {
      tmp.add(Slice(i.data()));
    }
    tmp.close();
    _vecBuf = *tmp.steal();
    _vecBufDirty = false;
  }
}
Example #15
0
Builder Collection::values(Slice const& slice) {
  Builder b;
  b.add(Value(ValueType::Array));

  ObjectIterator it(slice);

  while (it.valid()) {
    b.add(it.value());
    it.next();
  }

  b.close();
  return b;
}
Example #16
0
bool Node::handle<PREPEND>(VPackSlice const& slice) {
  if (!slice.hasKey("new")) {
    LOG_TOPIC(WARN, Logger::AGENCY)
        << "Operator prepend without new value: " << slice.toJson();
    return false;
  }
  Builder tmp;
  tmp.openArray();
  tmp.add(slice.get("new"));
  if (this->slice().isArray()) {
    for (auto const& old : VPackArrayIterator(this->slice())) tmp.add(old);
  }
  tmp.close();
  *this = tmp.slice();
  return true;
}
Example #17
0
bool Node::handle<POP>(VPackSlice const& slice) {
  Builder tmp;
  tmp.openArray();
  if (this->slice().isArray()) {
    VPackArrayIterator it(this->slice());
    if (it.size() > 1) {
      size_t j = it.size() - 1;
      for (auto old : it) {
        tmp.add(old);
        if (--j == 0) break;
      }
    }
  }
  tmp.close();
  *this = tmp.slice();
  return true;
}
Example #18
0
/// Update my term
void Constituent::term(term_t t) {

  term_t tmp;
  {
    MUTEX_LOCKER(guard, _castLock);
    tmp = _term;
    _term = t;
  }

  if (tmp != t) {
    
    LOG_TOPIC(DEBUG, Logger::AGENCY)
      << _id << ": " << roleStr[_role] << " term " << t;

    Builder body;
    body.add(VPackValue(VPackValueType::Object));
    std::ostringstream i_str;
    i_str << std::setw(20) << std::setfill('0') << t;
    body.add("_key", Value(i_str.str()));
    body.add("term", Value(t));
    body.add("voted_for", Value((uint32_t)_votedFor));
    body.close();

    TRI_ASSERT(_vocbase != nullptr);
    auto transactionContext =
        std::make_shared<StandaloneTransactionContext>(_vocbase);
    SingleCollectionTransaction trx(transactionContext, "election",
                                    TRI_TRANSACTION_WRITE);

    int res = trx.begin();

    if (res != TRI_ERROR_NO_ERROR) {
      THROW_ARANGO_EXCEPTION(res);
    }

    OperationOptions options;
    options.waitForSync = waitForSync();
    options.silent = true;

    OperationResult result = trx.insert("election", body.slice(), options);
    trx.finish(result.code);
    
  }
  
}
Example #19
0
Builder Collection::remove(Slice const& slice,
                           std::unordered_set<std::string> const& keys) {
  Builder b;
  b.add(Value(ValueType::Object));

  ObjectIterator it(slice);

  while (it.valid()) {
    auto key = it.key(true).copyString();
    if (keys.find(key) == keys.end()) {
      b.add(key, it.value());
    }
    it.next();
  }

  b.close();
  return b;
}
Example #20
0
/// Create collection by name
bool State::createCollection(std::string const& name) {
  Builder body;
  body.add(VPackValue(VPackValueType::Object));
  body.add("type", VPackValue(static_cast<int>(TRI_COL_TYPE_DOCUMENT))); 
  body.add("name", VPackValue(name));
  body.add("isSystem", VPackValue(LogicalCollection::IsSystemName(name)));
  body.close();

  TRI_voc_cid_t cid = 0;
  arangodb::LogicalCollection const* collection =
      _vocbase->createCollection(body.slice(), cid, true);

  if (collection == nullptr) {
    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_errno(), "cannot create collection");
  }

  return true;
}
Example #21
0
bool Node::handle<SHIFT>(VPackSlice const& slice) {
  Builder tmp;
  tmp.openArray();
  if (this->slice().isArray()) {  // If a
    VPackArrayIterator it(this->slice());
    bool first = true;
    for (auto const& old : it) {
      if (first) {
        first = false;
      } else {
        tmp.add(old);
      }
    }
  }
  tmp.close();
  *this = tmp.slice();
  return true;
}
Example #22
0
Builder Collection::filter(Slice const& slice, Predicate const& predicate) {
  // construct a new Array
  Builder b;
  b.add(Value(ValueType::Array));

  ArrayIterator it(slice);
  ValueLength index = 0;

  while (it.valid()) {
    Slice s = it.value();
    if (predicate(s, index)) {
      b.add(s);
    }
    it.next();
    ++index;
  }
  b.close();
  return b;
}
Example #23
0
Builder Collection::sort(
      Slice const& array,
      std::function<bool (Slice const&, Slice const&)> lessthan) {
  if (!array.isArray()) {
    throw Exception(Exception::InvalidValueType, "Expecting type Array");
  }
  std::vector<Slice> subValues;
  ValueLength len = array.length();
  subValues.reserve(checkOverflow(len));
  for (ValueLength i = 0; i < len; i++) {
    subValues.push_back(array[i]);
  }
  std::sort(subValues.begin(), subValues.end(), lessthan);
  Builder b;
  b.openArray();
  for (auto const&s : subValues) {
    b.add(s);
  }
  b.close();
  return b;
}
Example #24
0
Builder Collection::remove(Slice const& slice,
                           std::vector<std::string> const& keys) {
  // check if there are so many keys that we want to use the hash-based version
  // cut-off values are arbitrary...
  if (keys.size() >= 4 && slice.length() > 10) {
    return remove(slice, makeSet(keys));
  }

  Builder b;
  b.add(Value(ValueType::Object));

  ObjectIterator it(slice);

  while (it.valid()) {
    auto key = it.key(true).copyString();
    if (std::find(keys.begin(), keys.end(), key) == keys.end()) {
      b.add(key, it.value());
    }
    it.next();
  }

  b.close();
  return b;
}
Example #25
0
// Check all DB servers, guarded above doChecks
std::vector<check_t> Supervision::checkDBServers() {
  std::vector<check_t> ret;
  Node::Children const& machinesPlanned =
      _snapshot(planDBServersPrefix).children();
  Node::Children const serversRegistered =
      _snapshot(currentServersRegisteredPrefix).children();

  std::vector<std::string> todelete;
  for (auto const& machine : _snapshot(healthPrefix).children()) {
    if (machine.first.substr(0, 2) == "DB") {
      todelete.push_back(machine.first);
    }
  }

  for (auto const& machine : machinesPlanned) {
    bool good = false;
    std::string lastHeartbeatTime, lastHeartbeatAcked, lastStatus,
        heartbeatTime, heartbeatStatus, serverID;

    serverID = machine.first;
    heartbeatTime = _snapshot(syncPrefix + serverID + "/time").toJson();
    heartbeatStatus = _snapshot(syncPrefix + serverID + "/status").toJson();

    todelete.erase(std::remove(todelete.begin(), todelete.end(), serverID),
                   todelete.end());

    try {  // Existing
      lastHeartbeatTime =
          _snapshot(healthPrefix + serverID + "/LastHeartbeatSent").toJson();
      lastHeartbeatAcked =
          _snapshot(healthPrefix + serverID + "/LastHeartbeatAcked").toJson();
      lastStatus = _snapshot(healthPrefix + serverID + "/Status").toJson();
      if (lastHeartbeatTime != heartbeatTime) {  // Update
        good = true;
      }
    } catch (...) {  // New server
      good = true;
    }

    query_t report = std::make_shared<Builder>();
    report->openArray();
    report->openArray();
    report->openObject();
    report->add(_agencyPrefix + healthPrefix + serverID,
                VPackValue(VPackValueType::Object));
    report->add("LastHeartbeatSent", VPackValue(heartbeatTime));
    report->add("LastHeartbeatStatus", VPackValue(heartbeatStatus));
    report->add("Role", VPackValue("DBServer"));
    auto endpoint = serversRegistered.find(serverID);
    if (endpoint != serversRegistered.end()) {
      endpoint = endpoint->second->children().find("endpoint");
      if (endpoint != endpoint->second->children().end()) {
        if (endpoint->second->children().size() == 0) {
          VPackSlice epString = endpoint->second->slice();
          if (epString.isString()) {
            report->add("Endpoint", epString);
          }
        }
      }
    }

    if (good) {
      
      report->add(
          "LastHeartbeatAcked",
          VPackValue(timepointToString(std::chrono::system_clock::now())));
      report->add("Status", VPackValue(Supervision::HEALTH_STATUS_GOOD));
      
      std::string failedServerPath = failedServersPrefix + "/" + serverID;
      if (_snapshot.exists(failedServerPath).size() == 3) {
        Builder del;
        del.openArray();
        del.openObject();
        del.add(_agencyPrefix + failedServerPath,
                VPackValue(VPackValueType::Object));
        del.add("op", VPackValue("delete"));
        del.close();
        del.close();
        del.close();
        transact(_agent, del);
      }
      
    } else {
      
      std::chrono::seconds t{0};
      t = std::chrono::duration_cast<std::chrono::seconds>(
          std::chrono::system_clock::now() -
          stringToTimepoint(lastHeartbeatAcked));

      if (t.count() > _gracePeriod) {  // Failure
        if (lastStatus == "BAD") {
          report->add("Status", VPackValue("FAILED"));
          FailedServer fsj(_snapshot, _agent, std::to_string(_jobId++),
                           "supervision", _agencyPrefix, serverID);
        }
      } else {
        report->add("Status", VPackValue("BAD"));
      }
      
    }

    report->close();
    report->close();
    report->close();
    report->close();
    
    if (!this->isStopping()) {
      _agent->write(report);
    }
    
  }

  if (!todelete.empty()) {
    query_t del = std::make_shared<Builder>();
    del->openArray();
    del->openArray();
    del->openObject();
    for (auto const& srv : todelete) {
      del->add(_agencyPrefix + healthPrefix + srv,
               VPackValue(VPackValueType::Object));
      del->add("op", VPackValue("delete"));
      del->close();
    }
    del->close();
    del->close();
    del->close();
    _agent->write(del);
  }

  return ret;
}
Example #26
0
JOB_STATUS RemoveServer::status() {
  auto status = exists();

  if (status != NOTFOUND) {  // Get job details from agency

    try {
      _server = _snapshot(pos[status] + _jobId + "/server").getString();
    } catch (std::exception const& e) {
      std::stringstream err;
      err << "Failed to find job " << _jobId << " in agency: " << e.what();
      LOG_TOPIC(ERR, Logger::AGENCY) << err.str();
      finish("DBServers/" + _server, false, err.str());
      return FAILED;
    }
  }

  if (status == PENDING) {
    Node::Children const todos = _snapshot(toDoPrefix).children();
    Node::Children const pends = _snapshot(pendingPrefix).children();

    size_t found = 0;
    for (auto const& subJob : todos) {
      if (!subJob.first.compare(0, _jobId.size() + 1, _jobId + "-")) {
        found++;
      }
    }
    for (auto const& subJob : pends) {
      if (!subJob.first.compare(0, _jobId.size() + 1, _jobId + "-")) {
        found++;
      }
    }

    if (found > 0) {
      // mop: TODO check if the server has reappeared and abort shard moving if
      // server has reappeared
      return status;
    }

    // mop: all addfollower jobs have been finished. Forcefully remove the
    // server from everything
    Node::Children const& planDatabases =
        _snapshot("/Plan/Collections").children();

    Builder desiredPlanState;
    Builder preconditions;

    desiredPlanState.openObject();
    preconditions.openObject();

    Builder trx;
    trx.openArray();
    trx.openObject();
    for (auto const& database : planDatabases) {
      for (auto const& collptr : database.second->children()) {
        Node const& collection = *(collptr.second);
        for (auto const& shard : collection("shards").children()) {
          VPackArrayIterator dbsit(shard.second->slice());

          bool found = false;
          Builder desiredServers;
          desiredServers.openArray();
          // Only shards, which are affected
          for (auto const& dbserver : dbsit) {
            std::string server = dbserver.copyString();
            if (server != _server) {
              desiredServers.add(VPackValue(server));
            } else {
              found = true;
            }
          }
          desiredServers.close();
          if (found == false) {
            continue;
          }

          std::string const& key(_agencyPrefix + "/Plan/Collections/" +
                                 database.first + "/" + collptr.first +
                                 "/shards/" + shard.first);

          trx.add(key, desiredServers.slice());
          preconditions.add(VPackValue(key));
          preconditions.openObject();
          preconditions.add("old", shard.second->slice());
          preconditions.close();
        }
      }
    }
    preconditions.close();

    trx.add(VPackValue(_agencyPrefix + "/Target/CleanedServers"));
    trx.openObject();
    trx.add("op", VPackValue("push"));
    trx.add("new", VPackValue(_server));
    trx.close();
    trx.add(VPackValue(_agencyPrefix + planVersion));
    trx.openObject();
    trx.add("op", VPackValue("increment"));
    trx.close();

    trx.close();
    trx.add(preconditions.slice());
    trx.close();

    // Transact to agency
    write_ret_t res = transact(_agent, trx);

    if (res.accepted && res.indices.size() == 1 && res.indices[0] != 0) {
      LOG_TOPIC(INFO, Logger::AGENCY) << "Have reported " << _server
                                      << " in /Target/CleanedServers";
      if (finish("DBServers/" + _server)) {
        return FINISHED;
      }
      return status;
    }
  }

  return status;
}
Example #27
0
Builder Collection::merge(Slice const& left, Slice const& right,
                          bool mergeValues, bool nullMeansRemove) {
  if (!left.isObject() || !right.isObject()) {
    throw Exception(Exception::InvalidValueType, "Expecting type Object");
  }

  Builder b;
  b.add(Value(ValueType::Object));

  std::unordered_map<std::string, Slice> rightValues;
  {
    ObjectIterator it(right);
    while (it.valid()) {
      rightValues.emplace(it.key(true).copyString(), it.value());
      it.next();
    }
  }

  {
    ObjectIterator it(left);

    while (it.valid()) {
      auto key = it.key(true).copyString();
      auto found = rightValues.find(key);

      if (found == rightValues.end()) {
        // use left value
        b.add(key, it.value());
      } else if (mergeValues && it.value().isObject() &&
                 (*found).second.isObject()) {
        // merge both values
        auto& value = (*found).second;
        if (!nullMeansRemove || (!value.isNone() && !value.isNull())) {
          Builder sub = Collection::merge(it.value(), value, true, nullMeansRemove);
          b.add(key, sub.slice());
        }
        // clear the value in the map so its not added again
        (*found).second = Slice();
      } else {
        // use right value
        auto& value = (*found).second;
        if (!nullMeansRemove || (!value.isNone() && !value.isNull())) {
          b.add(key, value);
        }
        // clear the value in the map so its not added again
        (*found).second = Slice();
      }
      it.next();
    }
  }

  // add remaining values that were only in right
  for (auto& it : rightValues) {
    auto& s = it.second;
    if (s.isNone()) {
      continue;
    }
    if (nullMeansRemove && s.isNull()) {
      continue;
    }
    b.add(std::move(it.first), s);
  }

  b.close();
  return b;
}
Example #28
0
/// Load persisted configuration
bool State::loadOrPersistConfiguration() {
  auto bindVars = std::make_shared<VPackBuilder>();
  bindVars->openObject();
  bindVars->close();

  std::string const aql(
      std::string("FOR c in configuration FILTER c._key==\"0\" RETURN c.cfg"));

  arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars,
                             nullptr, arangodb::aql::PART_MAIN);

  auto queryResult = query.execute(QueryRegistryFeature::QUERY_REGISTRY);

  if (queryResult.code != TRI_ERROR_NO_ERROR) {
    THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details);
  }

  VPackSlice result = queryResult.result->slice();

  if (result.isArray() &&
      result.length()) {  // We already have a persisted conf

    try {
      LOG_TOPIC(DEBUG, Logger::AGENCY)
        << "Merging configuration " << result[0].resolveExternals().toJson();
      _agent->mergeConfiguration(result[0].resolveExternals());
      
    } catch (std::exception const& e) {
      LOG_TOPIC(ERR, Logger::AGENCY)
          << "Failed to merge persisted configuration into runtime "
             "configuration: "
          << e.what();
      FATAL_ERROR_EXIT();
    }

  } else {  // Fresh start

    LOG_TOPIC(DEBUG, Logger::AGENCY) << "New agency!";

    TRI_ASSERT(_agent != nullptr);
    _agent->id(to_string(boost::uuids::random_generator()()));

    auto transactionContext =
        std::make_shared<StandaloneTransactionContext>(_vocbase);
    SingleCollectionTransaction trx(transactionContext, "configuration",
                                    TRI_TRANSACTION_WRITE);

    int res = trx.begin();
    OperationResult result;

    if (res != TRI_ERROR_NO_ERROR) {
      THROW_ARANGO_EXCEPTION(res);
    }

    Builder doc;
    doc.openObject();
    doc.add("_key", VPackValue("0"));
    doc.add("cfg", _agent->config().toBuilder()->slice());
    doc.close();
    try {
      result = trx.insert("configuration", doc.slice(), _options);
    } catch (std::exception const& e) {
      LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to persist configuration entry:"
                                     << e.what();
      FATAL_ERROR_EXIT();
    }

    res = trx.finish(result.code);

    return (res == TRI_ERROR_NO_ERROR);
  }

  return true;
}
Example #29
0
HttpHandler::status_t RestAgencyHandler::handleWrite() {
  arangodb::velocypack::Options options;  // TODO: User not wait.
  if (_request->requestType() == GeneralRequest::RequestType::POST) {
    query_t query;

    try {
      query = _request->toVelocyPack(&options);
    } catch (std::exception const& e) {
      LOG_TOPIC(ERR, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__;
      Builder body;
      body.openObject();
      body.add("message", VPackValue(e.what()));
      body.close();
      generateResult(GeneralResponse::ResponseCode::BAD, body.slice());
      return HttpHandler::status_t(HANDLER_DONE);
    }

    if (!query->slice().isArray()) {
      Builder body;
      body.openObject();
      body.add("message",
               VPackValue("Excpecting array of arrays as body for writes"));
      body.close();
      generateResult(GeneralResponse::ResponseCode::BAD, body.slice());
      return HttpHandler::status_t(HANDLER_DONE);
    }

    if (query->slice().length() == 0) {
      Builder body;
      body.openObject();
      body.add(
        "message", VPackValue("Empty request."));
      body.close();
      generateResult(GeneralResponse::ResponseCode::BAD, body.slice());
      return HttpHandler::status_t(HANDLER_DONE);
    }

    while(_agent->size() > 1 && _agent->leaderID() > 100) {
      std::this_thread::sleep_for(duration_t(100));
    }

    write_ret_t ret = _agent->write(query);

    if (ret.accepted) {  // We're leading and handling the request

      bool found;
      std::string call_mode = _request->header("x-arangodb-agency-mode", found);
      if (!found) {
        call_mode = "waitForCommitted";
      }

      size_t errors = 0;
      Builder body;
      body.openObject();

      if (call_mode != "noWait") {
        // Note success/error
        body.add("results", VPackValue(VPackValueType::Array));
        for (auto const& index : ret.indices) {
          body.add(VPackValue(index));
          if (index == 0) {
            errors++;
          }
        }
        body.close();
        
        // Wait for commit of highest except if it is 0?
        if (!ret.indices.empty() && call_mode == "waitForCommitted") {
          arangodb::consensus::index_t max_index = 0;
          try {
            max_index =
              *std::max_element(ret.indices.begin(), ret.indices.end());
          } catch (std::exception const& e) {
            LOG_TOPIC(WARN, Logger::AGENCY)
              << e.what() << " " << __FILE__ << __LINE__;
          }

          if (max_index > 0) {
            if(!_agent->waitFor(max_index)) {
            LOG_TOPIC(WARN, Logger::AGENCY)
              << "Waiting for log index" << max_index << " timed out.";
            }
          }
        }
      }

      body.close();

      if (errors > 0) {  // Some/all requests failed
        generateResult(GeneralResponse::ResponseCode::PRECONDITION_FAILED,
                       body.slice());
      } else {  // All good
        generateResult(GeneralResponse::ResponseCode::OK, body.slice());
      }
    } else {  // Redirect to leader
      redirectRequest(ret.redirect);
    }
  } else {  // Unknown method
    generateError(GeneralResponse::ResponseCode::METHOD_NOT_ALLOWED, 405);
  }
  return HttpHandler::status_t(HANDLER_DONE);
}