void
    ensureObjectDoesNotExist(const fs::path& p,
                             const std::string& objname)
    {
        EXPECT_FALSE(fs::exists(p));

        EXPECT_FALSE(bi_(nspace_->ns())->objectExists(objname));

        EXPECT_THROW(bi_(nspace_->ns())->getSize(objname),
                     BackendObjectDoesNotExistException);

        if (BackendTestSetup::backend_config().backend_type.value() == BackendType::S3)
        {
            EXPECT_THROW(bi_(nspace_->ns())->getCheckSum(objname),
                         BackendNotImplementedException);
        }
        else
        {
            EXPECT_THROW(bi_(nspace_->ns())->getCheckSum(objname),
                         BackendObjectDoesNotExistException);
        }
        EXPECT_THROW(bi_(nspace_->ns())->read(p, objname, InsistOnLatestVersion::T),
                     BackendObjectDoesNotExistException);

        EXPECT_FALSE(fs::exists(p));
    }
    void
    createAndListObjects(size_t objsize,
                         const std::string& pattern,
                         size_t num_objs,
                         size_t check_every_n = 0)
    {
        std::list<std::string> objs;

        const fs::path src(path_ / "test.src");
        const yt::CheckSum chksum(createTestFile(src,
                                                 objsize,
                                                 pattern));


        for (size_t i = 0; i < num_objs; ++i)
        {
            if (check_every_n != 0 and i % check_every_n == 0)
            {
                bi_(nspace_->ns())->listObjects(objs);
                EXPECT_EQ(i, objs.size());
                objs.clear();
            }

            std::stringstream ss;
            ss << std::setw(8) << std::setfill('0') << i;

            bi_(nspace_->ns())->write(src,
                                      ss.str(),
                                      OverwriteObject::F,
                                      &chksum);
        }

        bi_(nspace_->ns())->listObjects(objs);
        EXPECT_EQ(num_objs, objs.size());
    }
示例#3
0
  void update_one_instance(const Instance& ins, double step_size)  {
    size_t uid = ins.get_feature_group_index(0, 0);
    size_t iid = ins.get_feature_group_index(1, 0);
    size_t jid;

    double pred = predict(ins);

    for (size_t idx = 0; idx < num_neg_; ++ idx) {
      jid = sample_negative_item(uid);
      
      double pred_neg = predict_user_item_rating(uid, jid);

      double grad = loss_->gradient(pred - pred_neg, 1.);

      auto bigrad = grad + lambda_ * bi_(iid);
      auto bjgrad = -grad + lambda_ * bi_(jid);

      if (using_adagrad_) {
        bi_grad_(iid) += bigrad * bigrad;
        bi_grad_(jid) += bjgrad * bjgrad;
        bigrad /= std::sqrt(bi_grad_(iid)); 
        bjgrad /= std::sqrt(bi_grad_(jid)); 
      }

      bi_(iid) -= step_size * bigrad;
      bi_(jid) -= step_size * bjgrad;

      DVector x_grad = DVector::Zero(num_dim_);

      auto fit = user_rated_items_.find(uid);
      CHECK(fit != user_rated_items_.end());
      double user_size = static_cast<double>(fit->second.size());
      for (auto& kid : fit->second) {
        if (kid == iid) continue;
        DVector pj_grad = grad * (q_.row(iid) - q_.row(jid)) / std::pow((user_size - 1.), alpha_) + lambda_ * p_.row(kid);
        if (using_adagrad_) {
          p_grad_.row(kid) += pj_grad.cwiseProduct(pj_grad); 
          for (size_t k = 0; k < num_dim_; ++k) 
            pj_grad(k) = pj_grad(k) / std::sqrt(p_grad_(kid, k));
        }
        p_.row(kid) -= step_size * pj_grad;
        x_grad += pj_grad;
      }

      DVector qi_grad = grad * (x_.row(uid) - p_.row(iid)) / std::pow((user_size - 1.), alpha_) + lambda_ * q_.row(iid);
      DVector qj_grad = - grad * (x_.row(uid) - p_.row(iid)) / std::pow((user_size - 1.), alpha_) + lambda_ * q_.row(jid);
      if (using_adagrad_) {
        q_grad_.row(iid) += qi_grad.cwiseProduct(qi_grad); 
        q_grad_.row(jid) += qj_grad.cwiseProduct(qj_grad); 
        for (size_t k = 0; k < num_dim_; ++k) { 
          qi_grad(k) = qi_grad(k) / std::sqrt(q_grad_(iid, k));
          qj_grad(k) = qj_grad(k) / std::sqrt(q_grad_(jid, k));
        }
      }
      q_.row(iid) -= step_size * qi_grad;
      q_.row(jid) -= step_size * qj_grad;
      x_.row(uid) -= step_size * x_grad; 
    }
  }
TEST_F(BackendObjectTest, basics)
{
    const std::string objname("test");
    const fs::path src(path_ / std::string(objname + ".src"));
    const fs::path dst(path_ / std::string(objname + ".dst"));

    ensureObjectDoesNotExist(dst,
                             objname);

    const std::string pattern("William, it was really nothing");

    const size_t size = 1 << 20;
    const yt::CheckSum chksum(createPutAndVerify(src,
                                                 size,
                                                 pattern,
                                                 cm_,
                                                 objname,
                                                 nspace_->ns(),
                                                 OverwriteObject::F));

    EXPECT_TRUE(retrieveAndVerify(dst,
                                  size,
                                  pattern,
                                  &chksum,
                                  cm_,
                                  objname,
                                  nspace_->ns()));

    bi_(nspace_->ns())->remove(objname);
    fs::remove_all(dst);

    ensureObjectDoesNotExist(dst, objname);
}
// This is not too interesting either; it's mainly a debugging / testing help
TEST_F(BackendObjectTest, cache_invalidation)
{
    const std::string oname("some-object");
    const fs::path src(path_ / (oname + ".src"));
    const fs::path dst(path_ / (oname + ".dst"));
    const size_t size = 1ULL << 20;

    BackendInterfacePtr bi(bi_(nspace_->ns()));

    const std::string pattern1("some pattern");

    const yt::CheckSum chksum1(createPutAndVerify(src,
                                                  size,
                                                  pattern1,
                                                  cm_,
                                                  oname,
                                                  nspace_->ns(),
                                                  OverwriteObject::F));

    auto check([&](const std::string& pattern,
                   const yt::CheckSum& chksum,
                   InsistOnLatestVersion insist)
               {
                   bi->read(dst,
                            oname,
                            insist);

                   ALWAYS_CLEANUP_FILE(dst);

                   EXPECT_TRUE(verifyTestFile(dst,
                                              size,
                                              pattern,
                                              &chksum));
               });

    check(pattern1,
          chksum1,
          InsistOnLatestVersion::F);

    const std::string pattern2("some other pattern");

    const yt::CheckSum chksum2(createAndPut(src,
                                            size,
                                            pattern2,
                                            cm_,
                                            oname,
                                            nspace_->ns(),
                                            OverwriteObject::T));

    check(pattern2,
          chksum2,
          InsistOnLatestVersion::T);

    bi->invalidate_cache();

    check(pattern2,
          chksum2,
          InsistOnLatestVersion::F);
}
示例#6
0
  virtual double predict(const Instance& ins) const {
    double ret = 0.;
    size_t user_id = ins.get_feature_group_index(0, 0);
    size_t item_id = ins.get_feature_group_index(1, 0);

    auto fit = user_rated_items_.find(user_id);
    CHECK(fit != user_rated_items_.end());
    auto& user_rated_items = fit->second;

    double user_size = static_cast<double>(fit->second.size());
    if (user_rated_items.count(item_id)) {
      ret += bu_(user_id) + bi_(item_id) + (x_.row(user_id) - p_.row(item_id)).dot(q_.row(item_id)) / std::pow((user_size - 1.), alpha_);
    } else {
      ret += bu_(user_id) + bi_(item_id) + x_.row(user_id).dot(q_.row(item_id)) / std::pow(user_size, alpha_);
    }

    return ret;
  }
TEST_F(BackendObjectTest, DISABLED_insist_on_latest_version)
{
    const std::string oname("insist-on-latest");
    const fs::path src(path_ / (oname + ".src"));
    const fs::path dst(path_ / (oname + ".dst"));

    createAndPut(src,
                 4096,
                 oname,
                 cm_,
                 oname,
                 nspace_->ns(),
                 OverwriteObject::F);

    bi_(nspace_->ns())->read(dst,
                             oname,
                             InsistOnLatestVersion::T);
}
// OVS-2204: for some reason the count of open fds goes up after an error, but
//  valgrind does not report a leak. Try to figure our what's going on here.
TEST_F(ConnectionManagerTest, limited_pool_on_errors)
{
    std::async(std::launch::async,
               [&]
               {
                   pin_to_cpu_0();
                   const std::string oname("some-object");
                   const fs::path opath(path_ / oname);

                   const yt::CheckSum cs(createTestFile(opath,
                                                        4096,
                                                        "some pattern"));

                   yt::CheckSum wrong_cs(1);

                   ASSERT_NE(wrong_cs,
                             cs);

                   std::unique_ptr<be::BackendTestSetup::WithRandomNamespace>
                       nspace(make_random_namespace());

                   be::BackendInterfacePtr bi(bi_(nspace->ns()));
                   const size_t count = 2 * cm_->capacity();

                   for (size_t i = 0; i < count; ++i)
                   {
                       ASSERT_THROW(bi->write(opath,
                                              oname,
                                              OverwriteObject::F,
                                              &wrong_cs),
                                    be::BackendInputException);
                   }

                   ASSERT_EQ(cm_->capacity() / cm_->shards(),
                             cm_->size());
               }).wait();
}
TEST_F(BackendObjectTest, checksum_mismatch)
{
    const std::string oname("some-object");
    const fs::path opath(path_ / oname);

    const yt::CheckSum cs(createTestFile(opath,
                                         4096,
                                         "some pattern"));

    BackendInterfacePtr bi(bi_(nspace_->ns()));

    yt::CheckSum wrong_cs(1);

    ASSERT_NE(wrong_cs,
              cs);

    ASSERT_THROW(bi->write(opath,
                           oname,
                           OverwriteObject::F,
                           &wrong_cs),
                 BackendInputException);

    ASSERT_FALSE(bi->objectExists(oname));
}
TEST_F(BackendObjectTest, overwrite)
{
    const std::string objname("test");
    const fs::path src(path_ / std::string(objname + ".src"));
    const fs::path dst(path_ / std::string(objname + ".dst"));

    const size_t size1 = 4 << 20;
    const std::string pattern1("Stop me if you think you've heard this one before");

    const yt::CheckSum chksum1(createPutAndVerify(src,
                                                  size1,
                                                  pattern1,
                                                  cm_,
                                                  objname,
                                                  nspace_->ns(),
                                                  OverwriteObject::F));

    EXPECT_TRUE(retrieveAndVerify(dst,
                                  size1,
                                  pattern1,
                                  &chksum1,
                                  cm_,
                                  objname,
                                  nspace_->ns()));

    fs::remove_all(src);
    fs::remove_all(dst);

    {
        const size_t size2 = size1 / 2;
        const std::string pattern2("Shoplifters of the world unite");
        const yt::CheckSum chksum2(createTestFile(src,
                                                  size2,
                                                  pattern2));
        ALWAYS_CLEANUP_FILE(src);
        EXPECT_THROW(bi_(nspace_->ns())->write(src,
                                               objname,
                                               OverwriteObject::F,
                                               &chksum2),
                     BackendAssertionFailedException);

        try
        {
            EXPECT_FALSE(chksum2 == bi_(nspace_->ns())->getCheckSum(objname));
            EXPECT_TRUE(chksum1 == bi_(nspace_->ns())->getCheckSum(objname));
        }
        catch(const BackendNotImplementedException& e)
        {}

        EXPECT_EQ(size1,
                  bi_(nspace_->ns())->getSize(objname));

        bi_(nspace_->ns())->read(dst,
                                 objname,
                                 InsistOnLatestVersion::T);
        ALWAYS_CLEANUP_FILE(dst);

        EXPECT_TRUE(retrieveAndVerify(dst,
                                      size1,
                                      pattern1,
                                      &chksum1,
                                      cm_,
                                      objname,
                                      nspace_->ns()));
    }

    size_t size2 = size1 / 2;
    const std::string pattern2("Bigmouth strikes again");
    const yt::CheckSum chksum2(createPutAndVerify(src,
                                                  size2,
                                                  pattern2,
                                                  cm_,
                                                  objname,
                                                  nspace_->ns(),
                                                  OverwriteObject::T));

    EXPECT_TRUE(retrieveAndVerify(dst,
                                  size2,
                                  pattern2,
                                  &chksum2,
                                  cm_,
                                  objname,
                                  nspace_->ns()));

}