void
ensureObjectDoesNotExist(const fs::path& p,
const std::string& objname)
{
EXPECT_FALSE(fs::exists(p));
EXPECT_FALSE(bi_(nspace_->ns())->objectExists(objname));
EXPECT_THROW(bi_(nspace_->ns())->getSize(objname),
BackendObjectDoesNotExistException);
if (BackendTestSetup::backend_config().backend_type.value() == BackendType::S3)
{
EXPECT_THROW(bi_(nspace_->ns())->getCheckSum(objname),
BackendNotImplementedException);
}
else
{
EXPECT_THROW(bi_(nspace_->ns())->getCheckSum(objname),
BackendObjectDoesNotExistException);
}
EXPECT_THROW(bi_(nspace_->ns())->read(p, objname, InsistOnLatestVersion::T),
BackendObjectDoesNotExistException);
EXPECT_FALSE(fs::exists(p));
}
void
createAndListObjects(size_t objsize,
const std::string& pattern,
size_t num_objs,
size_t check_every_n = 0)
{
std::list<std::string> objs;
const fs::path src(path_ / "test.src");
const yt::CheckSum chksum(createTestFile(src,
objsize,
pattern));
for (size_t i = 0; i < num_objs; ++i)
{
if (check_every_n != 0 and i % check_every_n == 0)
{
bi_(nspace_->ns())->listObjects(objs);
EXPECT_EQ(i, objs.size());
objs.clear();
}
std::stringstream ss;
ss << std::setw(8) << std::setfill('0') << i;
bi_(nspace_->ns())->write(src,
ss.str(),
OverwriteObject::F,
&chksum);
}
bi_(nspace_->ns())->listObjects(objs);
EXPECT_EQ(num_objs, objs.size());
}
void update_one_instance(const Instance& ins, double step_size) {
size_t uid = ins.get_feature_group_index(0, 0);
size_t iid = ins.get_feature_group_index(1, 0);
size_t jid;
double pred = predict(ins);
for (size_t idx = 0; idx < num_neg_; ++ idx) {
jid = sample_negative_item(uid);
double pred_neg = predict_user_item_rating(uid, jid);
double grad = loss_->gradient(pred - pred_neg, 1.);
auto bigrad = grad + lambda_ * bi_(iid);
auto bjgrad = -grad + lambda_ * bi_(jid);
if (using_adagrad_) {
bi_grad_(iid) += bigrad * bigrad;
bi_grad_(jid) += bjgrad * bjgrad;
bigrad /= std::sqrt(bi_grad_(iid));
bjgrad /= std::sqrt(bi_grad_(jid));
}
bi_(iid) -= step_size * bigrad;
bi_(jid) -= step_size * bjgrad;
DVector x_grad = DVector::Zero(num_dim_);
auto fit = user_rated_items_.find(uid);
CHECK(fit != user_rated_items_.end());
double user_size = static_cast<double>(fit->second.size());
for (auto& kid : fit->second) {
if (kid == iid) continue;
DVector pj_grad = grad * (q_.row(iid) - q_.row(jid)) / std::pow((user_size - 1.), alpha_) + lambda_ * p_.row(kid);
if (using_adagrad_) {
p_grad_.row(kid) += pj_grad.cwiseProduct(pj_grad);
for (size_t k = 0; k < num_dim_; ++k)
pj_grad(k) = pj_grad(k) / std::sqrt(p_grad_(kid, k));
}
p_.row(kid) -= step_size * pj_grad;
x_grad += pj_grad;
}
DVector qi_grad = grad * (x_.row(uid) - p_.row(iid)) / std::pow((user_size - 1.), alpha_) + lambda_ * q_.row(iid);
DVector qj_grad = - grad * (x_.row(uid) - p_.row(iid)) / std::pow((user_size - 1.), alpha_) + lambda_ * q_.row(jid);
if (using_adagrad_) {
q_grad_.row(iid) += qi_grad.cwiseProduct(qi_grad);
q_grad_.row(jid) += qj_grad.cwiseProduct(qj_grad);
for (size_t k = 0; k < num_dim_; ++k) {
qi_grad(k) = qi_grad(k) / std::sqrt(q_grad_(iid, k));
qj_grad(k) = qj_grad(k) / std::sqrt(q_grad_(jid, k));
}
}
q_.row(iid) -= step_size * qi_grad;
q_.row(jid) -= step_size * qj_grad;
x_.row(uid) -= step_size * x_grad;
}
}
TEST_F(BackendObjectTest, basics)
{
const std::string objname("test");
const fs::path src(path_ / std::string(objname + ".src"));
const fs::path dst(path_ / std::string(objname + ".dst"));
ensureObjectDoesNotExist(dst,
objname);
const std::string pattern("William, it was really nothing");
const size_t size = 1 << 20;
const yt::CheckSum chksum(createPutAndVerify(src,
size,
pattern,
cm_,
objname,
nspace_->ns(),
OverwriteObject::F));
EXPECT_TRUE(retrieveAndVerify(dst,
size,
pattern,
&chksum,
cm_,
objname,
nspace_->ns()));
bi_(nspace_->ns())->remove(objname);
fs::remove_all(dst);
ensureObjectDoesNotExist(dst, objname);
}
// This is not too interesting either; it's mainly a debugging / testing help
TEST_F(BackendObjectTest, cache_invalidation)
{
const std::string oname("some-object");
const fs::path src(path_ / (oname + ".src"));
const fs::path dst(path_ / (oname + ".dst"));
const size_t size = 1ULL << 20;
BackendInterfacePtr bi(bi_(nspace_->ns()));
const std::string pattern1("some pattern");
const yt::CheckSum chksum1(createPutAndVerify(src,
size,
pattern1,
cm_,
oname,
nspace_->ns(),
OverwriteObject::F));
auto check([&](const std::string& pattern,
const yt::CheckSum& chksum,
InsistOnLatestVersion insist)
{
bi->read(dst,
oname,
insist);
ALWAYS_CLEANUP_FILE(dst);
EXPECT_TRUE(verifyTestFile(dst,
size,
pattern,
&chksum));
});
check(pattern1,
chksum1,
InsistOnLatestVersion::F);
const std::string pattern2("some other pattern");
const yt::CheckSum chksum2(createAndPut(src,
size,
pattern2,
cm_,
oname,
nspace_->ns(),
OverwriteObject::T));
check(pattern2,
chksum2,
InsistOnLatestVersion::T);
bi->invalidate_cache();
check(pattern2,
chksum2,
InsistOnLatestVersion::F);
}
virtual double predict(const Instance& ins) const {
double ret = 0.;
size_t user_id = ins.get_feature_group_index(0, 0);
size_t item_id = ins.get_feature_group_index(1, 0);
auto fit = user_rated_items_.find(user_id);
CHECK(fit != user_rated_items_.end());
auto& user_rated_items = fit->second;
double user_size = static_cast<double>(fit->second.size());
if (user_rated_items.count(item_id)) {
ret += bu_(user_id) + bi_(item_id) + (x_.row(user_id) - p_.row(item_id)).dot(q_.row(item_id)) / std::pow((user_size - 1.), alpha_);
} else {
ret += bu_(user_id) + bi_(item_id) + x_.row(user_id).dot(q_.row(item_id)) / std::pow(user_size, alpha_);
}
return ret;
}
TEST_F(BackendObjectTest, DISABLED_insist_on_latest_version)
{
const std::string oname("insist-on-latest");
const fs::path src(path_ / (oname + ".src"));
const fs::path dst(path_ / (oname + ".dst"));
createAndPut(src,
4096,
oname,
cm_,
oname,
nspace_->ns(),
OverwriteObject::F);
bi_(nspace_->ns())->read(dst,
oname,
InsistOnLatestVersion::T);
}
// OVS-2204: for some reason the count of open fds goes up after an error, but
// valgrind does not report a leak. Try to figure our what's going on here.
TEST_F(ConnectionManagerTest, limited_pool_on_errors)
{
std::async(std::launch::async,
[&]
{
pin_to_cpu_0();
const std::string oname("some-object");
const fs::path opath(path_ / oname);
const yt::CheckSum cs(createTestFile(opath,
4096,
"some pattern"));
yt::CheckSum wrong_cs(1);
ASSERT_NE(wrong_cs,
cs);
std::unique_ptr<be::BackendTestSetup::WithRandomNamespace>
nspace(make_random_namespace());
be::BackendInterfacePtr bi(bi_(nspace->ns()));
const size_t count = 2 * cm_->capacity();
for (size_t i = 0; i < count; ++i)
{
ASSERT_THROW(bi->write(opath,
oname,
OverwriteObject::F,
&wrong_cs),
be::BackendInputException);
}
ASSERT_EQ(cm_->capacity() / cm_->shards(),
cm_->size());
}).wait();
}
TEST_F(BackendObjectTest, checksum_mismatch)
{
const std::string oname("some-object");
const fs::path opath(path_ / oname);
const yt::CheckSum cs(createTestFile(opath,
4096,
"some pattern"));
BackendInterfacePtr bi(bi_(nspace_->ns()));
yt::CheckSum wrong_cs(1);
ASSERT_NE(wrong_cs,
cs);
ASSERT_THROW(bi->write(opath,
oname,
OverwriteObject::F,
&wrong_cs),
BackendInputException);
ASSERT_FALSE(bi->objectExists(oname));
}
TEST_F(BackendObjectTest, overwrite)
{
const std::string objname("test");
const fs::path src(path_ / std::string(objname + ".src"));
const fs::path dst(path_ / std::string(objname + ".dst"));
const size_t size1 = 4 << 20;
const std::string pattern1("Stop me if you think you've heard this one before");
const yt::CheckSum chksum1(createPutAndVerify(src,
size1,
pattern1,
cm_,
objname,
nspace_->ns(),
OverwriteObject::F));
EXPECT_TRUE(retrieveAndVerify(dst,
size1,
pattern1,
&chksum1,
cm_,
objname,
nspace_->ns()));
fs::remove_all(src);
fs::remove_all(dst);
{
const size_t size2 = size1 / 2;
const std::string pattern2("Shoplifters of the world unite");
const yt::CheckSum chksum2(createTestFile(src,
size2,
pattern2));
ALWAYS_CLEANUP_FILE(src);
EXPECT_THROW(bi_(nspace_->ns())->write(src,
objname,
OverwriteObject::F,
&chksum2),
BackendAssertionFailedException);
try
{
EXPECT_FALSE(chksum2 == bi_(nspace_->ns())->getCheckSum(objname));
EXPECT_TRUE(chksum1 == bi_(nspace_->ns())->getCheckSum(objname));
}
catch(const BackendNotImplementedException& e)
{}
EXPECT_EQ(size1,
bi_(nspace_->ns())->getSize(objname));
bi_(nspace_->ns())->read(dst,
objname,
InsistOnLatestVersion::T);
ALWAYS_CLEANUP_FILE(dst);
EXPECT_TRUE(retrieveAndVerify(dst,
size1,
pattern1,
&chksum1,
cm_,
objname,
nspace_->ns()));
}
size_t size2 = size1 / 2;
const std::string pattern2("Bigmouth strikes again");
const yt::CheckSum chksum2(createPutAndVerify(src,
size2,
pattern2,
cm_,
objname,
nspace_->ns(),
OverwriteObject::T));
EXPECT_TRUE(retrieveAndVerify(dst,
size2,
pattern2,
&chksum2,
cm_,
objname,
nspace_->ns()));
}