TEST_P(SnapshotRestoreTest, TestFailOver)
{
auto foc_ctx(start_one_foc());
auto ns_ptr = make_random_namespace();
SharedVolumePtr v = newVolume(VolumeId("volume1"),
ns_ptr->ns(),
VolumeSize((1 << 18) * 512),
SCOMultiplier(1));
v->setFailOverCacheConfig(foc_ctx->config(GetParam().foc_mode()));
VolumeConfig cfg = v->get_config();
v->createSnapshot(SnapshotName("snap0"));
for(int i = 0; i < 5; ++i)
{
writeToVolume(*v,
0,
4096,
"a");
}
waitForThisBackendWrite(*v);
v->restoreSnapshot(SnapshotName("snap0"));
for(int i = 0; i < 7; ++i)
{
writeToVolume(*v,
8,
4096,
"d");
}
flushFailOverCache(*v);
destroyVolume(v,
DeleteLocalData::T,
RemoveVolumeCompletely::F);
SharedVolumePtr v1 = 0;
v1 = getVolume(VolumeId("volume1"));
ASSERT_FALSE(v1);
restartVolume(cfg);
v1 = getVolume(VolumeId("volume1"));
ASSERT_TRUE(v1 != nullptr);
checkVolume(*v1,0,4096, "\0");
checkVolume(*v1,8,4096, "d");
checkCurrentBackendSize(*v1);
}
TEST_P(SnapshotRestoreTest, RestoreAndWriteAgain1)
{
auto ns_ptr = make_random_namespace();
SharedVolumePtr v = newVolume(VolumeId("volume1"),
ns_ptr->ns(),
VolumeSize(1 << 26),
SCOMultiplier(1));
const std::string pattern("e-manual");
v->createSnapshot(SnapshotName("snap1"));
waitForThisBackendWrite(*v);
writeToVolume(*v, 0, 5 * 4096, pattern);
waitForThisBackendWrite(*v);
restoreSnapshot(*v,"snap1");
writeToVolume(*v, 0, 4*4096, pattern);
waitForThisBackendWrite(*v);
checkCurrentBackendSize(*v);
}
bool
Backup::create_backup_volume()
{
LOG_INFO(__FUNCTION__);
// Need and AssertLocked here
VERIFY(not target_volume_);
VERIFY(source_volume_config.get());
api::getManagementMutex().assertLocked();
// We check whether the source is a Normal volume
boost::optional<VolumeConfig::WanBackupVolumeRole> role;
VERIFY(not role);
switch(source_volume_config->wan_backup_volume_role_)
{
case VolumeConfig::WanBackupVolumeRole::WanBackupNormal:
if(start_snapshot_)
{
LOG_INFO("Incremental backup of a normal volume");
role = VolumeConfig::WanBackupVolumeRole::WanBackupIncremental;
}
else
{
LOG_INFO("Complete backup of a normal volume");
role = VolumeConfig::WanBackupVolumeRole::WanBackupBase;
}
break;
case VolumeConfig::WanBackupVolumeRole::WanBackupBase:
if(start_snapshot_)
{
LOG_INFO("Incremental backup of a backup volume");
role = VolumeConfig::WanBackupVolumeRole::WanBackupIncremental;
}
else
{
LOG_INFO("Complete backup of a normal volume");
role = VolumeConfig::WanBackupVolumeRole::WanBackupBase;
}
break;
case VolumeConfig::WanBackupVolumeRole::WanBackupIncremental:
if(start_snapshot_)
{
LOG_INFO("Incremental backup of a incremental volume");
role = VolumeConfig::WanBackupVolumeRole::WanBackupIncremental;
}
else
{
std::vector<SnapshotNum> snaps;
source_snapshot_persistor->getAllSnapshots(snaps);
VERIFY(snaps.size() >= 2);
start_snapshot_ = source_snapshot_persistor->getSnapshotName(snaps[0]);
LOG_INFO("Complete backup of a incremental volume, created start snapshot " << *start_snapshot_);
role = VolumeConfig::WanBackupVolumeRole::WanBackupIncremental;
}
break;
}
VERIFY(role);
LOG_INFO("Creating new volume in the target with role " << *role);
try
{
boost::this_thread::interruption_point();
TODO("AR: consider OwnerTag")
const auto params =
WriteOnlyVolumeConfigParameters(VolumeId(source_volume_config->getNS().str()),
Namespace(target_namespace),
VolumeSize(source_volume_config->lba_count() *
source_volume_config->lba_size_),
*role,
OwnerTag(1))
.lba_size(source_volume_config->lba_size_)
.cluster_multiplier(source_volume_config->cluster_mult_)
.sco_multiplier(source_volume_config->sco_mult_);
target_volume_.reset(api::createNewWriteOnlyVolume(params));
}
CATCH_STD_ALL_LOG_RETHROW("problem creating WriteOnlyVolume");
LOG_INFO("Created new volume in the target -- name is the namespace of the source" << source_volume_config->getNS());
if(start_snapshot_)
{
LOG_INFO("Just created a volume and start snapshot was given... we create a snapshot in the beginning of the volume");
const SnapshotNum start_snapshot_num =
source_snapshot_persistor->getSnapshotNum(*start_snapshot_);
const SnapshotName& start_snapshot_name = *start_snapshot_;
const UUID
start_snapshot_uuid(source_snapshot_persistor->getUUID(start_snapshot_num));
boost::this_thread::interruption_point();
api::createSnapshot(target_volume_.get(),
SnapshotMetaData(),
&start_snapshot_name,
start_snapshot_uuid);
WAIT_FOR_THIS_VOLUME_WRITE(target_volume_.get());
VERIFY(api::isVolumeSyncedUpTo(target_volume_.get(),
start_snapshot_name));
if(*start_snapshot_ == end_snapshot_name)
{
LOG_INFO("start_snapshot == end_snapshot " << end_snapshot_name << " so exiting now");
while(not api::isVolumeSyncedUpTo(target_volume_.get(),
*start_snapshot_))
{
LOG_INFO("Waiting for the volume to finish writing to the backend, sleeping another 10 seconds");
sleep(10);
}
status_.finish();
LOG_INFO("Finished the Backup");
return false;
}
}
else
{
LOG_INFO("Is a full backup to the newly created volume");
}
return true;
}
VImage
VDoMulticomp3d(VImage src, VImage multicomp, VBoolean verbose) {
VString str;
Volumes volumes;
Volume vol;
VImage tmp = NULL, label_image = NULL, dest = NULL;
VFloat *src_pp, *dst_pp, u, zthr = 0, zmax = 0, voxsize = 1, sym = 0;
VBit *bin_pp;
int i, nl, size, nbands, nrows, ncols, npixels;
int b, r, c, b0, r0, c0;
int nflag, pflag;
float x0, x1, x2;
float sign = 1;
float voxel[3], ca[3], extent[3];
/*
** read multicomp file header
*/
voxsize = 1;
if(VGetAttr(VImageAttrList(multicomp), "voxel_size", NULL,
VFloatRepn, (VPointer) & voxsize) != VAttrFound)
VError(" attribute 'voxel_size' not found");
if(VGetAttr(VImageAttrList(multicomp), "zthr", NULL, VFloatRepn, (VPointer) &zthr) != VAttrFound)
VError(" attribute 'zthr' not found");
/*
** read src header
*/
if(verbose) {
if(VGetAttr(VImageAttrList(src), "voxel", NULL,
VStringRepn, (VPointer) & str) != VAttrFound)
VError(" attribute 'voxel' not found");
sscanf(str, "%f %f %f", &x0, &x1, &x2);
if(ABS(x0 * x1 * x2 - voxsize) > 0.01)
VError(" voxel sizes do not match %.3f %.3f %.3f", x0, x1, x2);
voxel[0] = x0;
voxel[1] = x1;
voxel[2] = x2;
if(VGetAttr(VImageAttrList(src), "ca", NULL,
VStringRepn, (VPointer) & str) != VAttrFound)
VError(" attribute 'ca' not found");
sscanf(str, "%f %f %f", &x0, &x1, &x2);
ca[0] = x0;
ca[1] = x1;
ca[2] = x2;
if(VGetAttr(VImageAttrList(src), "extent", NULL,
VStringRepn, (VPointer) & str) != VAttrFound)
VError(" attribute 'extent' not found");
sscanf(str, "%f %f %f", &x0, &x1, &x2);
extent[0] = x0;
extent[1] = x1;
extent[2] = x2;
}
nbands = VImageNBands(src);
nrows = VImageNRows(src);
ncols = VImageNColumns(src);
npixels = VImageNPixels(src);
tmp = VCreateImage(nbands, nrows, ncols, VBitRepn);
dest = VCopyImage(src, NULL, VAllBands);
VFillImage(dest, VAllBands, 0);
/*
** positive threshold
*/
nflag = pflag = 0;
src_pp = VImageData(src);
bin_pp = VImageData(tmp);
for(i = 0; i < npixels; i++) {
*bin_pp = 0;
u = *src_pp;
if(u > 0 && u >= zthr)
*bin_pp = 1;
src_pp++;
bin_pp++;
}
label_image = VLabelImage3d(tmp, label_image, (int)26, VShortRepn, &nl);
if(nl < 1 && pflag == 0) {
VWarning(" no voxels above threshold %.3f", zthr);
goto next;
} else
pflag++;
volumes = VImage2Volumes(label_image);
for(vol = volumes->first; vol != NULL; vol = vol->next) {
sign = 1;
size = VolumeSize(vol);
zmax = VolumeZMax(vol, src, (float)1, &b, &r, &c);
sym = VolumeSym(vol, src, sign, zthr);
if(CheckValueSymm(multicomp, size, zmax, sym) == FALSE) {
VolumeZero(vol, tmp);
} else {
if(verbose) {
VPixel2Tal(ca, voxel, extent, b, r, c, &x0, &x1, &x2);
c0 = VRint(x0);
r0 = VRint(x1);
b0 = VRint(x2);
fprintf(stderr, " nvoxels: %5d, zmax: %7.3f, sym: %.3f, addr: %3d %3d %3d\n",
size, zmax, sym, c0, r0, b0);
}
}
}
src_pp = VImageData(src);
dst_pp = VImageData(dest);
bin_pp = VImageData(tmp);
for(i = 0; i < npixels; i++) {
if(*bin_pp > 0)
*dst_pp = *src_pp;
src_pp++;
dst_pp++;
bin_pp++;
}
/*
** negative threshold
*/
next:
src_pp = VImageData(src);
bin_pp = VImageData(tmp);
for(i = 0; i < npixels; i++) {
*bin_pp = 0;
u = *src_pp;
if(u < 0 && -u >= zthr)
*bin_pp = 1;
src_pp++;
bin_pp++;
}
label_image = VLabelImage3d(tmp, label_image, (int)26, VShortRepn, &nl);
if(nl < 1 && nflag == 0) {
/* VWarning(" no voxels below negative threshold %.3f",-zthr); */
goto ende;
} else
nflag++;
volumes = VImage2Volumes(label_image);
for(vol = volumes->first; vol != NULL; vol = vol->next) {
sign = -1;
size = VolumeSize(vol);
zmax = VolumeZMax(vol, src, sign, &b, &r, &c);
sym = VolumeSym(vol, src, sign, zthr);
if(CheckValueSymm(multicomp, size, zmax, sym) == FALSE) {
VolumeZero(vol, tmp);
} else {
if(verbose) {
VPixel2Tal(ca, voxel, extent, b, r, c, &x0, &x1, &x2);
c0 = VRint(x0);
r0 = VRint(x1);
b0 = VRint(x2);
fprintf(stderr, " nvoxels: %5d, zmax: %7.3f, sym: %.3f, addr: %3d %3d %3d\n",
size, zmax, sym, c0, r0, b0);
}
}
}
src_pp = VImageData(src);
dst_pp = VImageData(dest);
bin_pp = VImageData(tmp);
for(i = 0; i < npixels; i++) {
if(*bin_pp > 0)
*dst_pp = *src_pp;
src_pp++;
dst_pp++;
bin_pp++;
}
ende:
if(nflag == 0 && pflag == 0)
VError(" no voxels passed threshold");
return dest;
}
void
VROIpaired_ttest(VImage *src1, VImage *src2, VImage *mask, int n, int nmask, FILE *fp) {
VString str;
int i, j, id, b, r, c, c0, c1, nROI = 0;
float ave1 = 0, ave2 = 0, var1 = 0, var2 = 0;
float sum1 = 0, sum2 = 0, u, mx;
float t, z, p, df, sd, cov, *data1 = NULL, *data2 = NULL;
float tiny = 1.0e-8;
float xa, ya, za, xx, yy, zz, voxelsize = 1;
double mean[3];
Volumes *volumes;
Volume vol;
VTrack tc;
VBoolean found = FALSE;
gsl_set_error_handler_off();
/*
** get ROIs from mask
*/
fprintf(stderr, "\n List of ROIs:\n");
fprintf(fp, "\n List of ROIs:\n");
volumes = (Volumes *) VCalloc(nmask, sizeof(Volumes));
nROI = 0;
for(i = 0; i < nmask; i++) {
fprintf(stderr, "\n Mask %2d:\n", i + 1);
fprintf(fp, "\n Mask %2d:\n", i + 1);
volumes[i] = VImage2Volumes(mask[i]);
voxelsize = 1;
if(VGetAttr(VImageAttrList(mask[i]), "voxel", NULL,
VStringRepn, (VPointer) & str) == VAttrFound) {
sscanf(str, "%f %f %f", &xa, &ya, &za);
voxelsize = xa * ya * za;
}
fprintf(stderr, " ROI addr size(mm^3)\n");
fprintf(stderr, "-----------------------------------------------\n");
fprintf(fp, " ROI addr size(mm^3)\n");
fprintf(fp, "-----------------------------------------------\n");
for(vol = volumes[i]->first; vol != NULL; vol = vol->next) {
VolumeCentroid(vol, mean);
if(nROI < vol->label)
nROI = vol->label;
b = mean[0];
r = mean[1];
c = mean[2];
xx = mean[2];
yy = mean[1];
zz = mean[0];
VGetTalCoord(src1[0], zz, yy, xx, &xa, &ya, &za);
id = vol->label;
fprintf(stderr, " %2d %7.2f %7.2f %7.2f %7.0f\n",
id, xa, ya, za, voxelsize *(double)VolumeSize(vol));
fprintf(fp, " %2d %7.2f %7.2f %7.2f %7.0f\n",
id, xa, ya, za, voxelsize *(double)VolumeSize(vol));
}
}
fprintf(stderr, "\n\n");
fprintf(fp, "\n\n");
/* check consistency */
if(nROI < 1)
VError(" no ROIs found");
CheckROI(volumes, nmask, nROI);
/*
** process each ROI
*/
fprintf(stderr, "\n");
fprintf(stderr, " ROI mean t z p \n");
fprintf(stderr, " --------------------------------------------------\n");
if(fp) {
fprintf(fp, "\n");
fprintf(fp, " ROI mean t z p \n");
fprintf(fp, " --------------------------------------------------\n");
}
df = n - 1;
data1 = (float *) VCalloc(n, sizeof(float));
data2 = (float *) VCalloc(n, sizeof(float));
for(id = 1; id <= nROI; id++) {
for(i = 0; i < n; i++) {
j = 0;
if(nmask > 1)
j = i;
found = FALSE;
for(vol = volumes[j]->first; vol != NULL; vol = vol->next) {
if(vol->label != id)
continue;
found = TRUE;
sum1 = sum2 = mx = 0;
for(j = 0; j < VolumeNBuckets(vol); j++) {
for(tc = VFirstTrack(vol, j); VTrackExists(tc); tc = VNextTrack(tc)) {
b = tc->band;
r = tc->row;
c0 = tc->col;
c1 = c0 + tc->length;
for(c = c0; c < c1; c++) {
u = VPixel(src1[i], b, r, c, VFloat);
if(ABS(u) < tiny)
continue;
sum1 += u;
u = VPixel(src2[i], b, r, c, VFloat);
if(ABS(u) < tiny)
continue;
sum2 += u;
mx++;
}
}
}
if(mx < 1) {
VWarning(" no voxels in ROI %d of mask %d", id, i);
data1[i] = data2[i] = 0;
continue;
}
data1[i] = sum1 / mx;
data2[i] = sum2 / mx;
}
if(!found)
goto next;
}
avevar(data1, n, &ave1, &var1);
avevar(data2, n, &ave2, &var2);
if(var1 < tiny || var2 < tiny) {
VWarning(" no variance in ROI %d", id);
continue;
}
z = t = p = 0;
cov = 0;
for(j = 0; j < n; j++)
cov += (data1[j] - ave1) * (data2[j] - ave2);
cov /= df;
sd = sqrt((var1 + var2 - 2.0 * cov) / (df + 1.0));
if(sd < tiny)
continue;
t = (ave1 - ave2) / sd;
if(ABS(t) < tiny)
p = z = 0;
else {
p = t2p((double)t, (double)df);
z = t2z((double)t, (double)df);
if(t < 0)
z = -z;
}
fprintf(stderr, " %3d %8.4f (%.3f) %7.3f %7.3f %7.4f\n",
id, (ave1 - ave2), sd, t, z, p);
if(fp)
fprintf(fp, " %3d %8.4f (%.3f) %7.3f %7.3f %7.4f\n",
id, (ave1 - ave2), sd, t, z, p);
next:
;
}
fprintf(stderr, "\n");
if(fp) {
fprintf(fp, "\n");
fclose(fp);
}
}