static
void
domapread(unsigned offset, unsigned size)
{
unsigned pg_offset;
unsigned map_size;
char *p;
offset -= offset % readbdy;
if (size == 0) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping zero size read\n");
log4(OP_SKIPPED, OP_MAPREAD, offset, size);
return;
}
if (size + offset > file_size) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping seek/read past end of file\n");
log4(OP_SKIPPED, OP_MAPREAD, offset, size);
return;
}
log4(OP_MAPREAD, offset, size, 0);
if (testcalls <= simulatedopcount)
return;
if (!quiet && ((progressinterval &&
testcalls % progressinterval == 0) ||
(debug &&
(monitorstart == -1 ||
(offset + size > monitorstart &&
(monitorend == -1 || offset <= monitorend))))))
prt("%lu mapread\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
offset, offset + size - 1, size);
pg_offset = offset & page_mask;
map_size = pg_offset + size;
if ((p = (char *)mmap(0, map_size, PROT_READ, MAP_FILE | MAP_SHARED, fd,
(off_t)(offset - pg_offset))) == (char *)-1) {
prterr("domapread: mmap");
report_failure(190);
}
if (setjmp(jmpbuf) == 0) {
jmpbuf_good = 1;
memcpy(temp_buf, p + pg_offset, size);
check_eofpage("Read", offset, p, size);
jmpbuf_good = 0;
} else {
report_failure(1901);
}
if (munmap(p, map_size) != 0) {
prterr("domapread: munmap");
report_failure(191);
}
check_buffers(offset, size);
}
bool cpiPython::CallAll(int func, w_Targs* args) // the default handler: returns true unless the calback returns 0
{
if (!online) return true;
w_Targs* result;
bool ret = true;
long l;
if (func != W_OnTimer) log2("PY: CallAll %s: parameters %s\n", lib_hookname(func), lib_packprint(args)) // no ';' since this would break 'else'
else log4("PY: CallAll %s\n", lib_hookname(func));
if(Size())
{
tvPythonInterpreter::iterator it;
for(it = mPython.begin(); it != mPython.end(); ++it)
{
result = (*it)->CallFunction(func, args);
if(!result)
{
// callback doesn't exist or a failure
if (func != W_OnTimer) log4("PY: CallAll %s: returned NULL\n", lib_hookname(func));
continue;
}
if(lib_unpack(result, "l", &l)) // default return value is 1L meaning: further processing,
{
if (func != W_OnTimer) log3("PY: CallAll %s: returned l:%ld\n", lib_hookname(func), l);
if (!l) ret = false; // 0L means no more processing outside this plugin
}
else // something unknown was returned... we will let the hub call other plugins
{
log1("PY: CallAll %s: unexpected return value %s\n", lib_hookname(func), lib_packprint(result));
}
free(result);
}
}
free(args); // WARNING: args is freed, do not try to access it after calling CallAll!
return ret;
}
void
dowrite(unsigned offset, unsigned size)
{
ssize_t ret;
off_t newsize;
offset -= offset % writebdy;
if (o_direct)
size -= size % writebdy;
if (size == 0) {
if (!quiet && testcalls > simulatedopcount && !o_direct)
prt("skipping zero size write\n");
log4(OP_SKIPPED, OP_WRITE, offset, size);
return;
}
log4(OP_WRITE, offset, size, file_size);
gendata(original_buf, good_buf, offset, size);
if (file_size < offset + size) {
newsize = ceil(((double)offset + size) / truncbdy) * truncbdy;
if (file_size < newsize)
memset(good_buf + file_size, '\0', newsize - file_size);
file_size = newsize;
if (lite) {
warn("Lite file size bug in fsx!");
report_failure(149);
}
ret = rbd_resize(image, newsize);
if (ret < 0) {
prterrcode("dowrite: resize", ret);
report_failure(150);
}
}
if (testcalls <= simulatedopcount)
return;
if (!quiet &&
((progressinterval && testcalls % progressinterval == 0) ||
(debug &&
(monitorstart == -1 ||
(offset + size > monitorstart &&
(monitorend == -1 || offset <= monitorend))))))
prt("%lu write\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
offset, offset + size - 1, size);
ret = rbd_write(image, offset, size, good_buf + offset);
if (ret != size) {
if (ret < 0)
prterrcode("dowrite: rbd_write", ret);
else
prt("short write: 0x%x bytes instead of 0x%x\n",
ret, size);
report_failure(151);
}
if (flush) {
doflush(offset, size);
}
}
static
void
dowrite(unsigned offset, unsigned size)
{
off_t ret;
unsigned iret;
offset -= offset % writebdy;
if (size == 0) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping zero size write\n");
log4(OP_SKIPPED, OP_WRITE, offset, size);
return;
}
log4(OP_WRITE, offset, size, file_size);
gendata(original_buf, good_buf, offset, size);
if (file_size < offset + size) {
if (file_size < offset)
memset(good_buf + file_size, '\0', offset - file_size);
file_size = offset + size;
if (lite) {
warn("Lite file size bug in fsx!");
report_failure(149);
}
}
if (testcalls <= simulatedopcount)
return;
if (!quiet && ((progressinterval &&
testcalls % progressinterval == 0) ||
(debug &&
(monitorstart == -1 ||
(offset + size > monitorstart &&
(monitorend == -1 || offset <= monitorend))))))
prt("%lu write\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
offset, offset + size - 1, size);
ret = lseek(fd, (off_t)offset, SEEK_SET);
if (ret == (off_t)-1) {
prterr("dowrite: lseek");
report_failure(150);
}
iret = write(fd, good_buf + offset, size);
if (iret != size) {
if (iret == -1)
prterr("dowrite: write");
else
prt("short write: 0x%x bytes instead of 0x%x\n",
iret, size);
report_failure(151);
}
}
void
doread(unsigned offset, unsigned size)
{
off_t ret;
unsigned iret;
offset -= offset % readbdy;
if (o_direct)
size -= size % readbdy;
if (size == 0) {
if (!quiet && testcalls > simulatedopcount && !o_direct)
prt("skipping zero size read\n");
log4(OP_SKIPPED, OP_READ, offset, size);
return;
}
if (size + offset > file_size) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping seek/read past end of file\n");
log4(OP_SKIPPED, OP_READ, offset, size);
return;
}
log4(OP_READ, offset, size, 0);
if (testcalls <= simulatedopcount)
return;
if (!quiet &&
((progressinterval && testcalls % progressinterval == 0) ||
(fsx_debug &&
(monitorstart == -1 ||
(offset + size > monitorstart &&
(monitorend == -1 || offset <= monitorend))))))
prt("%lu read\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
offset, offset + size - 1, size);
ret = lseek(fd, (off_t)offset, SEEK_SET);
if (ret == (off_t)-1) {
prterr("doread: lseek");
report_failure(140);
}
iret = fsxread(fd, temp_buf, size, offset);
if (iret != size) {
if (iret == -1)
prterr("doread: read");
else
prt("short read: 0x%x bytes instead of 0x%x\n",
iret, size);
report_failure(141);
}
check_buffers(offset, size);
}
void
do_punch_hole(unsigned offset, unsigned length)
{
unsigned end_offset;
int max_offset = 0;
int max_len = 0;
int ret;
offset -= offset % holebdy;
length -= length % holebdy;
if (length == 0) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping zero length punch hole\n");
log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, length);
return;
}
if (file_size <= (loff_t)offset) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping hole punch off the end of the file\n");
log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, length);
return;
}
end_offset = offset + length;
log4(OP_PUNCH_HOLE, offset, length, 0);
if (testcalls <= simulatedopcount)
return;
if ((progressinterval && testcalls % progressinterval == 0) ||
(debug && (monitorstart == -1 || monitorend == -1 ||
end_offset <= monitorend))) {
prt("%lu punch\tfrom 0x%x to 0x%x, (0x%x bytes)\n", testcalls,
offset, offset+length, length);
}
ret = ops->discard(&ctx, (unsigned long long)offset,
(unsigned long long)length);
if (ret < 0) {
prterrcode("do_punch_hole: ops->discard", ret);
report_failure(161);
}
max_offset = offset < file_size ? offset : file_size;
max_len = max_offset + length <= file_size ? length :
file_size - max_offset;
memset(good_buf + max_offset, '\0', max_len);
}
/* fallocate is basically a no-op unless extending, then a lot like a truncate */
void
do_preallocate(unsigned offset, unsigned length)
{
unsigned end_offset;
int keep_size;
if (length == 0) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping zero length fallocate\n");
log4(OP_SKIPPED, OP_FALLOCATE, offset, length);
return;
}
keep_size = random() % 2;
end_offset = keep_size ? 0 : offset + length;
if (end_offset > biggest) {
biggest = end_offset;
if (!quiet && testcalls > simulatedopcount)
prt("fallocating to largest ever: 0x%x\n", end_offset);
}
/*
* last arg matches fallocate string array index in logdump:
* 0: allocate past EOF
* 1: extending prealloc
* 2: interior prealloc
*/
log4(OP_FALLOCATE, offset, length, (end_offset > file_size) ? (keep_size ? 0 : 1) : 2);
if (end_offset > file_size) {
memset(good_buf + file_size, '\0', end_offset - file_size);
file_size = end_offset;
}
if (testcalls <= simulatedopcount)
return;
if ((progressinterval && testcalls % progressinterval == 0) ||
(fsx_debug && (monitorstart == -1 || monitorend == -1 ||
end_offset <= monitorend)))
prt("%lu falloc\tfrom 0x%x to 0x%x (0x%x bytes)\n", testcalls,
offset, offset + length, length);
if (fallocate(fd, keep_size ? FALLOC_FL_KEEP_SIZE : 0, (loff_t)offset, (loff_t)length) == -1) {
prt("fallocate: %x to %x\n", offset, length);
prterr("do_preallocate: fallocate");
report_failure(161);
}
}
void
dotruncate(unsigned size)
{
int oldsize = file_size;
size -= size % truncbdy;
if (size > biggest) {
biggest = size;
if (!quiet && testcalls > simulatedopcount)
prt("truncating to largest ever: 0x%x\n", size);
}
log4(OP_TRUNCATE, size, (unsigned)file_size, 0);
if (size > file_size)
memset(good_buf + file_size, '\0', size - file_size);
file_size = size;
if (testcalls <= simulatedopcount)
return;
if ((progressinterval && testcalls % progressinterval == 0) ||
(fsx_debug && (monitorstart == -1 || monitorend == -1 ||
size <= monitorend)))
prt("%lu trunc\tfrom 0x%x to 0x%x\n", testcalls, oldsize, size);
if (ftruncate(fd, (off_t)size) == -1) {
prt("ftruncate1: %x\n", size);
prterr("dotruncate: ftruncate");
report_failure(160);
}
}
void
dotruncate(unsigned size)
{
int oldsize = file_size;
int ret;
size -= size % truncbdy;
if (size > biggest) {
biggest = size;
if (!quiet && testcalls > simulatedopcount)
prt("truncating to largest ever: 0x%x\n", size);
}
log4(OP_TRUNCATE, size, (unsigned)file_size, 0);
if (size > file_size)
memset(good_buf + file_size, '\0', size - file_size);
else if (size < file_size)
memset(good_buf + size, '\0', file_size - size);
file_size = size;
if (testcalls <= simulatedopcount)
return;
if ((progressinterval && testcalls % progressinterval == 0) ||
(debug && (monitorstart == -1 || monitorend == -1 ||
size <= monitorend)))
prt("%lu trunc\tfrom 0x%x to 0x%x\n", testcalls, oldsize, size);
if ((ret = rbd_resize(image, size)) < 0) {
prt("rbd_resize: %x\n", size);
prterrcode("dotruncate: ftruncate", ret);
report_failure(160);
}
}
/* @Core */
Mat PDCounter::detect(Mat& input) {
string log1("Begin detecting...\n");
Mat output = input.clone();
for (vector<PDDetector>::iterator it = Detectors.begin(); it != Detectors.end(); it++) {
(*it).detect(output, Trackers);// NO parallel
}
string log2("All areas are detected...\n");
string lost = i_to_s(Trackers.tracking(output)); // NO parallel
string log3("Tracking completed...\nThe trackers lost " + lost + " pedestrians in this frame.\n");
if (showPedestrian) {
drawFounds(output, Trackers.getCurrRects(), PEDESTRIAN_COLOR);
}
if (showArea) {
for (vector<PDDetector>::iterator it = Detectors.begin(); it != Detectors.end(); it++) {
drawArea(output, (*it).getArea(), AREA_COLOR);
}
}
if (showTrajectory) {
for (int i = 0; i < Trackers.getSize(); i++) {
drawTrajectory(output, Trackers[i].getTrajectory(), Trajectory_COLOR);
}
}
string log4("Drawing completed");
lastLog = log1 + log2 + log3 + log4;
return output;
}
w_Targs* cpiPython::SQL (int id, w_Targs* args) // (char *query)
{
char *query;
string q;
long limit;
if (!lib_begin || !lib_pack || !lib_unpack || !lib_packprint || !mQuery) return NULL;
if (!lib_unpack(args, "sl", &query, &limit)) return NULL;
if (!query) return NULL;
if (limit < 1) limit = 100;
log4("PY: SQL query: %s\n", query);
q = string() + query;
mQuery->OStream() << q;
if (mQuery->Query() < 0) { mQuery->Clear(); return lib_pack("lllp", (long)0, (long)0, (long)0, (void*)NULL); } // error
int rows = mQuery->StoreResult();
if (limit < rows) rows = limit;
if (rows < 1) { mQuery->Clear(); return lib_pack("lllp", (long)1, (long)0, (long)0, (void*)NULL); }
int cols = mQuery->Cols();
char * nil = (char*)"NULL";
char ** res = (char **) calloc (cols*rows, sizeof(char*));
if (!res) { log1("PY: SQL malloc failed\n"); mQuery->Clear(); return lib_pack("lllp", (long)0, (long)0, (long)0, (void*)NULL); }
for (int r = 0 ; r < rows; r++)
{
mQuery->DataSeek(r);
MYSQL_ROW row;
row = mQuery->Row();
if (!row)
{ log1("PY: SQL failed to fetch row: %d\n", r); mQuery->Clear(); free(res); return lib_pack("lllp", (long)0, (long)0, (long)0, (void*)NULL); }
for (int i = 0; i < cols; i++)
res[(r*cols)+i] = strdup( (row[i]) ? row[i] : nil );
}
mQuery->Clear();
return lib_pack("lllp", (long)1, (long)rows, (long)cols, (void*)res);
}
static unsigned int compute_subdivisions(const BezierPath::Point& p0,
const BezierPath::Point& p1,
const BezierPath::Point& p2,
const BezierPath::Point& p3,
BezierPath::Coord flat)
{
vector2d<BezierPath::Coord> d01(p1.x() - p0.x(), p1.y() - p0.y());
vector2d<BezierPath::Coord> d12(p2.x() - p1.x(), p2.y() - p1.y());
vector2d<BezierPath::Coord> d23(p3.x() - p2.x(), p3.y() - p2.y());
vector2d<BezierPath::Coord> d03(p3.x() - p0.x(), p3.y() - p0.y());
vector2d<BezierPath::Coord> d01_12 = d12 - d01;
vector2d<BezierPath::Coord> d12_23 = d23 - d12;
double dist = norm(d03);
double dist2;
if (dist != 0) {
dist2 = __myabs(cross(d01_12, d03) / dist);
double tmp = __myabs(cross(d12_23, d03) / dist);
if (tmp > dist2) {
dist2 = tmp;
}
} else {
vector2d<BezierPath::Coord> d0(p0.x(), p0.y());
dist2 = norm(vector2d<BezierPath::Coord>(d01_12 - d0));
double tmp = norm(vector2d<BezierPath::Coord>(d12_23 - d0));
if (tmp > dist2) {
dist2 = tmp;
}
}
unsigned int subdivisions = 0;
if (dist2 > 0) {
subdivisions = log4(long((3.0 / 4.0) * dist2 / flat + 0.5));
}
return subdivisions;
}
void
do_punch_hole(unsigned offset, unsigned length)
{
unsigned end_offset;
int max_offset = 0;
int max_len = 0;
int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
if (length == 0) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping zero length punch hole\n");
log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, length);
return;
}
if (file_size <= (loff_t)offset) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping hole punch off the end of the file\n");
log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, length);
return;
}
end_offset = offset + length;
log4(OP_PUNCH_HOLE, offset, length, 0);
if (testcalls <= simulatedopcount)
return;
if ((progressinterval && testcalls % progressinterval == 0) ||
(fsx_debug && (monitorstart == -1 || monitorend == -1 ||
end_offset <= monitorend))) {
prt("%lu punch\tfrom 0x%x to 0x%x, (0x%x bytes)\n", testcalls,
offset, offset+length, length);
}
if (fallocate(fd, mode, (loff_t)offset, (loff_t)length) == -1) {
prt("%punch hole: %x to %x\n", offset, length);
prterr("do_punch_hole: fallocate");
report_failure(161);
}
max_offset = offset < file_size ? offset : file_size;
max_len = max_offset + length <= file_size ? length :
file_size - max_offset;
memset(good_buf + max_offset, '\0', max_len);
}
void
doread(unsigned offset, unsigned size)
{
int ret;
offset -= offset % readbdy;
if (o_direct)
size -= size % readbdy;
if (size == 0) {
if (!quiet && testcalls > simulatedopcount && !o_direct)
prt("skipping zero size read\n");
log4(OP_SKIPPED, OP_READ, offset, size);
return;
}
if (size + offset > file_size) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping seek/read past end of file\n");
log4(OP_SKIPPED, OP_READ, offset, size);
return;
}
log4(OP_READ, offset, size, 0);
if (testcalls <= simulatedopcount)
return;
if (!quiet &&
((progressinterval && testcalls % progressinterval == 0) ||
(debug &&
(monitorstart == -1 ||
(offset + size > monitorstart &&
(monitorend == -1 || offset <= monitorend))))))
prt("%lu read\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
offset, offset + size - 1, size);
ret = ops->read(&ctx, offset, size, temp_buf);
if (ret != (int)size) {
if (ret < 0)
prterrcode("doread: ops->read", ret);
else
prt("short read: 0x%x bytes instead of 0x%x\n",
ret, size);
report_failure(141);
}
check_buffers(good_buf, temp_buf, offset, size);
}
void
do_flatten()
{
int ret;
if (!rbd_image_has_parent(&ctx)) {
log4(OP_SKIPPED, OP_FLATTEN, 0, 0);
return;
}
log4(OP_FLATTEN, 0, 0, 0);
prt("%lu flatten\n", testcalls);
ret = ops->flatten(&ctx);
if (ret < 0) {
prterrcode("writefileimage: ops->flatten", ret);
exit(177);
}
}
void
do_clone()
{
char filename[1024];
char imagename[1024];
char lastimagename[1024];
int ret, fd;
int order = 0, stripe_unit = 0, stripe_count = 0;
if (randomize_striping) {
order = 18 + rand() % 8;
stripe_unit = 1ull << (order - 1 - (rand() % 8));
stripe_count = 2 + rand() % 14;
}
log4(OP_CLONE, 0, 0, 0);
++num_clones;
prt("%lu clone\t%d order %d su %d sc %d\n", testcalls, num_clones, order, stripe_unit, stripe_count);
clone_filename(filename, sizeof(filename), num_clones);
if ((fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) {
simple_err("do_clone: open", -errno);
exit(162);
}
save_buffer(good_buf, file_size, fd);
if ((ret = close(fd)) < 0) {
simple_err("do_clone: close", -errno);
exit(163);
}
if ((ret = rbd_snap_create(image, "snap")) < 0) {
simple_err("do_clone: rbd create snap", ret);
exit(164);
}
if ((ret = rbd_snap_protect(image, "snap")) < 0) {
simple_err("do_clone: rbd protect snap", ret);
exit(164);
}
clone_imagename(imagename, sizeof(imagename), num_clones);
clone_imagename(lastimagename, sizeof(lastimagename),
num_clones - 1);
ret = rbd_clone2(ioctx, lastimagename, "snap", ioctx, imagename,
RBD_FEATURES_ALL, &order, stripe_unit, stripe_count);
if (ret < 0) {
simple_err("do_clone: rbd clone", ret);
exit(165);
}
rbd_close(image);
if ((ret = rbd_open(ioctx, imagename, &image, NULL)) < 0) {
simple_err("do_clone: rbd open", ret);
exit(166);
}
}
void
do_collapse_range(unsigned offset, unsigned length)
{
unsigned end_offset;
int mode = FALLOC_FL_COLLAPSE_RANGE;
if (length == 0) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping zero length collapse range\n");
log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, length);
return;
}
end_offset = offset + length;
if ((loff_t)end_offset >= file_size) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping collapse range behind EOF\n");
log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, length);
return;
}
log4(OP_COLLAPSE_RANGE, offset, length, 0);
if (testcalls <= simulatedopcount)
return;
if ((progressinterval && testcalls % progressinterval == 0) ||
(debug && (monitorstart == -1 || monitorend == -1 ||
end_offset <= monitorend))) {
prt("%lu collapse\tfrom 0x%x to 0x%x, (0x%x bytes)\n", testcalls,
offset, offset+length, length);
}
if (fallocate(fd, mode, (loff_t)offset, (loff_t)length) == -1) {
prt("collapse range: %x to %x\n", offset, length);
prterr("do_collapse_range: fallocate");
report_failure(161);
}
memmove(good_buf + offset, good_buf + end_offset,
file_size - end_offset);
file_size -= length;
}
double cwf(unsigned long long base_counts[], unsigned long long window)
{
assert(window != 0);
assert(base_counts != NULL);
unsigned long long i = 0;
unsigned long long count = 0;
mpfr_t val;
mpfr_init(val);
mpfr_set_ui(val, 1, RND);
mpfr_t loop_result;
mpfr_init(loop_result);
mpfr_set_ui(loop_result, 1, RND);
mpfr_t log_result;
mpfr_init(log_result);
mpfr_set_ui(log_result, 1, RND);
for (i = 0; i < NUM_BASES; i++) {
count = base_counts[i];
assert(count > 0);
fact(val, count);
/* loop_result *= val */
mpfr_mul(loop_result, loop_result, val, RND);
}
assert(mpfr_cmp_ui(loop_result, 0) != 0);
/* val = factorial(window) */
fact(val, window);
/* val = val / loop_result */
mpfr_div(val, val, loop_result, RND);
/* log_result = log4(val) */
log4(log_result, val);
/* val = log_result * (1.0 / window) */
mpfr_mul_d(val, log_result, (1.0 / window), RND);
double output = 0.0;
output = mpfr_get_d(val, RND);
mpfr_clear(val);
mpfr_clear(loop_result);
mpfr_clear(log_result);
return output;
}
/**
* Apply the given NAKs to the naklist
*/
void apply_naks(unsigned char *naklist, struct finfo_t *finfo,
int blocks_this_sec, int section_offset)
{
int nakidx, listidx, i;
for (i = 0; i < blocks_this_sec; i++) {
// Each bit represents a NAK; check each one
// Simplified: (naklist[listidx / 8] & (1 << (listidx % 8)))
nakidx = i + section_offset;
listidx = i;
if ((naklist[listidx >> 3] & (1 << (listidx & 7))) != 0) {
log4(0, 0, "Got NAK for %d", nakidx);
finfo->naklist[nakidx] = 1;
}
}
}
TEST(LoggerTest, LevelTest)
{
std::ostringstream ss;
Logger log1(ss, Logger::TRACE_);
Logger log2(ss, Logger::VERBOSE_);
Logger log3(ss, Logger::DEBUG_);
Logger log4(ss, Logger::INFO_);
Logger log5(ss, Logger::WARN_);
Logger log6(ss, Logger::ERROR_);
ASSERT_TRUE(log1.t());
ASSERT_TRUE(log1.v());
ASSERT_TRUE(log1.d());
ASSERT_TRUE(log1.i());
ASSERT_FALSE(log2.t());
ASSERT_TRUE(log2.v());
ASSERT_TRUE(log2.d());
ASSERT_TRUE(log2.i());
ASSERT_FALSE(log3.t());
ASSERT_FALSE(log3.v());
ASSERT_TRUE(log3.d());
ASSERT_TRUE(log3.i());
ASSERT_FALSE(log4.t());
ASSERT_FALSE(log4.v());
ASSERT_FALSE(log4.d());
ASSERT_TRUE(log4.i());
ASSERT_FALSE(log5.t());
ASSERT_FALSE(log5.v());
ASSERT_FALSE(log5.d());
ASSERT_FALSE(log5.i());
ASSERT_FALSE(log6.t());
ASSERT_FALSE(log6.v());
ASSERT_FALSE(log6.d());
ASSERT_FALSE(log6.i());
}
void
test(void)
{
unsigned long offset;
unsigned long size = maxoplen;
unsigned long rv = random();
unsigned long op;
if (simulatedopcount > 0 && testcalls == simulatedopcount)
writefileimage();
testcalls++;
if (closeprob)
closeopen = (rv >> 3) < (1u << 28) / (unsigned)closeprob;
if (debugstart > 0 && testcalls >= debugstart)
debug = 1;
if (!quiet && testcalls < simulatedopcount && testcalls % 100000 == 0)
prt("%lu...\n", testcalls);
offset = random();
if (randomoplen)
size = random() % (maxoplen + 1);
/* calculate appropriate op to run */
if (lite)
op = rv % OP_MAX_LITE;
else
op = rv % OP_MAX_FULL;
switch (op) {
case OP_MAPREAD:
if (!mapped_reads)
op = OP_READ;
break;
case OP_MAPWRITE:
if (!mapped_writes)
op = OP_WRITE;
break;
case OP_FALLOCATE:
if (!fallocate_calls) {
log4(OP_SKIPPED, OP_FALLOCATE, offset, size);
goto out;
}
break;
case OP_PUNCH_HOLE:
if (!punch_hole_calls) {
log4(OP_SKIPPED, OP_PUNCH_HOLE, offset, size);
goto out;
}
break;
}
switch (op) {
case OP_READ:
TRIM_OFF_LEN(offset, size, file_size);
doread(offset, size);
break;
case OP_WRITE:
TRIM_OFF_LEN(offset, size, maxfilelen);
dowrite(offset, size);
break;
case OP_MAPREAD:
TRIM_OFF_LEN(offset, size, file_size);
exit(183);
break;
case OP_MAPWRITE:
TRIM_OFF_LEN(offset, size, maxfilelen);
exit(182);
break;
case OP_TRUNCATE:
if (!style)
size = random() % maxfilelen;
dotruncate(size);
break;
case OP_PUNCH_HOLE:
TRIM_OFF_LEN(offset, size, file_size);
do_punch_hole(offset, size);
break;
default:
prterr("test: unknown operation");
report_failure(42);
break;
}
out:
if (sizechecks && testcalls > simulatedopcount)
check_size();
if (closeopen)
docloseopen();
}
static void DefParams(int MinHitLength, double MinId, int SeqLengthT, int SeqLengthQ,
bool Self, double MaxMem, FilterParams *ptrFP, DPParams *ptrDP)
{
if (MinId < 0 || MinId > 1.0)
Quit("DefParams: bad MinId=%g", MinId);
if (MinHitLength <= 4)
Quit("DefParams: bad MinHitLength=%d", MinHitLength);
const char *strTubeOffset = ValueOpt("tubeoffset");
const int TubeOffset = (0 == strTubeOffset) ? -1 : atoi(strTubeOffset);
// Lower bound on word length k by requiring manageable index.
// Given kmer occurs once every 4^k positions.
// Hence average number of index entries is i = N/(4^k) for random
// string of length N.
// Require i <= I, then k > log_4(N/i).
const double dSeqLengthA = (double) SeqLengthT;
const int MinWordSize = (int) (log4(dSeqLengthA) - log4(MAX_AVG_INDEX_LIST_LENGTH) + 0.5);
// First choice is that filter criteria are same as DP criteria,
// but this may not be possible.
int SeedLength = MinHitLength;
int SeedDiffs = (int) (MinHitLength*(1.0 - MinId));
// Find filter valid filter parameters,
// starting from preferred case.
int WordSize = -1;
for (;;)
{
int MinWords = -1;
for (WordSize = MAX_WORD_LENGTH; WordSize >= MinWordSize; --WordSize)
{
ptrFP->WordSize = WordSize;
ptrFP->SeedLength = SeedLength;
ptrFP->SeedDiffs = SeedDiffs;
ptrFP->TubeOffset = TubeOffset > 0 ? TubeOffset : ptrFP->SeedDiffs + TUBE_OFFSET_DELTA;
double Mem = TotalMemRequired(SeqLengthT, SeqLengthQ, Self, *ptrFP);
if (MaxMem > 0 && Mem > MaxMem)
{
Log("Parameters n=%d k=%d e=%d, mem=%.0f Mb > maxmem=%.0f Mb\n",
ptrFP->SeedLength,
ptrFP->WordSize,
ptrFP->SeedDiffs,
Mem/1e6,
MaxMem/1e6);
MinWords = -1;
continue;
}
MinWords = MinWordsPerFilterHit(SeedLength, WordSize, SeedDiffs);
if (MinWords <= 0)
{
Log("Parameters n=%d k=%d e=%d, B=%d\n",
ptrFP->SeedLength,
ptrFP->WordSize,
ptrFP->SeedDiffs,
MinWords);
MinWords = -1;
continue;
}
const double Len = AvgIndexListLength(SeqLengthT, *ptrFP);
if (Len > MAX_AVG_INDEX_LIST_LENGTH)
{
Log("Parameters n=%d k=%d e=%d, B=%d avgixlen=%g > max = %d\n",
ptrFP->SeedLength,
ptrFP->WordSize,
ptrFP->SeedDiffs,
MinWords,
Len,
MAX_AVG_INDEX_LIST_LENGTH);
MinWords = -1;
continue;
}
break;
}
if (MinWords > 0)
break;
// Failed to find filter parameters, try
// fewer errors and shorter seed.
if (SeedLength >= MinHitLength/4)
{
SeedLength /= 2;
continue;
}
if (SeedDiffs > 0)
{
--SeedDiffs;
continue;
}
Quit("Failed to find filter parameters");
}
ptrDP->MinHitLength = MinHitLength;
ptrDP->MinId = MinId;
}
void
do_clone()
{
char filename[1024];
char imagename[1024];
char lastimagename[1024];
int ret, fd;
int order = 0, stripe_unit = 0, stripe_count = 0;
uint64_t newsize = file_size;
log4(OP_CLONE, 0, 0, 0);
++num_clones;
if (randomize_striping) {
order = 18 + get_random() % 8;
stripe_unit = 1ull << (order - 1 - (get_random() % 8));
stripe_count = 2 + get_random() % 14;
}
prt("%lu clone\t%d order %d su %d sc %d\n", testcalls, num_clones,
order, stripe_unit, stripe_count);
clone_imagename(imagename, sizeof(imagename), num_clones);
clone_imagename(lastimagename, sizeof(lastimagename),
num_clones - 1);
assert(strcmp(lastimagename, ctx.name) == 0);
ret = ops->clone(&ctx, "snap", imagename, &order, stripe_unit,
stripe_count);
if (ret < 0) {
prterrcode("do_clone: ops->clone", ret);
exit(165);
}
if (randomize_parent_overlap && rbd_image_has_parent(&ctx)) {
int rand = get_random() % 16 + 1; // [1..16]
if (rand < 13) {
uint64_t overlap;
ret = rbd_get_overlap(ctx.image, &overlap);
if (ret < 0) {
prterrcode("do_clone: rbd_get_overlap", ret);
exit(1);
}
if (rand < 10) { // 9/16
newsize = overlap * ((double)rand / 10);
newsize -= newsize % truncbdy;
} else { // 3/16
newsize = 0;
}
assert(newsize != (uint64_t)file_size);
prt("truncating image %s from 0x%llx (overlap 0x%llx) to 0x%llx\n",
ctx.name, file_size, overlap, newsize);
ret = ops->resize(&ctx, newsize);
if (ret < 0) {
prterrcode("do_clone: ops->resize", ret);
exit(1);
}
} else if (rand < 15) { // 2/16
prt("flattening image %s\n", ctx.name);
ret = ops->flatten(&ctx);
if (ret < 0) {
prterrcode("do_clone: ops->flatten", ret);
exit(1);
}
} else { // 2/16
prt("leaving image %s intact\n", ctx.name);
}
}
clone_filename(filename, sizeof(filename), num_clones);
if ((fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) {
simple_err("do_clone: open", -errno);
exit(162);
}
save_buffer(good_buf, newsize, fd);
if ((ret = close(fd)) < 0) {
simple_err("do_clone: close", -errno);
exit(163);
}
/*
* Close parent.
*/
if ((ret = ops->close(&ctx)) < 0) {
prterrcode("do_clone: ops->close", ret);
exit(174);
}
/*
* Open freshly made clone.
*/
if ((ret = ops->open(imagename, &ctx)) < 0) {
prterrcode("do_clone: ops->open", ret);
exit(166);
}
if (num_clones > 1)
check_clone(num_clones - 2);
}
/**
* For messages that send a list of clients in the body, append all clients
* in the specified state to the packet, then send the message of the given
* type when either the body is full or the end of the client list has been
* reached. All header fields besides uftp_h.blsize and message.destcount
* must be populated before calling.
* Returns 1 on success, 0 on fail.
*/
int send_multiple(const struct finfo_t *finfo, unsigned char *packet,
int message, int attempt, uint32_t *addrlist, int state,
uint16_t *mes_destcount, int encrypt,
const struct sockaddr_in *destaddr, int regconf)
{
struct uftp_h *header;
int hsize, payloadlen;
int maxdest, packetcnt, dests, i;
unsigned char *mheader, *encpacket, *outpacket;
header = (struct uftp_h *)packet;
mheader = packet + sizeof(struct uftp_h);
hsize = (unsigned char *)addrlist - mheader;
maxdest = ((encrypt ? encpayloadsize : payloadsize) - hsize) /
sizeof(struct in_addr);
packetcnt = 1;
if (encrypt) {
encpacket = calloc(mtu + keylen, 1);
if (encpacket == NULL) {
syserror(0, 0, "calloc failed!");
exit(1);
}
} else {
encpacket = NULL;
}
for (i = 0, dests = 0; i < destcount; i++) {
if (message == REG_CONF) {
// Only send REG_CONF for a particular client if either it's
// behind a proxy or we're sending them to everyone.
// Also, don't send if we already sent one and we haven't
// gotten another REGISTER
if ((destlist[i].status == state) &&
(!finfo->deststate[i].conf_sent) &&
(regconf || (destlist[i].proxyidx != -1))) {
addrlist[dests++] = destlist[i].addr.s_addr;
finfo->deststate[i].conf_sent = 1;
}
} else if (message == DONE_CONF) {
// As with REG_CONF, don't send a DONE_CONF for a client
// if we already sent one and we haven't gotten another COMPLETE
if ((destlist[i].status == state) &&
(!finfo->deststate[i].conf_sent)) {
addrlist[dests++] = destlist[i].addr.s_addr;
finfo->deststate[i].conf_sent = 1;
}
} else if (destlist[i].status == state) {
addrlist[dests++] = destlist[i].addr.s_addr;
}
if ((dests >= maxdest) || ((i == destcount - 1) && (dests > 0))) {
payloadlen = hsize + (dests * sizeof(uint32_t));
*mes_destcount = htons(dests);
if (encrypt) {
if (!encrypt_and_sign(packet, &encpacket, payloadlen, mtu,
keytype, groupkey, groupsalt, ivlen, hashtype,
grouphmackey, hmaclen, sigtype, privkey, rsalen)) {
log0(0, 0, "Error encrypting %s", func_name(message));
free(encpacket);
return 0;
}
outpacket = encpacket;
payloadlen = ntohs(((struct uftp_h *)outpacket)->blsize);
} else {
outpacket = packet;
header->blsize = htons(payloadlen);
}
log2(0, 0, "Sending %s %d.%d", func_name(message),
attempt, packetcnt);
log4(0, 0, "Sending to %08X", ntohl(destaddr->sin_addr.s_addr));
if (nb_sendto(sock, outpacket, payloadlen + sizeof(struct uftp_h),0,
(struct sockaddr *)destaddr,
sizeof(*destaddr)) == SOCKET_ERROR) {
sockerror(0, 0, "Error sending %s", func_name(message));
sleep(1);
free(encpacket);
return 0;
}
if (packet_wait) usleep(packet_wait);
memset(addrlist, 0, maxdest * sizeof(uint32_t));
dests = 0;
packetcnt++;
}
}
free(encpacket);
return 1;
}
/**
* Performs the Transfer phase for a particular file
* Returns 1 if at least one client finished, 0 if all are dropped or aborted
*/
int transfer_phase(struct finfo_t *finfo)
{
unsigned char *packet, *encpacket, *data;
char path[MAXPATHNAME];
struct uftp_h *header;
struct fileseg_h *fileseg;
int max_time, alldone, numbytes, sent_blocks, current_naks;
unsigned int pass, section, numnaks, block;
struct timeval start_time, last_sent, current_sent;
int64_t avgwait, waitcnt, overage, tdiff;
f_offset_t offset, curr_offset;
int file, i;
if (finfo->file_id != 0) {
// First check to see if all clients are already done for this file.
// This can happen on a restart when the file finished on the
// last attempt and responded to the FILEINFO with a COMPLETE
for (i = 0, alldone = 1; (i < destcount) && alldone; i++) {
alldone = alldone && ((destlist[i].status == DEST_DONE) ||
(client_error(i)) || (destlist[i].clientcnt != -1));
}
if (alldone) {
gettimeofday(&start_time, NULL);
print_status(finfo, start_time);
return 1;
}
}
// If rate is -1, use 100Mbps for purpose of calculating max time
max_time = (int)floor(((double)weight / 100) *
((double)finfo->size / (((rate == -1) ? 100000 : rate) / 8) / 1024));
if (max_time < min_time) {
max_time = min_time;
}
if ((finfo->file_id != 0) && (finfo->ftype == FTYPE_REG)) {
log(0, 0, "Maximum file transfer time: %d seconds", max_time);
snprintf(path, sizeof(path), "%s%c%s", finfo->basedir, PATH_SEP,
finfo->filename);
if ((file = open(path, OPENREAD, 0)) == -1) {
syserror(0, 0, "Error opening file");
return 1;
}
} else {
// At end of group, all non-errored client are DEST_DONE from the
// last file, so reset them to DEST_ACTIVE to get the final COMPLETE.
for (i = 0; i < destcount; i++) {
if (!client_error(i)) {
destlist[i].status = DEST_ACTIVE;
}
}
}
packet = calloc(mtu, 1);
encpacket = calloc(mtu, 1);
if ((packet == NULL) || (encpacket == NULL)) {
syserror(0, 0, "calloc failed!");
exit(1);
}
header = (struct uftp_h *)packet;
fileseg = (struct fileseg_h *)(packet + sizeof(struct uftp_h));
data = (unsigned char *)fileseg + sizeof(struct fileseg_h);
set_uftp_header(header, FILESEG, finfo, &receive_dest);
pass = 1;
alldone = 0;
gettimeofday(&start_time, NULL);
do {
avgwait = 0;
waitcnt = 0;
numnaks = 0;
overage = 0;
section = 1;
curr_offset = 0;
sent_blocks = 0;
gettimeofday(&last_sent, NULL);
if (finfo->file_id != 0) {
log(0, 0, "Sending file...pass %d", pass);
lseek_func(file, 0, SEEK_SET);
} else {
log(0, 0, "Finishing group");
}
fileseg->func = FILESEG;
fileseg->file_id = htons(finfo->file_id);
fileseg->pass = pass;
fileseg->section = htons(section);
for (block = 0; block < finfo->blocks; block++) {
// If all clients received this file partially on a prior attempt
// and it's the first pass, request NAKs for all sections
// right away and don't send any data packets.
if (((pass == 1) || finfo->naklist[block]) &&
!((pass == 1) && finfo->partial)) {
if (diff_sec(last_sent, start_time) > max_time) {
log0(0, 0, "Max file transfer time exceeded");
send_abort(finfo, "Max file transfer time exceeded",
&receive_dest, NULL, (keytype != KEY_NONE),
!quit_on_error);
alldone = 1;
for (i = 0; i < destcount; i++) {
if (quit_on_error || ((destlist[i].status == DEST_ACTIVE) &&
destlist[i].clientcnt == -1 )) {
destlist[i].status = DEST_ABORT;
}
}
break;
}
// On the first pass, go straight through the file.
// On later passes, seek to the next packet.
if (pass != 1) {
log4(0, 0, "Resending %d", block);
if (!seek_block(file, block, &offset, curr_offset)) {
continue;
}
}
if ((numbytes = read(file, data, blocksize)) == -1) {
syserror(0, 0, "read failed");
continue;
}
if (pass != 1) {
curr_offset = offset + numbytes;
}
// Keep track of how long we really slept compared to how
// long we expected to sleep. If we went over, subtract the
// time over from the next sleep time. This way we maintain
// the proper average sleep time. This can result in multiple
// packets going out at once, potentially losing packets.
if (packet_wait > overage) {
usleep(packet_wait - (int32_t)overage);
}
gettimeofday(¤t_sent, NULL);
tdiff = diff_usec(current_sent, last_sent);
avgwait += tdiff;
waitcnt++;
if (packet_wait) overage += tdiff - packet_wait;
last_sent = current_sent;
fileseg->seq_num = htonl(block);
send_data(finfo, packet, numbytes, encpacket);
sent_blocks++;
}
if ((block % (blocksize * 8) == (blocksize * 8) - 1) ||
(block == finfo->blocks - 1)) {
if ((pass == 1) || sent_blocks) {
current_naks = get_naks(finfo, pass, section, &alldone);
numnaks += current_naks;
if ((rate != -1) && (cc_count > 0)) {
if (!read_cc_config(cc_config)) {
log1(0, 0, "Error rereading congestion control "
"config, using prior values");
}
adjust_rate(current_naks, sent_blocks);
}
overage = 0;
if (alldone) break;
}
sent_blocks = 0;
gettimeofday(&last_sent, NULL);
fileseg->section = htons(++section);
}
}
if ((finfo->size == 0) && !alldone) {
// If it's the end of the group, or an empty file, a DONE was
// never sent, so send it now
numnaks += get_naks(finfo, pass, section, &alldone);
}
if (finfo->file_id != 0) {
log(0, 0, "Average wait time = %.2f us",
(waitcnt == 0) ? 0 : (float)avgwait / waitcnt);
log(0, 0, "Received %d distinct NAKs for pass %d", numnaks, pass);
}
pass++;
} while (!alldone);
if ((finfo->file_id != 0) && (finfo->ftype == FTYPE_REG)) {
close(file);
}
print_status(finfo, start_time);
free(packet);
free(encpacket);
for (i = 0; i < destcount; i++) {
if (quit_on_error) {
// Check to see that all finished
if ((destlist[i].status != DEST_DONE) &&
(destlist[i].clientcnt == -1)) {
return 0;
}
} else {
// Check to see if at least one finished
if (destlist[i].status == DEST_DONE) {
return 1;
}
}
}
if (quit_on_error) {
return 1;
} else {
return 0;
}
}
void
domapwrite(unsigned offset, unsigned size)
{
unsigned pg_offset;
unsigned map_size;
off_t cur_filesize;
char *p;
offset -= offset % writebdy;
if (size == 0) {
if (!quiet && testcalls > simulatedopcount)
prt("skipping zero size write\n");
log4(OP_SKIPPED, OP_MAPWRITE, offset, size);
return;
}
cur_filesize = file_size;
log4(OP_MAPWRITE, offset, size, 0);
gendata(original_buf, good_buf, offset, size);
if (file_size < offset + size) {
if (file_size < offset)
memset(good_buf + file_size, '\0', offset - file_size);
file_size = offset + size;
if (lite) {
warn("Lite file size bug in fsx!");
report_failure(200);
}
}
if (testcalls <= simulatedopcount)
return;
if (!quiet &&
((progressinterval && testcalls % progressinterval == 0) ||
(fsx_debug &&
(monitorstart == -1 ||
(offset + size > monitorstart &&
(monitorend == -1 || offset <= monitorend))))))
prt("%lu mapwrite\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
offset, offset + size - 1, size);
if (file_size > cur_filesize) {
if (ftruncate(fd, file_size) == -1) {
prterr("domapwrite: ftruncate");
exit(201);
}
}
pg_offset = offset & PAGE_MASK;
map_size = pg_offset + size;
if ((p = (char *)mmap(0, map_size, PROT_READ | PROT_WRITE,
MAP_FILE | MAP_SHARED, fd,
(off_t)(offset - pg_offset))) == (char *)-1) {
prterr("domapwrite: mmap");
report_failure(202);
}
memcpy(p + pg_offset, good_buf + offset, size);
if (msync(p, map_size, MS_SYNC) != 0) {
prterr("domapwrite: msync");
report_failure(203);
}
check_eofpage("Write", offset, p, size);
if (munmap(p, map_size) != 0) {
prterr("domapwrite: munmap");
report_failure(204);
}
}
