static int produce_block_map(const char* path, const char* map_file, const char* blk_dev,
bool encrypted, bool f2fs_fs, int socket) {
std::string err;
if (!android::base::RemoveFileIfExists(map_file, &err)) {
LOG(ERROR) << "failed to remove the existing map file " << map_file << ": " << err;
return kUncryptFileRemoveError;
}
std::string tmp_map_file = std::string(map_file) + ".tmp";
android::base::unique_fd mapfd(open(tmp_map_file.c_str(),
O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR));
if (mapfd == -1) {
PLOG(ERROR) << "failed to open " << tmp_map_file;
return kUncryptFileOpenError;
}
// Make sure we can write to the socket.
if (!write_status_to_socket(0, socket)) {
LOG(ERROR) << "failed to write to socket " << socket;
return kUncryptSocketWriteError;
}
struct stat sb;
if (stat(path, &sb) != 0) {
LOG(ERROR) << "failed to stat " << path;
return kUncryptFileStatError;
}
LOG(INFO) << " block size: " << sb.st_blksize << " bytes";
int blocks = ((sb.st_size-1) / sb.st_blksize) + 1;
LOG(INFO) << " file size: " << sb.st_size << " bytes, " << blocks << " blocks";
std::vector<int> ranges;
std::string s = android::base::StringPrintf("%s\n%" PRId64 " %" PRId64 "\n",
blk_dev, static_cast<int64_t>(sb.st_size),
static_cast<int64_t>(sb.st_blksize));
if (!android::base::WriteStringToFd(s, mapfd)) {
PLOG(ERROR) << "failed to write " << tmp_map_file;
return kUncryptWriteError;
}
std::vector<std::vector<unsigned char>> buffers;
if (encrypted) {
buffers.resize(WINDOW_SIZE, std::vector<unsigned char>(sb.st_blksize));
}
int head_block = 0;
int head = 0, tail = 0;
android::base::unique_fd fd(open(path, O_RDONLY));
if (fd == -1) {
PLOG(ERROR) << "failed to open " << path << " for reading";
return kUncryptFileOpenError;
}
android::base::unique_fd wfd;
if (encrypted) {
wfd.reset(open(blk_dev, O_WRONLY));
if (wfd == -1) {
PLOG(ERROR) << "failed to open " << blk_dev << " for writing";
return kUncryptBlockOpenError;
}
}
// F2FS-specific ioctl
// It requires the below kernel commit merged in v4.16-rc1.
// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file")
// In android-4.4,
// 56ee1e817908 ("f2fs: updates on v4.16-rc1")
// In android-4.9,
// 2f17e34672a8 ("f2fs: updates on v4.16-rc1")
// In android-4.14,
// ce767d9a55bc ("f2fs: updates on v4.16-rc1")
#ifndef F2FS_IOC_SET_PIN_FILE
#ifndef F2FS_IOCTL_MAGIC
#define F2FS_IOCTL_MAGIC 0xf5
#endif
#define F2FS_IOC_SET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 13, __u32)
#define F2FS_IOC_GET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 14, __u32)
#endif
if (f2fs_fs) {
int error = ioctl(fd, F2FS_IOC_SET_PIN_FILE);
// Don't break the old kernels which don't support it.
if (error && errno != ENOTTY && errno != ENOTSUP) {
PLOG(ERROR) << "Failed to set pin_file for f2fs: " << path << " on " << blk_dev;
return kUncryptIoctlError;
}
}
off64_t pos = 0;
int last_progress = 0;
while (pos < sb.st_size) {
// Update the status file, progress must be between [0, 99].
int progress = static_cast<int>(100 * (double(pos) / double(sb.st_size)));
if (progress > last_progress) {
last_progress = progress;
write_status_to_socket(progress, socket);
}
if ((tail+1) % WINDOW_SIZE == head) {
// write out head buffer
int block = head_block;
if (ioctl(fd, FIBMAP, &block) != 0) {
PLOG(ERROR) << "failed to find block " << head_block;
return kUncryptIoctlError;
}
if (block == 0) {
LOG(ERROR) << "failed to find block " << head_block << ", retrying";
int error = retry_fibmap(fd, path, &block, head_block);
if (error != kUncryptNoError) {
return error;
}
}
add_block_to_ranges(ranges, block);
if (encrypted) {
if (write_at_offset(buffers[head].data(), sb.st_blksize, wfd,
static_cast<off64_t>(sb.st_blksize) * block) != 0) {
return kUncryptWriteError;
}
}
head = (head + 1) % WINDOW_SIZE;
++head_block;
}
// read next block to tail
if (encrypted) {
size_t to_read = static_cast<size_t>(
std::min(static_cast<off64_t>(sb.st_blksize), sb.st_size - pos));
if (!android::base::ReadFully(fd, buffers[tail].data(), to_read)) {
PLOG(ERROR) << "failed to read " << path;
return kUncryptReadError;
}
pos += to_read;
} else {
// If we're not encrypting; we don't need to actually read
// anything, just skip pos forward as if we'd read a
// block.
pos += sb.st_blksize;
}
tail = (tail+1) % WINDOW_SIZE;
}
while (head != tail) {
// write out head buffer
int block = head_block;
if (ioctl(fd, FIBMAP, &block) != 0) {
PLOG(ERROR) << "failed to find block " << head_block;
return kUncryptIoctlError;
}
if (block == 0) {
LOG(ERROR) << "failed to find block " << head_block << ", retrying";
int error = retry_fibmap(fd, path, &block, head_block);
if (error != kUncryptNoError) {
return error;
}
}
add_block_to_ranges(ranges, block);
if (encrypted) {
if (write_at_offset(buffers[head].data(), sb.st_blksize, wfd,
static_cast<off64_t>(sb.st_blksize) * block) != 0) {
return kUncryptWriteError;
}
}
head = (head + 1) % WINDOW_SIZE;
++head_block;
}
if (!android::base::WriteStringToFd(
android::base::StringPrintf("%zu\n", ranges.size() / 2), mapfd)) {
PLOG(ERROR) << "failed to write " << tmp_map_file;
return kUncryptWriteError;
}
for (size_t i = 0; i < ranges.size(); i += 2) {
if (!android::base::WriteStringToFd(
android::base::StringPrintf("%d %d\n", ranges[i], ranges[i+1]), mapfd)) {
PLOG(ERROR) << "failed to write " << tmp_map_file;
return kUncryptWriteError;
}
}
if (fsync(mapfd) == -1) {
PLOG(ERROR) << "failed to fsync \"" << tmp_map_file << "\"";
return kUncryptFileSyncError;
}
if (close(mapfd.release()) == -1) {
PLOG(ERROR) << "failed to close " << tmp_map_file;
return kUncryptFileCloseError;
}
if (encrypted) {
if (fsync(wfd) == -1) {
PLOG(ERROR) << "failed to fsync \"" << blk_dev << "\"";
return kUncryptFileSyncError;
}
if (close(wfd.release()) == -1) {
PLOG(ERROR) << "failed to close " << blk_dev;
return kUncryptFileCloseError;
}
}
if (rename(tmp_map_file.c_str(), map_file) == -1) {
PLOG(ERROR) << "failed to rename " << tmp_map_file << " to " << map_file;
return kUncryptFileRenameError;
}
// Sync dir to make rename() result written to disk.
std::string file_name = map_file;
std::string dir_name = dirname(&file_name[0]);
android::base::unique_fd dfd(open(dir_name.c_str(), O_RDONLY | O_DIRECTORY));
if (dfd == -1) {
PLOG(ERROR) << "failed to open dir " << dir_name;
return kUncryptFileOpenError;
}
if (fsync(dfd) == -1) {
PLOG(ERROR) << "failed to fsync " << dir_name;
return kUncryptFileSyncError;
}
if (close(dfd.release()) == -1) {
PLOG(ERROR) << "failed to close " << dir_name;
return kUncryptFileCloseError;
}
return 0;
}
static int produce_block_map(const char* path, const char* map_file, const char* blk_dev,
bool encrypted, int status_fd) {
std::string err;
if (!android::base::RemoveFileIfExists(map_file, &err)) {
ALOGE("failed to remove the existing map file %s: %s", map_file, err.c_str());
return -1;
}
std::string tmp_map_file = std::string(map_file) + ".tmp";
unique_fd mapfd(open(tmp_map_file.c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR));
if (!mapfd) {
ALOGE("failed to open %s: %s\n", tmp_map_file.c_str(), strerror(errno));
return -1;
}
// Make sure we can write to the status_file.
if (!android::base::WriteStringToFd("0\n", status_fd)) {
ALOGE("failed to update \"%s\"\n", STATUS_FILE.c_str());
return -1;
}
struct stat sb;
if (stat(path, &sb) != 0) {
ALOGE("failed to stat %s", path);
return -1;
}
ALOGI(" block size: %ld bytes", static_cast<long>(sb.st_blksize));
int blocks = ((sb.st_size-1) / sb.st_blksize) + 1;
ALOGI(" file size: %" PRId64 " bytes, %d blocks", sb.st_size, blocks);
std::vector<int> ranges;
std::string s = android::base::StringPrintf("%s\n%" PRId64 " %ld\n",
blk_dev, sb.st_size, static_cast<long>(sb.st_blksize));
if (!android::base::WriteStringToFd(s, mapfd.get())) {
ALOGE("failed to write %s: %s", tmp_map_file.c_str(), strerror(errno));
return -1;
}
std::vector<std::vector<unsigned char>> buffers;
if (encrypted) {
buffers.resize(WINDOW_SIZE, std::vector<unsigned char>(sb.st_blksize));
}
int head_block = 0;
int head = 0, tail = 0;
unique_fd fd(open(path, O_RDONLY));
if (!fd) {
ALOGE("failed to open %s for reading: %s", path, strerror(errno));
return -1;
}
unique_fd wfd(-1);
if (encrypted) {
wfd = open(blk_dev, O_WRONLY);
if (!wfd) {
ALOGE("failed to open fd for writing: %s", strerror(errno));
return -1;
}
}
off64_t pos = 0;
int last_progress = 0;
while (pos < sb.st_size) {
// Update the status file, progress must be between [0, 99].
int progress = static_cast<int>(100 * (double(pos) / double(sb.st_size)));
if (progress > last_progress) {
last_progress = progress;
android::base::WriteStringToFd(std::to_string(progress) + "\n", status_fd);
}
if ((tail+1) % WINDOW_SIZE == head) {
// write out head buffer
int block = head_block;
if (ioctl(fd.get(), FIBMAP, &block) != 0) {
ALOGE("failed to find block %d", head_block);
return -1;
}
add_block_to_ranges(ranges, block);
if (encrypted) {
if (write_at_offset(buffers[head].data(), sb.st_blksize, wfd.get(),
static_cast<off64_t>(sb.st_blksize) * block) != 0) {
return -1;
}
}
head = (head + 1) % WINDOW_SIZE;
++head_block;
}
// read next block to tail
if (encrypted) {
size_t to_read = static_cast<size_t>(
std::min(static_cast<off64_t>(sb.st_blksize), sb.st_size - pos));
if (!android::base::ReadFully(fd.get(), buffers[tail].data(), to_read)) {
ALOGE("failed to read: %s", strerror(errno));
return -1;
}
pos += to_read;
} else {
// If we're not encrypting; we don't need to actually read
// anything, just skip pos forward as if we'd read a
// block.
pos += sb.st_blksize;
}
tail = (tail+1) % WINDOW_SIZE;
}
while (head != tail) {
// write out head buffer
int block = head_block;
if (ioctl(fd.get(), FIBMAP, &block) != 0) {
ALOGE("failed to find block %d", head_block);
return -1;
}
add_block_to_ranges(ranges, block);
if (encrypted) {
if (write_at_offset(buffers[head].data(), sb.st_blksize, wfd.get(),
static_cast<off64_t>(sb.st_blksize) * block) != 0) {
return -1;
}
}
head = (head + 1) % WINDOW_SIZE;
++head_block;
}
if (!android::base::WriteStringToFd(
android::base::StringPrintf("%zu\n", ranges.size() / 2), mapfd.get())) {
ALOGE("failed to write %s: %s", tmp_map_file.c_str(), strerror(errno));
return -1;
}
for (size_t i = 0; i < ranges.size(); i += 2) {
if (!android::base::WriteStringToFd(
android::base::StringPrintf("%d %d\n", ranges[i], ranges[i+1]), mapfd.get())) {
ALOGE("failed to write %s: %s", tmp_map_file.c_str(), strerror(errno));
return -1;
}
}
if (fsync(mapfd.get()) == -1) {
ALOGE("failed to fsync \"%s\": %s", tmp_map_file.c_str(), strerror(errno));
return -1;
}
if (close(mapfd.get() == -1)) {
ALOGE("failed to close %s: %s", tmp_map_file.c_str(), strerror(errno));
return -1;
}
mapfd = -1;
if (encrypted) {
if (fsync(wfd.get()) == -1) {
ALOGE("failed to fsync \"%s\": %s", blk_dev, strerror(errno));
return -1;
}
if (close(wfd.get()) == -1) {
ALOGE("failed to close %s: %s", blk_dev, strerror(errno));
return -1;
}
wfd = -1;
}
if (rename(tmp_map_file.c_str(), map_file) == -1) {
ALOGE("failed to rename %s to %s: %s", tmp_map_file.c_str(), map_file, strerror(errno));
return -1;
}
// Sync dir to make rename() result written to disk.
std::string file_name = map_file;
std::string dir_name = dirname(&file_name[0]);
unique_fd dfd(open(dir_name.c_str(), O_RDONLY | O_DIRECTORY));
if (!dfd) {
ALOGE("failed to open dir %s: %s", dir_name.c_str(), strerror(errno));
return -1;
}
if (fsync(dfd.get()) == -1) {
ALOGE("failed to fsync %s: %s", dir_name.c_str(), strerror(errno));
return -1;
}
if (close(dfd.get() == -1)) {
ALOGE("failed to close %s: %s", dir_name.c_str(), strerror(errno));
return -1;
}
dfd = -1;
return 0;
}
/** Incomplete beta function for variable objects.
Evaluates the continued fraction for imcomplete beta function.
\param _a \f$a\f$
\param _b \f$b\f$
\param _x \f$x\f$
\param MAXIT Maximum number of iterations for the continued fraction approximation in betacf.
\return Incomplete beta function \f$I_x(a,b)\f$
\n\n The implementation of this algorithm was inspired by
"Numerical Recipes in C", 2nd edition,
Press, Teukolsky, Vetterling, Flannery, chapter 2
*/
dvariable betacf(const dvariable& _a, const dvariable& _b, const dvariable& _x,
int MAXIT)
{
double qab,qam,qap;
double a=value(_a);
double b=value(_b);
double x=value(_x);
qab=a+b;
qap=a+1.0;
qam=a-1.0;
dvector c1(0,MAXIT);
dvector c(1,MAXIT);
dvector d1(0,MAXIT);
dvector d(1,MAXIT);
dvector del(1,MAXIT);
dvector h1(0,MAXIT);
dvector h(1,MAXIT);
dvector aa(1,MAXIT);
dvector aa1(1,MAXIT);
c1(0)=1.0;
d1(0)=1.0/(1.0-qab*x/qap);
h1(0)=d1(0);
int m = 1;
for (; m <= MAXIT; m++)
{
int i=m;
int m2=2*m;
aa(i)=m*(b-m)*x/((qam+m2)*(a+m2));
d(i)=1.0/(1.0+aa(i)*d1(i-1));
c(i)=1.0+aa(i)/c1(i-1);
h(i) = h1(i-1)*d(i)*c(i);
aa1(i) = -(a+m)*(qab+m)*x/((a+m2)*(qap+m2));
d1(i)=1.0/(1.0+aa1(i)*d(i));
c1(i)=1.0+aa1(i)/c(i);
del(i)=d1(i)*c1(i);
h1(i) = h(i)*del(i);
if (fabs(del(i)-1.0) < EPS) break;
}
if (m > MAXIT)
{
cerr << "a or b too big, or MAXIT too small in cumulative beta function"
" routine" << endl;
m=MAXIT;
}
int mmax=m;
dvariable hh;
value(hh)=h1(mmax);
dvector dfc1(0,MAXIT);
dvector dfc(1,MAXIT);
dvector dfd1(0,MAXIT);
dvector dfd(1,MAXIT);
dvector dfh1(0,MAXIT);
dvector dfh(1,MAXIT);
dvector dfaa(1,MAXIT);
dvector dfaa1(1,MAXIT);
dvector dfdel(1,MAXIT);
dfc1.initialize();
dfc.initialize();
dfaa1.initialize();
dfaa.initialize();
dfd1.initialize();
dfd.initialize();
dfh1.initialize();
dfh.initialize();
dfdel.initialize();
dfh1(mmax)=1.0;
double dfqab=0.0;
double dfqam=0.0;
double dfqap=0.0;
double dfa=0.0;
double dfb=0.0;
double dfx=0.0;
for (m=mmax;m>=1;m--)
{
/*
int i=m;
m2=2*m;
aa(i)=m*(b-m)*x/((qam+m2)*(a+m2));
d(i)=1.0/(1.0+aa(i)*d1(i-1));
c(i)=1.0+aa(i)/c1(i-1);
h(i) = h1(i-1)*d(i)*c(i);
aa1(i) = -(a+m)*(qab+m)*x/((a+m2)*(qap+m2));
d1(i)=1.0/(1.0+aa1(i)*d(i));
c1(i)=1.0+aa1(i)/c(i);
del(i)=d1(i)*c1(i);
h1(i) = h(i)*del(i);
*/
int i=m;
int m2=2*m;
//h1(i) = h(i)*del(i);
dfh(i)+=dfh1(i)*del(i);
dfdel(i)+=dfh1(i)*h(i);
dfh1(i)=0.0;
//del(i)=d1(i)*c1(i);
dfd1(i)+=dfdel(i)*c1(i);
dfc1(i)+=dfdel(i)*d1(i);
dfdel(i)=0.0;
//c1(i)=1.0+aa1(i)/c(i);
dfaa1(i)+=dfc1(i)/c(i);
dfc(i)-=dfc1(i)*aa1(i)/(c(i)*c(i));
dfc1(i)=0.0;
//d1(i)=1.0/(1.0+aa1(i)*d(i));
double sq=square(d1(i));
dfaa1(i)-=dfd1(i)*sq*d(i);
dfd(i)-=dfd1(i)*sq*aa1(i);
dfd1(i)=0.0;
//aa1(i) = -(a+m)*(qab+m)*x/((a+m2)*(qap+m2));
dfx -= dfaa1(i) *
(a+m)*(qab+m)/((a+m2)*(qap+m2));
dfa += dfaa1(i) * aa1(i)* (1.0/(a+m) - 1.0/(a+m2));
dfqab += dfaa1(i) * aa1(i)/(qab+m);
dfqap += dfaa1(i) * aa1(i)* (-1.0/(qap+m2));
dfaa1(i)=0.0;
//h(i) = h1(i-1)*d(i)*c(i);
dfh1(i-1)+=dfh(i)*d(i)*c(i);
dfd(i)+=dfh(i)*h1(i-1)*c(i);
dfc(i)+=dfh(i)*h1(i-1)*d(i);
dfh(i)=0.0;
//c(i)=1.0+aa(i)/c1(i-1);
dfaa(i)+=dfc(i)/c1(i-1);
dfc1(i-1)-=dfc(i)*aa(i)/square(c1(i-1));
dfc(i)=0.0;
//d(i)=1.0/(1.0+aa(i)*d1(i-1));
dfaa(i)-=dfd(i)*square(d(i))*d1(i-1);
dfd1(i-1)-=dfd(i)*square(d(i))*aa(i);
dfd(i)=0.0;
//aa(i)=m*(b-m)*x/((qam+m2)*(a+m2));
dfx+=dfaa(i)*
m*(b-m)/((qam+m2)*(a+m2));
dfb+=dfaa(i)*
m*x/((qam+m2)*(a+m2));
dfa-=dfaa(i)*aa(i)/(a+m2);
dfqam-=dfaa(i)*aa(i)/(qam+m2);
dfaa(i)=0.0;
}
/*
c1(0)=1.0;
d1(0)=1.0/(1.0-qab*x/qap);
h1(0)=d1(0);
*/
//h1(0)=d1(0);
dfd1(0)+=dfh1(0);
dfh1(0)=0.0;
//d1(0)=1.0/(1.0-qab*x/qap);
double sq1=square(d1(0))/qap;
dfx+=dfd1(0)*sq1*qab;
dfqab+=dfd1(0)*sq1*x;
dfqap-=dfd1(0)*sq1*qab*x/qap;
dfd1(0)=0.0;
/*
qab=a+b;
qap=a+1.0;
qam=a-1.0;
*/
//qam=a-1.0;
dfa+=dfqam;
//qap=a+1.0;
dfa+=dfqap;
//qab=a+b;
dfa+=dfqab;
dfb+=dfqab;
gradient_structure::GRAD_STACK1->set_gradient_stack(default_evaluation3ind,
&(value(hh)) ,&(value(_a)),dfa ,&(value(_b)),dfb ,&(value(_x)),dfx);
return hh;
}
