void
calculate_projections(const std::string file_in,
const std::string file_out,
Matrix<double> eigenvecs,
std::size_t mem_buf_size,
bool use_correlation,
Matrix<double> stats) {
// calculating the projection, we need twice the space
// (original data + result)
mem_buf_size /= 4;
std::size_t n_variables = stats.n_rows();
std::vector<double> means(n_variables);
std::vector<double> inverse_sigmas(n_variables);
for (std::size_t i=0; i < n_variables; ++i) {
means[i] = stats(i,0);
inverse_sigmas[i] = 1.0 / stats(i,1);
}
bool append_to_file = false;
DataFileReader<double> fh_file_in(file_in, mem_buf_size);
DataFileWriter<double> fh_file_out(file_out);
read_blockwise(fh_file_in, [&](Matrix<double>& m) {
FastPCA::shift_matrix_columns_inplace(m, means);
if (use_correlation) {
FastPCA::scale_matrix_columns_inplace(m, inverse_sigmas);
}
fh_file_out.write(std::move(m*eigenvecs), append_to_file);
append_to_file = true;
});
}
static int test_read_blockwise(void)
{
void *buffer = NULL;
int fd = -1;
ssize_t ret = -EINVAL;
//printf("Entering test_read_blockwise ");
//printf("test_bsize: %zu, test_length: %zu\n", test_bsize, test_length);
if (posix_memalign(&buffer, test_mem_alignment, test_length)) {
fprintf(stderr, "Failed to allocate aligned buffer.\n");
goto out;
}
fd = open(test_file, O_RDONLY | O_DIRECT);
if (fd < 0) {
fprintf(stderr, "Failed to open %s.\n", test_file);
goto out;
}
ret = read_blockwise(fd, test_bsize, test_mem_alignment, buffer, test_length);
if (ret < 0)
goto out;
ret = (size_t) ret == test_length ? 0 : -EIO;
out:
if (fd >= 0)
close(fd);
free(buffer);
return ret;
}
int LUKS_read_phdr(struct luks_phdr *hdr,
int require_luks_device,
int repair,
struct crypt_device *ctx)
{
struct device *device = crypt_metadata_device(ctx);
ssize_t hdr_size = sizeof(struct luks_phdr);
int devfd = 0, r = 0;
/* LUKS header starts at offset 0, first keyslot on LUKS_ALIGN_KEYSLOTS */
assert(sizeof(struct luks_phdr) <= LUKS_ALIGN_KEYSLOTS);
/* Stripes count cannot be changed without additional code fixes yet */
assert(LUKS_STRIPES == 4000);
if (repair && !require_luks_device)
return -EINVAL;
log_dbg("Reading LUKS header of size %zu from device %s",
hdr_size, device_path(device));
devfd = device_open(device, O_RDONLY);
if (devfd < 0) {
log_err(ctx, _("Cannot open device %s."), device_path(device));
return -EINVAL;
}
if (read_blockwise(devfd, device_block_size(device), device_alignment(device),
hdr, hdr_size) < hdr_size)
r = -EIO;
else
r = _check_and_convert_hdr(device_path(device), hdr, require_luks_device,
repair, ctx);
if (!r)
r = LUKS_check_device_size(ctx, hdr, 0);
/*
* Cryptsetup 1.0.0 did not align keyslots to 4k (very rare version).
* Disable direct-io to avoid possible IO errors if underlying device
* has bigger sector size.
*/
if (!r && hdr->keyblock[0].keyMaterialOffset * SECTOR_SIZE < LUKS_ALIGN_KEYSLOTS) {
log_dbg("Old unaligned LUKS keyslot detected, disabling direct-io.");
device_disable_direct_io(device);
}
close(devfd);
return r;
}
void
calculate_projections(const std::string file_in,
const std::string file_out,
Matrix<double> eigenvecs,
std::size_t mem_buf_size,
bool use_correlation,
Matrix<double> stats) {
mem_buf_size /= 4;
std::size_t n_variables = stats.n_rows();
std::vector<double> means(n_variables);
std::vector<double> inverse_sigmas(n_variables);
std::vector<double> dih_shifts(n_variables);
std::vector<double> scaled_periodicities(n_variables);
for (std::size_t i=0; i < n_variables; ++i) {
means[i] = stats(i,0);
inverse_sigmas[i] = 1.0 / stats(i,1);
if (use_correlation) {
dih_shifts[i] = (stats(i,2) - means[i]) * inverse_sigmas[i];
scaled_periodicities[i] = 2*M_PI * inverse_sigmas[i];
} else {
dih_shifts[i] = stats(i,2);
scaled_periodicities[i] = 2*M_PI;
}
}
// projections
bool append_to_file = false;
DataFileReader<double> fh_file_in(file_in, mem_buf_size);
DataFileWriter<double> fh_file_out(file_out);
read_blockwise(fh_file_in, [&](Matrix<double>& m) {
// convert degrees to radians
FastPCA::deg2rad_inplace(m);
if (use_correlation) {
// shift by periodic means (necessary for scaling)
FastPCA::Periodic::shift_matrix_columns_inplace(m, means);
// scale data by sigmas for correlated projections
FastPCA::scale_matrix_columns_inplace(m, inverse_sigmas);
}
// shift dihedrals to minimize boundary jumps
// and correct for periodic boundary condition
FastPCA::Periodic::shift_matrix_columns_inplace(m, dih_shifts, scaled_periodicities);
// output
fh_file_out.write(m*eigenvecs, append_to_file);
append_to_file = true;
});
}
int LUKS_read_phdr(struct luks_phdr *hdr,
int require_luks_device,
int repair,
struct crypt_device *ctx)
{
struct device *device = crypt_metadata_device(ctx);
ssize_t hdr_size = sizeof(struct luks_phdr);
int devfd = 0, r = 0;
/* LUKS header starts at offset 0, first keyslot on LUKS_ALIGN_KEYSLOTS */
assert(sizeof(struct luks_phdr) <= LUKS_ALIGN_KEYSLOTS);
/* Stripes count cannot be changed without additional code fixes yet */
assert(LUKS_STRIPES == 4000);
if (repair && !require_luks_device)
return -EINVAL;
log_dbg("Reading LUKS header of size %zu from device %s",
hdr_size, device_path(device));
devfd = device_open(device, O_RDONLY);
if (devfd == -1) {
log_err(ctx, _("Cannot open device %s.\n"), device_path(device));
return -EINVAL;
}
if (read_blockwise(devfd, device_block_size(device), hdr, hdr_size) < hdr_size)
r = -EIO;
else
r = _check_and_convert_hdr(device_path(device), hdr, require_luks_device,
repair, ctx);
if (!r)
r = LUKS_check_device_size(ctx, hdr->keyBytes);
close(devfd);
return r;
}
int device_ready(struct crypt_device *cd, const char *device, int mode)
{
int devfd, r = 0;
ssize_t s;
struct stat st;
char buf[512];
if(stat(device, &st) < 0) {
log_err(cd, _("Device %s doesn't exist or access denied.\n"), device);
return -EINVAL;
}
if (!S_ISBLK(st.st_mode))
return -ENOTBLK;
log_dbg("Trying to open and read device %s.", device);
devfd = open(device, mode | O_DIRECT | O_SYNC);
if(devfd < 0) {
log_err(cd, _("Cannot open device %s for %s%s access.\n"), device,
(mode & O_EXCL) ? _("exclusive ") : "",
(mode & O_RDWR) ? _("writable") : _("read-only"));
return -EINVAL;
}
/* Try to read first sector */
s = read_blockwise(devfd, buf, sizeof(buf));
if (s < 0 || s != sizeof(buf)) {
log_verbose(cd, _("Cannot read device %s.\n"), device);
r = -EIO;
}
memset(buf, 0, sizeof(buf));
close(devfd);
return r;
}
int LUKS_hdr_backup(const char *backup_file, struct crypt_device *ctx)
{
struct device *device = crypt_metadata_device(ctx);
struct luks_phdr hdr;
int r = 0, devfd = -1;
ssize_t hdr_size;
ssize_t buffer_size;
char *buffer = NULL;
r = LUKS_read_phdr(&hdr, 1, 0, ctx);
if (r)
return r;
hdr_size = LUKS_device_sectors(hdr.keyBytes) << SECTOR_SHIFT;
buffer_size = size_round_up(hdr_size, crypt_getpagesize());
buffer = crypt_safe_alloc(buffer_size);
if (!buffer || hdr_size < LUKS_ALIGN_KEYSLOTS || hdr_size > buffer_size) {
r = -ENOMEM;
goto out;
}
log_dbg("Storing backup of header (%zu bytes) and keyslot area (%zu bytes).",
sizeof(hdr), hdr_size - LUKS_ALIGN_KEYSLOTS);
log_dbg("Output backup file size: %zu bytes.", buffer_size);
devfd = device_open(device, O_RDONLY);
if(devfd == -1) {
log_err(ctx, _("Device %s is not a valid LUKS device.\n"), device_path(device));
r = -EINVAL;
goto out;
}
if (read_blockwise(devfd, device_block_size(device), buffer, hdr_size) < hdr_size) {
r = -EIO;
goto out;
}
close(devfd);
/* Wipe unused area, so backup cannot contain old signatures */
if (hdr.keyblock[0].keyMaterialOffset * SECTOR_SIZE == LUKS_ALIGN_KEYSLOTS)
memset(buffer + sizeof(hdr), 0, LUKS_ALIGN_KEYSLOTS - sizeof(hdr));
devfd = open(backup_file, O_CREAT|O_EXCL|O_WRONLY, S_IRUSR);
if (devfd == -1) {
if (errno == EEXIST)
log_err(ctx, _("Requested header backup file %s already exists.\n"), backup_file);
else
log_err(ctx, _("Cannot create header backup file %s.\n"), backup_file);
r = -EINVAL;
goto out;
}
if (write(devfd, buffer, buffer_size) < buffer_size) {
log_err(ctx, _("Cannot write header backup file %s.\n"), backup_file);
r = -EIO;
goto out;
}
close(devfd);
r = 0;
out:
if (devfd != -1)
close(devfd);
crypt_memzero(&hdr, sizeof(hdr));
crypt_safe_free(buffer);
return r;
}
/* Read verity superblock from disk */
int VERITY_read_sb(struct crypt_device *cd,
uint64_t sb_offset,
char **uuid_string,
struct crypt_params_verity *params)
{
struct device *device = crypt_metadata_device(cd);
int bsize = device_block_size(device);
struct verity_sb sb = {};
ssize_t hdr_size = sizeof(struct verity_sb);
int devfd = 0, sb_version;
log_dbg("Reading VERITY header of size %zu on device %s, offset %" PRIu64 ".",
sizeof(struct verity_sb), device_path(device), sb_offset);
if (params->flags & CRYPT_VERITY_NO_HEADER) {
log_err(cd, _("Verity device %s doesn't use on-disk header.\n"),
device_path(device));
return -EINVAL;
}
if (sb_offset % 512) {
log_err(cd, _("Unsupported VERITY hash offset.\n"));
return -EINVAL;
}
devfd = device_open(device, O_RDONLY);
if(devfd == -1) {
log_err(cd, _("Cannot open device %s.\n"), device_path(device));
return -EINVAL;
}
if(lseek(devfd, sb_offset, SEEK_SET) < 0 ||
read_blockwise(devfd, bsize, &sb, hdr_size) < hdr_size) {
close(devfd);
return -EIO;
}
close(devfd);
if (memcmp(sb.signature, VERITY_SIGNATURE, sizeof(sb.signature))) {
log_err(cd, _("Device %s is not a valid VERITY device.\n"),
device_path(device));
return -EINVAL;
}
sb_version = le32_to_cpu(sb.version);
if (sb_version != 1) {
log_err(cd, _("Unsupported VERITY version %d.\n"), sb_version);
return -EINVAL;
}
params->hash_type = le32_to_cpu(sb.hash_type);
if (params->hash_type > VERITY_MAX_HASH_TYPE) {
log_err(cd, _("Unsupported VERITY hash type %d.\n"), params->hash_type);
return -EINVAL;
}
params->data_block_size = le32_to_cpu(sb.data_block_size);
params->hash_block_size = le32_to_cpu(sb.hash_block_size);
if (VERITY_BLOCK_SIZE_OK(params->data_block_size) ||
VERITY_BLOCK_SIZE_OK(params->hash_block_size)) {
log_err(cd, _("Unsupported VERITY block size.\n"));
return -EINVAL;
}
params->data_size = le64_to_cpu(sb.data_blocks);
params->hash_name = strndup((const char*)sb.algorithm, sizeof(sb.algorithm));
if (!params->hash_name)
return -ENOMEM;
if (crypt_hash_size(params->hash_name) <= 0) {
log_err(cd, _("Hash algorithm %s not supported.\n"),
params->hash_name);
free(CONST_CAST(char*)params->hash_name);
return -EINVAL;
}
params->salt_size = le16_to_cpu(sb.salt_size);
if (params->salt_size > sizeof(sb.salt)) {
log_err(cd, _("VERITY header corrupted.\n"));
free(CONST_CAST(char*)params->hash_name);
return -EINVAL;
}
params->salt = malloc(params->salt_size);
if (!params->salt) {
free(CONST_CAST(char*)params->hash_name);
return -ENOMEM;
}
memcpy(CONST_CAST(char*)params->salt, sb.salt, params->salt_size);
if ((*uuid_string = malloc(40)))
uuid_unparse(sb.uuid, *uuid_string);
params->hash_area_offset = sb_offset;
return 0;
}
