static void print_scrub_summary(struct btrfs_scrub_progress *p)
{
u64 err_cnt;
u64 err_cnt2;
err_cnt = p->read_errors +
p->csum_errors +
p->verify_errors +
p->super_errors;
err_cnt2 = p->corrected_errors + p->uncorrectable_errors;
if (p->malloc_errors)
printf("*** WARNING: memory allocation failed while scrubbing. "
"results may be inaccurate\n");
printf("\ttotal bytes scrubbed: %s with %llu errors\n",
pretty_size(p->data_bytes_scrubbed + p->tree_bytes_scrubbed),
max(err_cnt, err_cnt2));
if (err_cnt || err_cnt2) {
printf("\terror details:");
PRINT_SCRUB_ERROR(p->read_errors, "read");
PRINT_SCRUB_ERROR(p->super_errors, "super");
PRINT_SCRUB_ERROR(p->verify_errors, "verify");
PRINT_SCRUB_ERROR(p->csum_errors, "csum");
printf("\n");
printf("\tcorrected errors: %llu, uncorrectable errors: %llu, "
"unverified errors: %llu\n", p->corrected_errors,
p->uncorrectable_errors, p->unverified_errors);
}
}
TextureStore::~TextureStore()
{
RENDERER_LOG_DEBUG(
"texture store statistics:\n"
" cache %s\n"
" peak size %s\n",
format_cache_stats(m_tile_cache).c_str(),
pretty_size(m_tile_swapper.m_max_memory_size).c_str());
}
static void
print_bg(FILE *html, char *name, u64 start, u64 len, u64 used, u64 flags,
u64 areas)
{
double frag = (double)areas / (len / 4096) * 2;
fprintf(html, "<p>%s chunk starts at %lld, size is %s, %.2f%% used, "
"%.2f%% fragmented</p>\n", chunk_type(flags), start,
pretty_size(len), 100.0 * used / len, 100.0 * frag);
fprintf(html, "<img src=\"%s\" border=\"1\" />\n", name);
}
int main(int argc, char* argv[])
{
std::ios::sync_with_stdio(false);
if (argc < 3) {
std::cout << "usage: " << argv[0] << " path/to/file size (GB)" << std::endl;
return 1;
}
std::string file(argv[1]);
std::string size(argv[2]);
std::string::size_type sz;
double target_gb = std::stod(size, &sz);
double target = target_gb * GB_SIZE;
srand (time(NULL));
std::vector<std::thread> threads;
const size_t hardware_concurrency = std::thread::hardware_concurrency();
double segment_size = target / hardware_concurrency;
cstorage<std::vector<std::string>> storage;
for(size_t i = 0; i < hardware_concurrency; i++) {
threads.push_back(std::thread([&storage, segment_size](){
producer(storage, segment_size);
}));
}
bool done = false;
std::thread consumer_thread([&file, &storage, &done](){
consumer(file, storage, done);
});
for(size_t i = 0; i < hardware_concurrency; i++) {
threads[i].join();
}
done = true;
consumer_thread.join();
std::cout << "generated: " << file << " size: " << pretty_size(target) << std::endl;
return 0;
}
static int cmd_info(void)
{
if (scan_sys_cache_info () != 0)
error (EXIT_FAILURE, 0, "unable to get cache data");
printf ("%-6s %10s %10s %10s %10s\n", "level", "type", "size", "assoc", "colors");
for (int level = 0; level < nb_cache_levels; ++level) {
struct cache_info * i = &caches[level];
if (i->found) {
char sx; size_t sz;
sz = pretty_size (&sx, i->size);
printf ("L%-5d %10s %9zu%c %10d %10d\n", level, i->type, sz, sx, i->assoc, i->nb_colors);
}
}
return EXIT_SUCCESS;
}
int btrfs_prepare_device(int fd, char *file, int zero_end, u64 *block_count_ret,
u64 max_block_count, int *mixed, int discard)
{
u64 block_count;
u64 bytenr;
struct stat st;
int i, ret;
ret = fstat(fd, &st);
if (ret < 0) {
fprintf(stderr, "unable to stat %s\n", file);
exit(1);
}
block_count = btrfs_device_size(fd, &st);
if (block_count == 0) {
fprintf(stderr, "unable to find %s size\n", file);
exit(1);
}
if (max_block_count)
block_count = min(block_count, max_block_count);
zero_end = 1;
if (block_count < 1024 * 1024 * 1024 && !(*mixed)) {
printf("SMALL VOLUME: forcing mixed metadata/data groups\n");
*mixed = 1;
}
if (discard) {
/*
* We intentionally ignore errors from the discard ioctl. It
* is not necessary for the mkfs functionality but just an
* optimization.
*/
if (discard_blocks(fd, 0, 0) == 0) {
fprintf(stderr, "Performing full device TRIM (%s) ...\n",
pretty_size(block_count));
discard_blocks(fd, 0, block_count);
}
}
ret = zero_dev_start(fd);
if (ret) {
fprintf(stderr, "failed to zero device start %d\n", ret);
exit(1);
}
for (i = 0 ; i < BTRFS_SUPER_MIRROR_MAX; i++) {
bytenr = btrfs_sb_offset(i);
if (bytenr >= block_count)
break;
zero_blocks(fd, bytenr, BTRFS_SUPER_INFO_SIZE);
}
if (zero_end) {
ret = zero_dev_end(fd, block_count);
if (ret) {
fprintf(stderr, "failed to zero device end %d\n", ret);
exit(1);
}
}
*block_count_ret = block_count;
return 0;
}
// Initialize OSL's shading system.
bool initialize_osl_shading_system(
TextureStore& texture_store,
IAbortSwitch& abort_switch)
{
// Construct a search paths string from the project's search paths.
const string project_search_paths =
to_string(m_project.search_paths().to_string_reversed(SearchPaths::osl_path_separator()));
// Initialize OIIO.
const size_t texture_cache_size_bytes =
m_params.child("texture_store").get_optional<size_t>(
"max_size",
TextureStore::get_default_size());
RENDERER_LOG_INFO(
"setting oiio texture cache size to %s.",
pretty_size(texture_cache_size_bytes).c_str());
const float texture_cache_size_mb =
static_cast<float>(texture_cache_size_bytes) / (1024 * 1024);
m_texture_system->attribute("max_memory_MB", texture_cache_size_mb);
// Set OIIO search paths.
string prev_oiio_search_path;
m_texture_system->getattribute("searchpath", prev_oiio_search_path);
if (prev_oiio_search_path != project_search_paths)
{
RENDERER_LOG_INFO("setting oiio search paths to %s", project_search_paths.c_str());
m_texture_system->invalidate_all(true);
m_texture_system->attribute("searchpath", project_search_paths);
}
// Also use the project search paths to look for OpenImageIO plugins.
m_texture_system->attribute("plugin_searchpath", project_search_paths);
// Initialize OSL.
m_renderer_services->initialize(texture_store);
// Set OSL search paths.
string prev_osl_search_paths;
m_shading_system->getattribute("searchpath:shader", prev_osl_search_paths);
if (prev_osl_search_paths != project_search_paths)
{
RENDERER_LOG_INFO("setting osl shader search paths to %s", project_search_paths.c_str());
m_project.get_scene()->release_optimized_osl_shader_groups();
m_shading_system->attribute("searchpath:shader", project_search_paths);
}
// Initialize the shader compiler, if the OSL headers are found.
if (m_resource_search_paths.exist("stdosl.h"))
{
const APIString stdosl_path = m_resource_search_paths.qualify("stdosl.h");
RENDERER_LOG_INFO("found OSL headers in %s", stdosl_path.c_str());
m_osl_compiler = ShaderCompilerFactory::create(stdosl_path.c_str());
}
else
RENDERER_LOG_INFO("OSL headers not found.");
// Re-optimize shader groups that need updating.
return
m_project.get_scene()->create_optimized_osl_shader_groups(
*m_shading_system,
m_osl_compiler.get(),
&abort_switch);
}
int main(int ac, char **av)
{
char *file;
struct btrfs_root *root;
struct btrfs_trans_handle *trans;
char *label = NULL;
char *first_file;
u64 block_count = 0;
u64 dev_block_count = 0;
u64 blocks[7];
u64 alloc_start = 0;
u64 metadata_profile = 0;
u64 data_profile = 0;
u32 leafsize = sysconf(_SC_PAGESIZE);
u32 sectorsize = 4096;
u32 nodesize = leafsize;
u32 stripesize = 4096;
int zero_end = 1;
int option_index = 0;
int fd;
int ret;
int i;
int mixed = 0;
int data_profile_opt = 0;
int metadata_profile_opt = 0;
int discard = 1;
int ssd = 0;
int force_overwrite = 0;
char *source_dir = NULL;
int source_dir_set = 0;
u64 num_of_meta_chunks = 0;
u64 size_of_data = 0;
u64 source_dir_size = 0;
int dev_cnt = 0;
int saved_optind;
char estr[100];
u64 features = 0;
while(1) {
int c;
c = getopt_long(ac, av, "A:b:fl:n:s:m:d:L:O:r:VMK",
long_options, &option_index);
if (c < 0)
break;
switch(c) {
case 'A':
alloc_start = parse_size(optarg);
break;
case 'f':
force_overwrite = 1;
break;
case 'd':
data_profile = parse_profile(optarg);
data_profile_opt = 1;
break;
case 'l':
case 'n':
nodesize = parse_size(optarg);
leafsize = parse_size(optarg);
break;
case 'L':
label = parse_label(optarg);
break;
case 'm':
metadata_profile = parse_profile(optarg);
metadata_profile_opt = 1;
break;
case 'M':
mixed = 1;
break;
case 'O': {
char *orig = strdup(optarg);
char *tmp = orig;
tmp = parse_fs_features(tmp, &features);
if (tmp) {
fprintf(stderr,
"Unrecognized filesystem feature '%s'\n",
tmp);
free(orig);
exit(1);
}
free(orig);
if (features & BTRFS_FEATURE_LIST_ALL) {
list_all_fs_features();
exit(0);
}
break;
}
case 's':
sectorsize = parse_size(optarg);
break;
case 'b':
block_count = parse_size(optarg);
if (block_count <= 1024*1024*1024) {
printf("SMALL VOLUME: forcing mixed "
"metadata/data groups\n");
mixed = 1;
}
zero_end = 0;
break;
case 'V':
print_version();
break;
case 'r':
source_dir = optarg;
source_dir_set = 1;
break;
case 'K':
discard = 0;
break;
default:
print_usage();
}
}
sectorsize = max(sectorsize, (u32)sysconf(_SC_PAGESIZE));
if (check_leaf_or_node_size(leafsize, sectorsize))
exit(1);
if (check_leaf_or_node_size(nodesize, sectorsize))
exit(1);
saved_optind = optind;
dev_cnt = ac - optind;
if (dev_cnt == 0)
print_usage();
if (source_dir_set && dev_cnt > 1) {
fprintf(stderr,
"The -r option is limited to a single device\n");
exit(1);
}
while (dev_cnt-- > 0) {
file = av[optind++];
if (is_block_device(file))
if (test_dev_for_mkfs(file, force_overwrite, estr)) {
fprintf(stderr, "Error: %s", estr);
exit(1);
}
}
optind = saved_optind;
dev_cnt = ac - optind;
file = av[optind++];
ssd = is_ssd(file);
if (is_vol_small(file)) {
printf("SMALL VOLUME: forcing mixed metadata/data groups\n");
mixed = 1;
if (metadata_profile != data_profile) {
if (metadata_profile_opt || data_profile_opt) {
fprintf(stderr,
"With mixed block groups data and metadata profiles must be the same\n");
exit(1);
}
}
}
/*
* Set default profiles according to number of added devices.
* For mixed groups defaults are single/single.
*/
if (!mixed) {
if (!metadata_profile_opt) {
if (dev_cnt == 1 && ssd)
printf("Detected a SSD, turning off metadata "
"duplication. Mkfs with -m dup if you want to "
"force metadata duplication.\n");
metadata_profile = (dev_cnt > 1) ?
BTRFS_BLOCK_GROUP_RAID1 : (ssd) ?
0: BTRFS_BLOCK_GROUP_DUP;
}
if (!data_profile_opt) {
data_profile = (dev_cnt > 1) ?
BTRFS_BLOCK_GROUP_RAID0 : 0; /* raid0 or single */
}
} else {
metadata_profile = 0;
data_profile = 0;
}
ret = test_num_disk_vs_raid(metadata_profile, data_profile,
dev_cnt, mixed, estr);
if (ret) {
fprintf(stderr, "Error: %s\n", estr);
exit(1);
}
/* if we are here that means all devs are good to btrfsify */
printf("\nWARNING! - %s IS EXPERIMENTAL\n", BTRFS_BUILD_VERSION);
printf("WARNING! - see http://btrfs.wiki.kernel.org before using\n\n");
dev_cnt--;
if (!source_dir_set) {
/*
* open without O_EXCL so that the problem should not
* occur by the following processing.
* (btrfs_register_one_device() fails if O_EXCL is on)
*/
fd = open(file, O_RDWR);
if (fd < 0) {
fprintf(stderr, "unable to open %s: %s\n", file,
strerror(errno));
exit(1);
}
first_file = file;
ret = btrfs_prepare_device(fd, file, zero_end, &dev_block_count,
block_count, &mixed, discard);
if (block_count && block_count > dev_block_count) {
fprintf(stderr, "%s is smaller than requested size\n", file);
exit(1);
}
} else {
fd = open_target(file);
if (fd < 0) {
fprintf(stderr, "unable to open the %s\n", file);
exit(1);
}
first_file = file;
source_dir_size = size_sourcedir(source_dir, sectorsize,
&num_of_meta_chunks, &size_of_data);
if(block_count < source_dir_size)
block_count = source_dir_size;
ret = zero_output_file(fd, block_count, sectorsize);
if (ret) {
fprintf(stderr, "unable to zero the output file\n");
exit(1);
}
/* our "device" is the new image file */
dev_block_count = block_count;
}
/* To create the first block group and chunk 0 in make_btrfs */
if (dev_block_count < BTRFS_MKFS_SYSTEM_GROUP_SIZE) {
fprintf(stderr, "device is too small to make filesystem\n");
exit(1);
}
blocks[0] = BTRFS_SUPER_INFO_OFFSET;
for (i = 1; i < 7; i++) {
blocks[i] = BTRFS_SUPER_INFO_OFFSET + 1024 * 1024 +
leafsize * i;
}
/*
* FS features that can be set by other means than -O
* just set the bit here
*/
if (mixed)
features |= BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS;
if ((data_profile | metadata_profile) &
(BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6)) {
features |= BTRFS_FEATURE_INCOMPAT_RAID56;
}
process_fs_features(features);
ret = make_btrfs(fd, file, label, blocks, dev_block_count,
nodesize, leafsize,
sectorsize, stripesize, features);
if (ret) {
fprintf(stderr, "error during mkfs: %s\n", strerror(-ret));
exit(1);
}
root = open_ctree(file, 0, OPEN_CTREE_WRITES);
if (!root) {
fprintf(stderr, "Open ctree failed\n");
close(fd);
exit(1);
}
root->fs_info->alloc_start = alloc_start;
ret = make_root_dir(root, mixed);
if (ret) {
fprintf(stderr, "failed to setup the root directory\n");
exit(1);
}
trans = btrfs_start_transaction(root, 1);
if (dev_cnt == 0)
goto raid_groups;
btrfs_register_one_device(file);
zero_end = 1;
while (dev_cnt-- > 0) {
int old_mixed = mixed;
file = av[optind++];
/*
* open without O_EXCL so that the problem should not
* occur by the following processing.
* (btrfs_register_one_device() fails if O_EXCL is on)
*/
fd = open(file, O_RDWR);
if (fd < 0) {
fprintf(stderr, "unable to open %s: %s\n", file,
strerror(errno));
exit(1);
}
ret = btrfs_device_already_in_root(root, fd,
BTRFS_SUPER_INFO_OFFSET);
if (ret) {
fprintf(stderr, "skipping duplicate device %s in FS\n",
file);
close(fd);
continue;
}
ret = btrfs_prepare_device(fd, file, zero_end, &dev_block_count,
block_count, &mixed, discard);
mixed = old_mixed;
BUG_ON(ret);
ret = btrfs_add_to_fsid(trans, root, fd, file, dev_block_count,
sectorsize, sectorsize, sectorsize);
BUG_ON(ret);
btrfs_register_one_device(file);
}
raid_groups:
if (!source_dir_set) {
ret = create_raid_groups(trans, root, data_profile,
data_profile_opt, metadata_profile,
metadata_profile_opt, mixed, ssd);
BUG_ON(ret);
}
ret = create_data_reloc_tree(trans, root);
BUG_ON(ret);
printf("fs created label %s on %s\n\tnodesize %u leafsize %u "
"sectorsize %u size %s\n",
label, first_file, nodesize, leafsize, sectorsize,
pretty_size(btrfs_super_total_bytes(root->fs_info->super_copy)));
printf("%s\n", BTRFS_BUILD_VERSION);
btrfs_commit_transaction(trans, root);
if (source_dir_set) {
trans = btrfs_start_transaction(root, 1);
ret = create_chunks(trans, root,
num_of_meta_chunks, size_of_data);
BUG_ON(ret);
btrfs_commit_transaction(trans, root);
ret = make_image(source_dir, root, fd);
BUG_ON(ret);
}
ret = close_ctree(root);
BUG_ON(ret);
free(label);
return 0;
}
void TextureStore::TileSwapper::load(const TileKey& key, TileRecord& record)
{
// Fetch the texture container.
const TextureContainer& textures =
key.m_assembly_uid == UniqueID(~0)
? m_scene.textures()
: m_assemblies[key.m_assembly_uid]->textures();
// Fetch the texture.
Texture* texture = textures.get_by_uid(key.m_texture_uid);
if (m_params.m_track_tile_loading)
{
RENDERER_LOG_DEBUG(
"loading tile (" FMT_SIZE_T ", " FMT_SIZE_T ") "
"from texture \"%s\"...",
key.get_tile_x(),
key.get_tile_y(),
texture->get_name());
}
// Load the tile.
record.m_tile = texture->load_tile(key.get_tile_x(), key.get_tile_y());
record.m_owners = 0;
// Convert the tile to the linear RGB color space.
switch (texture->get_color_space())
{
case ColorSpaceLinearRGB:
break;
case ColorSpaceSRGB:
convert_tile_srgb_to_linear_rgb(*record.m_tile);
break;
case ColorSpaceCIEXYZ:
convert_tile_ciexyz_to_linear_rgb(*record.m_tile);
break;
assert_otherwise;
}
// Track the amount of memory used by the tile cache.
m_memory_size += record.m_tile->get_memory_size();
m_peak_memory_size = max(m_peak_memory_size, m_memory_size);
if (m_params.m_track_store_size)
{
if (m_memory_size > m_params.m_memory_limit)
{
RENDERER_LOG_DEBUG(
"texture store size is %s, exceeding capacity %s by %s",
pretty_size(m_memory_size).c_str(),
pretty_size(m_params.m_memory_limit).c_str(),
pretty_size(m_memory_size - m_params.m_memory_limit).c_str());
}
else
{
RENDERER_LOG_DEBUG(
"texture store size is %s, below capacity %s by %s",
pretty_size(m_memory_size).c_str(),
pretty_size(m_params.m_memory_limit).c_str(),
pretty_size(m_params.m_memory_limit - m_memory_size).c_str());
}
}
}
void TextureStore::TileSwapper::load(const TileKey& key, TileRecord& record)
{
// Fetch the texture container.
const TextureContainer& textures =
key.m_assembly_uid == ~0
? m_scene.textures()
: m_scene.assemblies().get_by_uid(key.m_assembly_uid)->textures();
// Fetch the texture.
const size_t texture_index = key.get_texture_index();
assert(texture_index < textures.size());
Texture* texture = textures.get_by_index(texture_index);
#ifdef TRACK_TILE_LOADING
RENDERER_LOG_DEBUG(
"loading tile (" FMT_SIZE_T ", " FMT_SIZE_T ") "
"from texture \"%s\"...",
key.get_tile_x(),
key.get_tile_y(),
texture->get_name());
#endif
// Load the tile.
record.m_tile = texture->load_tile(key.get_tile_x(), key.get_tile_y());
record.m_owners = 0;
// Convert the tile to the linear RGB color space.
switch (texture->get_color_space())
{
case ColorSpaceLinearRGB:
break;
case ColorSpaceSRGB:
convert_tile_srgb_to_linear_rgb(*record.m_tile);
break;
case ColorSpaceCIEXYZ:
convert_tile_ciexyz_to_linear_rgb(*record.m_tile);
break;
assert_otherwise;
}
// Track the amount of memory used by the tile cache.
m_memory_size += dynamic_sizeof(*record.m_tile);
m_max_memory_size = max(m_max_memory_size, m_memory_size);
#ifdef TRACK_CACHE_SIZE
if (m_memory_size > m_memory_limit)
{
RENDERER_LOG_DEBUG(
"texture store size is %s, exceeding capacity %s by %s",
pretty_size(m_memory_size).c_str(),
pretty_size(m_memory_limit).c_str(),
pretty_size(m_memory_size - m_memory_limit).c_str());
}
else
{
RENDERER_LOG_DEBUG(
"texture store size is %s, below capacity %s by %s",
pretty_size(m_memory_size).c_str(),
pretty_size(m_memory_limit).c_str(),
pretty_size(m_memory_size - m_memory_limit).c_str());
}
#endif
}
