static int
_write_buffer(const uint32_t key, const void *data, void *user_data)
{
if(!data) return 1;
struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)data;
// too small to write. no error, but don't write.
if(dsc->width <= 8 && dsc->height <= 8) return 0;
_iterate_data_t *d = (_iterate_data_t *)user_data;
int written = fwrite(&(d->mip), sizeof(dt_mipmap_size_t), 1, d->f);
if(written != 1) return 1;
written = fwrite(&key, sizeof(uint32_t), 1, d->f);
if(written != 1) return 1;
if(d->compression_type)
{
// write buffer size, wd, ht and the full blob, as it is in memory.
const int32_t length = compressed_buffer_size(d->compression_type, dsc->width, dsc->height);
written = fwrite(&length, sizeof(int32_t), 1, d->f);
if(written != 1) return 1;
written = fwrite(&dsc->width, sizeof(int32_t), 1, d->f);
if(written != 1) return 1;
written = fwrite(&dsc->height, sizeof(int32_t), 1, d->f);
if(written != 1) return 1;
written = fwrite(dsc+1, sizeof(uint8_t), length, d->f);
if(written != length) return 1;
}
else
{
dt_mipmap_buffer_t buf;
buf.width = dsc->width;
buf.height = dsc->height;
buf.imgid = get_imgid(key);
buf.size = get_size(key);
// skip to next 8-byte alignment, for sse buffers.
buf.buf = (uint8_t *)(dsc+1);
const int cache_quality = dt_conf_get_int("database_cache_quality");
const int32_t length = dt_imageio_jpeg_compress(buf.buf, d->blob, buf.width, buf.height, MIN(100, MAX(10, cache_quality)));
written = fwrite(&length, sizeof(int32_t), 1, d->f);
if(written != 1) return 1;
written = fwrite(d->blob, sizeof(uint8_t), length, d->f);
if(written != length) return 1;
}
return 0;
}
static int
dt_mipmap_cache_deserialize(dt_mipmap_cache_t *cache)
{
int32_t rd = 0;
const dt_mipmap_size_t mip = DT_MIPMAP_2;
uint8_t *blob = NULL;
FILE *f = NULL;
int file_width[mip+1], file_height[mip+1];
gchar dbfilename[DT_MAX_PATH_LEN];
if (dt_mipmap_cache_get_filename(dbfilename, sizeof(dbfilename)))
{
fprintf(stderr, "[mipmap_cache] could not retrieve cache filename; not deserializing\n");
return 1;
}
if (!strcmp(dbfilename, ":memory:"))
{
// fprintf(stderr, "[mipmap_cache] library is in memory; not deserializing\n");
return 0;
}
// drop any old cache if the database is new. in that case newly imported images will probably mapped to old thumbnails
if(dt_database_is_new(darktable.db) && g_file_test(dbfilename, G_FILE_TEST_IS_REGULAR))
{
fprintf(stderr, "[mipmap_cache] database is new, dropping old cache `%s'\n", dbfilename);
goto read_finalize;
}
f = fopen(dbfilename, "rb");
if(!f)
{
if (errno == ENOENT)
{
fprintf(stderr, "[mipmap_cache] cache is empty, file `%s' doesn't exist\n", dbfilename);
}
else
{
fprintf(stderr, "[mipmap_cache] failed to open the cache from `%s'\n", dbfilename);
}
goto read_finalize;
}
// read version info:
const int32_t magic = DT_MIPMAP_CACHE_FILE_MAGIC + DT_MIPMAP_CACHE_FILE_VERSION;
int32_t magic_file = 0;
rd = fread(&magic_file, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
if(magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22)
{
// same format, but compression was broken in 22 and below
}
else if(magic_file != magic)
{
if(magic_file > DT_MIPMAP_CACHE_FILE_MAGIC && magic_file < magic)
fprintf(stderr, "[mipmap_cache] cache version too old, dropping `%s' cache\n", dbfilename);
else
fprintf(stderr, "[mipmap_cache] invalid cache file, dropping `%s' cache\n", dbfilename);
goto read_finalize;
}
// also read compression type and yell out on missmatch.
int32_t compression = -1;
rd = fread(&compression, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
if(compression != cache->compression_type)
{
fprintf(stderr, "[mipmap_cache] cache is %s, but settings say we should use %s, dropping `%s' cache\n",
compression == 0 ? "uncompressed" : (compression == 1 ? "low quality compressed" : "high quality compressed"),
cache->compression_type == 0 ? "no compression" : (cache->compression_type == 1 ? "low quality compression" : "high quality compression"),
dbfilename);
goto read_finalize;
}
if(compression && (magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22))
{
// compression is enabled and we have the affected version. can't read that.
fprintf(stderr, "[mipmap_cache] dropping compressed cache v22 to regenerate mips without artifacts.\n");
goto read_finalize;
}
for (int i=0; i<=mip; i++)
{
rd = fread(&file_width[i], sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
rd = fread(&file_height[i], sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
if(file_width[i] != cache->mip[i].max_width ||
file_height[i] != cache->mip[i].max_height)
{
fprintf(stderr, "[mipmap_cache] cache settings changed, dropping `%s' cache\n", dbfilename);
goto read_finalize;
}
}
if(cache->compression_type) blob = NULL;
else blob = malloc(sizeof(uint32_t)*file_width[mip]*file_height[mip]);
while(!feof(f))
{
int level = 0;
rd = fread(&level, sizeof(int), 1, f);
if (level > mip) break;
int32_t key = 0;
rd = fread(&key, sizeof(int32_t), 1, f);
if(rd != 1) break; // first value is break only, goes to eof.
int32_t length = 0;
rd = fread(&length, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
uint8_t *data = (uint8_t *)dt_cache_read_get(&cache->mip[level].cache, key);
struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)data;
if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE)
{
if(cache->compression_type)
{
int32_t wd, ht;
rd = fread(&wd, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
rd = fread(&ht, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
dsc->width = wd;
dsc->height = ht;
if(length != compressed_buffer_size(cache->compression_type, wd, ht)) goto read_error;
// directly read from disk into cache:
rd = fread(data + sizeof(*dsc), 1, length, f);
if(rd != length) goto read_error;
}
else
{
// jpg too large?
if(length > sizeof(uint32_t)*file_width[mip]*file_height[mip]) goto read_error;
rd = fread(blob, sizeof(uint8_t), length, f);
if(rd != length) goto read_error;
// no compression, the image is still compressed on disk, as jpg
dt_imageio_jpeg_t jpg;
if(dt_imageio_jpeg_decompress_header(blob, length, &jpg) ||
(jpg.width > file_width[level] || jpg.height > file_height[level]) ||
dt_imageio_jpeg_decompress(&jpg, data+sizeof(*dsc)))
{
fprintf(stderr, "[mipmap_cache] failed to decompress thumbnail for image %d!\n", get_imgid(key));
}
dsc->width = jpg.width;
dsc->height = jpg.height;
}
dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
// these come write locked in case idata[3] == 1, so release that!
dt_cache_write_release(&cache->mip[level].cache, key);
}
dt_cache_read_release(&cache->mip[level].cache, key);
}
fclose(f);
free(blob);
return 0;
read_error:
fprintf(stderr, "[mipmap_cache] failed to recover the cache from `%s'\n", dbfilename);
read_finalize:
if(f) fclose(f);
free(blob);
g_unlink(dbfilename);
return 1;
}
void dt_mipmap_cache_deallocate_dynamic(void *data, dt_cache_entry_t *entry)
{
dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
const dt_mipmap_size_t mip = get_size(entry->key);
if(mip < DT_MIPMAP_F)
{
struct dt_mipmap_buffer_dsc *dsc = (struct dt_mipmap_buffer_dsc *)entry->data;
// don't write skulls:
if(dsc->width > 8 && dsc->height > 8)
{
if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE)
{
// also remove jpg backing (always try to do that, in case user just temporarily switched it off,
// to avoid inconsistencies.
// if(dt_conf_get_bool("cache_disk_backend"))
if(cache->cachedir[0])
{
char filename[PATH_MAX] = {0};
snprintf(filename, sizeof(filename), "%s.d/%d/%d.jpg", cache->cachedir, mip, get_imgid(entry->key));
g_unlink(filename);
}
}
else if(cache->cachedir[0] && dt_conf_get_bool("cache_disk_backend"))
{
// serialize to disk
char filename[PATH_MAX] = {0};
snprintf(filename, sizeof(filename), "%s.d/%d", cache->cachedir, mip);
int mkd = g_mkdir_with_parents(filename, 0750);
if(!mkd)
{
snprintf(filename, sizeof(filename), "%s.d/%d/%d.jpg", cache->cachedir, mip, get_imgid(entry->key));
// Don't write existing files as both performance and quality (lossy jpg) suffer
FILE *f = NULL;
if (!g_file_test(filename, G_FILE_TEST_EXISTS) && (f = fopen(filename, "wb")))
{
// first check the disk isn't full
struct statvfs vfsbuf;
if (!statvfs(filename, &vfsbuf))
{
int64_t free_mb = ((vfsbuf.f_frsize * vfsbuf.f_bavail) >> 20);
if (free_mb < 100)
{
fprintf(stderr, "Aborting image write as only %" PRId64 " MB free to write %s\n", free_mb, filename);
goto write_error;
}
}
else
{
fprintf(stderr, "Aborting image write since couldn't determine free space available to write %s\n", filename);
goto write_error;
}
const int cache_quality = dt_conf_get_int("database_cache_quality");
const uint8_t *exif = NULL;
int exif_len = 0;
if(dsc->color_space == DT_COLORSPACE_SRGB)
{
exif = dt_mipmap_cache_exif_data_srgb;
exif_len = dt_mipmap_cache_exif_data_srgb_length;
}
else if(dsc->color_space == DT_COLORSPACE_ADOBERGB)
{
exif = dt_mipmap_cache_exif_data_adobergb;
exif_len = dt_mipmap_cache_exif_data_adobergb_length;
}
if(dt_imageio_jpeg_write(filename, entry->data + sizeof(*dsc), dsc->width, dsc->height, MIN(100, MAX(10, cache_quality)), exif, exif_len))
{
write_error:
g_unlink(filename);
}
}
if(f) fclose(f);
}
// callback for the cache backend to initialize payload pointers
void dt_mipmap_cache_allocate_dynamic(void *data, dt_cache_entry_t *entry)
{
dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
// for full image buffers
struct dt_mipmap_buffer_dsc *dsc = entry->data;
const dt_mipmap_size_t mip = get_size(entry->key);
// alloc mere minimum for the header + broken image buffer:
if(!dsc)
{
if(mip <= DT_MIPMAP_F)
{
// these are fixed-size:
entry->data = dt_alloc_align(16, cache->buffer_size[mip]);
}
else
{
entry->data = dt_alloc_align(16, sizeof(*dsc) + sizeof(float) * 4 * 64);
}
// fprintf(stderr, "[mipmap cache] alloc dynamic for key %u %p\n", key, *buf);
if(!(entry->data))
{
fprintf(stderr, "[mipmap cache] memory allocation failed!\n");
exit(1);
}
dsc = entry->data;
if(mip <= DT_MIPMAP_F)
{
dsc->width = cache->max_width[mip];
dsc->height = cache->max_height[mip];
dsc->size = cache->buffer_size[mip];
dsc->color_space = DT_COLORSPACE_NONE;
}
else
{
dsc->width = 0;
dsc->height = 0;
dsc->color_space = DT_COLORSPACE_NONE;
dsc->size = sizeof(*dsc) + sizeof(float) * 4 * 64;
}
}
assert(dsc->size >= sizeof(*dsc));
int loaded_from_disk = 0;
if(mip < DT_MIPMAP_F)
{
if(cache->cachedir[0] && dt_conf_get_bool("cache_disk_backend"))
{
// try and load from disk, if successful set flag
char filename[PATH_MAX] = {0};
snprintf(filename, sizeof(filename), "%s.d/%d/%d.jpg", cache->cachedir, mip, get_imgid(entry->key));
FILE *f = fopen(filename, "rb");
if(f)
{
long len = 0;
uint8_t *blob = 0;
fseek(f, 0, SEEK_END);
len = ftell(f);
if(len <= 0) goto read_error; // coverity madness
blob = (uint8_t *)malloc(len);
if(!blob) goto read_error;
fseek(f, 0, SEEK_SET);
int rd = fread(blob, sizeof(uint8_t), len, f);
if(rd != len) goto read_error;
dt_colorspaces_color_profile_type_t color_space;
dt_imageio_jpeg_t jpg;
if(dt_imageio_jpeg_decompress_header(blob, len, &jpg)
|| (jpg.width > cache->max_width[mip] || jpg.height > cache->max_height[mip])
|| ((color_space = dt_imageio_jpeg_read_color_space(&jpg)) == DT_COLORSPACE_NONE) // pointless test to keep it in the if clause
|| dt_imageio_jpeg_decompress(&jpg, entry->data + sizeof(*dsc)))
{
fprintf(stderr, "[mipmap_cache] failed to decompress thumbnail for image %d from `%s'!\n", get_imgid(entry->key), filename);
goto read_error;
}
dsc->width = jpg.width;
dsc->height = jpg.height;
dsc->color_space = color_space;
loaded_from_disk = 1;
if(0)
{
read_error:
g_unlink(filename);
}
free(blob);
fclose(f);
}
}
}
if(!loaded_from_disk)
dsc->flags = DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
else dsc->flags = 0;
// cost is just flat one for the buffer, as the buffers might have different sizes,
// to make sure quota is meaningful.
if(mip >= DT_MIPMAP_F) entry->cost = 1;
else entry->cost = cache->buffer_size[mip];
}
