const dt_image_t*
dt_image_cache_read_get(
dt_image_cache_t *cache,
const uint32_t imgid)
{
if(imgid <= 0) return NULL;
return (const dt_image_t *)dt_cache_read_get(&cache->cache, imgid);
}
int main(int argc, char *arg[])
{
dt_cache_t cache;
// dt_cache_init(&cache, 110000, 16, 64, 100000);
// really hammer it, make quota insanely low:
dt_cache_init(&cache, 110000, 16, 64, 100);
dt_cache_set_allocate_callback(&cache, alloc_dummy, NULL);
#ifdef _OPENMP
# pragma omp parallel for default(none) schedule(guided) shared(cache, stderr) num_threads(16)
#endif
for(int k=0;k<100000;k++)
{
void *data = (void *)(long int)k;
const int size = 0;//dt_cache_size(&cache);
const int con1 = dt_cache_contains(&cache, k);
const int val1 = (int)(long int)dt_cache_read_get(&cache, k);
const int val2 = (int)(long int)dt_cache_read_get(&cache, k);
// fprintf(stderr, "\rinserted number %d, size %d, value %d - %d, contains %d - %d", k, size, val1, val2, con1, con2);
const int con2 = dt_cache_contains(&cache, k);
assert (con1 == 0);
assert (con2 == 1);
assert (val2 == k);
dt_cache_read_release(&cache, k);
dt_cache_read_release(&cache, k);
}
dt_cache_print_locked(&cache);
// fprintf(stderr, "\n");
fprintf(stderr, "[passed] inserting 100000 entries concurrently\n");
const int size = dt_cache_size(&cache);
const int lru_cnt = lru_check_consistency(&cache);
const int lru_cnt_r = lru_check_consistency_reverse(&cache);
// fprintf(stderr, "lru list contains %d|%d/%d entries\n", lru_cnt, lru_cnt_r, size);
assert(size == lru_cnt);
assert(lru_cnt_r == lru_cnt);
fprintf(stderr, "[passed] cache lru consistency after removals, have %d entries left.\n", size);
dt_cache_cleanup(&cache);
exit(0);
}
void
dt_mipmap_cache_read_get(
dt_mipmap_cache_t *cache,
dt_mipmap_buffer_t *buf,
const uint32_t imgid,
const dt_mipmap_size_t mip,
const dt_mipmap_get_flags_t flags)
{
const uint32_t key = get_key(imgid, mip);
if(flags == DT_MIPMAP_TESTLOCK)
{
// simple case: only get and lock if it's there.
struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)dt_cache_read_testget(&cache->mip[mip].cache, key);
if(dsc)
{
buf->width = dsc->width;
buf->height = dsc->height;
buf->imgid = imgid;
buf->size = mip;
// skip to next 8-byte alignment, for sse buffers.
buf->buf = (uint8_t *)(dsc+1);
}
else
{
// set to NULL if failed.
buf->width = buf->height = 0;
buf->imgid = 0;
buf->size = DT_MIPMAP_NONE;
buf->buf = NULL;
}
}
else if(flags == DT_MIPMAP_PREFETCH)
{
// and opposite: prefetch without locking
if(mip > DT_MIPMAP_FULL || mip < DT_MIPMAP_0) return;
dt_job_t j;
dt_image_load_job_init(&j, imgid, mip);
// if the job already exists, make it high-priority, if not, add it:
if(dt_control_revive_job(darktable.control, &j) < 0)
dt_control_add_job(darktable.control, &j);
}
else if(flags == DT_MIPMAP_BLOCKING)
{
// simple case: blocking get
struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)dt_cache_read_get(&cache->mip[mip].cache, key);
if(!dsc)
{
// should never happen for anything but full images which have been moved.
assert(mip == DT_MIPMAP_FULL || mip == DT_MIPMAP_F);
// fprintf(stderr, "[mipmap cache get] no data in cache for imgid %u size %d!\n", imgid, mip);
// sorry guys, no image for you :(
buf->width = buf->height = 0;
buf->imgid = 0;
buf->size = DT_MIPMAP_NONE;
buf->buf = NULL;
}
else
{
// fprintf(stderr, "[mipmap cache get] found data in cache for imgid %u size %d\n", imgid, mip);
// uninitialized?
//assert(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE || dsc->size == 0);
if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE)
{
__sync_fetch_and_add (&(cache->mip[mip].stats_fetches), 1);
// fprintf(stderr, "[mipmap cache get] now initializing buffer for img %u mip %d!\n", imgid, mip);
// we're write locked here, as requested by the alloc callback.
// now fill it with data:
if(mip == DT_MIPMAP_FULL)
{
// load the image:
// make sure we access the r/w lock as shortly as possible!
dt_image_t buffered_image;
const dt_image_t *cimg = dt_image_cache_read_get(darktable.image_cache, imgid);
buffered_image = *cimg;
// dt_image_t *img = dt_image_cache_write_get(darktable.image_cache, cimg);
// dt_image_cache_write_release(darktable.image_cache, img, DT_IMAGE_CACHE_RELAXED);
dt_image_cache_read_release(darktable.image_cache, cimg);
char filename[DT_MAX_PATH_LEN];
gboolean from_cache = TRUE;
dt_image_full_path(buffered_image.id, filename, DT_MAX_PATH_LEN, &from_cache);
dt_mipmap_cache_allocator_t a = (dt_mipmap_cache_allocator_t)&dsc;
struct dt_mipmap_buffer_dsc* prvdsc = dsc;
dt_imageio_retval_t ret = dt_imageio_open(&buffered_image, filename, a);
if(dsc != prvdsc)
{
// fprintf(stderr, "[mipmap cache] realloc %p\n", data);
// write back to cache, too.
// in case something went wrong, still keep the buffer and return it to the hashtable
// so we don't produce mem leaks or unnecessary mem fragmentation.
dt_cache_realloc(&cache->mip[mip].cache, key, 1, (void*)dsc);
}
if(ret != DT_IMAGEIO_OK)
{
// fprintf(stderr, "[mipmap read get] error loading image: %d\n", ret);
//
// we can only return a zero dimension buffer if the buffer has been allocated.
// in case dsc couldn't be allocated and points to the static buffer, it contains
// a dead image already.
if((void *)dsc != (void *)dt_mipmap_cache_static_dead_image) dsc->width = dsc->height = 0;
}
else
{
// swap back new image data:
cimg = dt_image_cache_read_get(darktable.image_cache, imgid);
dt_image_t *img = dt_image_cache_write_get(darktable.image_cache, cimg);
*img = buffered_image;
// fprintf(stderr, "[mipmap read get] initializing full buffer img %u with %u %u -> %d %d (%p)\n", imgid, data[0], data[1], img->width, img->height, data);
// don't write xmp for this (we only changed db stuff):
dt_image_cache_write_release(darktable.image_cache, img, DT_IMAGE_CACHE_RELAXED);
dt_image_cache_read_release(darktable.image_cache, img);
}
}
else if(mip == DT_MIPMAP_F)
{
_init_f((float *)(dsc+1), &dsc->width, &dsc->height, imgid);
}
else
{
// 8-bit thumbs, possibly need to be compressed:
if(cache->compression_type)
{
// get per-thread temporary storage without malloc from a separate cache:
const int key = dt_control_get_threadid();
// const void *cbuf =
dt_cache_read_get(&cache->scratchmem.cache, key);
uint8_t *scratchmem = (uint8_t *)dt_cache_write_get(&cache->scratchmem.cache, key);
_init_8(scratchmem, &dsc->width, &dsc->height, imgid, mip);
buf->width = dsc->width;
buf->height = dsc->height;
buf->imgid = imgid;
buf->size = mip;
buf->buf = (uint8_t *)(dsc+1);
dt_mipmap_cache_compress(buf, scratchmem);
dt_cache_write_release(&cache->scratchmem.cache, key);
dt_cache_read_release(&cache->scratchmem.cache, key);
}
else
{
_init_8((uint8_t *)(dsc+1), &dsc->width, &dsc->height, imgid, mip);
}
}
dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
// drop the write lock
dt_cache_write_release(&cache->mip[mip].cache, key);
/* raise signal that mipmaps has been flushed to cache */
dt_control_signal_raise(darktable.signals, DT_SIGNAL_DEVELOP_MIPMAP_UPDATED);
}
buf->width = dsc->width;
buf->height = dsc->height;
buf->imgid = imgid;
buf->size = mip;
buf->buf = (uint8_t *)(dsc+1);
if(dsc->width == 0 || dsc->height == 0)
{
// fprintf(stderr, "[mipmap cache get] got a zero-sized image for img %u mip %d!\n", imgid, mip);
if(mip < DT_MIPMAP_F) dead_image_8(buf);
else if(mip == DT_MIPMAP_F) dead_image_f(buf);
else buf->buf = NULL; // full images with NULL buffer have to be handled, indicates `missing image'
}
}
}
else if(flags == DT_MIPMAP_BEST_EFFORT)
{
__sync_fetch_and_add (&(cache->mip[mip].stats_requests), 1);
// best-effort, might also return NULL.
// never decrease mip level for float buffer or full image:
dt_mipmap_size_t min_mip = (mip >= DT_MIPMAP_F) ? mip : DT_MIPMAP_0;
for(int k=mip; k>=min_mip && k>=0; k--)
{
// already loaded?
dt_mipmap_cache_read_get(cache, buf, imgid, k, DT_MIPMAP_TESTLOCK);
if(buf->buf && buf->width > 0 && buf->height > 0)
{
if(mip != k) __sync_fetch_and_add (&(cache->mip[k].stats_standin), 1);
return;
}
// didn't succeed the first time? prefetch for later!
if(mip == k)
{
__sync_fetch_and_add (&(cache->mip[mip].stats_near_match), 1);
dt_mipmap_cache_read_get(cache, buf, imgid, mip, DT_MIPMAP_PREFETCH);
}
}
__sync_fetch_and_add (&(cache->mip[mip].stats_misses), 1);
// fprintf(stderr, "[mipmap cache get] image not found in cache: imgid %u mip %d!\n", imgid, mip);
// nothing found :(
buf->buf = NULL;
buf->imgid = 0;
buf->size = DT_MIPMAP_NONE;
buf->width = buf->height = 0;
}
}
static int
dt_mipmap_cache_deserialize(dt_mipmap_cache_t *cache)
{
int32_t rd = 0;
const dt_mipmap_size_t mip = DT_MIPMAP_2;
uint8_t *blob = NULL;
FILE *f = NULL;
int file_width[mip+1], file_height[mip+1];
gchar dbfilename[DT_MAX_PATH_LEN];
if (dt_mipmap_cache_get_filename(dbfilename, sizeof(dbfilename)))
{
fprintf(stderr, "[mipmap_cache] could not retrieve cache filename; not deserializing\n");
return 1;
}
if (!strcmp(dbfilename, ":memory:"))
{
// fprintf(stderr, "[mipmap_cache] library is in memory; not deserializing\n");
return 0;
}
// drop any old cache if the database is new. in that case newly imported images will probably mapped to old thumbnails
if(dt_database_is_new(darktable.db) && g_file_test(dbfilename, G_FILE_TEST_IS_REGULAR))
{
fprintf(stderr, "[mipmap_cache] database is new, dropping old cache `%s'\n", dbfilename);
goto read_finalize;
}
f = fopen(dbfilename, "rb");
if(!f)
{
if (errno == ENOENT)
{
fprintf(stderr, "[mipmap_cache] cache is empty, file `%s' doesn't exist\n", dbfilename);
}
else
{
fprintf(stderr, "[mipmap_cache] failed to open the cache from `%s'\n", dbfilename);
}
goto read_finalize;
}
// read version info:
const int32_t magic = DT_MIPMAP_CACHE_FILE_MAGIC + DT_MIPMAP_CACHE_FILE_VERSION;
int32_t magic_file = 0;
rd = fread(&magic_file, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
if(magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22)
{
// same format, but compression was broken in 22 and below
}
else if(magic_file != magic)
{
if(magic_file > DT_MIPMAP_CACHE_FILE_MAGIC && magic_file < magic)
fprintf(stderr, "[mipmap_cache] cache version too old, dropping `%s' cache\n", dbfilename);
else
fprintf(stderr, "[mipmap_cache] invalid cache file, dropping `%s' cache\n", dbfilename);
goto read_finalize;
}
// also read compression type and yell out on missmatch.
int32_t compression = -1;
rd = fread(&compression, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
if(compression != cache->compression_type)
{
fprintf(stderr, "[mipmap_cache] cache is %s, but settings say we should use %s, dropping `%s' cache\n",
compression == 0 ? "uncompressed" : (compression == 1 ? "low quality compressed" : "high quality compressed"),
cache->compression_type == 0 ? "no compression" : (cache->compression_type == 1 ? "low quality compression" : "high quality compression"),
dbfilename);
goto read_finalize;
}
if(compression && (magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22))
{
// compression is enabled and we have the affected version. can't read that.
fprintf(stderr, "[mipmap_cache] dropping compressed cache v22 to regenerate mips without artifacts.\n");
goto read_finalize;
}
for (int i=0; i<=mip; i++)
{
rd = fread(&file_width[i], sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
rd = fread(&file_height[i], sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
if(file_width[i] != cache->mip[i].max_width ||
file_height[i] != cache->mip[i].max_height)
{
fprintf(stderr, "[mipmap_cache] cache settings changed, dropping `%s' cache\n", dbfilename);
goto read_finalize;
}
}
if(cache->compression_type) blob = NULL;
else blob = malloc(sizeof(uint32_t)*file_width[mip]*file_height[mip]);
while(!feof(f))
{
int level = 0;
rd = fread(&level, sizeof(int), 1, f);
if (level > mip) break;
int32_t key = 0;
rd = fread(&key, sizeof(int32_t), 1, f);
if(rd != 1) break; // first value is break only, goes to eof.
int32_t length = 0;
rd = fread(&length, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
uint8_t *data = (uint8_t *)dt_cache_read_get(&cache->mip[level].cache, key);
struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)data;
if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE)
{
if(cache->compression_type)
{
int32_t wd, ht;
rd = fread(&wd, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
rd = fread(&ht, sizeof(int32_t), 1, f);
if(rd != 1) goto read_error;
dsc->width = wd;
dsc->height = ht;
if(length != compressed_buffer_size(cache->compression_type, wd, ht)) goto read_error;
// directly read from disk into cache:
rd = fread(data + sizeof(*dsc), 1, length, f);
if(rd != length) goto read_error;
}
else
{
// jpg too large?
if(length > sizeof(uint32_t)*file_width[mip]*file_height[mip]) goto read_error;
rd = fread(blob, sizeof(uint8_t), length, f);
if(rd != length) goto read_error;
// no compression, the image is still compressed on disk, as jpg
dt_imageio_jpeg_t jpg;
if(dt_imageio_jpeg_decompress_header(blob, length, &jpg) ||
(jpg.width > file_width[level] || jpg.height > file_height[level]) ||
dt_imageio_jpeg_decompress(&jpg, data+sizeof(*dsc)))
{
fprintf(stderr, "[mipmap_cache] failed to decompress thumbnail for image %d!\n", get_imgid(key));
}
dsc->width = jpg.width;
dsc->height = jpg.height;
}
dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
// these come write locked in case idata[3] == 1, so release that!
dt_cache_write_release(&cache->mip[level].cache, key);
}
dt_cache_read_release(&cache->mip[level].cache, key);
}
fclose(f);
free(blob);
return 0;
read_error:
fprintf(stderr, "[mipmap_cache] failed to recover the cache from `%s'\n", dbfilename);
read_finalize:
if(f) fclose(f);
free(blob);
g_unlink(dbfilename);
return 1;
}
