const dt_image_t* dt_image_cache_read_get( dt_image_cache_t *cache, const uint32_t imgid) { if(imgid <= 0) return NULL; return (const dt_image_t *)dt_cache_read_get(&cache->cache, imgid); }
int main(int argc, char *arg[]) { dt_cache_t cache; // dt_cache_init(&cache, 110000, 16, 64, 100000); // really hammer it, make quota insanely low: dt_cache_init(&cache, 110000, 16, 64, 100); dt_cache_set_allocate_callback(&cache, alloc_dummy, NULL); #ifdef _OPENMP # pragma omp parallel for default(none) schedule(guided) shared(cache, stderr) num_threads(16) #endif for(int k=0;k<100000;k++) { void *data = (void *)(long int)k; const int size = 0;//dt_cache_size(&cache); const int con1 = dt_cache_contains(&cache, k); const int val1 = (int)(long int)dt_cache_read_get(&cache, k); const int val2 = (int)(long int)dt_cache_read_get(&cache, k); // fprintf(stderr, "\rinserted number %d, size %d, value %d - %d, contains %d - %d", k, size, val1, val2, con1, con2); const int con2 = dt_cache_contains(&cache, k); assert (con1 == 0); assert (con2 == 1); assert (val2 == k); dt_cache_read_release(&cache, k); dt_cache_read_release(&cache, k); } dt_cache_print_locked(&cache); // fprintf(stderr, "\n"); fprintf(stderr, "[passed] inserting 100000 entries concurrently\n"); const int size = dt_cache_size(&cache); const int lru_cnt = lru_check_consistency(&cache); const int lru_cnt_r = lru_check_consistency_reverse(&cache); // fprintf(stderr, "lru list contains %d|%d/%d entries\n", lru_cnt, lru_cnt_r, size); assert(size == lru_cnt); assert(lru_cnt_r == lru_cnt); fprintf(stderr, "[passed] cache lru consistency after removals, have %d entries left.\n", size); dt_cache_cleanup(&cache); exit(0); }
void dt_mipmap_cache_read_get( dt_mipmap_cache_t *cache, dt_mipmap_buffer_t *buf, const uint32_t imgid, const dt_mipmap_size_t mip, const dt_mipmap_get_flags_t flags) { const uint32_t key = get_key(imgid, mip); if(flags == DT_MIPMAP_TESTLOCK) { // simple case: only get and lock if it's there. struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)dt_cache_read_testget(&cache->mip[mip].cache, key); if(dsc) { buf->width = dsc->width; buf->height = dsc->height; buf->imgid = imgid; buf->size = mip; // skip to next 8-byte alignment, for sse buffers. buf->buf = (uint8_t *)(dsc+1); } else { // set to NULL if failed. buf->width = buf->height = 0; buf->imgid = 0; buf->size = DT_MIPMAP_NONE; buf->buf = NULL; } } else if(flags == DT_MIPMAP_PREFETCH) { // and opposite: prefetch without locking if(mip > DT_MIPMAP_FULL || mip < DT_MIPMAP_0) return; dt_job_t j; dt_image_load_job_init(&j, imgid, mip); // if the job already exists, make it high-priority, if not, add it: if(dt_control_revive_job(darktable.control, &j) < 0) dt_control_add_job(darktable.control, &j); } else if(flags == DT_MIPMAP_BLOCKING) { // simple case: blocking get struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)dt_cache_read_get(&cache->mip[mip].cache, key); if(!dsc) { // should never happen for anything but full images which have been moved. assert(mip == DT_MIPMAP_FULL || mip == DT_MIPMAP_F); // fprintf(stderr, "[mipmap cache get] no data in cache for imgid %u size %d!\n", imgid, mip); // sorry guys, no image for you :( buf->width = buf->height = 0; buf->imgid = 0; buf->size = DT_MIPMAP_NONE; buf->buf = NULL; } else { // fprintf(stderr, "[mipmap cache get] found data in cache for imgid %u size %d\n", imgid, mip); // uninitialized? //assert(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE || dsc->size == 0); if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE) { __sync_fetch_and_add (&(cache->mip[mip].stats_fetches), 1); // fprintf(stderr, "[mipmap cache get] now initializing buffer for img %u mip %d!\n", imgid, mip); // we're write locked here, as requested by the alloc callback. // now fill it with data: if(mip == DT_MIPMAP_FULL) { // load the image: // make sure we access the r/w lock as shortly as possible! dt_image_t buffered_image; const dt_image_t *cimg = dt_image_cache_read_get(darktable.image_cache, imgid); buffered_image = *cimg; // dt_image_t *img = dt_image_cache_write_get(darktable.image_cache, cimg); // dt_image_cache_write_release(darktable.image_cache, img, DT_IMAGE_CACHE_RELAXED); dt_image_cache_read_release(darktable.image_cache, cimg); char filename[DT_MAX_PATH_LEN]; gboolean from_cache = TRUE; dt_image_full_path(buffered_image.id, filename, DT_MAX_PATH_LEN, &from_cache); dt_mipmap_cache_allocator_t a = (dt_mipmap_cache_allocator_t)&dsc; struct dt_mipmap_buffer_dsc* prvdsc = dsc; dt_imageio_retval_t ret = dt_imageio_open(&buffered_image, filename, a); if(dsc != prvdsc) { // fprintf(stderr, "[mipmap cache] realloc %p\n", data); // write back to cache, too. // in case something went wrong, still keep the buffer and return it to the hashtable // so we don't produce mem leaks or unnecessary mem fragmentation. dt_cache_realloc(&cache->mip[mip].cache, key, 1, (void*)dsc); } if(ret != DT_IMAGEIO_OK) { // fprintf(stderr, "[mipmap read get] error loading image: %d\n", ret); // // we can only return a zero dimension buffer if the buffer has been allocated. // in case dsc couldn't be allocated and points to the static buffer, it contains // a dead image already. if((void *)dsc != (void *)dt_mipmap_cache_static_dead_image) dsc->width = dsc->height = 0; } else { // swap back new image data: cimg = dt_image_cache_read_get(darktable.image_cache, imgid); dt_image_t *img = dt_image_cache_write_get(darktable.image_cache, cimg); *img = buffered_image; // fprintf(stderr, "[mipmap read get] initializing full buffer img %u with %u %u -> %d %d (%p)\n", imgid, data[0], data[1], img->width, img->height, data); // don't write xmp for this (we only changed db stuff): dt_image_cache_write_release(darktable.image_cache, img, DT_IMAGE_CACHE_RELAXED); dt_image_cache_read_release(darktable.image_cache, img); } } else if(mip == DT_MIPMAP_F) { _init_f((float *)(dsc+1), &dsc->width, &dsc->height, imgid); } else { // 8-bit thumbs, possibly need to be compressed: if(cache->compression_type) { // get per-thread temporary storage without malloc from a separate cache: const int key = dt_control_get_threadid(); // const void *cbuf = dt_cache_read_get(&cache->scratchmem.cache, key); uint8_t *scratchmem = (uint8_t *)dt_cache_write_get(&cache->scratchmem.cache, key); _init_8(scratchmem, &dsc->width, &dsc->height, imgid, mip); buf->width = dsc->width; buf->height = dsc->height; buf->imgid = imgid; buf->size = mip; buf->buf = (uint8_t *)(dsc+1); dt_mipmap_cache_compress(buf, scratchmem); dt_cache_write_release(&cache->scratchmem.cache, key); dt_cache_read_release(&cache->scratchmem.cache, key); } else { _init_8((uint8_t *)(dsc+1), &dsc->width, &dsc->height, imgid, mip); } } dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE; // drop the write lock dt_cache_write_release(&cache->mip[mip].cache, key); /* raise signal that mipmaps has been flushed to cache */ dt_control_signal_raise(darktable.signals, DT_SIGNAL_DEVELOP_MIPMAP_UPDATED); } buf->width = dsc->width; buf->height = dsc->height; buf->imgid = imgid; buf->size = mip; buf->buf = (uint8_t *)(dsc+1); if(dsc->width == 0 || dsc->height == 0) { // fprintf(stderr, "[mipmap cache get] got a zero-sized image for img %u mip %d!\n", imgid, mip); if(mip < DT_MIPMAP_F) dead_image_8(buf); else if(mip == DT_MIPMAP_F) dead_image_f(buf); else buf->buf = NULL; // full images with NULL buffer have to be handled, indicates `missing image' } } } else if(flags == DT_MIPMAP_BEST_EFFORT) { __sync_fetch_and_add (&(cache->mip[mip].stats_requests), 1); // best-effort, might also return NULL. // never decrease mip level for float buffer or full image: dt_mipmap_size_t min_mip = (mip >= DT_MIPMAP_F) ? mip : DT_MIPMAP_0; for(int k=mip; k>=min_mip && k>=0; k--) { // already loaded? dt_mipmap_cache_read_get(cache, buf, imgid, k, DT_MIPMAP_TESTLOCK); if(buf->buf && buf->width > 0 && buf->height > 0) { if(mip != k) __sync_fetch_and_add (&(cache->mip[k].stats_standin), 1); return; } // didn't succeed the first time? prefetch for later! if(mip == k) { __sync_fetch_and_add (&(cache->mip[mip].stats_near_match), 1); dt_mipmap_cache_read_get(cache, buf, imgid, mip, DT_MIPMAP_PREFETCH); } } __sync_fetch_and_add (&(cache->mip[mip].stats_misses), 1); // fprintf(stderr, "[mipmap cache get] image not found in cache: imgid %u mip %d!\n", imgid, mip); // nothing found :( buf->buf = NULL; buf->imgid = 0; buf->size = DT_MIPMAP_NONE; buf->width = buf->height = 0; } }
static int dt_mipmap_cache_deserialize(dt_mipmap_cache_t *cache) { int32_t rd = 0; const dt_mipmap_size_t mip = DT_MIPMAP_2; uint8_t *blob = NULL; FILE *f = NULL; int file_width[mip+1], file_height[mip+1]; gchar dbfilename[DT_MAX_PATH_LEN]; if (dt_mipmap_cache_get_filename(dbfilename, sizeof(dbfilename))) { fprintf(stderr, "[mipmap_cache] could not retrieve cache filename; not deserializing\n"); return 1; } if (!strcmp(dbfilename, ":memory:")) { // fprintf(stderr, "[mipmap_cache] library is in memory; not deserializing\n"); return 0; } // drop any old cache if the database is new. in that case newly imported images will probably mapped to old thumbnails if(dt_database_is_new(darktable.db) && g_file_test(dbfilename, G_FILE_TEST_IS_REGULAR)) { fprintf(stderr, "[mipmap_cache] database is new, dropping old cache `%s'\n", dbfilename); goto read_finalize; } f = fopen(dbfilename, "rb"); if(!f) { if (errno == ENOENT) { fprintf(stderr, "[mipmap_cache] cache is empty, file `%s' doesn't exist\n", dbfilename); } else { fprintf(stderr, "[mipmap_cache] failed to open the cache from `%s'\n", dbfilename); } goto read_finalize; } // read version info: const int32_t magic = DT_MIPMAP_CACHE_FILE_MAGIC + DT_MIPMAP_CACHE_FILE_VERSION; int32_t magic_file = 0; rd = fread(&magic_file, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; if(magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22) { // same format, but compression was broken in 22 and below } else if(magic_file != magic) { if(magic_file > DT_MIPMAP_CACHE_FILE_MAGIC && magic_file < magic) fprintf(stderr, "[mipmap_cache] cache version too old, dropping `%s' cache\n", dbfilename); else fprintf(stderr, "[mipmap_cache] invalid cache file, dropping `%s' cache\n", dbfilename); goto read_finalize; } // also read compression type and yell out on missmatch. int32_t compression = -1; rd = fread(&compression, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; if(compression != cache->compression_type) { fprintf(stderr, "[mipmap_cache] cache is %s, but settings say we should use %s, dropping `%s' cache\n", compression == 0 ? "uncompressed" : (compression == 1 ? "low quality compressed" : "high quality compressed"), cache->compression_type == 0 ? "no compression" : (cache->compression_type == 1 ? "low quality compression" : "high quality compression"), dbfilename); goto read_finalize; } if(compression && (magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22)) { // compression is enabled and we have the affected version. can't read that. fprintf(stderr, "[mipmap_cache] dropping compressed cache v22 to regenerate mips without artifacts.\n"); goto read_finalize; } for (int i=0; i<=mip; i++) { rd = fread(&file_width[i], sizeof(int32_t), 1, f); if(rd != 1) goto read_error; rd = fread(&file_height[i], sizeof(int32_t), 1, f); if(rd != 1) goto read_error; if(file_width[i] != cache->mip[i].max_width || file_height[i] != cache->mip[i].max_height) { fprintf(stderr, "[mipmap_cache] cache settings changed, dropping `%s' cache\n", dbfilename); goto read_finalize; } } if(cache->compression_type) blob = NULL; else blob = malloc(sizeof(uint32_t)*file_width[mip]*file_height[mip]); while(!feof(f)) { int level = 0; rd = fread(&level, sizeof(int), 1, f); if (level > mip) break; int32_t key = 0; rd = fread(&key, sizeof(int32_t), 1, f); if(rd != 1) break; // first value is break only, goes to eof. int32_t length = 0; rd = fread(&length, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; uint8_t *data = (uint8_t *)dt_cache_read_get(&cache->mip[level].cache, key); struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)data; if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE) { if(cache->compression_type) { int32_t wd, ht; rd = fread(&wd, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; rd = fread(&ht, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; dsc->width = wd; dsc->height = ht; if(length != compressed_buffer_size(cache->compression_type, wd, ht)) goto read_error; // directly read from disk into cache: rd = fread(data + sizeof(*dsc), 1, length, f); if(rd != length) goto read_error; } else { // jpg too large? if(length > sizeof(uint32_t)*file_width[mip]*file_height[mip]) goto read_error; rd = fread(blob, sizeof(uint8_t), length, f); if(rd != length) goto read_error; // no compression, the image is still compressed on disk, as jpg dt_imageio_jpeg_t jpg; if(dt_imageio_jpeg_decompress_header(blob, length, &jpg) || (jpg.width > file_width[level] || jpg.height > file_height[level]) || dt_imageio_jpeg_decompress(&jpg, data+sizeof(*dsc))) { fprintf(stderr, "[mipmap_cache] failed to decompress thumbnail for image %d!\n", get_imgid(key)); } dsc->width = jpg.width; dsc->height = jpg.height; } dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE; // these come write locked in case idata[3] == 1, so release that! dt_cache_write_release(&cache->mip[level].cache, key); } dt_cache_read_release(&cache->mip[level].cache, key); } fclose(f); free(blob); return 0; read_error: fprintf(stderr, "[mipmap_cache] failed to recover the cache from `%s'\n", dbfilename); read_finalize: if(f) fclose(f); free(blob); g_unlink(dbfilename); return 1; }