// drop a write lock, read will still remain. void dt_mipmap_cache_write_release( dt_mipmap_cache_t *cache, dt_mipmap_buffer_t *buf) { if(buf->size == DT_MIPMAP_NONE || buf->buf == NULL) return; assert(buf->imgid > 0); assert(buf->size >= DT_MIPMAP_0); assert(buf->size < DT_MIPMAP_NONE); dt_cache_write_release(&cache->mip[buf->size].cache, get_key(buf->imgid, buf->size)); buf->size = DT_MIPMAP_NONE; buf->buf = NULL; }
// drops the write priviledges on an image struct. // this triggers a write-through to sql, and if the setting // is present, also to xmp sidecar files (safe setting). void dt_image_cache_write_release( dt_image_cache_t *cache, dt_image_t *img, dt_image_cache_write_mode_t mode) { if(img->id <= 0) return; sqlite3_stmt *stmt; DT_DEBUG_SQLITE3_PREPARE_V2(dt_database_get(darktable.db), "update images set width = ?1, height = ?2, maker = ?3, model = ?4, " "lens = ?5, exposure = ?6, aperture = ?7, iso = ?8, focal_length = ?9, " "focus_distance = ?10, film_id = ?11, datetime_taken = ?12, flags = ?13, " "crop = ?14, orientation = ?15, raw_parameters = ?16, group_id = ?17, longitude = ?18, " "latitude = ?19, color_matrix = ?20 where id = ?21", -1, &stmt, NULL); DT_DEBUG_SQLITE3_BIND_INT(stmt, 1, img->width); DT_DEBUG_SQLITE3_BIND_INT(stmt, 2, img->height); DT_DEBUG_SQLITE3_BIND_TEXT(stmt, 3, img->exif_maker, strlen(img->exif_maker), SQLITE_STATIC); DT_DEBUG_SQLITE3_BIND_TEXT(stmt, 4, img->exif_model, strlen(img->exif_model), SQLITE_STATIC); DT_DEBUG_SQLITE3_BIND_TEXT(stmt, 5, img->exif_lens, strlen(img->exif_lens), SQLITE_STATIC); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 6, img->exif_exposure); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 7, img->exif_aperture); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 8, img->exif_iso); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 9, img->exif_focal_length); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 10, img->exif_focus_distance); DT_DEBUG_SQLITE3_BIND_INT(stmt, 11, img->film_id); DT_DEBUG_SQLITE3_BIND_TEXT(stmt, 12, img->exif_datetime_taken, strlen(img->exif_datetime_taken), SQLITE_STATIC); DT_DEBUG_SQLITE3_BIND_INT(stmt, 13, img->flags); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 14, img->exif_crop); DT_DEBUG_SQLITE3_BIND_INT(stmt, 15, img->orientation); DT_DEBUG_SQLITE3_BIND_INT(stmt, 16, *(uint32_t*)(&img->legacy_flip)); DT_DEBUG_SQLITE3_BIND_INT(stmt, 17, img->group_id); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 18, img->longitude); DT_DEBUG_SQLITE3_BIND_DOUBLE(stmt, 19, img->latitude); DT_DEBUG_SQLITE3_BIND_BLOB(stmt, 20, &img->d65_color_matrix, sizeof(img->d65_color_matrix), SQLITE_STATIC); DT_DEBUG_SQLITE3_BIND_INT(stmt, 21, img->id); int rc = sqlite3_step(stmt); if (rc != SQLITE_DONE) fprintf(stderr, "[image_cache_write_release] sqlite3 error %d\n", rc); sqlite3_finalize(stmt); // TODO: make this work in relaxed mode, too. if(mode == DT_IMAGE_CACHE_SAFE) { // rest about sidecars: // also synch dttags file: dt_image_write_sidecar_file(img->id); } dt_cache_write_release(&cache->cache, img->id); }
void dt_mipmap_cache_read_get( dt_mipmap_cache_t *cache, dt_mipmap_buffer_t *buf, const uint32_t imgid, const dt_mipmap_size_t mip, const dt_mipmap_get_flags_t flags) { const uint32_t key = get_key(imgid, mip); if(flags == DT_MIPMAP_TESTLOCK) { // simple case: only get and lock if it's there. struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)dt_cache_read_testget(&cache->mip[mip].cache, key); if(dsc) { buf->width = dsc->width; buf->height = dsc->height; buf->imgid = imgid; buf->size = mip; // skip to next 8-byte alignment, for sse buffers. buf->buf = (uint8_t *)(dsc+1); } else { // set to NULL if failed. buf->width = buf->height = 0; buf->imgid = 0; buf->size = DT_MIPMAP_NONE; buf->buf = NULL; } } else if(flags == DT_MIPMAP_PREFETCH) { // and opposite: prefetch without locking if(mip > DT_MIPMAP_FULL || mip < DT_MIPMAP_0) return; dt_job_t j; dt_image_load_job_init(&j, imgid, mip); // if the job already exists, make it high-priority, if not, add it: if(dt_control_revive_job(darktable.control, &j) < 0) dt_control_add_job(darktable.control, &j); } else if(flags == DT_MIPMAP_BLOCKING) { // simple case: blocking get struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)dt_cache_read_get(&cache->mip[mip].cache, key); if(!dsc) { // should never happen for anything but full images which have been moved. assert(mip == DT_MIPMAP_FULL || mip == DT_MIPMAP_F); // fprintf(stderr, "[mipmap cache get] no data in cache for imgid %u size %d!\n", imgid, mip); // sorry guys, no image for you :( buf->width = buf->height = 0; buf->imgid = 0; buf->size = DT_MIPMAP_NONE; buf->buf = NULL; } else { // fprintf(stderr, "[mipmap cache get] found data in cache for imgid %u size %d\n", imgid, mip); // uninitialized? //assert(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE || dsc->size == 0); if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE) { __sync_fetch_and_add (&(cache->mip[mip].stats_fetches), 1); // fprintf(stderr, "[mipmap cache get] now initializing buffer for img %u mip %d!\n", imgid, mip); // we're write locked here, as requested by the alloc callback. // now fill it with data: if(mip == DT_MIPMAP_FULL) { // load the image: // make sure we access the r/w lock as shortly as possible! dt_image_t buffered_image; const dt_image_t *cimg = dt_image_cache_read_get(darktable.image_cache, imgid); buffered_image = *cimg; // dt_image_t *img = dt_image_cache_write_get(darktable.image_cache, cimg); // dt_image_cache_write_release(darktable.image_cache, img, DT_IMAGE_CACHE_RELAXED); dt_image_cache_read_release(darktable.image_cache, cimg); char filename[DT_MAX_PATH_LEN]; gboolean from_cache = TRUE; dt_image_full_path(buffered_image.id, filename, DT_MAX_PATH_LEN, &from_cache); dt_mipmap_cache_allocator_t a = (dt_mipmap_cache_allocator_t)&dsc; struct dt_mipmap_buffer_dsc* prvdsc = dsc; dt_imageio_retval_t ret = dt_imageio_open(&buffered_image, filename, a); if(dsc != prvdsc) { // fprintf(stderr, "[mipmap cache] realloc %p\n", data); // write back to cache, too. // in case something went wrong, still keep the buffer and return it to the hashtable // so we don't produce mem leaks or unnecessary mem fragmentation. dt_cache_realloc(&cache->mip[mip].cache, key, 1, (void*)dsc); } if(ret != DT_IMAGEIO_OK) { // fprintf(stderr, "[mipmap read get] error loading image: %d\n", ret); // // we can only return a zero dimension buffer if the buffer has been allocated. // in case dsc couldn't be allocated and points to the static buffer, it contains // a dead image already. if((void *)dsc != (void *)dt_mipmap_cache_static_dead_image) dsc->width = dsc->height = 0; } else { // swap back new image data: cimg = dt_image_cache_read_get(darktable.image_cache, imgid); dt_image_t *img = dt_image_cache_write_get(darktable.image_cache, cimg); *img = buffered_image; // fprintf(stderr, "[mipmap read get] initializing full buffer img %u with %u %u -> %d %d (%p)\n", imgid, data[0], data[1], img->width, img->height, data); // don't write xmp for this (we only changed db stuff): dt_image_cache_write_release(darktable.image_cache, img, DT_IMAGE_CACHE_RELAXED); dt_image_cache_read_release(darktable.image_cache, img); } } else if(mip == DT_MIPMAP_F) { _init_f((float *)(dsc+1), &dsc->width, &dsc->height, imgid); } else { // 8-bit thumbs, possibly need to be compressed: if(cache->compression_type) { // get per-thread temporary storage without malloc from a separate cache: const int key = dt_control_get_threadid(); // const void *cbuf = dt_cache_read_get(&cache->scratchmem.cache, key); uint8_t *scratchmem = (uint8_t *)dt_cache_write_get(&cache->scratchmem.cache, key); _init_8(scratchmem, &dsc->width, &dsc->height, imgid, mip); buf->width = dsc->width; buf->height = dsc->height; buf->imgid = imgid; buf->size = mip; buf->buf = (uint8_t *)(dsc+1); dt_mipmap_cache_compress(buf, scratchmem); dt_cache_write_release(&cache->scratchmem.cache, key); dt_cache_read_release(&cache->scratchmem.cache, key); } else { _init_8((uint8_t *)(dsc+1), &dsc->width, &dsc->height, imgid, mip); } } dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE; // drop the write lock dt_cache_write_release(&cache->mip[mip].cache, key); /* raise signal that mipmaps has been flushed to cache */ dt_control_signal_raise(darktable.signals, DT_SIGNAL_DEVELOP_MIPMAP_UPDATED); } buf->width = dsc->width; buf->height = dsc->height; buf->imgid = imgid; buf->size = mip; buf->buf = (uint8_t *)(dsc+1); if(dsc->width == 0 || dsc->height == 0) { // fprintf(stderr, "[mipmap cache get] got a zero-sized image for img %u mip %d!\n", imgid, mip); if(mip < DT_MIPMAP_F) dead_image_8(buf); else if(mip == DT_MIPMAP_F) dead_image_f(buf); else buf->buf = NULL; // full images with NULL buffer have to be handled, indicates `missing image' } } } else if(flags == DT_MIPMAP_BEST_EFFORT) { __sync_fetch_and_add (&(cache->mip[mip].stats_requests), 1); // best-effort, might also return NULL. // never decrease mip level for float buffer or full image: dt_mipmap_size_t min_mip = (mip >= DT_MIPMAP_F) ? mip : DT_MIPMAP_0; for(int k=mip; k>=min_mip && k>=0; k--) { // already loaded? dt_mipmap_cache_read_get(cache, buf, imgid, k, DT_MIPMAP_TESTLOCK); if(buf->buf && buf->width > 0 && buf->height > 0) { if(mip != k) __sync_fetch_and_add (&(cache->mip[k].stats_standin), 1); return; } // didn't succeed the first time? prefetch for later! if(mip == k) { __sync_fetch_and_add (&(cache->mip[mip].stats_near_match), 1); dt_mipmap_cache_read_get(cache, buf, imgid, mip, DT_MIPMAP_PREFETCH); } } __sync_fetch_and_add (&(cache->mip[mip].stats_misses), 1); // fprintf(stderr, "[mipmap cache get] image not found in cache: imgid %u mip %d!\n", imgid, mip); // nothing found :( buf->buf = NULL; buf->imgid = 0; buf->size = DT_MIPMAP_NONE; buf->width = buf->height = 0; } }
static int dt_mipmap_cache_deserialize(dt_mipmap_cache_t *cache) { int32_t rd = 0; const dt_mipmap_size_t mip = DT_MIPMAP_2; uint8_t *blob = NULL; FILE *f = NULL; int file_width[mip+1], file_height[mip+1]; gchar dbfilename[DT_MAX_PATH_LEN]; if (dt_mipmap_cache_get_filename(dbfilename, sizeof(dbfilename))) { fprintf(stderr, "[mipmap_cache] could not retrieve cache filename; not deserializing\n"); return 1; } if (!strcmp(dbfilename, ":memory:")) { // fprintf(stderr, "[mipmap_cache] library is in memory; not deserializing\n"); return 0; } // drop any old cache if the database is new. in that case newly imported images will probably mapped to old thumbnails if(dt_database_is_new(darktable.db) && g_file_test(dbfilename, G_FILE_TEST_IS_REGULAR)) { fprintf(stderr, "[mipmap_cache] database is new, dropping old cache `%s'\n", dbfilename); goto read_finalize; } f = fopen(dbfilename, "rb"); if(!f) { if (errno == ENOENT) { fprintf(stderr, "[mipmap_cache] cache is empty, file `%s' doesn't exist\n", dbfilename); } else { fprintf(stderr, "[mipmap_cache] failed to open the cache from `%s'\n", dbfilename); } goto read_finalize; } // read version info: const int32_t magic = DT_MIPMAP_CACHE_FILE_MAGIC + DT_MIPMAP_CACHE_FILE_VERSION; int32_t magic_file = 0; rd = fread(&magic_file, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; if(magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22) { // same format, but compression was broken in 22 and below } else if(magic_file != magic) { if(magic_file > DT_MIPMAP_CACHE_FILE_MAGIC && magic_file < magic) fprintf(stderr, "[mipmap_cache] cache version too old, dropping `%s' cache\n", dbfilename); else fprintf(stderr, "[mipmap_cache] invalid cache file, dropping `%s' cache\n", dbfilename); goto read_finalize; } // also read compression type and yell out on missmatch. int32_t compression = -1; rd = fread(&compression, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; if(compression != cache->compression_type) { fprintf(stderr, "[mipmap_cache] cache is %s, but settings say we should use %s, dropping `%s' cache\n", compression == 0 ? "uncompressed" : (compression == 1 ? "low quality compressed" : "high quality compressed"), cache->compression_type == 0 ? "no compression" : (cache->compression_type == 1 ? "low quality compression" : "high quality compression"), dbfilename); goto read_finalize; } if(compression && (magic_file == DT_MIPMAP_CACHE_FILE_MAGIC + 22)) { // compression is enabled and we have the affected version. can't read that. fprintf(stderr, "[mipmap_cache] dropping compressed cache v22 to regenerate mips without artifacts.\n"); goto read_finalize; } for (int i=0; i<=mip; i++) { rd = fread(&file_width[i], sizeof(int32_t), 1, f); if(rd != 1) goto read_error; rd = fread(&file_height[i], sizeof(int32_t), 1, f); if(rd != 1) goto read_error; if(file_width[i] != cache->mip[i].max_width || file_height[i] != cache->mip[i].max_height) { fprintf(stderr, "[mipmap_cache] cache settings changed, dropping `%s' cache\n", dbfilename); goto read_finalize; } } if(cache->compression_type) blob = NULL; else blob = malloc(sizeof(uint32_t)*file_width[mip]*file_height[mip]); while(!feof(f)) { int level = 0; rd = fread(&level, sizeof(int), 1, f); if (level > mip) break; int32_t key = 0; rd = fread(&key, sizeof(int32_t), 1, f); if(rd != 1) break; // first value is break only, goes to eof. int32_t length = 0; rd = fread(&length, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; uint8_t *data = (uint8_t *)dt_cache_read_get(&cache->mip[level].cache, key); struct dt_mipmap_buffer_dsc* dsc = (struct dt_mipmap_buffer_dsc*)data; if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE) { if(cache->compression_type) { int32_t wd, ht; rd = fread(&wd, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; rd = fread(&ht, sizeof(int32_t), 1, f); if(rd != 1) goto read_error; dsc->width = wd; dsc->height = ht; if(length != compressed_buffer_size(cache->compression_type, wd, ht)) goto read_error; // directly read from disk into cache: rd = fread(data + sizeof(*dsc), 1, length, f); if(rd != length) goto read_error; } else { // jpg too large? if(length > sizeof(uint32_t)*file_width[mip]*file_height[mip]) goto read_error; rd = fread(blob, sizeof(uint8_t), length, f); if(rd != length) goto read_error; // no compression, the image is still compressed on disk, as jpg dt_imageio_jpeg_t jpg; if(dt_imageio_jpeg_decompress_header(blob, length, &jpg) || (jpg.width > file_width[level] || jpg.height > file_height[level]) || dt_imageio_jpeg_decompress(&jpg, data+sizeof(*dsc))) { fprintf(stderr, "[mipmap_cache] failed to decompress thumbnail for image %d!\n", get_imgid(key)); } dsc->width = jpg.width; dsc->height = jpg.height; } dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE; // these come write locked in case idata[3] == 1, so release that! dt_cache_write_release(&cache->mip[level].cache, key); } dt_cache_read_release(&cache->mip[level].cache, key); } fclose(f); free(blob); return 0; read_error: fprintf(stderr, "[mipmap_cache] failed to recover the cache from `%s'\n", dbfilename); read_finalize: if(f) fclose(f); free(blob); g_unlink(dbfilename); return 1; }