static void _mapcache_cache_mbtiles_multi_set(mapcache_context *ctx, mapcache_tile *tiles, int ntiles)
{
struct sqlite_conn *conn = NULL;
int i;
/* decode/encode image data before going into the sqlite write lock */
for (i = 0; i < ntiles; i++) {
mapcache_tile *tile = &tiles[i];
if(!tile->raw_image) {
tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
GC_CHECK_ERROR(ctx);
}
/* only encode to image format if tile is not blank */
if (mapcache_image_blank_color(tile->raw_image) != MAPCACHE_TRUE && !tile->encoded_data) {
tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
GC_CHECK_ERROR(ctx);
}
}
conn = _sqlite_get_conn(ctx, &tiles[0], 0);
GC_CHECK_ERROR(ctx);
sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0);
for (i = 0; i < ntiles; i++) {
mapcache_tile *tile = &tiles[i];
_single_mbtile_set(ctx,tile,conn);
if(GC_HAS_ERROR(ctx)) break;
}
if (GC_HAS_ERROR(ctx)) {
sqlite3_exec(conn->handle, "ROLLBACK TRANSACTION", 0, 0, 0);
} else {
sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0);
}
_sqlite_release_conn(ctx, &tiles[0], conn);
}
/**
* \brief push tile data to memcached
*
* writes the content of mapcache_tile::data to the configured memcached instance(s)
* \returns MAPCACHE_FAILURE if there is no data to write, or if the tile isn't locked
* \returns MAPCACHE_SUCCESS if the tile has been successfully written
* \private \memberof mapcache_cache_memcache
* \sa mapcache_cache::tile_set()
*/
static void _mapcache_cache_memcache_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
{
char *key;
int rv;
/* set no expiration if not configured */
int expires =0;
mapcache_buffer *encoded_data = NULL;
mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache;
mapcache_pooled_connection *pc;
struct mapcache_memcache_pooled_connection *mpc;
pc = _mapcache_memcache_get_conn(ctx,cache,tile);
GC_CHECK_ERROR(ctx);
mpc = pc->connection;
key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#");
if(GC_HAS_ERROR(ctx)) goto cleanup;
if(tile->tileset->auto_expire)
expires = tile->tileset->auto_expire;
if(cache->detect_blank) {
if(!tile->raw_image) {
tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
GC_CHECK_ERROR(ctx);
}
if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
encoded_data = mapcache_buffer_create(5,ctx->pool);
((char*)encoded_data->buf)[0] = '#';
memcpy(((char*)encoded_data->buf)+1,tile->raw_image->data,4);
encoded_data->size = 5;
}
}
if(!encoded_data) {
if(!tile->encoded_data) {
tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
if(GC_HAS_ERROR(ctx)) goto cleanup;
}
encoded_data = tile->encoded_data;
}
/* concatenate the current time to the end of the memcache data so we can extract it out
* when we re-get the tile */
char *data = calloc(1,encoded_data->size+sizeof(apr_time_t));
apr_time_t now = apr_time_now();
apr_pool_cleanup_register(ctx->pool, data, (void*)free, apr_pool_cleanup_null);
memcpy(data,encoded_data->buf,encoded_data->size);
memcpy(&(data[encoded_data->size]),&now,sizeof(apr_time_t));
rv = apr_memcache_set(mpc->memcache,key,data,encoded_data->size+sizeof(apr_time_t),expires,0);
if(rv != APR_SUCCESS) {
ctx->set_error(ctx,500,"failed to store tile %d %d %d to memcache cache %s",
tile->x,tile->y,tile->z,cache->cache.name);
goto cleanup;
}
cleanup:
_mapcache_memcache_release_conn(ctx,pc);
}
static void _single_mbtile_set(mapcache_context *ctx, mapcache_tile *tile, struct sqlite_conn *conn)
{
sqlite3_stmt *stmt1,*stmt2;
mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)tile->tileset->cache;
int ret;
if(!tile->raw_image) {
tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
GC_CHECK_ERROR(ctx);
}
if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
stmt1 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT1_IDX];
stmt2 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT2_IDX];
if(!stmt1) {
sqlite3_prepare(conn->handle,
"insert or ignore into images(tile_id,tile_data) values (:color,:data);",
-1, &conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT1_IDX], NULL);
sqlite3_prepare(conn->handle,
"insert or replace into map(tile_column,tile_row,zoom_level,tile_id) values (:x,:y,:z,:color);",
-1, &conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT2_IDX], NULL);
stmt1 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT1_IDX];
stmt2 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT2_IDX];
}
cache->bind_stmt(ctx, stmt1, tile);
cache->bind_stmt(ctx, stmt2, tile);
} else {
stmt1 = conn->prepared_statements[MBTILES_SET_TILE_STMT1_IDX];
stmt2 = conn->prepared_statements[MBTILES_SET_TILE_STMT2_IDX];
if(!stmt1) {
sqlite3_prepare(conn->handle,
"insert or replace into images(tile_id,tile_data) values (:key,:data);",
-1, &conn->prepared_statements[MBTILES_SET_TILE_STMT1_IDX], NULL);
sqlite3_prepare(conn->handle,
"insert or replace into map(tile_column,tile_row,zoom_level,tile_id) values (:x,:y,:z,:key);",
-1, &conn->prepared_statements[MBTILES_SET_TILE_STMT2_IDX], NULL);
stmt1 = conn->prepared_statements[MBTILES_SET_TILE_STMT1_IDX];
stmt2 = conn->prepared_statements[MBTILES_SET_TILE_STMT2_IDX];
}
cache->bind_stmt(ctx, stmt1, tile);
cache->bind_stmt(ctx, stmt2, tile);
}
do {
ret = sqlite3_step(stmt1);
if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) {
ctx->set_error(ctx, 500, "mbtiles backend failed on image set: %s (%d)", sqlite3_errmsg(conn->handle), ret);
break;
}
if (ret == SQLITE_BUSY) {
sqlite3_reset(stmt1);
}
} while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED);
if(ret == SQLITE_DONE) {
do {
ret = sqlite3_step(stmt2);
if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) {
ctx->set_error(ctx, 500, "mbtiles backend failed on map set: %s (%d)", sqlite3_errmsg(conn->handle), ret);
break;
}
if (ret == SQLITE_BUSY) {
sqlite3_reset(stmt2);
}
} while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED);
}
sqlite3_reset(stmt1);
sqlite3_reset(stmt2);
}
/**
* \brief apply appropriate tile properties to the sqlite statement */
static void _bind_sqlite_params(mapcache_context *ctx, void *vstmt, mapcache_tile *tile)
{
sqlite3_stmt *stmt = vstmt;
int paramidx;
/* tile->x */
paramidx = sqlite3_bind_parameter_index(stmt, ":x");
if (paramidx) sqlite3_bind_int(stmt, paramidx, tile->x);
/* tile->y */
paramidx = sqlite3_bind_parameter_index(stmt, ":y");
if (paramidx) sqlite3_bind_int(stmt, paramidx, tile->y);
/* tile->y */
paramidx = sqlite3_bind_parameter_index(stmt, ":z");
if (paramidx) sqlite3_bind_int(stmt, paramidx, tile->z);
/* eventual dimensions */
paramidx = sqlite3_bind_parameter_index(stmt, ":dim");
if (paramidx) {
if (tile->dimensions) {
char *dim = mapcache_util_get_tile_dimkey(ctx, tile, NULL, NULL);
sqlite3_bind_text(stmt, paramidx, dim, -1, SQLITE_STATIC);
} else {
sqlite3_bind_text(stmt, paramidx, "", -1, SQLITE_STATIC);
}
}
/* grid */
paramidx = sqlite3_bind_parameter_index(stmt, ":grid");
if (paramidx) sqlite3_bind_text(stmt, paramidx, tile->grid_link->grid->name, -1, SQLITE_STATIC);
/* tileset */
paramidx = sqlite3_bind_parameter_index(stmt, ":tileset");
if (paramidx) sqlite3_bind_text(stmt, paramidx, tile->tileset->name, -1, SQLITE_STATIC);
/* tile blob data */
paramidx = sqlite3_bind_parameter_index(stmt, ":data");
if (paramidx) {
int written = 0;
if(((mapcache_cache_sqlite*)tile->tileset->cache)->detect_blank && tile->grid_link->grid->tile_sx == 256 &&
tile->grid_link->grid->tile_sy == 256) {
if(!tile->raw_image) {
tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
GC_CHECK_ERROR(ctx);
}
if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
char *buf = apr_palloc(ctx->pool, 5* sizeof(char));
buf[0] = '#';
memcpy(buf+1,tile->raw_image->data,4);
written = 1;
sqlite3_bind_blob(stmt, paramidx, buf, 5, SQLITE_STATIC);
}
}
if(!written) {
if (!tile->encoded_data) {
tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
GC_CHECK_ERROR(ctx);
}
if (tile->encoded_data && tile->encoded_data->size) {
sqlite3_bind_blob(stmt, paramidx, tile->encoded_data->buf, tile->encoded_data->size, SQLITE_STATIC);
} else {
sqlite3_bind_text(stmt, paramidx, "", -1, SQLITE_STATIC);
}
}
}
}
/**
* \brief write tile data to disk
*
* writes the content of mapcache_tile::data to disk.
* \returns MAPCACHE_FAILURE if there is no data to write, or if the tile isn't locked
* \returns MAPCACHE_SUCCESS if the tile has been successfully written to disk
* \private \memberof mapcache_cache_disk
* \sa mapcache_cache::tile_set()
*/
static void _mapcache_cache_disk_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile)
{
apr_size_t bytes;
apr_file_t *f;
apr_status_t ret;
char errmsg[120];
char *filename, *hackptr1, *hackptr2=NULL;
mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache;
const int creation_retry = cache->creation_retry;
int retry_count_create_file = 0;
#ifdef DEBUG
/* all this should be checked at a higher level */
if(!tile->encoded_data && !tile->raw_image) {
ctx->set_error(ctx,500,"attempting to write empty tile to disk");
return;
}
if(!tile->encoded_data && !tile->tileset->format) {
ctx->set_error(ctx,500,"received a raw tile image for a tileset with no format");
return;
}
#endif
cache->tile_key(ctx, cache, tile, &filename);
GC_CHECK_ERROR(ctx);
/* find the location of the last '/' in the string */
hackptr1 = filename;
while(*hackptr1) {
if(*hackptr1 == '/')
hackptr2 = hackptr1;
hackptr1++;
}
*hackptr2 = '\0';
if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) {
/*
* apr_dir_make_recursive sometimes sends back this error, although it should not.
* ignore this one
*/
if(!APR_STATUS_IS_EEXIST(ret)) {
ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(ret,errmsg,120));
return;
}
}
*hackptr2 = '/';
ret = apr_file_remove(filename,ctx->pool);
if(ret != APR_SUCCESS && !APR_STATUS_IS_ENOENT(ret)) {
ctx->set_error(ctx, 500, "failed to remove file %s: %s",filename, apr_strerror(ret,errmsg,120));
}
#ifdef HAVE_SYMLINK
if(cache->symlink_blank) {
if(!tile->raw_image) {
tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data);
GC_CHECK_ERROR(ctx);
}
if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) {
char *blankname;
int retry_count_create_symlink = 0;
char *blankname_rel = NULL;
_mapcache_cache_disk_blank_tile_key(ctx,cache,tile,tile->raw_image->data,&blankname);
if(apr_file_open(&f, blankname, APR_FOPEN_READ, APR_OS_DEFAULT, ctx->pool) != APR_SUCCESS) {
int isLocked;
void *lock;
char *blankdirname;
if(!tile->encoded_data) {
tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
GC_CHECK_ERROR(ctx);
}
/* create the blank file */
blankdirname = apr_psprintf(ctx->pool, "%s/%s/%s/blanks",
cache->base_directory,
tile->tileset->name,
tile->grid_link->grid->name);
if(APR_SUCCESS != (ret = apr_dir_make_recursive(
blankdirname, APR_OS_DEFAULT,ctx->pool))) {
if(!APR_STATUS_IS_EEXIST(ret)) {
ctx->set_error(ctx, 500, "failed to create directory %s for blank tiles",blankdirname, apr_strerror(ret,errmsg,120));
return;
}
}
/* aquire a lock on the blank file */
isLocked = mapcache_lock_or_wait_for_resource(ctx,ctx->config->locker,blankname, &lock);
if(isLocked == MAPCACHE_TRUE) {
if((ret = apr_file_open(&f, blankname,
APR_FOPEN_CREATE|APR_FOPEN_WRITE|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,
APR_OS_DEFAULT, ctx->pool)) != APR_SUCCESS) {
ctx->set_error(ctx, 500, "failed to create file %s: %s",blankname, apr_strerror(ret,errmsg,120));
mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock);
return; /* we could not create the file */
}
bytes = (apr_size_t)tile->encoded_data->size;
ret = apr_file_write(f,(void*)tile->encoded_data->buf,&bytes);
if(ret != APR_SUCCESS) {
ctx->set_error(ctx, 500, "failed to write data to file %s (wrote %d of %d bytes): %s",blankname, (int)bytes, (int)tile->encoded_data->size, apr_strerror(ret,errmsg,120));
mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock);
return; /* we could not create the file */
}
if(bytes != tile->encoded_data->size) {
ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", blankname, (int)bytes, (int)tile->encoded_data->size);
mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock);
return;
}
apr_file_close(f);
mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock);
#ifdef DEBUG
ctx->log(ctx,MAPCACHE_DEBUG,"created blank tile %s",blankname);
#endif
}
} else {
apr_file_close(f);
}
/*
* compute the relative path between tile and blank tile
*/
blankname_rel = relative_path(ctx,filename, blankname);
GC_CHECK_ERROR(ctx);
/*
* depending on configuration symlink creation will retry if it fails.
* this can happen on nfs mounted network storage.
* the solution is to create the containing directory again and retry the symlink creation.
*/
while(symlink(blankname_rel, filename) != 0) {
retry_count_create_symlink++;
if(retry_count_create_symlink > creation_retry) {
char *error = strerror(errno);
ctx->set_error(ctx, 500, "failed to link tile %s to %s: %s",filename, blankname_rel, error);
return; /* we could not create the file */
}
*hackptr2 = '\0';
if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) {
if(!APR_STATUS_IS_EEXIST(ret)) {
ctx->set_error(ctx, 500, "failed to create symlink, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120));
return; /* we could not create the file */
}
}
*hackptr2 = '/';
}
#ifdef DEBUG
ctx->log(ctx, MAPCACHE_DEBUG, "linked blank tile %s to %s",filename,blankname);
#endif
return;
}
}
#endif /*HAVE_SYMLINK*/
/* go the normal way: either we haven't configured blank tile detection, or the tile was not blank */
if(!tile->encoded_data) {
tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format);
GC_CHECK_ERROR(ctx);
}
/*
* depending on configuration file creation will retry if it fails.
* this can happen on nfs mounted network storage.
* the solution is to create the containing directory again and retry the file creation.
*/
while((ret = apr_file_open(&f, filename,
APR_FOPEN_CREATE|APR_FOPEN_WRITE|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,
APR_OS_DEFAULT, ctx->pool)) != APR_SUCCESS) {
retry_count_create_file++;
if(retry_count_create_file > creation_retry) {
ctx->set_error(ctx, 500, "failed to create file %s: %s",filename, apr_strerror(ret,errmsg,120));
return; /* we could not create the file */
}
*hackptr2 = '\0';
if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) {
if(!APR_STATUS_IS_EEXIST(ret)) {
ctx->set_error(ctx, 500, "failed to create file, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120));
return; /* we could not create the file */
}
}
*hackptr2 = '/';
}
bytes = (apr_size_t)tile->encoded_data->size;
ret = apr_file_write(f,(void*)tile->encoded_data->buf,&bytes);
if(ret != APR_SUCCESS) {
ctx->set_error(ctx, 500, "failed to write data to file %s (wrote %d of %d bytes): %s",filename, (int)bytes, (int)tile->encoded_data->size, apr_strerror(ret,errmsg,120));
return; /* we could not create the file */
}
if(bytes != tile->encoded_data->size) {
ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", filename, (int)bytes, (int)tile->encoded_data->size);
}
ret = apr_file_close(f);
if(ret != APR_SUCCESS) {
ctx->set_error(ctx, 500, "failed to close file %s:%s",filename, apr_strerror(ret,errmsg,120));
return; /* we could not create the file */
}
}
