clean up bdb connections on process exits

jmckenna · Apr 26, 2012 · 8b79c1a · 8b79c1a
1 parent c9ae879
commit 8b79c1a
Show file tree

Hide file tree

Showing 7 changed files with 121 additions and 55 deletions.
diff --git a/apache/mod_mapcache.c b/apache/mod_mapcache.c
@@ -40,6 +40,10 @@
 #include <ap_mpm.h>
 #include <http_log.h>
 #include "mapcache.h"
+#ifdef APR_HAS_THREADS
+#include <apr_thread_mutex.h>
+apr_thread_mutex_t *thread_mutex = NULL;
+#endif
 
 #ifndef _WIN32
 #include <unistd.h>
@@ -55,6 +59,7 @@ typedef struct mapcache_context_apache mapcache_context_apache;
 typedef struct mapcache_context_apache_request mapcache_context_apache_request;
 typedef struct mapcache_context_apache_server mapcache_context_apache_server;
 
+apr_pool_t *pchild = NULL;
 
 struct mapcache_context_apache {
    mapcache_context ctx;
@@ -177,6 +182,11 @@ static mapcache_context_apache_request* apache_request_context_create(request_re
    mapcache_cfg *config = NULL;
 
    ctx->ctx.ctx.pool = r->pool;
+   ctx->ctx.ctx.process_pool = pchild;
+#ifdef APR_HAS_THREADS
+   ctx->ctx.ctx.threadlock = thread_mutex;
+#endif
+
    /* lookup the configuration object given the configuration file name */
    cfg = ap_get_module_config(r->server->module_config, &mapcache_module);
    config = apr_hash_get(cfg->aliases,(void*)r->filename,APR_HASH_KEY_STRING);
@@ -231,6 +241,17 @@ static int write_http_response(mapcache_context_apache_request *ctx, mapcache_ht
 
 }
 
+static void mod_mapcache_child_init(apr_pool_t *pool, server_rec *s) {
+   pchild = pool;
+#ifdef APR_HAS_THREADS
+   int threaded;
+   ap_mpm_query(AP_MPMQ_IS_THREADED,&threaded);
+   if(threaded) {
+      apr_thread_mutex_create(&thread_mutex,APR_THREAD_MUTEX_DEFAULT,pool);
+   }
+#endif
+}
+
 static int mod_mapcache_request_handler(request_rec *r) {
    apr_table_t *params;
    mapcache_request *request = NULL;
@@ -401,6 +422,7 @@ static void mod_mapcache_register_hooks(apr_pool_t *p) {
    static const char * const p1[] = { "mod_alias.c", "mod_rewrite.c", NULL };
    static const char * const n1[]= { "mod_userdir.c",
                                       "mod_vhost_alias.c", NULL };
+   ap_hook_child_init(mod_mapcache_child_init, NULL, NULL, APR_HOOK_MIDDLE);
    ap_hook_post_config(mod_mapcache_post_config, NULL, NULL, APR_HOOK_MIDDLE);
    ap_hook_handler(mod_mapcache_request_handler, NULL, NULL, APR_HOOK_MIDDLE);
    ap_hook_translate_name(mapcache_hook_intercept, p1, n1, APR_HOOK_MIDDLE);

diff --git a/cgi/mapcache.c b/cgi/mapcache.c
@@ -266,6 +266,8 @@ int main(int argc, const char **argv) {
          }
       }
       apr_pool_create(&(ctx->pool),config_pool);
+      ctx->process_pool = config_pool;
+      ctx->threadlock = NULL;
       request = NULL;
       pathInfo = getenv("PATH_INFO");
 

diff --git a/include/mapcache.h b/include/mapcache.h
@@ -191,6 +191,8 @@ struct mapcache_context {
     const char* (*get_instance_id)(mapcache_context * ctx);
     mapcache_context* (*clone)(mapcache_context *ctx);
     apr_pool_t *pool;
+    apr_pool_t *process_pool;
+    void *threadlock;
     char *_contenttype;
     char *_errmsg;
     int _errcode;
@@ -456,11 +458,8 @@ mapcache_cache* mapcache_cache_mbtiles_create(mapcache_context *ctx);
 typedef struct mapcache_cache_bdb mapcache_cache_bdb;
 struct mapcache_cache_bdb {
    mapcache_cache cache;
-   apr_reslist_t *rw_connection_pool;
-   apr_reslist_t *ro_connection_pool;
    char *basedir;
    char *key_template;
-   mapcache_context *ctx;
 };
 mapcache_cache *mapcache_cache_bdb_create(mapcache_context *ctx);
 #endif

diff --git a/lib/cache_bdb.c b/lib/cache_bdb.c
@@ -36,6 +36,9 @@
 #include <string.h>
 #include <errno.h>
 #include <time.h>
+#ifdef APR_HAS_THREADS
+#include <apr_thread_mutex.h>
+#endif
 
 #ifndef _WIN32
 #include <unistd.h>
@@ -46,58 +49,64 @@
 #define PAGESIZE 64*1024
 #define CACHESIZE 1024*1024
 
+apr_reslist_t *ro_connection_pool = NULL;
+apr_reslist_t *rw_connection_pool = NULL;
+
 struct bdb_env {
    DB* db;
    DB_ENV *env;
    int readonly;
+   char *errmsg;
 };
 
 static apr_status_t _bdb_reslist_get_connection(void **conn_, void *params, apr_pool_t *pool) {
    int ret;
    mapcache_cache_bdb *cache = (mapcache_cache_bdb*)params;
    char *dbfile = apr_pstrcat(pool,cache->basedir,"/",cache->cache.name,".db",NULL);
-   struct bdb_env *benv = apr_pcalloc(pool,sizeof(struct bdb_env));
+   struct bdb_env *benv = calloc(1,sizeof(struct bdb_env));
+   *conn_ = benv;
 
    ret = db_env_create(&benv->env, 0);
    if(ret) { 
-      cache->ctx->set_error(cache->ctx,500,"bdb cache failure for db_env_create: %s", db_strerror(ret));
+      benv->errmsg = apr_psprintf(pool,"bdb cache failure for db_env_create: %s", db_strerror(ret));
       return APR_EGENERAL;
    }
    ret = benv->env->set_cachesize(benv->env,0,CACHESIZE,1); /* set a larger cache size than default */
    if(ret) { 
-      cache->ctx->set_error(cache->ctx,500,"bdb cache failure for db->set_cachesize: %s", db_strerror(ret));
+      benv->errmsg = apr_psprintf(pool, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret));
       return APR_EGENERAL;
    }
    int env_flags = DB_INIT_CDB|DB_INIT_MPOOL|DB_CREATE;
    ret = benv->env->open(benv->env,cache->basedir,env_flags,0);
    if(ret) { 
-      cache->ctx->set_error(cache->ctx,500,"bdb cache failure for env->open: %s", db_strerror(ret));
+      benv->errmsg = apr_psprintf(pool,"bdb cache failure for env->open: %s", db_strerror(ret));
       return APR_EGENERAL;
    }
 
    if ((ret = db_create(&benv->db, benv->env, 0)) != 0) {
-      cache->ctx->set_error(cache->ctx,500,"bdb cache failure for db_create: %s", db_strerror(ret));
+      benv->errmsg = apr_psprintf(pool,"bdb cache failure for db_create: %s", db_strerror(ret));
       return APR_EGENERAL;
    }
    int mode = DB_BTREE;
    ret = benv->db->set_pagesize(benv->db,PAGESIZE); /* set pagesize to maximum allowed, as tile data is usually pretty large */
    if(ret) { 
-      cache->ctx->set_error(cache->ctx,500,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret));
+      benv->errmsg = apr_psprintf(pool,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret));
       return APR_EGENERAL;
    }
 
    if ((ret = benv->db->open(benv->db, NULL, dbfile, NULL, mode, DB_CREATE, 0664)) != 0) {
-      cache->ctx->set_error(cache->ctx,500,"bdb cache failure 1 for db->open: %s", db_strerror(ret));
+      benv->errmsg = apr_psprintf(pool,"bdb cache failure 1 for db->open: %s", db_strerror(ret));
       return APR_EGENERAL;
    }
-   *conn_ = benv;
    return APR_SUCCESS;
 }
 
 static apr_status_t _bdb_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) {
    struct bdb_env *benv = (struct bdb_env*)conn_;
    benv->db->close(benv->db,0);
    benv->env->close(benv->env,0);
+   free(benv);
+
    return APR_SUCCESS; 
 }
 
@@ -108,27 +117,65 @@ static struct bdb_env* _bdb_get_conn(mapcache_context *ctx, mapcache_tile* tile,
    mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache;
    struct bdb_env *benv;
    apr_reslist_t *pool;
+   if(!ro_connection_pool) {
+#ifdef APR_HAS_THREADS
+      if(ctx->threadlock)
+         apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock);
+#endif
+      if(!ro_connection_pool) {
+         rv = apr_reslist_create(&ro_connection_pool,
+               0 /* min */,
+               10 /* soft max */,
+               200 /* hard max */,
+               60*1000000 /*60 seconds, ttl*/,
+               _bdb_reslist_get_connection, /* resource constructor */
+               _bdb_reslist_free_connection, /* resource destructor */
+               cache, ctx->process_pool);
+         if(rv != APR_SUCCESS) {
+            ctx->set_error(ctx,500,"failed to create bdb ro connection pool");
+            if(ctx->threadlock)
+               apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock);
+            return NULL;
+         }
+         rv = apr_reslist_create(&rw_connection_pool,
+               0 /* min */,
+               1 /* soft max */,
+               1 /* hard max */,
+               60*1000000 /*60 seconds, ttl*/,
+               _bdb_reslist_get_connection, /* resource constructor */
+               _bdb_reslist_free_connection, /* resource destructor */
+               cache, ctx->process_pool);
+         if(rv != APR_SUCCESS) {
+            ctx->set_error(ctx,500,"failed to create bdb rw connection pool");
+            if(ctx->threadlock)
+               apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock);
+            return NULL;
+         }
+      }
+#ifdef APR_HAS_THREADS
+      if(ctx->threadlock)
+         apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock);
+#endif
+   }
    if(readonly) 
-      pool = cache->ro_connection_pool;
+      pool = ro_connection_pool;
    else
-      pool = cache->rw_connection_pool;
+      pool = rw_connection_pool;
    rv = apr_reslist_acquire(pool, (void **)&benv);
    if(rv != APR_SUCCESS) {
-      ctx->set_error(ctx,500,"failed to aquire connection to bdb backend: %s", cache->ctx->get_error_message(cache->ctx));
-      cache->ctx->clear_errors(cache->ctx);
+      ctx->set_error(ctx,500,"failed to aquire connection to bdb backend");
       return NULL;
    }
    benv->readonly = readonly;
    return benv;
 }
 
 static void _bdb_release_conn(mapcache_context *ctx, mapcache_tile *tile, struct bdb_env *benv) {
-   mapcache_cache_bdb* cache = (mapcache_cache_bdb*)tile->tileset->cache;
    apr_reslist_t *pool;
    if(benv->readonly) 
-      pool = cache->ro_connection_pool;
+      pool = ro_connection_pool;
    else
-      pool = cache->rw_connection_pool;
+      pool = rw_connection_pool;
    if(GC_HAS_ERROR(ctx)) {
       apr_reslist_invalidate(pool,(void*)benv);  
    } else {
@@ -306,31 +353,6 @@ static void _mapcache_cache_bdb_configuration_parse_xml(mapcache_context *ctx, e
       ctx->set_error(ctx,500,"dbd cache \"%s\" is missing <base> entry",cache->name);
       return;
    }
-   dcache->ctx = ctx;
-   rv = apr_reslist_create(&(dcache->ro_connection_pool),
-         0 /* min */,
-         10 /* soft max */,
-         200 /* hard max */,
-         60*1000000 /*60 seconds, ttl*/,
-         _bdb_reslist_get_connection, /* resource constructor */
-         _bdb_reslist_free_connection, /* resource destructor */
-         dcache, ctx->pool);
-   if(rv != APR_SUCCESS) {
-      ctx->set_error(ctx,500,"failed to create bdb connection pool");
-      return;
-   }
-   rv = apr_reslist_create(&(dcache->rw_connection_pool),
-         0 /* min */,
-         1 /* soft max */,
-         1 /* hard max */,
-         60*1000000 /*60 seconds, ttl*/,
-         _bdb_reslist_get_connection, /* resource constructor */
-         _bdb_reslist_free_connection, /* resource destructor */
-         dcache, ctx->pool);
-   if(rv != APR_SUCCESS) {
-      ctx->set_error(ctx,500,"failed to create bdb connection pool");
-      return;
-   }
 }
 
 /**
@@ -368,8 +390,6 @@ mapcache_cache* mapcache_cache_bdb_create(mapcache_context *ctx) {
    cache->cache.configuration_parse_xml = _mapcache_cache_bdb_configuration_parse_xml;
    cache->basedir = NULL;
    cache->key_template = NULL;
-   cache->ro_connection_pool = NULL;
-   cache->rw_connection_pool = NULL;
    return (mapcache_cache*)cache;
 }
 

diff --git a/lib/util.c b/lib/util.c
@@ -213,6 +213,8 @@ void mapcache_context_copy(mapcache_context *src, mapcache_context *dst) {
    dst->set_exception = src->set_exception;
    dst->service = src->service;
    dst->exceptions = src->exceptions;
+   dst->threadlock = src->threadlock;
+   dst->process_pool = src->process_pool;
 }
 
 char* mapcache_util_get_tile_dimkey(mapcache_context *ctx, mapcache_tile *tile, char* sanitized_chars, char *sanitize_to) {

diff --git a/nginx/ngx_http_mapcache_module.c b/nginx/ngx_http_mapcache_module.c
@@ -3,8 +3,11 @@
 #include <ngx_http.h>
 #include "../include/mapcache.h"
 #include <apr_date.h>
+#include <apr_strings.h>
+#include <apr_pools.h>
 
 
+apr_pool_t *process_pool = NULL;
 static char *ngx_http_mapcache(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
 
@@ -49,13 +52,14 @@ static mapcache_context* ngx_mapcache_context_clone(mapcache_context *ctx) {
 static void *
 ngx_http_mapcache_create_conf(ngx_conf_t *cf)
 {
-    apr_pool_t  *pool;
     apr_initialize();
     atexit(apr_terminate);
     apr_pool_initialize();
-    apr_pool_create(&pool,NULL);
-    mapcache_context *ctx = apr_pcalloc(pool, sizeof(mapcache_ngx_context));
-    ctx->pool = pool;
+    apr_pool_create(&process_pool,NULL);
+    mapcache_context *ctx = apr_pcalloc(process_pool, sizeof(mapcache_ngx_context));
+    ctx->pool = process_pool;
+    ctx->process_pool = process_pool;
+    ctx->threadlock = NULL;
     mapcache_context_init(ctx);
     ctx->log = ngx_mapcache_context_log;
     ctx->clone = ngx_mapcache_context_clone;
@@ -158,18 +162,30 @@ static ngx_http_module_t  ngx_http_mapcache_module_ctx = {
     NULL                           /* merge location configuration */
 };
 
+static ngx_int_t ngx_mapcache_init_process(ngx_cycle_t *cycle) {
+    apr_initialize();
+    atexit(apr_terminate);
+    apr_pool_initialize();
+    apr_pool_create(&process_pool,NULL);
+    return NGX_OK;
+}
+
+static void ngx_mapcache_exit_process(ngx_cycle_t *cycle) {
+    apr_pool_destroy(process_pool);
+}
+
 ngx_module_t  ngx_http_mapcache_module = {
     NGX_MODULE_V1,
     &ngx_http_mapcache_module_ctx, /* module context */
     ngx_http_mapcache_commands,   /* module directives */
     NGX_HTTP_MODULE,               /* module type */
     NULL,                          /* init master */
     NULL,                          /* init module */
-    NULL,/* init process */
+    ngx_mapcache_init_process,/* init process */
     NULL,                          /* init thread */
     NULL,                          /* exit thread */
-    NULL,                          /* exit process */
-    NULL,                          /* exit master */
+    ngx_mapcache_exit_process,                          /* exit process */
+    ngx_mapcache_exit_process,                          /* exit master */
     NGX_MODULE_V1_PADDING
 };
 
@@ -186,8 +202,8 @@ ngx_http_mapcache_handler(ngx_http_request_t *r)
     }
     mapcache_ngx_context *ngctx = ngx_http_get_module_loc_conf(r, ngx_http_mapcache_module);
     mapcache_context *ctx = (mapcache_context*)ngctx;
-    apr_pool_t *main_pool = ctx->pool;
-    apr_pool_create(&(ctx->pool),main_pool);
+    apr_pool_create(&(ctx->pool),process_pool);
+    ctx->process_pool = process_pool;
     ngctx->r = r;
     mapcache_request *request = NULL;
     mapcache_http_response *http_response;
@@ -248,7 +264,6 @@ ngx_http_mapcache_handler(ngx_http_request_t *r)
 cleanup:
       ctx->clear_errors(ctx);
       apr_pool_destroy(ctx->pool);
-      ctx->pool = main_pool;
       return NGX_HTTP_OK;
 }
 

diff --git a/util/mapcache_seed.c b/util/mapcache_seed.c
@@ -710,6 +710,12 @@ int main(int argc, const char **argv) {
     (void) signal(SIGINT,handle_sig_int);
     apr_pool_create(&ctx.pool,NULL);
     mapcache_context_init(&ctx);
+    ctx.process_pool = ctx.pool;
+#ifndef USE_FORK
+    apr_thread_mutex_create((apr_thread_mutex_t**)&ctx.threadlock,APR_THREAD_MUTEX_DEFAULT,ctx.pool);
+#else
+    ctx.threadlock = NULL;
+#endif
     cfg = mapcache_configuration_create(ctx.pool);
     ctx.config = cfg;
     ctx.log= mapcache_context_seeding_log;