LMDB: Robust mutexes and platform detection

[Kristian Amlie <kristian.amlie@cfengine.com>]

Hey,

In CFEngine's custom version of LMDB we have started using the
pthread_mutexattr_setrobust interface in order to recover from the
deadlock that can occur if a process is killed while holding the writer
lock on LMDB. If a different process is waiting for the same lock at
that time, it will wait forever, but with the robustness attribute, the
attempted lock will return with an error instead.

I think this would be a useful feature to have in LMDB in general, since
you can always risk a deadlock if there are at least two processes and
one of them receives SIGKILL. It requires using the attribute and a
little bit of extra error handling inside the part of LMDB where locking
is done.

One complication is that the interface is not available everywhere,
particularly the commercial Unixes and older Linux releases. In CFEngine
we have solved this by replacing the Makefile that LMDB comes with, with
a set of autotools files which can handle platform detection for us.

Robust mutexes have been briefly discussed before on the mailing list,
but it seems that no action was taken at that time. I hope this is
interesting to the LMDB community, and we'd like to work with you to get
the patches accepted.

I have attached the current versions of the patches we use. They will
definitely need to be adapted before they can be applied in the openldap
repository, for one thing they are made against LMDB 0.9.9, but they are
enough to get the idea.

-- 
Kristian Amlie
Software Engineer
CFEngine

>From f000ab84862a3a6397810508df4dd743c8a46f64 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@cfengine.com>
Date: Tue, 24 Jun 2014 14:25:55 +0200
Subject: [PATCH 1/4] Autoconf files.

---
 libraries/liblmdb/Makefile.am  |   11 +++++++++++
 libraries/liblmdb/configure.ac |   26 ++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 libraries/liblmdb/Makefile.am
 create mode 100644 libraries/liblmdb/configure.ac

diff --git a/libraries/liblmdb/Makefile.am b/libraries/liblmdb/Makefile.am
new file mode 100644
index 0000000..d3a4a4c
--- /dev/null
+++ b/libraries/liblmdb/Makefile.am
@@ -0,0 +1,11 @@
+lib_LTLIBRARIES = liblmdb.la
+liblmdb_la_SOURCES = mdb.c midl.c
+liblmdb_la_LDFLAGS=-no-undefined -avoid-version
+
+bin_PROGRAMS = mdb_stat mdb_copy lmdump lmmgr
+mdb_stat_LDADD = liblmdb.la
+mdb_copy_LDADD = liblmdb.la
+lmdump_LDADD = liblmdb.la
+lmmgr_LDADD = liblmdb.la
+
+include_HEADERS = lmdb.h
diff --git a/libraries/liblmdb/configure.ac b/libraries/liblmdb/configure.ac
new file mode 100644
index 0000000..d90e002
--- /dev/null
+++ b/libraries/liblmdb/configure.ac
@@ -0,0 +1,26 @@
+AC_INIT([hello], [1.0]) # good
+AM_INIT_AUTOMAKE([foreign])
+
+AC_PROG_CC
+
+# Use either new LT_INIT or old AC_DISABLE_STATIC/AC_PROG_LIBTOOL macros
+m4_ifdef([LT_INIT],
+  [LT_INIT([disable-static])],
+  [AC_DISABLE_STATIC
+   AC_PROG_LIBTOOL])
+
+AC_ARG_WITH([pthread], [Path to pthread])
+AS_IF([test -n "$with_pthread"],
+    [LDFLAGS+=" -L$with_pthread/lib"]
+)
+AC_CHECK_LIB([pthread], [pthread_mutex_lock])
+AC_CHECK_LIB([pthreadGC2], [pthread_mutex_lock])
+AS_IF([test "$ac_cv_lib_pthread" = "no" && test "$ac_cv_lib_pthreadGC2" = "no"],
+[
+  AC_MSG_ERROR([Could not find pthreads library])
+])
+AC_CHECK_FUNCS([pthread_mutexattr_settype pthread_mutexattr_setrobust])
+
+AC_CONFIG_SRCDIR([mdb.c])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
-- 
1.7.9.5

>From 49984f4be99198c811d7fbfbf300f8671bd59c20 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@cfengine.com>
Date: Mon, 28 Jul 2014 10:31:49 +0200
Subject: [PATCH 3/4] Patch for LMDB to use robust mutexes.

---
 libraries/liblmdb/mdb.c |   39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 33be876..2faa9d3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -143,6 +143,7 @@
 #ifdef _WIN32
 #define MDB_USE_HASH	1
 #define MDB_PIDLOCK	0
+#define EOWNERDEAD      130
 #define pthread_t	DWORD
 #define pthread_mutex_t	HANDLE
 #define pthread_key_t	DWORD
@@ -153,7 +154,7 @@
 #define pthread_getspecific(x)	TlsGetValue(x)
 #define pthread_setspecific(x,y)	(TlsSetValue(x,y) ? 0 : ErrCode())
 #define pthread_mutex_unlock(x)	ReleaseMutex(x)
-#define pthread_mutex_lock(x)	WaitForSingleObject(x, INFINITE)
+#define pthread_mutex_lock(x)	(WaitForSingleObject(x, INFINITE) == WAIT_ABANDONED ? EOWNERDEAD : 0) // masks FAIL and TIMEOUT error, but acceptable
 #define LOCK_MUTEX_R(env)	pthread_mutex_lock((env)->me_rmutex)
 #define UNLOCK_MUTEX_R(env)	pthread_mutex_unlock((env)->me_rmutex)
 #define LOCK_MUTEX_W(env)	pthread_mutex_lock((env)->me_wmutex)
@@ -2155,7 +2156,20 @@ mdb_txn_renew0(MDB_txn *txn)
 					env->me_flags |= MDB_LIVE_READER;
 				}
 
-				LOCK_MUTEX_R(env);
+				rc = LOCK_MUTEX_R(env);
+                                switch (rc)
+				{
+				case 0:
+					break;
+#ifdef HAVE_PTHREAD_MUTEXATTR_SETROBUST
+				case EOWNERDEAD:
+					// we cannot recover the state, so mark mutex as unusable
+					UNLOCK_MUTEX_R(env);
+					// FALLTHROUGH
+#endif
+				default:
+					return MDB_PANIC;
+				}
 				nr = ti->mti_numreaders;
 				for (i=0; i<nr; i++)
 					if (ti->mti_readers[i].mr_pid == 0)
@@ -2185,7 +2199,20 @@ mdb_txn_renew0(MDB_txn *txn)
 		}
 	} else {
 		if (ti) {
-			LOCK_MUTEX_W(env);
+			rc = LOCK_MUTEX_W(env);
+                        switch (rc)
+                        {
+                        case 0:
+                                break;
+#ifdef HAVE_PTHREAD_MUTEXATTR_SETROBUST
+                        case EOWNERDEAD:
+                                // we cannot recover the state, so mark mutex as unusable
+                                UNLOCK_MUTEX_W(env);
+                                // FALLTHROUGH
+#endif
+                        default:
+                                return MDB_PANIC;
+                        }
 
 			txn->mt_txnid = ti->mti_txnid;
 			meta = env->me_metas[txn->mt_txnid & 1];
@@ -3870,6 +3897,12 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		pthread_mutexattr_t mattr;
 
 		if ((rc = pthread_mutexattr_init(&mattr))
+#ifdef HAVE_PTHREAD_MUTEXATTR_SETTYPE
+                        || (rc = pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_ERRORCHECK))
+#endif
+#ifdef HAVE_PTHREAD_MUTEXATTR_SETROBUST
+                        || (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST))
+#endif
 			|| (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED))
 			|| (rc = pthread_mutex_init(&env->me_txns->mti_mutex, &mattr))
 			|| (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr)))
-- 
1.7.9.5