Merge branch 'master-experimental-patches/atomics_support_on_fringe_32bit_platforms' into master-experimental

11 files changed, 438 insertions, 333 deletions

diff --git a/configure.in b/configure.in
index d6421070aa..623171165a 100755
--- a/configure.in
+++ b/configure.in
@@ -2780,7 +2780,7 @@ esac
 dnl Use GCC atomic ops if they work on the target.
 if test x$GCC = "xyes"; then
 	case $TARGET in
-	X86 | AMD64 | ARM | POWERPC | POWERPC64 | MIPS)
+	X86 | AMD64 | ARM | POWERPC | POWERPC64 | MIPS | S390X)
 		AC_DEFINE(USE_GCC_ATOMIC_OPS, 1, [...])
 		;;
 	esac
diff --git a/docs/current-api b/docs/current-api
index d0efffcaba..1bbddca612 100644
--- a/docs/current-api
+++ b/docs/current-api
@@ -307,6 +307,7 @@ mono_gc_out_of_memory
 mono_gc_wbarrier_arrayref_copy
 mono_gc_wbarrier_generic_nostore
 mono_gc_wbarrier_generic_store
+mono_gc_wbarrier_generic_store_atomic
 mono_gc_wbarrier_object_copy
 mono_gc_wbarrier_set_arrayref
 mono_gc_wbarrier_set_field
diff --git a/docs/public-api b/docs/public-api
index 082bfd9fdd..231ecf171a 100644
--- a/docs/public-api
+++ b/docs/public-api
@@ -307,6 +307,7 @@ mono_gc_out_of_memory
 mono_gc_wbarrier_arrayref_copy
 mono_gc_wbarrier_generic_nostore
 mono_gc_wbarrier_generic_store
+mono_gc_wbarrier_generic_store_atomic
 mono_gc_wbarrier_object_copy
 mono_gc_wbarrier_set_arrayref
 mono_gc_wbarrier_set_field
diff --git a/mono/metadata/boehm-gc.c b/mono/metadata/boehm-gc.c
index 26665e50f5..0cb8292dcb 100644
--- a/mono/metadata/boehm-gc.c
+++ b/mono/metadata/boehm-gc.c
@@ -22,6 +22,7 @@
 #include <mono/metadata/marshal.h>
 #include <mono/metadata/runtime.h>
 #include <mono/utils/mono-logger-internal.h>
+#include <mono/utils/mono-memory-model.h>
 #include <mono/utils/mono-time.h>
 #include <mono/utils/mono-threads.h>
 #include <mono/utils/dtrace.h>
@@ -624,6 +625,12 @@ mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
 }
 
 void
+mono_gc_wbarrier_generic_store_atomic (gpointer ptr, MonoObject *value)
+{
+	mono_atomic_store_release ((volatile MonoObject **) ptr, value);
+}
+
+void
 mono_gc_wbarrier_generic_nostore (gpointer ptr)
 {
 }
diff --git a/mono/metadata/null-gc.c b/mono/metadata/null-gc.c
index 13446ee0ec..0430dd5cbf 100644
--- a/mono/metadata/null-gc.c
+++ b/mono/metadata/null-gc.c
@@ -193,6 +193,12 @@ mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
 }
 
 void
+mono_gc_wbarrier_generic_store_atomic (gpointer ptr, MonoObject *value)
+{
+	mono_atomic_store_release ((volatile MonoObject **) ptr, value);
+}
+
+void
 mono_gc_wbarrier_generic_nostore (gpointer ptr)
 {
 }
diff --git a/mono/metadata/object.h b/mono/metadata/object.h
index 70ff5814b3..959c09e49c 100644
--- a/mono/metadata/object.h
+++ b/mono/metadata/object.h
@@ -316,6 +316,7 @@ MONO_API void mono_gc_wbarrier_set_field     (MonoObject *obj, void* field_ptr,
 MONO_API void mono_gc_wbarrier_set_arrayref  (MonoArray *arr, void* slot_ptr, MonoObject* value);
 MONO_API void mono_gc_wbarrier_arrayref_copy (void* dest_ptr, void* src_ptr, int count);
 MONO_API void mono_gc_wbarrier_generic_store (void* ptr, MonoObject* value);
+MONO_API void mono_gc_wbarrier_generic_store_atomic (void *ptr, MonoObject *value);
 MONO_API void mono_gc_wbarrier_generic_nostore (void* ptr);
 MONO_API void mono_gc_wbarrier_value_copy    (void* dest, void* src, int count, MonoClass *klass);
 MONO_API void mono_gc_wbarrier_object_copy   (MonoObject* obj, MonoObject *src);
diff --git a/mono/metadata/sgen-gc.c b/mono/metadata/sgen-gc.c
index 8ce0994b50..2b6d1e3963 100644
--- a/mono/metadata/sgen-gc.c
+++ b/mono/metadata/sgen-gc.c
@@ -317,6 +317,7 @@ static int stat_wbarrier_set_field = 0;
 static int stat_wbarrier_set_arrayref = 0;
 static int stat_wbarrier_arrayref_copy = 0;
 static int stat_wbarrier_generic_store = 0;
+static int stat_wbarrier_generic_store_atomic = 0;
 static int stat_wbarrier_set_root = 0;
 static int stat_wbarrier_value_copy = 0;
 static int stat_wbarrier_object_copy = 0;
@@ -2217,6 +2218,7 @@ init_stats (void)
 	mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
 	mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
 	mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
+	mono_counters_register ("WBarrier generic atomic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_atomic);
 	mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
 	mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
 	mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
@@ -4424,6 +4426,24 @@ mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
 	sgen_dummy_use (value);
 }
 
+/* Same as mono_gc_wbarrier_generic_store () but performs the store
+ * as an atomic operation with release semantics.
+ */
+void
+mono_gc_wbarrier_generic_store_atomic (gpointer ptr, MonoObject *value)
+{
+	HEAVY_STAT (++stat_wbarrier_generic_store_atomic);
+
+	SGEN_LOG (8, "Wbarrier atomic store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
+
+	mono_atomic_store_release ((volatile MonoObject **) ptr, value);
+
+	if (ptr_in_nursery (value))
+		mono_gc_wbarrier_generic_nostore (ptr);
+
+	sgen_dummy_use (value);
+}
+
 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
 {
 	mword *dest = _dest;
diff --git a/mono/metadata/threads.c b/mono/metadata/threads.c
index a8720ba5bf..59f8293f86 100755
--- a/mono/metadata/threads.c
+++ b/mono/metadata/threads.c
@@ -48,6 +48,7 @@
 #include <mono/utils/hazard-pointer.h>
 #include <mono/utils/mono-tls.h>
 #include <mono/utils/atomic.h>
+#include <mono/utils/mono-memory-model.h>
 
 #include <mono/metadata/gc-internal.h>
 
@@ -1182,8 +1183,6 @@ HANDLE ves_icall_System_Threading_Thread_Thread_internal(MonoThread *this,
 
 void ves_icall_System_Threading_InternalThread_Thread_free_internal (MonoInternalThread *this, HANDLE thread)
 {
-	MONO_ARCH_SAVE_REGS;
-
 	THREAD_DEBUG (g_message ("%s: Closing thread %p, handle %p", __func__, this, thread));
 
 	if (thread)
@@ -1273,8 +1272,6 @@ void ves_icall_System_Threading_Thread_SpinWait_nop (void)
 gint32
 ves_icall_System_Threading_Thread_GetDomainID (void) 
 {
-	MONO_ARCH_SAVE_REGS;
-
 	return mono_domain_get()->domain_id;
 }
 
@@ -1640,8 +1637,6 @@ ves_icall_System_Threading_WaitHandle_SignalAndWait_Internal (HANDLE toSignal, H
 	guint32 ret;
 	MonoInternalThread *thread = mono_thread_internal_current ();
 
-	MONO_ARCH_SAVE_REGS;
-
 	if (ms == -1)
 		ms = INFINITE;
 
@@ -1660,8 +1655,6 @@ HANDLE ves_icall_System_Threading_Mutex_CreateMutex_internal (MonoBoolean owned,
 { 
 	HANDLE mutex;
 	
-	MONO_ARCH_SAVE_REGS;
-   
 	*created = TRUE;
 	
 	if (name == NULL) {
@@ -1678,8 +1671,6 @@ HANDLE ves_icall_System_Threading_Mutex_CreateMutex_internal (MonoBoolean owned,
 }                                                                   
 
 MonoBoolean ves_icall_System_Threading_Mutex_ReleaseMutex_internal (HANDLE handle ) { 
-	MONO_ARCH_SAVE_REGS;
-
 	return(ReleaseMutex (handle));
 }
 
@@ -1689,8 +1680,6 @@ HANDLE ves_icall_System_Threading_Mutex_OpenMutex_internal (MonoString *name,
 {
 	HANDLE ret;
 	
-	MONO_ARCH_SAVE_REGS;
-	
 	*error = ERROR_SUCCESS;
 	
 	ret = OpenMutex (rights, FALSE, mono_string_chars (name));
@@ -1706,8 +1695,6 @@ HANDLE ves_icall_System_Threading_Semaphore_CreateSemaphore_internal (gint32 ini
 { 
 	HANDLE sem;
 	
-	MONO_ARCH_SAVE_REGS;
-   
 	*created = TRUE;
 	
 	if (name == NULL) {
@@ -1728,8 +1715,6 @@ gint32 ves_icall_System_Threading_Semaphore_ReleaseSemaphore_internal (HANDLE ha
 { 
 	gint32 prevcount;
 	
-	MONO_ARCH_SAVE_REGS;
-
 	*fail = !ReleaseSemaphore (handle, releaseCount, &prevcount);
 
 	return (prevcount);
@@ -1739,8 +1724,6 @@ HANDLE ves_icall_System_Threading_Semaphore_OpenSemaphore_internal (MonoString *
 {
 	HANDLE ret;
 	
-	MONO_ARCH_SAVE_REGS;
-	
 	*error = ERROR_SUCCESS;
 	
 	ret = OpenSemaphore (rights, FALSE, mono_string_chars (name));
@@ -1755,8 +1738,6 @@ HANDLE ves_icall_System_Threading_Events_CreateEvent_internal (MonoBoolean manua
 {
 	HANDLE event;
 	
-	MONO_ARCH_SAVE_REGS;
-
 	*created = TRUE;
 
 	if (name == NULL) {
@@ -1774,21 +1755,15 @@ HANDLE ves_icall_System_Threading_Events_CreateEvent_internal (MonoBoolean manua
 }
 
 gboolean ves_icall_System_Threading_Events_SetEvent_internal (HANDLE handle) {
-	MONO_ARCH_SAVE_REGS;
-
 	return (SetEvent(handle));
 }
 
 gboolean ves_icall_System_Threading_Events_ResetEvent_internal (HANDLE handle) {
-	MONO_ARCH_SAVE_REGS;
-
 	return (ResetEvent(handle));
 }
 
 void
 ves_icall_System_Threading_Events_CloseEvent_internal (HANDLE handle) {
-	MONO_ARCH_SAVE_REGS;
-
 	CloseHandle (handle);
 }
 
@@ -1798,8 +1773,6 @@ HANDLE ves_icall_System_Threading_Events_OpenEvent_internal (MonoString *name,
 {
 	HANDLE ret;
 	
-	MONO_ARCH_SAVE_REGS;
-	
 	*error = ERROR_SUCCESS;
 	
 	ret = OpenEvent (rights, FALSE, mono_string_chars (name));
@@ -1812,53 +1785,26 @@ HANDLE ves_icall_System_Threading_Events_OpenEvent_internal (MonoString *name,
 
 gint32 ves_icall_System_Threading_Interlocked_Increment_Int (gint32 *location)
 {
-	MONO_ARCH_SAVE_REGS;
-
 	return InterlockedIncrement (location);
 }
 
 gint64 ves_icall_System_Threading_Interlocked_Increment_Long (gint64 *location)
 {
-	gint64 ret;
-
-	MONO_ARCH_SAVE_REGS;
-
-	mono_interlocked_lock ();
-
-	ret = ++ *location;
-	
-	mono_interlocked_unlock ();
-
-	
-	return ret;
+	return InterlockedIncrement64 (location);
 }
 
 gint32 ves_icall_System_Threading_Interlocked_Decrement_Int (gint32 *location)
 {
-	MONO_ARCH_SAVE_REGS;
-
 	return InterlockedDecrement(location);
 }
 
 gint64 ves_icall_System_Threading_Interlocked_Decrement_Long (gint64 * location)
 {
-	gint64 ret;
-
-	MONO_ARCH_SAVE_REGS;
-
-	mono_interlocked_lock ();
-
-	ret = -- *location;
-	
-	mono_interlocked_unlock ();
-
-	return ret;
+	return InterlockedDecrement64 (location);
 }
 
 gint32 ves_icall_System_Threading_Interlocked_Exchange_Int (gint32 *location, gint32 value)
 {
-	MONO_ARCH_SAVE_REGS;
-
 	return InterlockedExchange(location, value);
 }
 
@@ -1879,8 +1825,6 @@ gfloat ves_icall_System_Threading_Interlocked_Exchange_Single (gfloat *location,
 {
 	IntFloatUnion val, ret;
 
-	MONO_ARCH_SAVE_REGS;
-
 	val.fval = value;
 	ret.ival = InterlockedExchange((gint32 *) location, val.ival);
 
@@ -1890,54 +1834,22 @@ gfloat ves_icall_System_Threading_Interlocked_Exchange_Single (gfloat *location,
 gint64 
 ves_icall_System_Threading_Interlocked_Exchange_Long (gint64 *location, gint64 value)
 {
-#if SIZEOF_VOID_P == 8
-	return (gint64) InterlockedExchangePointer((gpointer *) location, (gpointer)value);
-#else
-	gint64 res;
-
-	/* 
-	 * According to MSDN, this function is only atomic with regards to the 
-	 * other Interlocked functions on 32 bit platforms.
-	 */
-	mono_interlocked_lock ();
-	res = *location;
-	*location = value;
-	mono_interlocked_unlock ();
-
-	return res;
-#endif
+	return InterlockedExchange64 (location, value);
 }
 
 gdouble 
 ves_icall_System_Threading_Interlocked_Exchange_Double (gdouble *location, gdouble value)
 {
-#if SIZEOF_VOID_P == 8
 	LongDoubleUnion val, ret;
 
 	val.fval = value;
-	ret.ival = (gint64)InterlockedExchangePointer((gpointer *) location, (gpointer)val.ival);
+	ret.ival = (gint64)InterlockedExchange64((gint64 *) location, val.ival);
 
 	return ret.fval;
-#else
-	gdouble res;
-
-	/* 
-	 * According to MSDN, this function is only atomic with regards to the 
-	 * other Interlocked functions on 32 bit platforms.
-	 */
-	mono_interlocked_lock ();
-	res = *location;
-	*location = value;
-	mono_interlocked_unlock ();
-
-	return res;
-#endif
 }
 
 gint32 ves_icall_System_Threading_Interlocked_CompareExchange_Int(gint32 *location, gint32 value, gint32 comparand)
 {
-	MONO_ARCH_SAVE_REGS;
-
 	return InterlockedCompareExchange(location, value, comparand);
 }
 
@@ -1958,8 +1870,6 @@ gfloat ves_icall_System_Threading_Interlocked_CompareExchange_Single (gfloat *lo
 {
 	IntFloatUnion val, ret, cmp;
 
-	MONO_ARCH_SAVE_REGS;
-
 	val.fval = value;
 	cmp.fval = comparand;
 	ret.ival = InterlockedCompareExchange((gint32 *) location, val.ival, cmp.ival);
@@ -2034,14 +1944,7 @@ ves_icall_System_Threading_Interlocked_Add_Int (gint32 *location, gint32 value)
 	mono_raise_exception (mono_get_exception_not_implemented (NULL));
 	return 0;
 #else
-	gint32 orig;
-
-	mono_interlocked_lock ();
-	orig = *location;
-	*location = orig + value;
-	mono_interlocked_unlock ();
-
-	return orig + value;
+	return InterlockedAdd (location, value);
 #endif
 }
 
@@ -2053,39 +1956,20 @@ ves_icall_System_Threading_Interlocked_Add_Long (gint64 *location, gint64 value)
 	mono_raise_exception (mono_get_exception_not_implemented (NULL));
 	return 0;
 #else
-	gint64 orig;
-
-	mono_interlocked_lock ();
-	orig = *location;
-	*location = orig + value;
-	mono_interlocked_unlock ();
-
-	return orig + value;
+	return InterlockedAdd64 (location, value);
 #endif
 }
 
 gint64 
 ves_icall_System_Threading_Interlocked_Read_Long (gint64 *location)
 {
-#if SIZEOF_VOID_P == 8
-	/* 64 bit reads are already atomic */
-	return *location;
-#else
-	gint64 res;
-
-	mono_interlocked_lock ();
-	res = *location;
-	mono_interlocked_unlock ();
-
-	return res;
-#endif
+	return InterlockedRead64 (location);
 }
 
 void
 ves_icall_System_Threading_Thread_MemoryBarrier (void)
 {
-	mono_threads_lock ();
-	mono_threads_unlock ();
+	mono_memory_barrier ();
 }
 
 void
@@ -2512,108 +2396,119 @@ void mono_thread_stop (MonoThread *thread)
 gint8
 ves_icall_System_Threading_Thread_VolatileRead1 (void *ptr)
 {
-	return *((volatile gint8 *) (ptr));
+	gint8 tmp;
+	mono_atomic_load_acquire (tmp, gint8, (volatile gint8 *) ptr);
+	return tmp;
 }
 
 gint16
 ves_icall_System_Threading_Thread_VolatileRead2 (void *ptr)
 {
-	return *((volatile gint16 *) (ptr));
+	gint16 tmp;
+	mono_atomic_load_acquire (tmp, gint16, (volatile gint16 *) ptr);
+	return tmp;
 }
 
 gint32
 ves_icall_System_Threading_Thread_VolatileRead4 (void *ptr)
 {
-	return *((volatile gint32 *) (ptr));
+	gint32 tmp;
+	mono_atomic_load_acquire (tmp, gint32, (volatile gint32 *) ptr);
+	return tmp;
 }
 
 gint64
 ves_icall_System_Threading_Thread_VolatileRead8 (void *ptr)
 {
-#if SIZEOF_VOID_P == 8
-	return *((volatile gint64 *) (ptr));
-#else
-	return InterlockedCompareExchange64 (ptr, 0, 0); /*Must ensure atomicity of the operation. */
-#endif
+	gint64 tmp;
+	mono_atomic_load_acquire (tmp, gint64, (volatile gint64 *) ptr);
+	return tmp;
 }
 
 void *
 ves_icall_System_Threading_Thread_VolatileReadIntPtr (void *ptr)
 {
-	return (void *)  *((volatile void **) ptr);
+	volatile void *tmp;
+	mono_atomic_load_acquire (tmp, volatile void *, (volatile void **) ptr);
+	return (void *) tmp;
 }
 
 double
 ves_icall_System_Threading_Thread_VolatileReadDouble (void *ptr)
 {
-	return *((volatile double *) (ptr));
+	double tmp;
+	mono_atomic_load_acquire (tmp, double, (volatile double *) ptr);
+	return tmp;
 }
 
 float
 ves_icall_System_Threading_Thread_VolatileReadFloat (void *ptr)
 {
-	return *((volatile float *) (ptr));
+	float tmp;
+	mono_atomic_load_acquire (tmp, float, (volatile float *) ptr);
+	return tmp;
 }
 
 MonoObject*
 ves_icall_System_Threading_Volatile_Read_T (void *ptr)
 {
-	return (MonoObject*)*((volatile MonoObject**)ptr);
+	volatile MonoObject *tmp;
+	mono_atomic_load_acquire (tmp, volatile MonoObject *, (volatile MonoObject **) ptr);
+	return (MonoObject *) tmp;
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWrite1 (void *ptr, gint8 value)
 {
-	*((volatile gint8 *) ptr) = value;
+	mono_atomic_store_release ((volatile gint8 *) ptr, value);
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWrite2 (void *ptr, gint16 value)
 {
-	*((volatile gint16 *) ptr) = value;
+	mono_atomic_store_release ((volatile gint16 *) ptr, value);
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWrite4 (void *ptr, gint32 value)
 {
-	*((volatile gint32 *) ptr) = value;
+	mono_atomic_store_release ((volatile gint32 *) ptr, value);
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWrite8 (void *ptr, gint64 value)
 {
-	*((volatile gint64 *) ptr) = value;
+	mono_atomic_store_release ((volatile gint64 *) ptr, value);
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWriteIntPtr (void *ptr, void *value)
 {
-	*((volatile void **) ptr) = value;
+	mono_atomic_store_release ((volatile void **) ptr, value);
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWriteObject (void *ptr, void *value)
 {
-	mono_gc_wbarrier_generic_store (ptr, value);
+	mono_gc_wbarrier_generic_store_atomic (ptr, value);
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWriteDouble (void *ptr, double value)
 {
-	*((volatile double *) ptr) = value;
+	mono_atomic_store_release ((volatile double *) ptr, value);
 }
 
 void
 ves_icall_System_Threading_Thread_VolatileWriteFloat (void *ptr, float value)
 {
-	*((volatile float *) ptr) = value;
+	mono_atomic_store_release ((volatile float *) ptr, value);
 }
 
 void
 ves_icall_System_Threading_Volatile_Write_T (void *ptr, MonoObject *value)
 {
-	*((volatile MonoObject **) ptr) = value;
-	mono_gc_wbarrier_generic_nostore (ptr);
+	mono_gc_wbarrier_generic_store_atomic (ptr, value);
 }
 
 void
diff --git a/mono/utils/atomic.c b/mono/utils/atomic.c
index da3290a518..4d59ea356c 100755
--- a/mono/utils/atomic.c
+++ b/mono/utils/atomic.c
@@ -13,12 +13,14 @@
 
 #include <mono/utils/atomic.h>
 
-#if defined (WAPI_NO_ATOMIC_ASM) || !defined (HAS_64BITS_ATOMIC)
+#if defined (WAPI_NO_ATOMIC_ASM) || defined (BROKEN_64BIT_ATOMICS_INTRINSIC)
 
 #include <pthread.h>
 
 static pthread_mutex_t spin = PTHREAD_MUTEX_INITIALIZER;
 
+#define NEED_64BIT_CMPXCHG_FALLBACK
+
 #endif
 
 #ifdef WAPI_NO_ATOMIC_ASM
@@ -82,6 +84,52 @@ gpointer InterlockedCompareExchangePointer(volatile gpointer *dest,
 	return(old);
 }
 
+gint32 InterlockedAdd(volatile gint32 *dest, gint32 add)
+{
+	gint32 ret;
+	int thr_ret;
+
+	mono_once(&spin_once, spin_init);
+
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	*dest += add;
+	ret= *dest;
+
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+
+	pthread_cleanup_pop (0);
+
+	return(ret);
+}
+
+gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
+{
+	gint64 ret;
+	int thr_ret;
+
+	mono_once(&spin_once, spin_init);
+
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	*dest += add;
+	ret= *dest;
+
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+
+	pthread_cleanup_pop (0);
+
+	return(ret);
+}
+
 gint32 InterlockedIncrement(volatile gint32 *dest)
 {
 	gint32 ret;
@@ -105,6 +153,29 @@ gint32 InterlockedIncrement(volatile gint32 *dest)
 	return(ret);
 }
 
+gint64 InterlockedIncrement64(volatile gint64 *dest)
+{
+	gint64 ret;
+	int thr_ret;
+
+	mono_once(&spin_once, spin_init);
+
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	(*dest)++;
+	ret= *dest;
+
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+
+	pthread_cleanup_pop (0);
+
+	return(ret);
+}
+
 gint32 InterlockedDecrement(volatile gint32 *dest)
 {
 	gint32 ret;
@@ -128,6 +199,29 @@ gint32 InterlockedDecrement(volatile gint32 *dest)
 	return(ret);
 }
 
+gint64 InterlockedDecrement64(volatile gint64 *dest)
+{
+	gint64 ret;
+	int thr_ret;
+
+	mono_once(&spin_once, spin_init);
+
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	(*dest)--;
+	ret= *dest;
+
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+
+	pthread_cleanup_pop (0);
+
+	return(ret);
+}
+
 gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 {
 	gint32 ret;
@@ -151,6 +245,29 @@ gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 	return(ret);
 }
 
+gint64 InterlockedExchange64(volatile gint64 *dest, gint64 exch)
+{
+	gint64 ret;
+	int thr_ret;
+
+	mono_once(&spin_once, spin_init);
+
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	ret=*dest;
+	*dest=exch;
+
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+
+	pthread_cleanup_pop (0);
+
+	return(ret);
+}
+
 gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
 {
 	gpointer ret;
@@ -197,40 +314,116 @@ gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 	return(ret);
 }
 
-#endif
+gint64 InterlockedExchangeAdd64(volatile gint64 *dest, gint64 add)
+{
+	gint64 ret;
+	int thr_ret;
+	
+	mono_once(&spin_once, spin_init);
+	
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
 
-#ifndef HAS_64BITS_ATOMICS
+	ret= *dest;
+	*dest+=add;
+	
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
 
-#if defined (TARGET_MACH) && defined (TARGET_ARM) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7S__))
+	pthread_cleanup_pop (0);
 
-gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)  __attribute__ ((naked));
+	return(ret);
+}
 
-gint64
-InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
+gint32 InterlockedRead(volatile gint32 *src)
 {
-	__asm__ (
-	"push {r4, r5, r6, r7}\n"
-	"ldr r4, [sp, #16]\n"
-	"dmb\n"
-"1:\n"
-	"ldrexd	r6, r7, [r0]\n"
-	"cmp	r7, r4\n"
-	"bne 2f\n"
-	"cmp	r6, r3\n"
-	"bne	2f\n"
-	"strexd	r5, r1, r2, [r0]\n"
-	"cmp	r5, #0\n"
-	"bne	1b\n"
-"2:\n"
-	"dmb\n"
-	"mov	r0, r6\n"
-	"mov	r1, r7\n"
-	"pop {r4, r5, r6, r7}\n"
-	"bx	lr\n"
-	);
+	gint32 ret;
+	int thr_ret;
+	
+	mono_once(&spin_once, spin_init);
+	
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	ret= *src;
+	
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+
+	pthread_cleanup_pop (0);
+
+	return(ret);
 }
 
-#elif defined (TARGET_MACH) && (defined (TARGET_X86) || defined (TARGET_AMD64))
+gint64 InterlockedRead64(volatile gint64 *src)
+{
+	gint64 ret;
+	int thr_ret;
+	
+	mono_once(&spin_once, spin_init);
+	
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	ret= *src;
+	
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+
+	pthread_cleanup_pop (0);
+
+	return(ret);
+}
+
+void InterlockedWrite(volatile gint32 *dst, gint32 val)
+{
+	int thr_ret;
+	
+	mono_once(&spin_once, spin_init);
+	
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	*dst=val;
+	
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+	
+	pthread_cleanup_pop (0);
+}
+
+void InterlockedWrite64(volatile gint64 *dst, gint64 val)
+{
+	int thr_ret;
+	
+	mono_once(&spin_once, spin_init);
+	
+	pthread_cleanup_push ((void(*)(void *))pthread_mutex_unlock,
+			      (void *)&spin);
+	thr_ret = pthread_mutex_lock(&spin);
+	g_assert (thr_ret == 0);
+
+	*dst=val;
+	
+	thr_ret = pthread_mutex_unlock(&spin);
+	g_assert (thr_ret == 0);
+	
+	pthread_cleanup_pop (0);
+}
+
+#endif
+
+#if defined (NEED_64BIT_CMPXCHG_FALLBACK)
+
+#if defined (TARGET_OSX)
 
 gint64
 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
@@ -238,6 +431,12 @@ InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
 	return __sync_val_compare_and_swap (dest, comp, exch);
 }
 
+#elif defined (HAVE_64BIT_CMPXCHG_FALLBACK)
+
+#ifdef ENABLE_EXTENSION_MODULE
+#include "../../../mono-extensions/mono/utils/atomic.c"
+#endif
+
 #else
 
 gint64
@@ -256,4 +455,5 @@ InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
 }
 
 #endif
+
 #endif
diff --git a/mono/utils/atomic.h b/mono/utils/atomic.h
index 8e57f5d116..151248ae14 100755
--- a/mono/utils/atomic.h
+++ b/mono/utils/atomic.h
@@ -11,16 +11,6 @@
 #ifndef _WAPI_ATOMIC_H_
 #define _WAPI_ATOMIC_H_
 
-#if defined(__NetBSD__)
-#include <sys/param.h>
-
-#if __NetBSD_Version__ > 499004000
-#include <sys/atomic.h>
-#define HAVE_ATOMIC_OPS
-#endif
-
-#endif
-
 #include "config.h"
 #include <glib.h>
 
@@ -32,7 +22,6 @@
 #if defined(__WIN32__) || defined(_WIN32)
 
 #include <windows.h>
-#define HAS_64BITS_ATOMICS 1
 
 /* mingw is missing InterlockedCompareExchange64 () from winbase.h */
 #ifdef __MINGW32__
@@ -42,6 +31,30 @@ static inline gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64
 }
 #endif
 
+/* And now for some dirty hacks... The Windows API doesn't
+ * provide any useful primitives for this (other than getting
+ * into architecture-specific madness), so use CAS. */
+
+static inline gint32 InterlockedRead(volatile gint32 *src)
+{
+	return InterlockedCompareExchange (src, 0, 0);
+}
+
+static inline gint64 InterlockedRead64(volatile gint64 *src)
+{
+	return InterlockedCompareExchange64 (src, 0, 0);
+}
+
+static inline void InterlockedWrite(volatile gint32 *dst, gint32 val)
+{
+	InterlockedExchange (dst, val);
+}
+
+static inline void InterlockedWrite64(volatile gint64 *dst, gint64 val)
+{
+	InterlockedExchange64 (dst, val);
+}
+
 /* Prefer GCC atomic ops if the target supports it (see configure.in). */
 #elif defined(USE_GCC_ATOMIC_OPS)
 
@@ -56,6 +69,11 @@ static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest
 	return __sync_val_compare_and_swap (dest, comp, exch);
 }
 
+static inline gint32 InterlockedAdd(volatile gint32 *dest, gint32 add)
+{
+	return __sync_add_and_fetch (dest, add);
+}
+
 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 {
 	return __sync_add_and_fetch (val, 1);
@@ -63,7 +81,7 @@ static inline gint32 InterlockedIncrement(volatile gint32 *val)
 
 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 {
-	return __sync_add_and_fetch (val, -1);
+	return __sync_sub_and_fetch (val, 1);
 }
 
 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
@@ -90,59 +108,127 @@ static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 	return __sync_fetch_and_add (val, add);
 }
 
-#if defined (TARGET_OSX)
+static inline gint32 InterlockedRead(volatile gint32 *src)
+{
+	/* Kind of a hack, but GCC doesn't give us anything better, and it's
+	   certainly not as bad as using a CAS loop. */
+	return __sync_fetch_and_add (src, 0);
+}
+
+static inline void InterlockedWrite(volatile gint32 *dst, gint32 val)
+{
+	/* Nothing useful from GCC at all, so fall back to CAS. */
+	InterlockedExchange (dst, val);
+}
+
+#if defined (TARGET_OSX) || defined (__arm__) || (defined (__mips__) && !defined (__mips64)) || (defined (__powerpc__) && !defined (__powerpc64__))
 #define BROKEN_64BIT_ATOMICS_INTRINSIC 1
 #endif
 
-
 #if !defined (BROKEN_64BIT_ATOMICS_INTRINSIC)
-#define HAS_64BITS_ATOMICS 1
 
 static inline gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
 {
 	return __sync_val_compare_and_swap (dest, comp, exch);
 }
 
-#endif
+static inline gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
+{
+	return __sync_add_and_fetch (dest, add);
+}
 
+static inline gint64 InterlockedIncrement64(volatile gint64 *val)
+{
+	return __sync_add_and_fetch (val, 1);
+}
 
-#elif defined(__NetBSD__) && defined(HAVE_ATOMIC_OPS)
+static inline gint64 InterlockedDecrement64(volatile gint64 *val)
+{
+	return __sync_sub_and_fetch (val, 1);
+}
 
-static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
-       gint32 exch, gint32 comp)
+static inline gint64 InterlockedExchangeAdd64(volatile gint64 *val, gint64 add)
 {
-       return atomic_cas_32((uint32_t*)dest, comp, exch);
+	return __sync_fetch_and_add (val, add);
 }
 
-static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
+static inline gint64 InterlockedRead64(volatile gint64 *src)
 {
-       return atomic_cas_ptr(dest, comp, exch);
+	/* Kind of a hack, but GCC doesn't give us anything better. */
+	return __sync_fetch_and_add (src, 0);
 }
 
-static inline gint32 InterlockedIncrement(volatile gint32 *val)
+#else
+
+/* Implement 64-bit cmpxchg by hand or emulate it. */
+extern gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp);
+
+/* Implement all other 64-bit atomics in terms of a specialized CAS
+ * in this case, since chances are that the other 64-bit atomic
+ * intrinsics are broken too.
+ */
+
+static inline gint64 InterlockedExchangeAdd64(volatile gint64 *dest, gint64 add)
 {
-       return atomic_inc_32_nv((uint32_t*)val);
+	gint64 old_val;
+	do {
+		old_val = *dest;
+	} while (InterlockedCompareExchange64 (dest, old_val + add, old_val) != old_val);
+	return old_val;
 }
 
-static inline gint32 InterlockedDecrement(volatile gint32 *val)
+static inline gint64 InterlockedIncrement64(volatile gint64 *val)
 {
-       return atomic_dec_32_nv((uint32_t*)val);
+	gint64 get, set;
+	do {
+		get = *val;
+		set = get + 1;
+	} while (InterlockedCompareExchange64 (val, set, get) != get);
+	return set;
 }
 
-static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
+static inline gint64 InterlockedDecrement64(volatile gint64 *val)
+{
+	gint64 get, set;
+	do {
+		get = *val;
+		set = get - 1;
+	} while (InterlockedCompareExchange64 (val, set, get) != get);
+	return set;
+}
+
+static inline gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
 {
-       return atomic_swap_32((uint32_t*)val, new_val);
+	gint64 get, set;
+	do {
+		get = *dest;
+		set = get + add;
+	} while (InterlockedCompareExchange64 (dest, set, get) != get);
+	return set;
 }
 
-static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
-               gpointer new_val)
+static inline gint64 InterlockedRead64(volatile gint64 *src)
 {
-       return atomic_swap_ptr(val, new_val);
+	return InterlockedCompareExchange64 (src, 0, 0);
 }
 
-static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
+#endif
+
+/* We always implement this in terms of a 64-bit cmpxchg since
+ * GCC doesn't have an intrisic to model it anyway. */
+static inline gint64 InterlockedExchange64(volatile gint64 *val, gint64 new_val)
+{
+	gint64 old_val;
+	do {
+		old_val = *val;
+	} while (InterlockedCompareExchange64 (val, new_val, old_val) != old_val);
+	return old_val;
+}
+
+static inline void InterlockedWrite64(volatile gint64 *dst, gint64 val)
 {
-       return atomic_add_32_nv((uint32_t*)val, add) - add;
+	/* Nothing useful from GCC at all, so fall back to CAS. */
+	InterlockedExchange64 (dst, val);
 }
 
 #elif (defined(sparc) || defined (__sparc__)) && defined(__GNUC__)
@@ -305,126 +391,6 @@ static inline gint32 InterlockedExchangeAdd(volatile gint32 *_dest, gint32 add)
         return ret;
 }
 
-#elif __s390x__
-
-static inline gint32 
-InterlockedCompareExchange(volatile gint32 *dest,
-			   gint32 exch, gint32 comp)
-{
-	gint32 old;
-
-	__asm__ __volatile__ ("\tLA\t1,%0\n"
-			      "\tLR\t%1,%3\n"
-			      "\tCS\t%1,%2,0(1)\n"
-			      : "+m" (*dest), "=&r" (old)
-			      : "r" (exch), "r" (comp)
-			      : "1", "cc");	
-	return(old);
-}
-
-static inline gpointer 
-InterlockedCompareExchangePointer(volatile gpointer *dest, 
-				  gpointer exch, 
-			          gpointer comp)
-{
-	gpointer old;
-
-	__asm__ __volatile__ ("\tLA\t1,%0\n"
-			      "\tLGR\t%1,%3\n"
-			      "\tCSG\t%1,%2,0(1)\n"
-			      : "+m" (*dest), "=&r" (old)
-			      : "r" (exch), "r" (comp)
-			      : "1", "cc");
-
-	return(old);
-}
-
-static inline gint32 
-InterlockedIncrement(volatile gint32 *val)
-{
-	gint32 tmp;
-	
-	__asm__ __volatile__ ("\tLA\t2,%1\n"
-			      "0:\tLGF\t%0,%1\n"
-			      "\tLGFR\t1,%0\n"
-			      "\tAGHI\t1,1\n"
-			      "\tCS\t%0,1,0(2)\n"
-			      "\tJNZ\t0b\n"
-			      "\tLGFR\t%0,1"
-			      : "=r" (tmp), "+m" (*val)
-			      : : "1", "2", "cc");
-
-	return(tmp);
-}
-
-static inline gint32 
-InterlockedDecrement(volatile gint32 *val)
-{
-	gint32 tmp;
-	
-	__asm__ __volatile__ ("\tLA\t2,%1\n"
-			      "0:\tLGF\t%0,%1\n"
-			      "\tLGFR\t1,%0\n"
-			      "\tAGHI\t1,-1\n"
-			      "\tCS\t%0,1,0(2)\n"
-			      "\tJNZ\t0b\n"
-			      "\tLGFR\t%0,1"
-			      : "=r" (tmp), "+m" (*val)
-			      : : "1", "2", "cc");
-
-	return(tmp);
-}
-
-static inline gint32 
-InterlockedExchange(volatile gint32 *val, gint32 new_val)
-{
-	gint32 ret;
-	
-	__asm__ __volatile__ ("\tLA\t1,%0\n"
-			      "0:\tL\t%1,%0\n"
-			      "\tCS\t%1,%2,0(1)\n"
-			      "\tJNZ\t0b"
-			      : "+m" (*val), "=&r" (ret)
-			      : "r" (new_val)
-			      : "1", "cc");
-
-	return(ret);
-}
-
-static inline gpointer
-InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
-{
-	gpointer ret;
-	
-	__asm__ __volatile__ ("\tLA\t1,%0\n"
-			      "0:\tLG\t%1,%0\n"
-			      "\tCSG\t%1,%2,0(1)\n"
-			      "\tJNZ\t0b"
-			      : "+m" (*val), "=&r" (ret)
-			      : "r" (new_val)
-			      : "1", "cc");
-
-	return(ret);
-}
-
-static inline gint32 
-InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
-{
-	gint32 ret;
-
-	__asm__ __volatile__ ("\tLA\t2,%1\n"
-			      "0:\tLGF\t%0,%1\n"
-			      "\tLGFR\t1,%0\n"
-			      "\tAGR\t1,%2\n"
-			      "\tCS\t%0,1,0(2)\n"
-			      "\tJNZ\t0b"
-			      : "=&r" (ret), "+m" (*val)
-			      : "r" (add) 
-			      : "1", "2", "cc");
-	
-	return(ret);
-}
-
 #elif defined(__ia64__)
 
 #ifdef __INTEL_COMPILER
@@ -546,17 +512,24 @@ static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
 #define WAPI_NO_ATOMIC_ASM
 
 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
+extern gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp);
 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
+extern gint32 InterlockedAdd(volatile gint32 *dest, gint32 add);
+extern gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add);
 extern gint32 InterlockedIncrement(volatile gint32 *dest);
+extern gint64 InterlockedIncrement64(volatile gint64 *dest);
 extern gint32 InterlockedDecrement(volatile gint32 *dest);
+extern gint64 InterlockedDecrement64(volatile gint64 *dest);
 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
+extern gint64 InterlockedExchange64(volatile gint64 *dest, gint64 exch);
 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
+extern gint64 InterlockedExchangeAdd64(volatile gint64 *dest, gint64 add);
+extern gint32 InterlockedRead(volatile gint32 *src);
+extern gint64 InterlockedRead64(volatile gint64 *src);
+extern void InterlockedWrite(volatile gint32 *dst, gint32 val);
+extern void InterlockedWrite64(volatile gint64 *dst, gint64 val);
 
 #endif
 
-#ifndef HAS_64BITS_ATOMICS
-extern gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp);
-#endif
-
 #endif /* _WAPI_ATOMIC_H_ */
diff --git a/msvc/mono.def b/msvc/mono.def
index 345a71e08b..7a264b225f 100644
--- a/msvc/mono.def
+++ b/msvc/mono.def
@@ -331,6 +331,7 @@ mono_gc_walk_heap
 mono_gc_wbarrier_arrayref_copy
 mono_gc_wbarrier_generic_nostore
 mono_gc_wbarrier_generic_store
+mono_gc_wbarrier_generic_store_atomic
 mono_gc_wbarrier_object_copy
 mono_gc_wbarrier_set_arrayref
 mono_gc_wbarrier_set_field