Add type-specialized vectorized hash table container (#100386)

This PR adds a type-specialized, vectorized (using 128-bit SIMD) hash table container, and migrates one part of the mono runtime to use it instead of GHashTable. It also adds a basic test suite and basic benchmark suite. Vectorization is not enabled for it in the WASM build yet because we need to make changes to the build there. It is also not vectorized for ARM MSVC.

src/mono/mono/metadata/CMakeLists.txt

@@ -42,6 +42,11 @@ else()
 set(metadata_platform_sources ${metadata_unix_sources})
 endif()
 
+set(imported_native_sources
+    ../../../native/containers/dn-simdhash.c
+    ../../../native/containers/dn-simdhash-string-ptr.c
+    ../../../native/containers/dn-simdhash-u32-ptr.c)
+
 set(metadata_common_sources
     appdomain.c
     domain.c
@@ -195,7 +200,7 @@ elseif(MONO_GC STREQUAL "boehm")
     set(metadata_compile_definitions "HAVE_BOEHM_GC")
 endif()
 
-set(metadata_sources "${metadata_platform_sources};${metadata_common_sources};${metadata_gc_dependent_sources};${metadata_gc_sources};${ilgen_sources}")
+set(metadata_sources "${metadata_platform_sources};${metadata_common_sources};${metadata_gc_dependent_sources};${metadata_gc_sources};${ilgen_sources};${imported_native_sources}")
 
 if(HOST_WIN32 AND NOT DISABLE_SHARED_LIBS)
     add_library(metadata_objects_shared OBJECT ${metadata_sources})

src/mono/mono/metadata/class.c

@@ -3037,18 +3037,22 @@ mono_image_init_name_cache (MonoImage *image)
 	const char *name;
 	const char *nspace;
 	guint32 visib, nspace_index;
-	GHashTable *name_cache2, *nspace_table, *the_name_cache;
+	dn_simdhash_u32_ptr_t *name_cache2;
+	dn_simdhash_string_ptr_t *nspace_table, *the_name_cache;
 
 	if (image->name_cache)
 		return;
 
-	the_name_cache = g_hash_table_new (g_str_hash, g_str_equal);
+	// TODO: Figure out a good initial capacity for this table by doing a scan,
+	//  or just pre-reserve a reasonable amount of space based on how many nspaces
+	//  an image typically has
+	the_name_cache = dn_simdhash_string_ptr_new (0, NULL);
 
 	if (image_is_dynamic (image)) {
 		mono_image_lock (image);
 		if (image->name_cache) {
 			/* Somebody initialized it before us */
-			g_hash_table_destroy (the_name_cache);
+			dn_simdhash_free (the_name_cache);
 		} else {
 			mono_atomic_store_release (&image->name_cache, the_name_cache);
 		}
@@ -3057,7 +3061,7 @@ mono_image_init_name_cache (MonoImage *image)
 	}
 
 	/* Temporary hash table to avoid lookups in the nspace_table */
-	name_cache2 = g_hash_table_new (NULL, NULL);
+	name_cache2 = dn_simdhash_u32_ptr_new (0, NULL);
 
 	/* FIXME: metadata-update */
 	int rows = table_info_get_rows (t);
@@ -3074,14 +3078,13 @@ mono_image_init_name_cache (MonoImage *image)
 		nspace = mono_metadata_string_heap (image, cols [MONO_TYPEDEF_NAMESPACE]);
 
 		nspace_index = cols [MONO_TYPEDEF_NAMESPACE];
-		nspace_table = (GHashTable *)g_hash_table_lookup (name_cache2, GUINT_TO_POINTER (nspace_index));
-		if (!nspace_table) {
-			nspace_table = g_hash_table_new (g_str_hash, g_str_equal);
-			g_hash_table_insert (the_name_cache, (char*)nspace, nspace_table);
-			g_hash_table_insert (name_cache2, GUINT_TO_POINTER (nspace_index),
-								 nspace_table);
+		if (!dn_simdhash_u32_ptr_try_get_value (name_cache2, nspace_index, (void **)&nspace_table)) {
+			// FIXME: Compute an appropriate capacity for this table to avoid growing it
+			nspace_table = dn_simdhash_string_ptr_new (0, NULL);
+			dn_simdhash_string_ptr_try_add (the_name_cache, nspace, nspace_table);
+			dn_simdhash_u32_ptr_try_add (name_cache2, nspace_index, nspace_table);
 		}
-		g_hash_table_insert (nspace_table, (char *) name, GUINT_TO_POINTER (i));
+		dn_simdhash_string_ptr_try_add (nspace_table, name, GUINT_TO_POINTER (i));
 	}
 
 	/* Load type names from EXPORTEDTYPES table */
@@ -3102,23 +3105,22 @@ mono_image_init_name_cache (MonoImage *image)
 			nspace = mono_metadata_string_heap (image, exptype_cols [MONO_EXP_TYPE_NAMESPACE]);
 
 			nspace_index = exptype_cols [MONO_EXP_TYPE_NAMESPACE];
-			nspace_table = (GHashTable *)g_hash_table_lookup (name_cache2, GUINT_TO_POINTER (nspace_index));
-			if (!nspace_table) {
-				nspace_table = g_hash_table_new (g_str_hash, g_str_equal);
-				g_hash_table_insert (the_name_cache, (char*)nspace, nspace_table);
-				g_hash_table_insert (name_cache2, GUINT_TO_POINTER (nspace_index),
-									 nspace_table);
+			if (!dn_simdhash_u32_ptr_try_get_value (name_cache2, nspace_index, (void **)&nspace_table)) {
+				// FIXME: Compute an appropriate capacity for this table to avoid growing it
+				nspace_table = dn_simdhash_string_ptr_new (0, NULL);
+				dn_simdhash_string_ptr_try_add (the_name_cache, nspace, nspace_table);
+				dn_simdhash_u32_ptr_try_add (name_cache2, nspace_index, nspace_table);
 			}
-			g_hash_table_insert (nspace_table, (char *) name, GUINT_TO_POINTER (mono_metadata_make_token (MONO_TABLE_EXPORTEDTYPE, i + 1)));
+			dn_simdhash_string_ptr_try_add (nspace_table, name, GUINT_TO_POINTER (mono_metadata_make_token (MONO_TABLE_EXPORTEDTYPE, i + 1)));
 		}
 	}
 
-	g_hash_table_destroy (name_cache2);
+	dn_simdhash_free (name_cache2);
 
 	mono_image_lock (image);
 	if (image->name_cache) {
 		/* Somebody initialized it before us */
-		g_hash_table_destroy (the_name_cache);
+		dn_simdhash_free (the_name_cache);
 	} else {
 		mono_atomic_store_release (&image->name_cache, the_name_cache);
 	}
@@ -3133,23 +3135,19 @@ void
 mono_image_add_to_name_cache (MonoImage *image, const char *nspace,
 							  const char *name, guint32 index)
 {
-	GHashTable *nspace_table;
-	GHashTable *name_cache;
-	guint32 old_index;
+	dn_simdhash_string_ptr_t *nspace_table, *name_cache;
 
 	mono_image_init_name_cache (image);
 	mono_image_lock (image);
 
 	name_cache = image->name_cache;
-	if (!(nspace_table = (GHashTable *)g_hash_table_lookup (name_cache, nspace))) {
-		nspace_table = g_hash_table_new (g_str_hash, g_str_equal);
-		g_hash_table_insert (name_cache, (char *)nspace, (char *)nspace_table);
+	if (!dn_simdhash_string_ptr_try_get_value (name_cache, nspace, (void **)&nspace_table)) {
+		nspace_table = dn_simdhash_string_ptr_new (0, NULL);
+		dn_simdhash_string_ptr_try_add (name_cache, nspace, nspace_table);
 	}
 
-	if ((old_index = GPOINTER_TO_UINT (g_hash_table_lookup (nspace_table, (char*) name))))
-		g_error ("overrwritting old token %x on image %s for type %s::%s", old_index, image->name, nspace, name);
-
-	g_hash_table_insert (nspace_table, (char *) name, GUINT_TO_POINTER (index));
+	if (!dn_simdhash_string_ptr_try_add (nspace_table, name, GUINT_TO_POINTER (index)))
+		g_error ("overrwritting old token ? on image %s for type %s::%s", image->name, nspace, name);
 
 	mono_image_unlock (image);
 }
@@ -3160,9 +3158,8 @@ typedef struct {
 } FindAllUserData;
 
 static void
-find_all_nocase (gpointer key, gpointer value, gpointer user_data)
+find_all_nocase (const char *name, gpointer value, gpointer user_data)
 {
-	char *name = (char*)key;
 	FindAllUserData *data = (FindAllUserData*)user_data;
 	if (mono_utf8_strcasecmp (name, (char*)data->key) == 0)
 		data->values = g_slist_prepend (data->values, value);
@@ -3174,9 +3171,8 @@ typedef struct {
 } FindUserData;
 
 static void
-find_nocase (gpointer key, gpointer value, gpointer user_data)
+find_nocase (const char *name, gpointer value, gpointer user_data)
 {
-	char *name = (char*)key;
 	FindUserData *data = (FindUserData*)user_data;
 
 	if (!data->value && (mono_utf8_strcasecmp (name, (char*)data->key) == 0))
@@ -3303,7 +3299,7 @@ search_modules (MonoImage *image, const char *name_space, const char *name, gboo
 static MonoClass *
 mono_class_from_name_checked_aux (MonoImage *image, const char* name_space, const char *name, GHashTable* visited_images, gboolean case_sensitive, MonoError *error)
 {
-	GHashTable *nspace_table = NULL;
+	dn_simdhash_string_ptr_t *nspace_table = NULL;
 	MonoImage *loaded_image = NULL;
 	guint32 token = 0;
 	MonoClass *klass;
@@ -3350,23 +3346,24 @@ mono_class_from_name_checked_aux (MonoImage *image, const char* name_space, cons
 	mono_image_lock (image);
 
 	if (case_sensitive) {
-		nspace_table = (GHashTable *)g_hash_table_lookup (image->name_cache, name_space);
-
-		if (nspace_table)
-			token = GPOINTER_TO_UINT (g_hash_table_lookup (nspace_table, name));
+		if (dn_simdhash_string_ptr_try_get_value (image->name_cache, name_space, (void **)&nspace_table)) {
+			void * temp;
+			if (dn_simdhash_string_ptr_try_get_value (nspace_table, name, &temp))
+				token = GPOINTER_TO_UINT(temp);
+		}
 	} else {
 		FindAllUserData all_user_data = { name_space, NULL };
 		FindUserData user_data = { name, NULL };
 		GSList *values;
 
 		// We're forced to check all matching namespaces, not just the first one found,
 		// because our desired type could be in any of the ones that match case-insensitively.
-		g_hash_table_foreach (image->name_cache, find_all_nocase, &all_user_data);
+		dn_simdhash_string_ptr_foreach (image->name_cache, find_all_nocase, &all_user_data);
 
 		values = all_user_data.values;
 		while (values && !user_data.value) {
-			nspace_table = (GHashTable*)values->data;
-			g_hash_table_foreach (nspace_table, find_nocase, &user_data);
+			nspace_table = (dn_simdhash_string_ptr_t *)values->data;
+			dn_simdhash_string_ptr_foreach (nspace_table, find_nocase, &user_data);
 			values = values->next;
 		}
 

src/mono/mono/metadata/image.c

@@ -1969,6 +1969,12 @@ free_hash_table (gpointer key, gpointer val, gpointer user_data)
 	g_hash_table_destroy ((GHashTable*)val);
 }
 
+static void
+free_simdhash_table (const char *key, gpointer val, gpointer user_data)
+{
+	dn_simdhash_free ((dn_simdhash_t*)val);
+}
+
 /*
 static void
 free_mr_signatures (gpointer key, gpointer val, gpointer user_data)
@@ -2128,8 +2134,8 @@ mono_image_close_except_pools (MonoImage *image)
 	if (image->ptr_cache)
 		g_hash_table_destroy (image->ptr_cache);
 	if (image->name_cache) {
-		g_hash_table_foreach (image->name_cache, free_hash_table, NULL);
-		g_hash_table_destroy (image->name_cache);
+		dn_simdhash_string_ptr_foreach (image->name_cache, free_simdhash_table, NULL);
+		dn_simdhash_free (image->name_cache);
 	}
 
 	free_hash (image->icall_wrapper_cache);

src/mono/mono/metadata/metadata-internals.h

@@ -19,6 +19,8 @@
 #include <mono/utils/mono-error.h>
 #include "mono/utils/mono-conc-hashtable.h"
 #include "mono/utils/refcount.h"
+// for dn_simdhash_string_ptr_t and dn_simdhash_u32_ptr_t
+#include "../native/containers/dn-simdhash-specializations.h"
 
 struct _MonoType {
 	union {
@@ -438,7 +440,7 @@ struct _MonoImage {
 	/*
 	 * Indexes namespaces to hash tables that map class name to typedef token.
 	 */
-	GHashTable *name_cache;  /*protected by the image lock*/
+	dn_simdhash_string_ptr_t *name_cache;  /*protected by the image lock*/
 
 	/*
 	 * Indexed by MonoClass

src/mono/mono/metadata/metadata.c

@@ -997,10 +997,10 @@ mono_metadata_table_bounds_check_slow (MonoImage *image, int table_index, int to
 	if (G_LIKELY (GINT_TO_UINT32(token_index) <= table_info_get_rows (&image->tables [table_index])))
 		return FALSE;
 
-        if (G_LIKELY (!image->has_updates))
-                return TRUE;
+	if (G_LIKELY (!image->has_updates))
+		return TRUE;
 
-        return mono_metadata_update_table_bounds_check (image, table_index, token_index);
+	return mono_metadata_update_table_bounds_check (image, table_index, token_index);
 }
 
 void
@@ -1094,7 +1094,7 @@ get_blob_heap (MonoImage *image)
 static gboolean
 mono_delta_heap_lookup (MonoImage *base_image, MetadataHeapGetterFunc get_heap, guint32 orig_index, MonoImage **image_out, guint32 *index_out)
 {
-        return mono_metadata_update_delta_heap_lookup (base_image, get_heap, orig_index, image_out, index_out);
+	return mono_metadata_update_delta_heap_lookup (base_image, get_heap, orig_index, image_out, index_out);
 }
 
 /**
@@ -6451,12 +6451,12 @@ mono_metadata_events_from_typedef (MonoImage *meta, guint32 index, guint *end_id
 	}
 
 	start = mono_metadata_decode_row_col (tdef, loc.result, MONO_EVENT_MAP_EVENTLIST);
-        /*
-         * metadata-update: note this next line needs block needs to look at the number of rows in
-         * EventMap and Event of the base image.  Updates will add rows for new properties,
-         * but they won't be contiguous.  if we set end to the number of rows in the updated
-         * Property table, the range will include properties from some other class
-         */
+	/*
+	 * metadata-update: note this next line needs block needs to look at the number of rows in
+	 * EventMap and Event of the base image.  Updates will add rows for new properties,
+	 * but they won't be contiguous.  if we set end to the number of rows in the updated
+	 * Property table, the range will include properties from some other class
+	 */
 	if (loc.result + 1 < table_info_get_rows (tdef)) {
 		end = mono_metadata_decode_row_col (tdef, loc.result + 1, MONO_EVENT_MAP_EVENTLIST) - 1;
 	} else {
@@ -6569,12 +6569,12 @@ mono_metadata_properties_from_typedef (MonoImage *meta, guint32 index, guint *en
 	}
 
 	start = mono_metadata_decode_row_col (tdef, loc.result, MONO_PROPERTY_MAP_PROPERTY_LIST);
-        /*
-         * metadata-update: note this next line needs block needs to look at the number of rows in
-         * PropertyMap and Property of the base image.  Updates will add rows for new properties,
-         * but they won't be contiguous.  if we set end to the number of rows in the updated
-         * Property table, the range will include properties from some other class
-         */
+	/*
+	 * metadata-update: note this next line needs block needs to look at the number of rows in
+	 * PropertyMap and Property of the base image.  Updates will add rows for new properties,
+	 * but they won't be contiguous.  if we set end to the number of rows in the updated
+	 * Property table, the range will include properties from some other class
+	 */
 	if (loc.result + 1 < table_info_get_rows (&meta->tables [MONO_TABLE_PROPERTYMAP])) {
 		end = mono_metadata_decode_row_col (tdef, loc.result + 1, MONO_PROPERTY_MAP_PROPERTY_LIST) - 1;
 	} else {
@@ -7088,10 +7088,10 @@ mono_metadata_get_marshal_info (MonoImage *meta, guint32 idx, gboolean is_field)
 
 	gboolean found = tdef->base && mono_binary_search (&loc, tdef->base, table_info_get_rows (tdef), tdef->row_size, table_locator);
 
-        if (G_UNLIKELY (meta->has_updates)) {
-                if (!found && !mono_metadata_update_metadata_linear_search (meta, tdef, &loc, table_locator))
-                        return NULL;
-        }
+	if (G_UNLIKELY (meta->has_updates)) {
+		if (!found && !mono_metadata_update_metadata_linear_search (meta, tdef, &loc, table_locator))
+			return NULL;
+	}
 
 	return mono_metadata_blob_heap (meta, mono_metadata_decode_row_col (tdef, loc.result, MONO_FIELD_MARSHAL_NATIVE_TYPE));
 }
@@ -8055,3 +8055,12 @@ mono_metadata_get_method_params (MonoImage *image, uint32_t method_idx, uint32_t
 
 	return param_index;
 }
+
+// Required by dn_simdhash
+void
+dn_simdhash_assert_fail (const char *file, int line, const char *condition);
+
+void
+dn_simdhash_assert_fail (const char *file, int line, const char *condition) {
+	mono_assertion_message (file, line, condition);
+}

src/mono/mono/utils/atomic.h

@@ -115,7 +115,7 @@ mono_atomic_cas_i64 (volatile gint64 *dest, gint64 exch, gint64 comp)
 	(void)atomic_compare_exchange_strong ((volatile atomic_llong *)dest, (long long*)&comp, exch);
 	return comp;
 #else
-#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
+#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
 #endif
 }
 
@@ -188,7 +188,7 @@ mono_atomic_xchg_i64 (volatile gint64 *dest, gint64 exch)
 	g_static_assert (sizeof (atomic_llong) == sizeof (*dest) && ATOMIC_LLONG_LOCK_FREE == 2);
 	return atomic_exchange ((volatile atomic_llong *)dest, exch);
 #else
-#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
+#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
 #endif
 }
 
@@ -216,7 +216,7 @@ mono_atomic_fetch_add_i64 (volatile gint64 *dest, gint64 add)
 	g_static_assert (sizeof (atomic_llong) == sizeof (*dest) && ATOMIC_LLONG_LOCK_FREE == 2);
 	return atomic_fetch_add ((volatile atomic_llong *)dest, add);
 #else
-#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
+#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
 #endif
 }
 
@@ -250,7 +250,7 @@ mono_atomic_load_i64 (volatile gint64 *src)
 	g_static_assert (sizeof (atomic_llong) == sizeof (*src) && ATOMIC_LLONG_LOCK_FREE == 2);
 	return atomic_load ((volatile atomic_llong *)src);
 #else
-#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
+#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
 #endif
 }
 
@@ -292,7 +292,7 @@ mono_atomic_store_i64 (volatile gint64 *dst, gint64 val)
 	g_static_assert (sizeof (atomic_llong) == sizeof (*dst) && ATOMIC_LLONG_LOCK_FREE == 2);
 	atomic_store ((volatile atomic_llong *)dst, val);
 #else
-#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
+#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
 #endif
 }