Backported my_atomic from 6.0-codebase and added support for 64-bit atomics to enable removal of LOCK_thread_count from every query, removed LOCK_thread_count from use in dispatch_command and close of query which is used in every query, now uses atomic increments/decrements instead

Mikael Ronstrom · Mikael Ronstrom · commit c4ce446f1925 · 2009-10-12T11:00:39.000+02:00
diff --git a/include/Makefile.am b/include/Makefile.am
@@ -38,7 +38,7 @@ noinst_HEADERS =	config-win.h config-netware.h my_bit.h \
 			thr_lock.h t_ctype.h violite.h my_md5.h base64.h \
 			my_handler.h my_time.h \
 			my_vle.h my_user.h my_atomic.h atomic/nolock.h \
-			atomic/rwlock.h atomic/x86-gcc.h atomic/x86-msvc.h \
+			atomic/rwlock.h atomic/x86-gcc.h atomic/generic-msvc.h \
                         atomic/solaris.h \
 			atomic/gcc_builtins.h my_libwrap.h my_stacktrace.h
 
diff --git a/include/atomic/rwlock.h b/include/atomic/rwlock.h
@@ -1,3 +1,6 @@
+#ifndef ATOMIC_RWLOCK_INCLUDED
+#define ATOMIC_RWLOCK_INCLUDED
+
 /* Copyright (C) 2006 MySQL AB
 
    This program is free software; you can redistribute it and/or modify
@@ -13,7 +16,8 @@
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
 
-typedef struct {pthread_rwlock_t rw;} my_atomic_rwlock_t;
+typedef struct {pthread_mutex_t rw;} my_atomic_rwlock_t;
+#define MY_ATOMIC_MODE_RWLOCKS 1
 
 #ifdef MY_ATOMIC_MODE_DUMMY
 /*
@@ -31,18 +35,27 @@ typedef struct {pthread_rwlock_t rw;} my_atomic_rwlock_t;
 #define my_atomic_rwlock_wrunlock(name)
 #define MY_ATOMIC_MODE "dummy (non-atomic)"
 #else
-#define my_atomic_rwlock_destroy(name)     pthread_rwlock_destroy(& (name)->rw)
-#define my_atomic_rwlock_init(name)        pthread_rwlock_init(& (name)->rw, 0)
-#define my_atomic_rwlock_rdlock(name)      pthread_rwlock_rdlock(& (name)->rw)
-#define my_atomic_rwlock_wrlock(name)      pthread_rwlock_wrlock(& (name)->rw)
-#define my_atomic_rwlock_rdunlock(name)    pthread_rwlock_unlock(& (name)->rw)
-#define my_atomic_rwlock_wrunlock(name)    pthread_rwlock_unlock(& (name)->rw)
-#define MY_ATOMIC_MODE "rwlocks"
+/*
+  we're using read-write lock macros but map them to mutex locks, and they're
+  faster. Still, having semantically rich API we can change the
+  underlying implementation, if necessary.
+*/
+#define my_atomic_rwlock_destroy(name)     pthread_mutex_destroy(& (name)->rw)
+#define my_atomic_rwlock_init(name)        pthread_mutex_init(& (name)->rw, 0)
+#define my_atomic_rwlock_rdlock(name)      pthread_mutex_lock(& (name)->rw)
+#define my_atomic_rwlock_wrlock(name)      pthread_mutex_lock(& (name)->rw)
+#define my_atomic_rwlock_rdunlock(name)    pthread_mutex_unlock(& (name)->rw)
+#define my_atomic_rwlock_wrunlock(name)    pthread_mutex_unlock(& (name)->rw)
+#define MY_ATOMIC_MODE "mutex"
+#ifndef MY_ATOMIC_MODE_RWLOCKS
+#define MY_ATOMIC_MODE_RWLOCKS 1
+#endif
 #endif
 
 #define make_atomic_add_body(S)     int ## S sav; sav= *a; *a+= v; v=sav;
-#define make_atomic_swap_body(S)    int ## S sav; sav= *a; *a= v; v=sav;
+#define make_atomic_fas_body(S)     int ## S sav; sav= *a; *a= v; v=sav;
 #define make_atomic_cas_body(S)     if ((ret= (*a == *cmp))) *a= set; else *cmp=*a;
 #define make_atomic_load_body(S)    ret= *a;
 #define make_atomic_store_body(S)   *a= v;
 
+#endif /* ATOMIC_RWLOCK_INCLUDED */
diff --git a/include/atomic/x86-gcc.h b/include/atomic/x86-gcc.h
@@ -42,15 +42,37 @@
 #endif
 
 #ifndef MY_ATOMIC_NO_XADD
-#define make_atomic_add_body(S)					\
-  asm volatile (LOCK_prefix "; xadd %0, %1;" : "+r" (v) , "+m" (*a))
+#define make_atomic_add_body(S)         make_atomic_add_body ## S
+#define make_atomic_cas_body(S)         make_atomic_cas_body ## S
 #endif
-#define make_atomic_fas_body(S)				\
-  asm volatile ("xchg %0, %1;" : "+q" (v) , "+m" (*a))
-#define make_atomic_cas_body(S)					\
+
+#define make_atomic_add_body32                                  \
+  asm volatile (LOCK_prefix "; xadd %0, %1;" : "+r" (v) , "+m" (*a))
+
+#define make_atomic_cas_body32                                  \
   asm volatile (LOCK_prefix "; cmpxchg %3, %0; setz %2;"	\
                : "+m" (*a), "+a" (*cmp), "=q" (ret): "r" (set))
 
+#define make_atomic_cas_bodyptr make_atomic_cas_body32
+
+#ifndef __x86_64__
+#define make_atomic_add_body64 make_atomic_add_body32
+#define make_atomic_cas_body64 make_atomic_cas_body32
+#else
+#define make_atomic_add_body64                                  \
+  int64 tmp=*a;                                                 \
+  while (!my_atomic_cas64(a, &tmp, tmp+v));                     \
+  v=tmp;
+#define make_atomic_cas_body64                                  \
+  int32 ebx=(set & 0xFFFFFFFF), ecx=(set >> 32);                \
+  asm volatile (LOCK_prefix "; cmpxchg8b %0; setz %2;"         \
+               : "+m" (*a), "+A" (*cmp), "=q" (ret)             \
+               :"b" (ebx), "c" (ecx))
+#endif
+
+#define make_atomic_fas_body(S)                                        \
+  asm volatile ("xchg %0, %1;" : "+r" (v) , "+m" (*a))
+
 #ifdef MY_ATOMIC_MODE_DUMMY
 #define make_atomic_load_body(S)   ret=*a
 #define make_atomic_store_body(S)  *a=v
diff --git a/include/my_atomic.h b/include/my_atomic.h
@@ -37,7 +37,7 @@
   my_atomic_store#(&var, what)
     store 'what' in *var
 
-  '#' is substituted by a size suffix - 8, 16, 32, or ptr
+  '#' is substituted by a size suffix - 8, 16, 32, 64, or ptr
   (e.g. my_atomic_add8, my_atomic_fas32, my_atomic_casptr).
 
   NOTE This operations are not always atomic, so they always must be
@@ -129,6 +129,7 @@
 make_transparent_unions(8)
 make_transparent_unions(16)
 make_transparent_unions(32)
+make_transparent_unions(64)
 make_transparent_unions(ptr)
 #undef uintptr
 #undef make_transparent_unions
@@ -140,10 +141,12 @@ make_transparent_unions(ptr)
 #define U_8    int8
 #define U_16   int16
 #define U_32   int32
+#define U_64   int64
 #define U_ptr  intptr
 #define Uv_8   int8
 #define Uv_16  int16
 #define Uv_32  int32
+#define Uv_64  int64
 #define Uv_ptr intptr
 #define U_a    volatile *a
 #define U_cmp  *cmp
@@ -217,33 +220,38 @@ make_atomic_cas(8)
 make_atomic_cas(16)
 #endif
 make_atomic_cas(32)
+make_atomic_cas(64)
 make_atomic_cas(ptr)
 
 #ifdef MY_ATOMIC_HAS_8_16
 make_atomic_add(8)
 make_atomic_add(16)
 #endif
 make_atomic_add(32)
+make_atomic_add(64)
 
 #ifdef MY_ATOMIC_HAS_8_16
 make_atomic_load(8)
 make_atomic_load(16)
 #endif
 make_atomic_load(32)
+make_atomic_load(64)
 make_atomic_load(ptr)
 
 #ifdef MY_ATOMIC_HAS_8_16
 make_atomic_fas(8)
 make_atomic_fas(16)
 #endif
 make_atomic_fas(32)
+make_atomic_fas(64)
 make_atomic_fas(ptr)
 
 #ifdef MY_ATOMIC_HAS_8_16
 make_atomic_store(8)
 make_atomic_store(16)
 #endif
 make_atomic_store(32)
+make_atomic_store(64)
 make_atomic_store(ptr)
 
 #ifdef _atomic_h_cleanup_
@@ -254,10 +262,12 @@ make_atomic_store(ptr)
 #undef U_8
 #undef U_16
 #undef U_32
+#undef U_64
 #undef U_ptr
 #undef Uv_8
 #undef Uv_16
 #undef Uv_32
+#undef Uv_64
 #undef Uv_ptr
 #undef a
 #undef cmp
diff --git a/include/my_global.h b/include/my_global.h
@@ -832,6 +832,8 @@ typedef SOCKET_SIZE_TYPE size_socket;
 #endif
 #endif /* defined (HAVE_LONG_LONG) && !defined(ULONGLONG_MAX)*/
 
+#define INT_MIN64       (~0x7FFFFFFFFFFFFFFFLL)
+#define INT_MAX64       0x7FFFFFFFFFFFFFFFLL
 #define INT_MIN32       (~0x7FFFFFFFL)
 #define INT_MAX32       0x7FFFFFFFL
 #define UINT_MAX32      0xFFFFFFFFL
diff --git a/sql/event_scheduler.cc b/sql/event_scheduler.cc
@@ -132,9 +132,10 @@ post_init_event_thread(THD *thd)
   pthread_mutex_lock(&LOCK_thread_count);
   threads.append(thd);
   thread_count++;
-  thread_running++;
   pthread_mutex_unlock(&LOCK_thread_count);
-
+  my_atomic_rwlock_wrlock(&global_query_id_lock);
+  inc_thread_running();
+  my_atomic_rwlock_wrunlock(&global_query_id_lock);
   return FALSE;
 }
 
@@ -156,10 +157,12 @@ deinit_event_thread(THD *thd)
   DBUG_PRINT("exit", ("Event thread finishing"));
   pthread_mutex_lock(&LOCK_thread_count);
   thread_count--;
-  thread_running--;
   delete thd;
   pthread_cond_broadcast(&COND_thread_count);
   pthread_mutex_unlock(&LOCK_thread_count);
+  my_atomic_rwlock_wrlock(&global_query_id_lock);
+  dec_thread_running();
+  my_atomic_rwlock_wrunlock(&global_query_id_lock);
 }
 
 
@@ -417,10 +420,12 @@ Event_scheduler::start()
     net_end(&new_thd->net);
     pthread_mutex_lock(&LOCK_thread_count);
     thread_count--;
-    thread_running--;
     delete new_thd;
     pthread_cond_broadcast(&COND_thread_count);
     pthread_mutex_unlock(&LOCK_thread_count);
+    my_atomic_rwlock_wrlock(&global_query_id_lock);
+    dec_thread_running();
+    my_atomic_rwlock_wrunlock(&global_query_id_lock);
   }
 end:
   UNLOCK_DATA();
@@ -550,10 +555,12 @@ Event_scheduler::execute_top(Event_queue_element_for_exec *event_name)
     net_end(&new_thd->net);
     pthread_mutex_lock(&LOCK_thread_count);
     thread_count--;
-    thread_running--;
     delete new_thd;
     pthread_cond_broadcast(&COND_thread_count);
     pthread_mutex_unlock(&LOCK_thread_count);
+    my_atomic_rwlock_wrlock(&global_query_id_lock);
+    dec_thread_running();
+    my_atomic_rwlock_wrunlock(&global_query_id_lock);
   }
   delete event_name;
   DBUG_RETURN(TRUE);
diff --git a/sql/log_event.cc b/sql/log_event.cc
@@ -3026,9 +3026,9 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli,
   {
     thd->set_time((time_t)when);
     thd->set_query((char*)query_arg, q_len_arg);
-    VOID(pthread_mutex_lock(&LOCK_thread_count));
+    my_atomic_rwlock_wrlock(&global_query_id_lock);
     thd->query_id = next_query_id();
-    VOID(pthread_mutex_unlock(&LOCK_thread_count));
+    my_atomic_rwlock_wrunlock(&global_query_id_lock);
     thd->variables.pseudo_thread_id= thread_id;		// for temp tables
     DBUG_PRINT("query",("%s",thd->query));
 
@@ -4504,9 +4504,9 @@ int Load_log_event::do_apply_event(NET* net, Relay_log_info const *rli,
   if (rpl_filter->db_ok(thd->db))
   {
     thd->set_time((time_t)when);
-    VOID(pthread_mutex_lock(&LOCK_thread_count));
+    my_atomic_rwlock_wrlock(&global_query_id_lock);
     thd->query_id = next_query_id();
-    VOID(pthread_mutex_unlock(&LOCK_thread_count));
+    my_atomic_rwlock_wrunlock(&global_query_id_lock);
     /*
       Initing thd->row_count is not necessary in theory as this variable has no
       influence in the case of the slave SQL thread (it is used to generate a
@@ -8025,9 +8025,9 @@ int Table_map_log_event::do_apply_event(Relay_log_info const *rli)
   DBUG_ASSERT(rli->sql_thd == thd);
 
   /* Step the query id to mark what columns that are actually used. */
-  pthread_mutex_lock(&LOCK_thread_count);
+  my_atomic_rwlock_wrlock(&global_query_id_lock);
   thd->query_id= next_query_id();
-  pthread_mutex_unlock(&LOCK_thread_count);
+  my_atomic_rwlock_wrunlock(&global_query_id_lock);
 
   if (!(memory= my_multi_malloc(MYF(MY_WME),
                                 &table_list, (uint) sizeof(RPL_TABLE_LIST),
diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h
@@ -43,6 +43,7 @@
 #include "sql_array.h"
 #include "sql_plugin.h"
 #include "scheduler.h"
+#include <my_atomic.h>
 
 class Parser_state;
 
@@ -75,11 +76,49 @@ typedef ulong nesting_map;  /* Used for flags of nesting constructs */
 typedef ulonglong nested_join_map;
 
 /* query_id */
-typedef ulonglong query_id_t;
+typedef int64 query_id_t;
 extern query_id_t global_query_id;
+extern int32 thread_running;
+extern my_atomic_rwlock_t global_query_id_lock;
 
 /* increment query_id and return it.  */
-inline query_id_t next_query_id() { return global_query_id++; }
+inline query_id_t next_query_id()
+{
+  query_id_t id;
+  id= my_atomic_add64(&global_query_id, 1);
+  return (id+1);
+}
+
+inline query_id_t get_query_id()
+{
+  query_id_t id;
+  id= my_atomic_load64(&global_query_id);
+  return id;
+}
+
+inline int32
+inc_thread_running()
+{
+  int32 num_thread_running;
+  num_thread_running= my_atomic_add32(&thread_running, 1);
+  return (num_thread_running+1);
+}
+
+inline int32
+dec_thread_running()
+{
+  int32 num_thread_running;
+  num_thread_running= my_atomic_add32(&thread_running, -1);
+  return (num_thread_running-1);
+}
+
+inline int32
+get_thread_running()
+{
+  int32 num_thread_running;
+  num_thread_running= my_atomic_load32(&thread_running);
+  return num_thread_running;
+}
 
 /* useful constants */
 extern MYSQL_PLUGIN_IMPORT const key_map key_map_empty;
@@ -1994,7 +2033,7 @@ extern bool opt_ignore_builtin_innodb;
 extern my_bool opt_character_set_client_handshake;
 extern bool volatile abort_loop, shutdown_in_progress;
 extern bool in_bootstrap;
-extern uint volatile thread_count, thread_running, global_read_lock;
+extern uint volatile thread_count, global_read_lock;
 extern uint connection_count;
 extern my_bool opt_sql_bin_update, opt_safe_user_create, opt_no_mix_types;
 extern my_bool opt_safe_show_db, opt_local_infile, opt_myisam_use_mmap;
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
@@ -523,7 +523,8 @@ uint mysqld_port_timeout;
 uint delay_key_write_options, protocol_version;
 uint lower_case_table_names;
 uint tc_heuristic_recover= 0;
-uint volatile thread_count, thread_running;
+uint volatile thread_count;
+int32 thread_running;
 ulonglong thd_startup_options;
 ulong back_log, connect_timeout, concurrency, server_id;
 ulong table_cache_size, table_def_size;
@@ -539,6 +540,7 @@ ulonglong  max_binlog_cache_size=0;
 ulong query_cache_size=0;
 ulong refresh_version;  /* Increments on each reload */
 query_id_t global_query_id;
+my_atomic_rwlock_t global_query_id_lock;
 ulong aborted_threads, aborted_connects;
 ulong delayed_insert_timeout, delayed_insert_limit, delayed_queue_size;
 ulong delayed_insert_threads, delayed_insert_writes, delayed_rows_in_use;
@@ -1349,6 +1351,7 @@ void clean_up(bool print_message)
   DBUG_PRINT("quit", ("Error messages freed"));
   /* Tell main we are ready */
   logger.cleanup_end();
+  my_atomic_rwlock_destroy(&global_query_id_lock);
   (void) pthread_mutex_lock(&LOCK_thread_count);
   DBUG_PRINT("quit", ("got thread count lock"));
   ready_to_exit=1;
@@ -7730,6 +7733,7 @@ static int mysql_init_variables(void)
   what_to_log= ~ (1L << (uint) COM_TIME);
   refresh_version= 1L;	/* Increments on each reload */
   global_query_id= thread_id= 1L;
+  my_atomic_rwlock_init(&global_query_id_lock);
   strmov(server_version, MYSQL_SERVER_VERSION);
   myisam_recover_options_str= sql_mode_str= "OFF";
   myisam_stats_method_str= "nulls_unequal";
diff --git a/sql/sp_head.cc b/sql/sp_head.cc
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
diff --git a/unittest/mysys/my_atomic-t.c b/unittest/mysys/my_atomic-t.c
diff --git a/unittest/mysys/thr_template.c b/unittest/mysys/thr_template.c
