Skip to content

Commit fffe0c3

Browse files
AliSQLAliSQL
AliSQL
authored and
AliSQL
committed
[Feature] Issue#4 THD MEMORY USAGE MONITOR
Summary: -------- This feature supply a way to monitor thd or query memory usage. Memory_used display current thread memory that allocated. Query_memory_used display current thd->query() memory that allocated. Notes: Memory_used is added when malloc and substracted when free. Query_memory_used is added when malloc and reset when query end. Usage: ------ 1. show status like '%memory_used%' 2. show full processlist;
1 parent 5d84f89 commit fffe0c3

11 files changed

Lines changed: 204 additions & 28 deletions

File tree

‎include/my_sys.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,18 @@ typedef struct my_aio_result {
148148
#define GETDATE_SHORT_DATE 2
149149
#define GETDATE_HHMMSSTIME 4
150150
#define GETDATE_GMT 8
151+
152+
/* Thread Memory Usage */
153+
#if defined(__APPLE__)
154+
#define _msize(p) malloc_size(p)
155+
#elif !defined(_WIN32)
156+
#define _msize(p) malloc_usable_size(p)
157+
#endif
158+
159+
typedef void (*THD_MALLOC_SIZE_CB) (size_t size, int flag, MEM_ROOT *root);
160+
extern void set_thd_mem_size_cb(THD_MALLOC_SIZE_CB func);
161+
extern void set_thd_query_size_cb(THD_MALLOC_SIZE_CB func);
162+
151163
#define GETDATE_FIXEDLENGTH 16
152164

153165
/* defines when allocating data */

‎mysql-test/r/thread_memory_usage.result‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
create table test.t1(id int);
2+
insert into test.t1 values(1);
3+
select * from test.t1;
4+
id
5+
1
6+
show full processlist;
7+
Id User Host db Command Time State Info Memory_used Memory_used_by_query
8+
# root localhost test # xxx # # # #
9+
# root localhost test # xxx # # # #
10+
drop table test.t1;

‎mysql-test/suite/funcs_1/datadict/processlist_val.inc‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ echo
9696
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
9797
--sorted_result
9898
SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
99-
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
99+
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 # 10 #
100100
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
101101
--sorted_result
102102
SHOW FULL PROCESSLIST;
@@ -173,7 +173,7 @@ let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST
173173
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
174174
--sorted_result
175175
SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
176-
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
176+
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 # 10 #
177177
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
178178
--sorted_result
179179
SHOW FULL PROCESSLIST;
@@ -222,7 +222,7 @@ connection con1;
222222
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
223223
--sorted_result
224224
SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
225-
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
225+
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 # 10 #
226226
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
227227
--sorted_result
228228
SHOW FULL PROCESSLIST;
@@ -260,7 +260,7 @@ connection con2;
260260
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
261261
--sorted_result
262262
SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
263-
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
263+
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 # 10 #
264264
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
265265
--sorted_result
266266
SHOW FULL PROCESSLIST;
@@ -324,7 +324,7 @@ WHERE ID = @test_user_con2_id AND Command IN('Query','Execute')
324324
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
325325
--sorted_result
326326
SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
327-
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME>
327+
--replace_column 1 <ID> 3 <HOST_NAME> 6 <TIME> 9 # 10 #
328328
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
329329
--sorted_result
330330
SHOW FULL PROCESSLIST;
@@ -460,7 +460,7 @@ echo
460460
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
461461
--sorted_result
462462
SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;
463-
--replace_column 1 <ID> 3 <HOST_NAME> 5 <COMMAND> 6 <TIME> 7 <STATE>
463+
--replace_column 1 <ID> 3 <HOST_NAME> 5 <COMMAND> 6 <TIME> 7 <STATE> 9 # 10 #
464464
--replace_result "init" STATE "starting" STATE "cleaning up" STATE
465465
--sorted_result
466466
SHOW FULL PROCESSLIST;

‎mysql-test/suite/funcs_1/r/processlist_val_ps.result‎

Lines changed: 18 additions & 18 deletions
Large diffs are not rendered by default.

‎mysql-test/t/thread_memory_usage.test‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
connect (conn1, localhost, root,,);
2+
3+
connection conn1;
4+
create table test.t1(id int);
5+
insert into test.t1 values(1);
6+
select * from test.t1;
7+
8+
connection default;
9+
10+
--replace_column 1 # 5 # 6 xxx 7 # 8 # 9 # 10 # 11 # 12 # 13 # 14 # 15 # 16 #
11+
show full processlist;
12+
13+
connection conn1;
14+
drop table test.t1;
15+
16+
disconnect conn1;

‎mysys/my_alloc.c‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,18 @@
2121
#undef EXTRA_DEBUG
2222
#define EXTRA_DEBUG
2323

24+
static THD_MALLOC_SIZE_CB query_size_cb_func= NULL;
25+
26+
void set_thd_query_size_cb(THD_MALLOC_SIZE_CB func)
27+
{
28+
query_size_cb_func= func;
29+
}
30+
31+
static void update_query_size(MEM_ROOT *root, size_t length, int flag)
32+
{
33+
if (query_size_cb_func)
34+
query_size_cb_func(length, flag, root);
35+
}
2436

2537
/*
2638
Initialize memory root
@@ -66,6 +78,7 @@ void init_alloc_root(MEM_ROOT *mem_root, size_t block_size,
6678
mem_root->free->size= pre_alloc_size+ALIGN_SIZE(sizeof(USED_MEM));
6779
mem_root->free->left= pre_alloc_size;
6880
mem_root->free->next= 0;
81+
update_query_size(mem_root, mem_root->free->size, 0);
6982
}
7083
}
7184
#endif
@@ -176,6 +189,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length)
176189
(*mem_root->error_handler)();
177190
DBUG_RETURN((uchar*) 0); /* purecov: inspected */
178191
}
192+
update_query_size(mem_root, length, 1);
179193
next->next= mem_root->used;
180194
next->size= length;
181195
mem_root->used= next;
@@ -227,6 +241,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length)
227241
(*mem_root->error_handler)();
228242
DBUG_RETURN((void*) 0); /* purecov: inspected */
229243
}
244+
update_query_size(mem_root, length, 1);
230245
mem_root->block_num++;
231246
next->next= *prev;
232247
next->size= get_size;
@@ -352,6 +367,7 @@ static inline void mark_blocks_free(MEM_ROOT* root)
352367
void free_root(MEM_ROOT *root, myf MyFlags)
353368
{
354369
reg1 USED_MEM *next,*old;
370+
ulonglong mem_size= 0;
355371
DBUG_ENTER("free_root");
356372
DBUG_PRINT("enter",("root: 0x%lx flags: %u", (long) root, (uint) MyFlags));
357373

@@ -390,7 +406,9 @@ void free_root(MEM_ROOT *root, myf MyFlags)
390406
root->free->left=root->pre_alloc->size-ALIGN_SIZE(sizeof(USED_MEM));
391407
TRASH_MEM(root->pre_alloc);
392408
root->free->next=0;
409+
mem_size= root->pre_alloc->size;
393410
}
411+
update_query_size(root, mem_size, 0);
394412
root->block_num= 4;
395413
root->first_block_usage= 0;
396414
DBUG_VOID_RETURN;

‎mysys/my_malloc.c‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,22 @@
1616
#include "mysys_priv.h"
1717
#include "mysys_err.h"
1818
#include <m_string.h>
19+
#include <malloc.h>
1920

21+
/* Thread Memory Used */
22+
static THD_MALLOC_SIZE_CB malloc_size_cb_func= NULL;
23+
24+
void set_thd_mem_size_cb(THD_MALLOC_SIZE_CB func)
25+
{
26+
malloc_size_cb_func= func;
27+
}
28+
29+
30+
static void update_malloc_size(size_t size)
31+
{
32+
if (malloc_size_cb_func)
33+
malloc_size_cb_func(size, -1, NULL);
34+
}
2035
/**
2136
Allocate a sized block of memory.
2237
@@ -60,8 +75,17 @@ void *my_malloc(size_t size, myf my_flags)
6075
if (my_flags & MY_FAE)
6176
exit(1);
6277
}
63-
else if (my_flags & MY_ZEROFILL)
64-
memset(point, 0, size);
78+
else
79+
{
80+
update_malloc_size(_msize(point));
81+
DBUG_EXECUTE_IF("simulate_out_of_memory",
82+
{
83+
my_free(point);
84+
point= NULL;
85+
});
86+
if (my_flags & MY_ZEROFILL)
87+
memset(point, 0, size);
88+
}
6589
DBUG_PRINT("exit",("ptr: %p", point));
6690
DBUG_RETURN(point);
6791
}
@@ -80,6 +104,7 @@ void *my_malloc(size_t size, myf my_flags)
80104
void *my_realloc(void *oldpoint, size_t size, myf my_flags)
81105
{
82106
void *point;
107+
size_t old_size;
83108
DBUG_ENTER("my_realloc");
84109
DBUG_PRINT("my",("ptr: %p size: %lu my_flags: %d", oldpoint,
85110
(ulong) size, my_flags));
@@ -93,6 +118,8 @@ void *my_realloc(void *oldpoint, size_t size, myf my_flags)
93118
goto end;);
94119
if (!oldpoint && (my_flags & MY_ALLOW_ZERO_PTR))
95120
DBUG_RETURN(my_malloc(size, my_flags));
121+
122+
old_size= _msize(oldpoint);
96123
#ifdef USE_HALLOC
97124
point= malloc(size);
98125
#else
@@ -113,10 +140,12 @@ void *my_realloc(void *oldpoint, size_t size, myf my_flags)
113140
size);
114141
DBUG_EXECUTE_IF("simulate_out_of_memory",
115142
DBUG_SET("-d,simulate_out_of_memory"););
143+
old_size+= 0;
116144
}
117145
#ifdef USE_HALLOC
118146
else
119147
{
148+
update_malloc_size(_msize(point) - (longlong)old_size);
120149
memcpy(point,oldpoint,size);
121150
free(oldpoint);
122151
}
@@ -137,7 +166,11 @@ void my_free(void *ptr)
137166
{
138167
DBUG_ENTER("my_free");
139168
DBUG_PRINT("my",("ptr: %p", ptr));
140-
free(ptr);
169+
if (ptr)
170+
{
171+
update_malloc_size(- _msize(ptr));
172+
free(ptr);
173+
}
141174
DBUG_VOID_RETURN;
142175
}
143176

‎sql/mysqld.cc‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ pthread_attr_t connection_attrib;
752752
mysql_mutex_t LOCK_server_started;
753753
mysql_cond_t COND_server_started;
754754

755-
int mysqld_server_started= 0;
755+
int mysqld_server_started= 0, mysqld_server_initialized= 0;
756756

757757
File_parser_dummy_hook file_parser_dummy_hook;
758758

@@ -3590,6 +3590,55 @@ SHOW_VAR com_status_vars[]= {
35903590
{NullS, NullS, SHOW_LONG}
35913591
};
35923592

3593+
/* Thread Memory Used */
3594+
extern "C" {
3595+
/*
3596+
Callback Function for memory allocation.
3597+
flag = -1 mean malloc/free
3598+
flag = 0 mean MEM mem_root
3599+
flag = 1 mean EXTEND mem_root
3600+
*/
3601+
3602+
void my_malloc_size_cb_func(size_t size, int flag, MEM_ROOT *root)
3603+
{
3604+
THD *thd= current_thd;
3605+
3606+
if (flag == -1)
3607+
{
3608+
if (mysqld_server_initialized || thd)
3609+
{
3610+
/*
3611+
THD may not be set if we are called from my_net_init() before THD
3612+
thread has started.
3613+
However, this should never happen, so better to assert and
3614+
fix this.
3615+
*/
3616+
//DBUG_ASSERT(thd);
3617+
3618+
if (thd)
3619+
{
3620+
DBUG_PRINT("info", ("memory_used: %lld size: %ld",
3621+
(longlong) thd->status_var.memory_used, size));
3622+
thd->status_var.memory_used+= size;
3623+
}
3624+
3625+
}
3626+
int64 volatile *volatile ptr= &global_status_var.memory_used;
3627+
my_atomic_add64(ptr, size);
3628+
}
3629+
else
3630+
{
3631+
if (!thd || !thd->mem_root || root != thd->mem_root)
3632+
return;
3633+
if (flag == 0)
3634+
thd->status_var.query_memory_used= size;
3635+
else if (flag == 1)
3636+
thd->status_var.query_memory_used+= size;
3637+
}
3638+
3639+
}
3640+
}
3641+
35933642
LEX_CSTRING sql_statement_names[(uint) SQLCOM_END + 1];
35943643

35953644
void init_sql_statement_names()
@@ -3720,6 +3769,10 @@ rpl_make_log_name(const char *opt,
37203769
int init_common_variables()
37213770
{
37223771
umask(((~my_umask) & 0666));
3772+
3773+
// Set memory size callback function
3774+
set_thd_mem_size_cb(my_malloc_size_cb_func);
3775+
set_thd_query_size_cb(my_malloc_size_cb_func);
37233776
connection_errors_select= 0;
37243777
connection_errors_accept= 0;
37253778
connection_errors_tcpwrap= 0;
@@ -5233,6 +5286,8 @@ int mysqld_main(int argc, char **argv)
52335286
*/
52345287
my_progname= argv[0];
52355288

5289+
mysqld_server_started= mysqld_server_initialized= 0;
5290+
52365291
#ifndef _WIN32
52375292
// For windows, my_init() is called from the win specific mysqld_main
52385293
if (my_init()) // init my_sys library & pthreads
@@ -5670,6 +5725,9 @@ int mysqld_main(int argc, char **argv)
56705725
unireg_abort(1);
56715726
}
56725727

5728+
/* It is now safe to use thread specific memory */
5729+
mysqld_server_initialized= 1;
5730+
56735731
create_shutdown_thread();
56745732
start_handle_manager();
56755733

@@ -7978,6 +8036,8 @@ SHOW_VAR status_vars[]= {
79788036
{"Last_query_cost", (char*) offsetof(STATUS_VAR, last_query_cost), SHOW_DOUBLE_STATUS},
79798037
{"Last_query_partial_plans", (char*) offsetof(STATUS_VAR, last_query_partial_plans), SHOW_LONGLONG_STATUS},
79808038
{"Max_used_connections", (char*) &max_used_connections, SHOW_LONG},
8039+
{"Memory_used", (char*) offsetof(STATUS_VAR, memory_used), SHOW_LONGLONG_STATUS},
8040+
{"Query_memory_used", (char*) offsetof(STATUS_VAR, query_memory_used), SHOW_LONGLONG_STATUS},
79818041
{"Not_flushed_delayed_rows", (char*) &delayed_rows_in_use, SHOW_LONG_NOFLUSH},
79828042
{"Open_files", (char*) &my_file_opened, SHOW_LONG_NOFLUSH},
79838043
{"Open_streams", (char*) &my_stream_opened, SHOW_LONG_NOFLUSH},

‎sql/sql_class.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -933,6 +933,8 @@ THD::THD(bool enable_plugins)
933933
ulong tmp;
934934

935935
mdl_context.init(this);
936+
status_var.memory_used= 0;
937+
status_var.query_memory_used= 0;
936938
/*
937939
Pass nominal parameters to init_alloc_root only to ensure that
938940
the destructor works OK in case of an error. The main_mem_root

‎sql/sql_class.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -639,6 +639,11 @@ typedef struct system_status_var
639639
automatically by add_to_status()/add_diff_to_status().
640640
*/
641641
double last_query_cost;
642+
643+
/* Thread Memory Used */
644+
volatile int64 memory_used;
645+
volatile int64 query_memory_used;
646+
642647
ulonglong last_query_partial_plans;
643648
} STATUS_VAR;
644649

@@ -649,6 +654,7 @@ typedef struct system_status_var
649654
*/
650655

651656
#define last_system_status_var questions
657+
#define last_cleared_system_status_var memory_used
652658

653659

654660
/**

0 commit comments

Comments
 (0)