BUG#22594514: HANDLE_FATAL_SIGNAL (SIG=11) IN

Nisha Gopalakrishnan · Nisha Gopalakrishnan · commit 6608f84158d3 · 2016-03-17T08:49:37.000+05:30
UNIQUE::~UNIQUE | SQL/UNIQUES.CC:355

Analysis
========

Enabling the sort_buffer_size with a large value
can cause operations utilizing the sort buffer
like DELETE as mentioned in the bug report to
fail. 5.5 and 5.6 versions reports OOM error
while in 5.7+, the server crashes.

While initializing the mem_root for the sort buffer
tree, the block size for the mem_root is determined
from the 'sort_buffer_size' value. This unsigned
long value is typecasted to unsigned int, hence
it becomes zero. Further block_size computation
while initializing the mem_root results in a very
large block_size value. Hence while trying to
allocate a block during the DELETE operation,
an OOM error is reported. In case of 5.7+, the PFS
instrumentation for memory allocation, overshoots
the unsigned value and allocates a block of just
one byte. While trying to free the block of the
mem_root, the original block_size is used. This
triggers the crash since the server tries to free
unallocated memory.

Fix:
====
In order to restrict usage of such unreasonable
sort_buffer_size, the typecast of block size
to 'unsigned int' is removed and hence reports
OOM error across all versions for sizes
exceeding unsigned int range.
diff --git a/include/my_tree.h b/include/my_tree.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -62,7 +62,7 @@ typedef struct st_tree {
 } TREE;
 
 	/* Functions on whole tree */
-void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,
+void init_tree(TREE *tree, size_t default_alloc_size, ulong memory_limit,
                int size, qsort_cmp2 compare, my_bool with_delete,
 	       tree_element_free free_element, void *custom_arg);
 void delete_tree(TREE*);
diff --git a/mysys/tree.c b/mysys/tree.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -83,7 +83,7 @@ static void rb_delete_fixup(TREE *tree,TREE_ELEMENT ***parent);
 static int test_rb_tree(TREE_ELEMENT *element);
 #endif
 
-void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,
+void init_tree(TREE *tree, size_t default_alloc_size, ulong memory_limit,
                int size, qsort_cmp2 compare, my_bool with_delete,
 	       tree_element_free free_element, void *custom_arg)
 {
@@ -127,7 +127,7 @@ void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,
   }
   if (!(tree->with_delete=with_delete))
   {
-    init_alloc_root(&tree->mem_root, (uint) default_alloc_size, 0);
+    init_alloc_root(&tree->mem_root, default_alloc_size, 0);
     tree->mem_root.min_malloc=(sizeof(TREE_ELEMENT)+tree->size_of_element);
   }
   DBUG_VOID_RETURN;

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.`
	`1`	`+/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.`
`2`	`2`
`3`	`3`	`This program is free software; you can redistribute it and/or modify`
`4`	`4`	`it under the terms of the GNU General Public License as published by`
`@@ -83,7 +83,7 @@ static void rb_delete_fixup(TREE tree,TREE_ELEMENT **parent);`
`83`	`83`	`static int test_rb_tree(TREE_ELEMENT *element);`
`84`	`84`	`#endif`
`85`	`85`
`86`		`-void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,`
	`86`	`+void init_tree(TREE *tree, size_t default_alloc_size, ulong memory_limit,`
`87`	`87`	`int size, qsort_cmp2 compare, my_bool with_delete,`
`88`	`88`	`tree_element_free free_element, void *custom_arg)`
`89`	`89`	`{`
`@@ -127,7 +127,7 @@ void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,`
`127`	`127`	`}`
`128`	`128`	`if (!(tree->with_delete=with_delete))`
`129`	`129`	`{`
`130`		`- init_alloc_root(&tree->mem_root, (uint) default_alloc_size, 0);`
	`130`	`+ init_alloc_root(&tree->mem_root, default_alloc_size, 0);`
`131`	`131`	`tree->mem_root.min_malloc=(sizeof(TREE_ELEMENT)+tree->size_of_element);`
`132`	`132`	`}`
`133`	`133`	`DBUG_VOID_RETURN;`