* Avoid undefined behaviors found by gcc -fsanitize=undefined.

akr · akr · commit 1efb3c31b731 · 2014-10-17T08:50:01.000Z
gcc (Debian 4.9.1-16) 4.9.1

* include/ruby/ruby.h (INT2FIX): Avoid undefined behavior.

* node.h (nd_set_line): Ditto.

* pack.c (encodes): Ditto.
  (pack_unpack): Ditto.

* regint.h (BIT_STATUS_AT): Ditto.
  (BS_BIT): Ditto.

* time.c (time_mdump): Ditto.
  (time_mload): Ditto.

* vm_core.h (VM_FRAME_MAGIC_MASK): Ditto.

* vm_trace.c (recalc_add_ruby_vm_event_flags): Ditto.



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@47996 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
diff --git a/ChangeLog b/ChangeLog
@@ -1,3 +1,25 @@
+Fri Oct 17 17:43:50 2014  Tanaka Akira  <akr@fsij.org>
+
+	* Avoid undefined behaviors found by gcc -fsanitize=undefined.
+	  gcc (Debian 4.9.1-16) 4.9.1
+
+	* include/ruby/ruby.h (INT2FIX): Avoid undefined behavior.
+
+	* node.h (nd_set_line): Ditto.
+
+	* pack.c (encodes): Ditto.
+	  (pack_unpack): Ditto.
+
+	* regint.h (BIT_STATUS_AT): Ditto.
+	  (BS_BIT): Ditto.
+
+	* time.c (time_mdump): Ditto.
+	  (time_mload): Ditto.
+
+	* vm_core.h (VM_FRAME_MAGIC_MASK): Ditto.
+
+	* vm_trace.c (recalc_add_ruby_vm_event_flags): Ditto.
+
 Fri Oct 17 15:06:49 2014  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* re.c (unescape_nonascii): make dynamically compiled US-ASCII
diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h
@@ -230,7 +230,7 @@ typedef char ruby_check_sizeof_voidp[SIZEOF_VOIDP == sizeof(void*) ? 1 : -1];
 #define FIXNUM_MAX (LONG_MAX>>1)
 #define FIXNUM_MIN RSHIFT((long)LONG_MIN,1)
 
-#define INT2FIX(i) ((VALUE)(((SIGNED_VALUE)(i))<<1 | FIXNUM_FLAG))
+#define INT2FIX(i) (((VALUE)(i))<<1 | FIXNUM_FLAG)
 #define LONG2FIX(i) INT2FIX(i)
 #define rb_fix_new(v) INT2FIX(v)
 VALUE rb_int2inum(SIGNED_VALUE);
diff --git a/node.h b/node.h
@@ -287,7 +287,7 @@ typedef struct RNode {
 #define NODE_LMASK  (((SIGNED_VALUE)1<<(sizeof(VALUE)*CHAR_BIT-NODE_LSHIFT))-1)
 #define nd_line(n) (int)(RNODE(n)->flags>>NODE_LSHIFT)
 #define nd_set_line(n,l) \
-    RNODE(n)->flags=((RNODE(n)->flags&~(-1<<NODE_LSHIFT))|(((l)&NODE_LMASK)<<NODE_LSHIFT))
+    RNODE(n)->flags=((RNODE(n)->flags&~((VALUE)(-1)<<NODE_LSHIFT))|((VALUE)((l)&NODE_LMASK)<<NODE_LSHIFT))
 
 #define nd_refinements  nd_reserved
 
diff --git a/pack.c b/pack.c
@@ -943,13 +943,14 @@ static const char b64_table[] =
 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 static void
-encodes(VALUE str, const char *s, long len, int type, int tail_lf)
+encodes(VALUE str, const char *s0, long len, int type, int tail_lf)
 {
     enum {buff_size = 4096, encoded_unit = 4, input_unit = 3};
     char buff[buff_size + 1];	/* +1 for tail_lf */
     long i = 0;
     const char *const trans = type == 'u' ? uu_table : b64_table;
     char padding;
+    const unsigned char *s = (const unsigned char *)s0;
 
     if (type == 'u') {
 	buff[i++] = (char)len + ' ';
@@ -1362,7 +1363,7 @@ pack_unpack(VALUE str, VALUE fmt)
 		t = RSTRING_PTR(bitstr);
 		for (i=0; i<len; i++) {
 		    if (i & 7) bits <<= 1;
-		    else bits = *s++;
+		    else bits = (unsigned char)*s++;
 		    *t++ = (bits & 128) ? '1' : '0';
 		}
 	    }
@@ -1406,7 +1407,7 @@ pack_unpack(VALUE str, VALUE fmt)
 		    if (i & 1)
 			bits <<= 4;
 		    else
-			bits = *s++;
+			bits = (unsigned char)*s++;
 		    *t++ = hexdigits[(bits >> 4) & 15];
 		}
 	    }
diff --git a/regint.h b/regint.h
@@ -392,7 +392,7 @@ typedef unsigned int  BitStatusType;
 #define BIT_STATUS_CLEAR(stats)      (stats) = 0
 #define BIT_STATUS_ON_ALL(stats)     (stats) = ~((BitStatusType )0)
 #define BIT_STATUS_AT(stats,n) \
-  ((n) < (int )BIT_STATUS_BITS_NUM  ?  ((stats) & (1 << n)) : ((stats) & 1))
+  ((n) < (int )BIT_STATUS_BITS_NUM  ?  ((stats) & ((BitStatusType)1 << n)) : ((stats) & 1))
 
 #define BIT_STATUS_ON_AT(stats,n) do {\
     if ((n) < (int )BIT_STATUS_BITS_NUM)	\
@@ -468,7 +468,7 @@ typedef Bits*          BitSetRef;
 } while (0)
 
 #define BS_ROOM(bs,pos)            (bs)[(int )(pos) / BITS_IN_ROOM]
-#define BS_BIT(pos)                (1 << ((int )(pos) % BITS_IN_ROOM))
+#define BS_BIT(pos)                (1U << ((int )(pos) % BITS_IN_ROOM))
 
 #define BITSET_AT(bs, pos)         (BS_ROOM(bs,pos) & BS_BIT(pos))
 #define BITSET_SET_BIT(bs, pos)     BS_ROOM(bs,pos) |= BS_BIT(pos)
diff --git a/time.c b/time.c
@@ -4653,7 +4653,7 @@ time_mdump(VALUE time)
 	(vtm.mon-1)      << 10 | /*  4 */
 	vtm.mday         <<  5 | /*  5 */
 	vtm.hour;                /*  5 */
-    s = vtm.min          << 26 | /*  6 */
+    s = (unsigned long)vtm.min << 26 | /*  6 */
 	vtm.sec          << 20 | /*  6 */
 	usec;    /* 20 */
 
@@ -4766,10 +4766,10 @@ time_mload(VALUE time, VALUE str)
 
     p = s = 0;
     for (i=0; i<4; i++) {
-	p |= buf[i]<<(8*i);
+	p |= (unsigned long)buf[i]<<(8*i);
     }
     for (i=4; i<8; i++) {
-	s |= buf[i]<<(8*(i-4));
+	s |= (unsigned long)buf[i]<<(8*(i-4));
     }
 
     if ((p & (1UL<<31)) == 0) {
diff --git a/vm_core.h b/vm_core.h
@@ -811,7 +811,7 @@ enum vm_special_object_type {
 #define VM_FRAME_MAGIC_LAMBDA 0xa1
 #define VM_FRAME_MAGIC_RESCUE 0xb1
 #define VM_FRAME_MAGIC_MASK_BITS 8
-#define VM_FRAME_MAGIC_MASK   (~(~0<<VM_FRAME_MAGIC_MASK_BITS))
+#define VM_FRAME_MAGIC_MASK   (~(~(VALUE)0<<VM_FRAME_MAGIC_MASK_BITS))
 
 #define VM_FRAME_TYPE(cfp) ((cfp)->flag & VM_FRAME_MAGIC_MASK)
 
diff --git a/vm_trace.c b/vm_trace.c
@@ -67,7 +67,7 @@ recalc_add_ruby_vm_event_flags(rb_event_flag_t events)
     ruby_vm_event_flags = 0;
 
     for (i=0; i<MAX_EVENT_NUM; i++) {
-	if (events & (1 << i)) {
+	if (events & ((rb_event_flag_t)1 << i)) {
 	    ruby_event_flag_count[i]++;
 	}
 	ruby_vm_event_flags |= ruby_event_flag_count[i] ? (1<<i) : 0;

Original file line number	Diff line number	Diff line change
`@@ -67,7 +67,7 @@ recalc_add_ruby_vm_event_flags(rb_event_flag_t events)`
`67`	`67`	`ruby_vm_event_flags = 0;`
`68`	`68`
`69`	`69`	`for (i=0; i<MAX_EVENT_NUM; i++) {`
`70`		`- if (events & (1 << i)) {`
	`70`	`+ if (events & ((rb_event_flag_t)1 << i)) {`
`71`	`71`	`ruby_event_flag_count[i]++;`
`72`	`72`	`}`
`73`	`73`	`ruby_vm_event_flags \|= ruby_event_flag_count[i] ? (1<<i) : 0;`