Clean up LIKELY/UNLIKELY definitions, making them upper-case to impro…

…ve visibility. Add LIKELY_NULL hint. Add PREFETCH_L1, PREFETCH_L2 and PREFETCH_RW for GCC, Clang, ICC and MSVC.
zlib-ng · Oct 22, 2019 · cf5959e · cf5959e
1 parent 72c9ed1
commit cf5959e
Show file tree

Hide file tree

Showing 8 changed files with 34 additions and 26 deletions.
diff --git a/arch/arm/fill_window_arm.c b/arch/arm/fill_window_arm.c
@@ -112,7 +112,7 @@ void fill_window_arm(deflate_state *s) {
 
             s->ins_h = s->window[str];
 
-            if (unlikely(s->lookahead < MIN_MATCH))
+            if (UNLIKELY(s->lookahead < MIN_MATCH))
                 insert_cnt += s->lookahead - MIN_MATCH;
             slen = insert_cnt;
             if (str >= (MIN_MATCH - 2))

diff --git a/arch/arm/insert_string_acle.c b/arch/arm/insert_string_acle.c
@@ -21,7 +21,7 @@
 Pos insert_string_acle(deflate_state *const s, const Pos str, unsigned int count) {
     Pos p, lp, ret;
 
-    if (unlikely(count == 0)) {
+    if (UNLIKELY(count == 0)) {
         return s->prev[str & s->w_mask];
     }
 

diff --git a/arch/x86/fill_window_sse.c b/arch/x86/fill_window_sse.c
@@ -95,7 +95,7 @@ ZLIB_INTERNAL void fill_window_sse(deflate_state *s) {
             }
 #else
             unsigned int count;
-            if (unlikely(s->lookahead == 1)){
+            if (UNLIKELY(s->lookahead == 1)){
                 count = s->insert - 1;
             }else{
                 count = s->insert;

diff --git a/deflate.c b/deflate.c
@@ -1335,7 +1335,7 @@ void ZLIB_INTERNAL fill_window_c(deflate_state *s) {
             }
 #else
             unsigned int count;
-            if (unlikely(s->lookahead == 1)){
+            if (UNLIKELY(s->lookahead == 1)){
                 count = s->insert - 1;
             }else{
                 count = s->insert;

diff --git a/deflate_medium.c b/deflate_medium.c
@@ -53,7 +53,7 @@ static int emit_match(deflate_state *s, struct match match) {
 }
 
 static void insert_match(deflate_state *s, struct match match) {
-    if (unlikely(s->lookahead <= match.match_length + MIN_MATCH))
+    if (UNLIKELY(s->lookahead <= match.match_length + MIN_MATCH))
         return;
 
     /* matches that are not long enough we need to emit as literals */
@@ -95,7 +95,7 @@ static void insert_match(deflate_state *s, struct match match) {
         match.strstart++;
 #ifdef NOT_TWEAK_COMPILER
         do {
-            if (likely(match.strstart >= match.orgstart)) {
+            if (LIKELY(match.strstart >= match.orgstart)) {
                 functable.insert_string(s, match.strstart, 1);
             }
             match.strstart++;
@@ -104,8 +104,8 @@ static void insert_match(deflate_state *s, struct match match) {
              */
         } while (--match.match_length != 0);
 #else
-        if (likely(match.strstart >= match.orgstart)) {
-            if (likely(match.strstart + match.match_length - 1 >= match.orgstart)) {
+        if (LIKELY(match.strstart >= match.orgstart)) {
+            if (LIKELY(match.strstart + match.match_length - 1 >= match.orgstart)) {
                 functable.insert_string(s, match.strstart, match.match_length);
             } else {
                 functable.insert_string(s, match.strstart, match.orgstart - match.strstart + 1);
@@ -145,17 +145,17 @@ static void fizzle_matches(deflate_state *s, struct match *current, struct match
     if (current->match_length <= 1)
         return;
 
-    if (unlikely(current->match_length > 1 + next->match_start))
+    if (UNLIKELY(current->match_length > 1 + next->match_start))
         return;
 
-    if (unlikely(current->match_length > 1 + next->strstart))
+    if (UNLIKELY(current->match_length > 1 + next->strstart))
         return;
 
     match = s->window - current->match_length + 1 + next->match_start;
     orig  = s->window - current->match_length + 1 + next->strstart;
 
     /* quick exit check.. if this fails then don't bother with anything else */
-    if (likely(*match != *orig))
+    if (LIKELY(*match != *orig))
         return;
 
     c = *current;

diff --git a/deflate_slow.c b/deflate_slow.c
@@ -110,7 +110,7 @@ ZLIB_INTERNAL block_state deflate_slow(deflate_state *s, int flush) {
                 unsigned int mov_fwd = s->prev_length - 2;
                 if (max_insert > s->strstart) {
                     unsigned int insert_cnt = mov_fwd;
-                    if (unlikely(insert_cnt > max_insert - s->strstart))
+                    if (UNLIKELY(insert_cnt > max_insert - s->strstart))
                         insert_cnt = max_insert - s->strstart;
 
                     functable.insert_string(s, s->strstart + 1, insert_cnt);

diff --git a/match_p.h b/match_p.h
@@ -226,7 +226,7 @@ static inline unsigned longest_match(deflate_state *const s, IPos cur_match) {
          */
         uint16_t val;
         memcpy(&val, match + best_len - 1, sizeof(val));
-        if (likely(val != scan_end))
+        if (LIKELY(val != scan_end))
             continue;
 
         memcpy(&val, match, sizeof(val));
@@ -432,7 +432,7 @@ static inline unsigned longest_match(deflate_state *const s, IPos cur_match) {
         int cont = 1;
         do {
             match = window + cur_match;
-            if (likely(memcmp(match+best_len-1, &scan_end, sizeof(scan_end)) != 0
+            if (LIKELY(memcmp(match+best_len-1, &scan_end, sizeof(scan_end)) != 0
                 || memcmp(match, &scan_start, sizeof(scan_start)) != 0)) {
                 if ((cur_match = prev[cur_match & wmask]) > limit
                     && --chain_length != 0) {

diff --git a/zutil.h b/zutil.h
@@ -215,19 +215,27 @@ void ZLIB_INTERNAL   zng_cfree(void *opaque, void *ptr);
 
 /* Only enable likely/unlikely if the compiler is known to support it */
 #if (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__INTEL_COMPILER) || defined(__Clang__)
-#  ifndef likely
-#    define likely(x)      __builtin_expect(!!(x), 1)
-#  endif
-#  ifndef unlikely
-#    define unlikely(x)    __builtin_expect(!!(x), 0)
-#  endif
+#    define LIKELY_NULL(x)      __builtin_expect((x) != 0, 0)
+#    define LIKELY(x)           __builtin_expect(!!(x), 1)
+#    define UNLIKELY(x)         __builtin_expect(!!(x), 0)
+#    define PREFETCH_L1(addr)    __builtin_prefetch(addr, 0, 3)
+#    define PREFETCH_L2(addr)    __builtin_prefetch(addr, 0, 2)
+#    define PREFETCH_RW(addr)    __builtin_prefetch(addr, 1, 2)
+#elif defined(__WIN__)
+#    include <xmmintrin.h>
+#    define LIKELY_NULL(x) x
+#    define LIKELY(x)      x
+#    define UNLIKELY(x)    x
+#    define PREFETCH_L1(addr)    _mm_prefetch((char *) addr, _MM_HINT_T0)
+#    define PREFETCH_L2(addr)    _mm_prefetch((char *) addr, _MM_HINT_T1)
+#    define PREFETCH_RW(addr)    _mm_prefetch((char *) addr, _MM_HINT_T1)
 #else
-#  ifndef likely
-#    define likely(x)      x
-#  endif
-#  ifndef unlikely
-#    define unlikely(x)    x
-#  endif
+#    define LIKELY_NULL(x) x
+#    define LIKELY(x)      x
+#    define UNLIKELY(x)    x
+#    define PREFETCH_L1(addr)    addr
+#    define PREFETCH_L2(addr)    addr
+#    define PREFETCH_RW(addr)    addr
 #endif /* (un)likely */
 
 #if defined(_MSC_VER)