Add "reproducible" deflate parameter

IBM Z DEFLATE CONVERSION CALL may produce different (but valid) compressed data for the same uncompressed data. This behavior might be unacceptable for certain use cases (e.g. reproducible builds). This patch introduces Z_DEFLATE_REPRODUCIBLE parameter, which can be used to indicate that this is the case, and turn off IBM Z DEFLATE CONVERSION CALL.
zlib-ng · Jul 18, 2019 · d746d23 · d746d23
1 parent b927c6a
commit d746d23
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 7 deletions.
diff --git a/arch/s390/README.md b/arch/s390/README.md
@@ -20,7 +20,10 @@ Two DFLTCC compression calls produce the same results only when they
 both are made on machines of the same generation, and when the
 respective buffers have the same offset relative to the start of the
 page. Therefore care should be taken when using hardware compression
-when reproducible results are desired.
+when reproducible results are desired. In particular, zlib-ng-specific
+zng_deflateSetParams call allows setting Z_DEFLATE_REPRODUCIBLE
+parameter, which would disable DFLTCC if reproducible results are
+required.
 
 DFLTCC does not support every single zlib-ng feature, in particular:
 
@@ -67,3 +70,11 @@ In addition to compression, DFLTCC computes CRC-32 and Adler-32
 checksums, therefore, whenever it's used, software checksumming is
 suppressed using DEFLATE_NEED_CHECKSUM and INFLATE_NEED_CHECKSUM
 macros.
+
+While software always produces reproducible compression results, this
+is not the case for DFLTCC. Therefore, zlib-ng users are given the
+ability to specify whether or not reproducible compression results
+are required. While it is always possible to specify this setting
+before the compression begins, it is not always possible to do so in
+the middle of a deflate stream - the exact conditions for that are
+determined by DEFLATE_CAN_SET_REPRODUCIBLE macro.
diff --git a/arch/s390/dfltcc_deflate.c b/arch/s390/dfltcc_deflate.c
@@ -19,11 +19,13 @@
 #include "dfltcc_deflate.h"
 #include "dfltcc_detail.h"
 
-static inline int dfltcc_are_params_ok(int level, uInt window_bits, int strategy, uint16_t level_mask)
+static inline int dfltcc_are_params_ok(int level, uInt window_bits, int strategy, uint16_t level_mask,
+                                       int reproducible)
 {
     return (level_mask & ((uint16_t)1 << level)) != 0 &&
         (window_bits == HB_BITS) &&
-        (strategy == Z_FIXED || strategy == Z_DEFAULT_STRATEGY);
+        (strategy == Z_FIXED || strategy == Z_DEFAULT_STRATEGY) &&
+        !reproducible;
 }
 
 
@@ -33,7 +35,8 @@ int ZLIB_INTERNAL dfltcc_can_deflate(PREFIX3(streamp) strm)
     struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
 
     /* Unsupported compression settings */
-    if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, dfltcc_state->level_mask))
+    if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, dfltcc_state->level_mask,
+                              state->reproducible))
         return 0;
 
     /* Unsupported hardware */
@@ -277,26 +280,41 @@ int ZLIB_INTERNAL dfltcc_deflate(PREFIX3(streamp) strm, int flush, block_state *
    fly with deflateParams, we need to convert between hardware and software
    window formats.
 */
+static int dfltcc_was_deflate_used(PREFIX3(streamp) strm)
+{
+    deflate_state *state = (deflate_state *)strm->state;
+    struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param;
+
+    return strm->total_in > 0 || param->nt == 0 || param->hl > 0;
+}
+
 int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int strategy)
 {
     deflate_state *state = (deflate_state *)strm->state;
     struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
-    struct dfltcc_param_v0 *param = &dfltcc_state->param;
     int could_deflate = dfltcc_can_deflate(strm);
-    int can_deflate = dfltcc_are_params_ok(level, state->w_bits, strategy, dfltcc_state->level_mask);
+    int can_deflate = dfltcc_are_params_ok(level, state->w_bits, strategy, dfltcc_state->level_mask,
+                                           state->reproducible);
 
     if (can_deflate == could_deflate)
         /* We continue to work in the same mode - no changes needed */
         return Z_OK;
 
-    if (strm->total_in == 0 && param->nt == 1 && param->hl == 0)
+    if (!dfltcc_was_deflate_used(strm))
         /* DFLTCC was not used yet - no changes needed */
         return Z_OK;
 
     /* Switching between hardware and software is not implemented */
     return Z_STREAM_ERROR;
 }
 
+int ZLIB_INTERNAL dfltcc_can_set_reproducible(PREFIX3(streamp) strm, int reproducible)
+{
+    deflate_state *state = (deflate_state *)strm->state;
+
+    return reproducible != state->reproducible && !dfltcc_was_deflate_used(strm);
+}
+
 /*
    Preloading history.
 */

diff --git a/arch/s390/dfltcc_deflate.h b/arch/s390/dfltcc_deflate.h
@@ -6,6 +6,7 @@
 int ZLIB_INTERNAL dfltcc_can_deflate(PREFIX3(streamp) strm);
 int ZLIB_INTERNAL dfltcc_deflate(PREFIX3(streamp) strm, int flush, block_state *result);
 int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int strategy);
+int ZLIB_INTERNAL dfltcc_can_set_reproducible(PREFIX3(streamp) strm, int reproducible);
 int ZLIB_INTERNAL dfltcc_deflate_set_dictionary(PREFIX3(streamp) strm,
                                                 const unsigned char *dictionary, uInt dict_length);
 int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(PREFIX3(streamp) strm, unsigned char *dictionary, uInt* dict_length);
@@ -47,4 +48,6 @@ int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(PREFIX3(streamp) strm, unsigned
 
 #define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm)))
 
+#define DEFLATE_CAN_SET_REPRODUCIBLE dfltcc_can_set_reproducible
+
 #endif
diff --git a/deflate.c b/deflate.c
@@ -94,6 +94,8 @@ const char deflate_copyright[] = " deflate 1.2.11.f Copyright 1995-2016 Jean-lou
 #  define DEFLATE_HOOK(strm, flush, bstate) 0
 /* Returns whether zlib-ng should compute a checksum. Set to 0 if arch-specific deflation code already does that. */
 #  define DEFLATE_NEED_CHECKSUM(strm) 1
+/* Returns whether reproducibility parameter can be set to a given value. */
+#  define DEFLATE_CAN_SET_REPRODUCIBLE(strm, reproducible) 1
 #endif
 
 /* ===========================================================================
@@ -409,6 +411,7 @@ int ZEXPORT PREFIX(deflateInit2_)(PREFIX3(stream) *strm, int level, int method,
     s->strategy = strategy;
     s->method = (unsigned char)method;
     s->block_open = 0;
+    s->reproducible = 0;
 
     return PREFIX(deflateReset)(strm);
 }
@@ -1732,11 +1735,13 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para
     deflate_state *s;
     zng_deflate_param_value *new_level = NULL;
     zng_deflate_param_value *new_strategy = NULL;
+    zng_deflate_param_value *new_reproducible = NULL;
     int param_buf_error;
     int version_error = 0;
     int buf_error = 0;
     int stream_error = 0;
     int ret;
+    int val;
 
     /* Initialize the statuses. */
     for (i = 0; i < count; i++)
@@ -1756,6 +1761,9 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para
             case Z_DEFLATE_STRATEGY:
                 param_buf_error = deflateSetParamPre(&new_strategy, sizeof(int), &params[i]);
                 break;
+            case Z_DEFLATE_REPRODUCIBLE:
+                param_buf_error = deflateSetParamPre(&new_reproducible, sizeof(int), &params[i]);
+                break;
             default:
                 params[i].status = Z_VERSION_ERROR;
                 version_error = 1;
@@ -1783,6 +1791,15 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para
             stream_error = 1;
         }
     }
+    if (new_reproducible != NULL) {
+        val = *(int *)new_reproducible->buf;
+        if (DEFLATE_CAN_SET_REPRODUCIBLE(strm, val))
+            s->reproducible = val;
+        else {
+            new_reproducible->status = Z_STREAM_ERROR;
+            stream_error = 1;
+        }
+    }
 
     /* Report version errors only if there are no real errors. */
     return stream_error ? Z_STREAM_ERROR : (version_error ? Z_VERSION_ERROR : Z_OK);
@@ -1818,6 +1835,12 @@ int ZEXPORT zng_deflateGetParams(zng_stream *strm, zng_deflate_param_value *para
                 else
                     *(int *)params[i].buf = s->strategy;
                 break;
+            case Z_DEFLATE_REPRODUCIBLE:
+                if (params[i].size < sizeof(int))
+                    params[i].status = Z_BUF_ERROR;
+                else
+                    *(int *)params[i].buf = s->reproducible;
+                break;
             default:
                 params[i].status = Z_VERSION_ERROR;
                 version_error = 1;

diff --git a/deflate.h b/deflate.h
@@ -284,6 +284,9 @@ typedef struct internal_state {
      * This is set to 1 if there is an active block, or 0 if the block was just
      * closed.
      */
+    int reproducible;
+    /* Whether reproducible compression results are required.
+     */
 
 } deflate_state;
 

diff --git a/zlib-ng.h b/zlib-ng.h
@@ -1793,6 +1793,13 @@ ZEXTERN int ZEXPORT zng_gzgetc_(gzFile file);  /* backward compatibility */
 typedef enum {
     Z_DEFLATE_LEVEL = 0,         /* compression level, represented as an int */
     Z_DEFLATE_STRATEGY = 1,      /* compression strategy, represented as an int */
+    Z_DEFLATE_REPRODUCIBLE = 2,
+    /*
+         Whether reproducible compression results are required. Represented as an int, where 0 means that it is allowed
+       to trade reproducibility for e.g. improved performance or compression ratio, and non-0 means that
+       reproducibility is strictly required. Reproducibility is guaranteed only when using an identical zlib-ng build.
+       Default is 0.
+    */
 } zng_deflate_param;
 
 typedef struct {