CRC32C Checksum support

.gitignore

@@ -10,3 +10,4 @@
 *.bundle
 *.so
 *.dll
+*.idea

ext/zlib/zlib.c

@@ -74,9 +74,13 @@ static ID id_dictionaries, id_read, id_buffer;
 static NORETURN(void raise_zlib_error(int, const char*));
 static VALUE rb_zlib_version(VALUE);
 static VALUE do_checksum(int, VALUE*, checksum_func);
+static VALUE crc32c(unsigned long, const unsigned char*, size_t);
 static VALUE rb_zlib_adler32(int, VALUE*, VALUE);
 static VALUE rb_zlib_crc32(int, VALUE*, VALUE);
-static VALUE rb_zlib_crc_table(VALUE);
+static VALUE rb_zlib_crc32_table(VALUE);
+static VALUE rb_zlib_crc32c(int, VALUE*, VALUE);
+static const unsigned long crc32ctbl[256];
+static VALUE rb_zlib_crc32c_table(VALUE);
 static voidpf zlib_mem_alloc(voidpf, uInt, uInt);
 static void zlib_mem_free(voidpf, voidpf);
 static void finalizer_warn(const char*);
@@ -495,10 +499,10 @@ rb_zlib_adler32_combine(VALUE klass, VALUE adler1, VALUE adler2, VALUE len2)
  *
  * call-seq: Zlib.crc32(string, crc)
  *
- * Calculates CRC checksum for +string+, and returns updated value of +crc+. If
- * +string+ is omitted, it returns the CRC initial value. If +crc+ is omitted, it
+ * Calculates CRC32 checksum for +string+, and returns updated value of +crc+. If
+ * +string+ is omitted, it returns the CRC32 initial value. If +crc+ is omitted, it
  * assumes that the initial value is given to +crc+. If +string+ is an IO instance,
- * reads from the IO until the IO returns nil and returns CRC checksum of all read
+ * reads from the IO until the IO returns nil and returns CRC32 checksum of all read
  * data.
  *
  * FIXME: expression.
@@ -515,8 +519,8 @@ rb_zlib_crc32(int argc, VALUE *argv, VALUE klass)
  *
  * call-seq: Zlib.crc32_combine(crc1, crc2, len2)
  *
- * Combine two CRC-32 check values in to one.  +crc1+ is the first CRC-32
- * value, +crc2+ is the second CRC-32 value.  +len2+ is the length of the
+ * Combine two CRC32 check values in to one.  +crc1+ is the first CRC32
+ * value, +crc2+ is the second CRC32 value.  +len2+ is the length of the
  * string used to generate +crc2+.
  *
  */
@@ -531,12 +535,12 @@ rb_zlib_crc32_combine(VALUE klass, VALUE crc1, VALUE crc2, VALUE len2)
 #endif
 
 /*
- * Document-method: Zlib.crc_table
+ * Document-method: Zlib.crc32_table
  *
- * Returns the table for calculating CRC checksum as an array.
+ * Returns the table for calculating CRC32 checksum as an array.
  */
 static VALUE
-rb_zlib_crc_table(VALUE obj)
+rb_zlib_crc32_table(VALUE obj)
 {
 #if !defined(HAVE_TYPE_Z_CRC_T)
     /* z_crc_t is defined since zlib-1.2.7. */
@@ -555,7 +559,233 @@ rb_zlib_crc_table(VALUE obj)
     return dst;
 }
 
+static VALUE
+crc32c(unsigned long crc, const unsigned char *buf, size_t size)
+{
+    const uint8_t *p = buf;
+
+    crc = crc ^ 0xFFFFFFFF;
+    while (size--)
+        crc = crc32ctbl[(crc ^ *p++) & 0xff] ^ (crc >> 8);
+	return crc ^ 0xFFFFFFFF;
+}
+
+/*
+ * Document-method: Zlib.crc32c
+ *
+ * call-seq: Zlib.crc32c(string, crc)
+ *
+ * Calculates CRC32C checksum for +string+, and returns updated value of +crc+. If
+ * +string+ is omitted, it returns the CRC32C initial value. If +crc+ is omitted, it
+ * assumes that the initial value is given to +crc+. If +string+ is an IO instance,
+ * reads from the IO until the IO returns nil and returns CRC32C checksum of all read
+ * data.
+ *
+ * FIXME: expression.
+ */
+static VALUE
+rb_zlib_crc32c(int argc, VALUE *argv, VALUE klass)
+{
+    return do_checksum(argc, argv, &crc32c);
+}
+
+/*
+ * The following CRC32C code was adapted from the CRC32 code in zlib, including
+ * the surrounding functions.
+ * https://github.com/luvit/zlib/blob/8de57bce969eb9dafc1f1f5c256ac608d0a73ec4/crc32.c#L355
+ */
+#define GF2_DIM 32      /* dimension of GF(2) vectors (length of CRC) */
+void gf2_matrix_square OF((unsigned long *, unsigned long *));
+unsigned long gf2_matrix_times OF((unsigned long *, unsigned long));
+static uLong crc32c_combine_(unsigned long, unsigned long, z_off64_t);
+
+void gf2_matrix_square(unsigned long *square, unsigned long *mat)
+{
+    int n;
+
+    for (n = 0; n < GF2_DIM; n++)
+        square[n] = gf2_matrix_times(mat, mat[n]);
+}
+
+unsigned long gf2_matrix_times(unsigned long *mat, unsigned long vec)
+{
+    unsigned long sum;
+
+    sum = 0;
+    while (vec) {
+        if (vec & 1)
+            sum ^= *mat;
+        vec >>= 1;
+        mat++;
+    }
+    return sum;
+}
+
+static uLong crc32c_combine_(unsigned long crc1, unsigned long crc2, z_off64_t len2)
+{
+   int n;
+   unsigned long row;
+   unsigned long even[GF2_DIM];    /* even-power-of-two zeros operator */
+   unsigned long odd[GF2_DIM];     /* odd-power-of-two zeros operator */
+
+   /* degenerate case (also disallow negative lengths) */
+   if (len2 <= 0)
+       return crc1;
+
+   /* put operator for one zero bit in odd */
+   odd[0] = 0x82f63b78UL;          /* CRC-32C polynomial */
+   row = 1;
+   for (n = 1; n < GF2_DIM; n++) {
+       odd[n] = row;
+       row <<= 1;
+   }
+
+   /* put operator for two zero bits in even */
+   gf2_matrix_square(even, odd);
+
+   /* put operator for four zero bits in odd */
+   gf2_matrix_square(odd, even);
+
+   /* apply len2 zeros to crc1 (first square will put the operator for one
+      zero byte, eight zero bits, in even) */
+   do {
+       /* apply zeros operator for this bit of len2 */
+       gf2_matrix_square(even, odd);
+       if (len2 & 1)
+           crc1 = gf2_matrix_times(even, crc1);
+       len2 >>= 1;
+
+       /* if no more bits set, then done */
+       if (len2 == 0)
+           break;
+
+       /* another iteration of the loop with odd and even swapped */
+       gf2_matrix_square(odd, even);
+       if (len2 & 1)
+           crc1 = gf2_matrix_times(odd, crc1);
+       len2 >>= 1;
 
+       /* if no more bits set, then done */
+   } while (len2 != 0);
+
+   /* return combined crc */
+   crc1 ^= crc2;
+   return crc1;
+}
+/* End of adapted CRC32C combine code */
+
+#ifdef HAVE_CRC32_COMBINE
+/*
+ * Document-method: Zlib.crc32c_combine
+ *
+ * call-seq: Zlib.crc32c_combine(crc1, crc2, len2)
+ *
+ * Combine two CRC32C check values in to one.  +crc1+ is the first CRC32C
+ * value, +crc2+ is the second CRC32C value.  +len2+ is the length of the
+ * string used to generate +crc2+.
+ *
+ */
+static VALUE
+rb_zlib_crc32c_combine(VALUE klass, VALUE crc1, VALUE crc2, VALUE len2)
+{
+    return ULONG2NUM(
+    crc32c_combine_(NUM2ULONG(crc1), NUM2ULONG(crc2), NUM2LONG(len2)));
+}
+#else
+#define rb_zlib_crc32c_combine rb_f_notimplement
+#endif
+
+static const unsigned long crc32ctbl[256] = {
+    0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
+    0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
+    0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
+    0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
+    0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
+    0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
+    0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
+    0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
+    0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
+    0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
+    0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
+    0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
+    0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
+    0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
+    0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
+    0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
+    0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
+    0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
+    0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
+    0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
+    0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
+    0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
+    0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
+    0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
+    0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
+    0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
+    0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
+    0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
+    0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
+    0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
+    0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
+    0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
+    0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
+    0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
+    0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
+    0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
+    0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
+    0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
+    0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
+    0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
+    0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
+    0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
+    0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
+    0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
+    0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
+    0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
+    0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
+    0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
+    0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
+    0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
+    0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
+    0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
+    0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
+    0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
+    0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
+    0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
+    0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
+    0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
+    0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
+    0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
+    0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
+    0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
+    0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
+    0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
+};
+
+/*
+ * Document-method: Zlib.crc32c_table
+ *
+ * Returns the table for calculating CRC32C checksum as an array.
+ */
+static VALUE
+rb_zlib_crc32c_table(VALUE obj)
+{
+#if !defined(HAVE_TYPE_Z_CRC_T)
+    /* z_crc_t is defined since zlib-1.2.7. */
+    typedef unsigned long z_crc_t;
+#endif
+    const z_crc_t *crctbl;
+    VALUE dst;
+    int i;
+
+    crctbl = crc32ctbl;
+    dst = rb_ary_new2(256);
+
+    for (i = 0; i < 256; i++) {
+	rb_ary_push(dst, rb_uint2inum(crctbl[i]));
+    }
+    return dst;
+}
 
 /*-------- zstream - internal APIs --------*/
 
@@ -4682,7 +4912,11 @@ Init_zlib(void)
     rb_define_module_function(mZlib, "adler32_combine", rb_zlib_adler32_combine, 3);
     rb_define_module_function(mZlib, "crc32", rb_zlib_crc32, -1);
     rb_define_module_function(mZlib, "crc32_combine", rb_zlib_crc32_combine, 3);
-    rb_define_module_function(mZlib, "crc_table", rb_zlib_crc_table, 0);
+    rb_define_module_function(mZlib, "crc_table", rb_zlib_crc32_table, 0);
+    rb_define_module_function(mZlib, "crc32_table", rb_zlib_crc32_table, 0);
+    rb_define_module_function(mZlib, "crc32c", rb_zlib_crc32c, -1);
+    rb_define_module_function(mZlib, "crc32c_combine", rb_zlib_crc32c_combine, 3);
+    rb_define_module_function(mZlib, "crc32c_table", rb_zlib_crc32c_table, 0);
 
     /* The Ruby/zlib version string. */
     rb_define_const(mZlib, "VERSION", rb_str_new2(RUBY_ZLIB_VERSION));

test/zlib/test_zlib.rb

@@ -1442,6 +1442,54 @@ def test_crc_table
       t.each {|x| assert_kind_of(Integer, x) }
     end
 
+    def test_crc32_table
+      t = Zlib.crc32_table
+      assert_instance_of(Array, t)
+      t.each {|x| assert_kind_of(Integer, x) }
+    end
+
+    def test_crc32c
+      assert_equal(0x00000000, Zlib.crc32c)
+      assert_equal(0xcfc4ae1d, Zlib.crc32c("foo"))
+      assert_equal(0xcfc4ae1d, Zlib.crc32c("o", Zlib.crc32c("fo")))
+      assert_equal(0x88a0fa6c, Zlib.crc32c("abc\x01\x02\x03" * 10000))
+      assert_equal(0x75349366, Zlib.crc32c("p", -305419897))
+      Tempfile.create("test_zlib_gzip_file_to_io") {|t|
+        File.binwrite(t.path, "foo")
+        t.rewind
+        assert_equal(0xcfc4ae1d, Zlib.crc32c(t))
+
+        t.rewind
+        crc = Zlib.crc32c(t.read(2))
+        assert_equal(0xcfc4ae1d, Zlib.crc32c(t, crc))
+
+        File.binwrite(t.path, "abc\x01\x02\x03" * 10000)
+        t.rewind
+        assert_equal(0x88a0fa6c, Zlib.crc32c(t))
+      }
+    end
+
+    def test_crc32c_combine
+      one = Zlib.crc32c("fo")
+      two = Zlib.crc32c("o")
+      begin
+        assert_equal(0xcfc4ae1d, Zlib.crc32c_combine(one, two, 1))
+      rescue NotImplementedError
+        omit "crc32_combine is not implemented"
+      rescue Test::Unit::AssertionFailedError
+        if /aix/ =~ RUBY_PLATFORM
+          omit "zconf.h in zlib does not handle _LARGE_FILES in AIX. Skip until it is fixed"
+        end
+        raise $!
+      end
+    end
+
+    def test_crc32c_table
+      t = Zlib.crc32c_table
+      assert_instance_of(Array, t)
+      t.each {|x| assert_kind_of(Integer, x) }
+    end
+
     def test_inflate
       s = Zlib::Deflate.deflate("foo")
       z = Zlib::Inflate.new