mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-09-05 15:28:39 +09:00
zlib: adapted chromium zlib
This commit is contained in:
Roy Tam
parent
b0e724dc85
commit
df51e6d171
27 changed files with 4084 additions and 51 deletions
|
|
@ -1,4 +1,4 @@
|
|||
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
|
||||
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
|
||||
# vim: set filetype=python:
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
|
|
|
|||
|
|
@ -59,6 +59,13 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2));
|
|||
# define MOD63(a) a %= BASE
|
||||
#endif
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
#include "adler32_simd.h"
|
||||
#include "x86.h"
|
||||
#elif defined(ADLER32_SIMD_NEON)
|
||||
#include "adler32_simd.h"
|
||||
#endif
|
||||
|
||||
/* ========================================================================= */
|
||||
uLong ZEXPORT adler32_z(adler, buf, len)
|
||||
uLong adler;
|
||||
|
|
@ -68,6 +75,14 @@ uLong ZEXPORT adler32_z(adler, buf, len)
|
|||
unsigned long sum2;
|
||||
unsigned n;
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
if (x86_cpu_enable_ssse3 && buf && len >= 64)
|
||||
return adler32_simd_(adler, buf, len);
|
||||
#elif defined(ADLER32_SIMD_NEON)
|
||||
if (buf && len >= 64)
|
||||
return adler32_simd_(adler, buf, len);
|
||||
#endif
|
||||
|
||||
/* split Adler-32 into component sums */
|
||||
sum2 = (adler >> 16) & 0xffff;
|
||||
adler &= 0xffff;
|
||||
|
|
@ -83,9 +98,24 @@ uLong ZEXPORT adler32_z(adler, buf, len)
|
|||
return adler | (sum2 << 16);
|
||||
}
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
/*
|
||||
* Use SSSE3 to compute the adler32. Since this routine can be
|
||||
* freely used, check CPU features here. zlib convention is to
|
||||
* call adler32(0, NULL, 0), before making calls to adler32().
|
||||
* So this is a good early (and infrequent) place to cache CPU
|
||||
* features for those later, more interesting adler32() calls.
|
||||
*/
|
||||
if (buf == Z_NULL) {
|
||||
if (!len) /* Assume user is calling adler32(0, NULL, 0); */
|
||||
x86_check_features();
|
||||
return 1L;
|
||||
}
|
||||
#else
|
||||
/* initial Adler-32 value (deferred check for len == 1 speed) */
|
||||
if (buf == Z_NULL)
|
||||
return 1L;
|
||||
#endif
|
||||
|
||||
/* in case short lengths are provided, keep it somewhat fast */
|
||||
if (len < 16) {
|
||||
|
|
|
|||
366
modules/zlib/src/adler32_simd.c
Normal file
366
modules/zlib/src/adler32_simd.c
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
/* adler32_simd.c
|
||||
*
|
||||
* Copyright 2017 The Chromium Authors. All rights reserved.
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the Chromium source repository LICENSE file.
|
||||
*
|
||||
* Per http://en.wikipedia.org/wiki/Adler-32 the adler32 A value (aka s1) is
|
||||
* the sum of N input data bytes D1 ... DN,
|
||||
*
|
||||
* A = A0 + D1 + D2 + ... + DN
|
||||
*
|
||||
* where A0 is the initial value.
|
||||
*
|
||||
* SSE2 _mm_sad_epu8() can be used for byte sums (see http://bit.ly/2wpUOeD,
|
||||
* for example) and accumulating the byte sums can use SSE shuffle-adds (see
|
||||
* the "Integer" section of http://bit.ly/2erPT8t for details). Arm NEON has
|
||||
* similar instructions.
|
||||
*
|
||||
* The adler32 B value (aka s2) sums the A values from each step:
|
||||
*
|
||||
* B0 + (A0 + D1) + (A0 + D1 + D2) + ... + (A0 + D1 + D2 + ... + DN) or
|
||||
*
|
||||
* B0 + N.A0 + N.D1 + (N-1).D2 + (N-2).D3 + ... + (N-(N-1)).DN
|
||||
*
|
||||
* B0 being the initial value. For 32 bytes (ideal for garden-variety SIMD):
|
||||
*
|
||||
* B = B0 + 32.A0 + [D1 D2 D3 ... D32] x [32 31 30 ... 1].
|
||||
*
|
||||
* Adjacent blocks of 32 input bytes can be iterated with the expressions to
|
||||
* compute the adler32 s1 s2 of M >> 32 input bytes [1].
|
||||
*
|
||||
* As M grows, the s1 s2 sums grow. If left unchecked, they would eventually
|
||||
* overflow the precision of their integer representation (bad). However, s1
|
||||
* and s2 also need to be computed modulo the adler BASE value (reduced). If
|
||||
* at most NMAX bytes are processed before a reduce, s1 s2 _cannot_ overflow
|
||||
* a uint32_t type (the NMAX constraint) [2].
|
||||
*
|
||||
* [1] the iterative equations for s2 contain constant factors; these can be
|
||||
* hoisted from the n-blocks do loop of the SIMD code.
|
||||
*
|
||||
* [2] zlib adler32_z() uses this fact to implement NMAX-block-based updates
|
||||
* of the adler s1 s2 of uint32_t type (see adler32.c).
|
||||
*/
|
||||
|
||||
#include "adler32_simd.h"
|
||||
|
||||
/* Definitions from adler32.c: largest prime smaller than 65536 */
|
||||
#define BASE 65521U
|
||||
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
|
||||
#define NMAX 5552
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
|
||||
#include <tmmintrin.h>
|
||||
|
||||
uint32_t ZLIB_INTERNAL adler32_simd_( /* SSSE3 */
|
||||
uint32_t adler,
|
||||
const unsigned char *buf,
|
||||
z_size_t len)
|
||||
{
|
||||
/*
|
||||
* Split Adler-32 into component sums.
|
||||
*/
|
||||
uint32_t s1 = adler & 0xffff;
|
||||
uint32_t s2 = adler >> 16;
|
||||
|
||||
/*
|
||||
* Process the data in blocks.
|
||||
*/
|
||||
const unsigned BLOCK_SIZE = 1 << 5;
|
||||
|
||||
z_size_t blocks = len / BLOCK_SIZE;
|
||||
len -= blocks * BLOCK_SIZE;
|
||||
|
||||
while (blocks)
|
||||
{
|
||||
unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */
|
||||
if (n > blocks)
|
||||
n = (unsigned) blocks;
|
||||
blocks -= n;
|
||||
|
||||
const __m128i tap1 =
|
||||
_mm_setr_epi8(32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17);
|
||||
const __m128i tap2 =
|
||||
_mm_setr_epi8(16,15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1);
|
||||
const __m128i zero =
|
||||
_mm_setr_epi8( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
|
||||
const __m128i ones =
|
||||
_mm_set_epi16( 1, 1, 1, 1, 1, 1, 1, 1);
|
||||
|
||||
/*
|
||||
* Process n blocks of data. At most NMAX data bytes can be
|
||||
* processed before s2 must be reduced modulo BASE.
|
||||
*/
|
||||
__m128i v_ps = _mm_set_epi32(0, 0, 0, s1 * n);
|
||||
__m128i v_s2 = _mm_set_epi32(0, 0, 0, s2);
|
||||
__m128i v_s1 = _mm_set_epi32(0, 0, 0, 0);
|
||||
|
||||
do {
|
||||
/*
|
||||
* Load 32 input bytes.
|
||||
*/
|
||||
const __m128i bytes1 = _mm_loadu_si128((__m128i*)(buf));
|
||||
const __m128i bytes2 = _mm_loadu_si128((__m128i*)(buf + 16));
|
||||
|
||||
/*
|
||||
* Add previous block byte sum to v_ps.
|
||||
*/
|
||||
v_ps = _mm_add_epi32(v_ps, v_s1);
|
||||
|
||||
/*
|
||||
* Horizontally add the bytes for s1, multiply-adds the
|
||||
* bytes by [ 32, 31, 30, ... ] for s2.
|
||||
*/
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes1, zero));
|
||||
const __m128i mad1 = _mm_maddubs_epi16(bytes1, tap1);
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad1, ones));
|
||||
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes2, zero));
|
||||
const __m128i mad2 = _mm_maddubs_epi16(bytes2, tap2);
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad2, ones));
|
||||
|
||||
buf += BLOCK_SIZE;
|
||||
|
||||
} while (--n);
|
||||
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_slli_epi32(v_ps, 5));
|
||||
|
||||
/*
|
||||
* Sum epi32 ints v_s1(s2) and accumulate in s1(s2).
|
||||
*/
|
||||
|
||||
#define S23O1 _MM_SHUFFLE(2,3,0,1) /* A B C D -> B A D C */
|
||||
#define S1O32 _MM_SHUFFLE(1,0,3,2) /* A B C D -> C D A B */
|
||||
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S23O1));
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S1O32));
|
||||
|
||||
s1 += _mm_cvtsi128_si32(v_s1);
|
||||
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S23O1));
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S1O32));
|
||||
|
||||
s2 = _mm_cvtsi128_si32(v_s2);
|
||||
|
||||
#undef S23O1
|
||||
#undef S1O32
|
||||
|
||||
/*
|
||||
* Reduce.
|
||||
*/
|
||||
s1 %= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Handle leftover data.
|
||||
*/
|
||||
if (len) {
|
||||
if (len >= 16) {
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
while (len--) {
|
||||
s2 += (s1 += *buf++);
|
||||
}
|
||||
|
||||
if (s1 >= BASE)
|
||||
s1 -= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the recombined sums.
|
||||
*/
|
||||
return s1 | (s2 << 16);
|
||||
}
|
||||
|
||||
#elif defined(ADLER32_SIMD_NEON)
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
uint32_t ZLIB_INTERNAL adler32_simd_( /* NEON */
|
||||
uint32_t adler,
|
||||
const unsigned char *buf,
|
||||
z_size_t len)
|
||||
{
|
||||
/*
|
||||
* Split Adler-32 into component sums.
|
||||
*/
|
||||
uint32_t s1 = adler & 0xffff;
|
||||
uint32_t s2 = adler >> 16;
|
||||
|
||||
/*
|
||||
* Serially compute s1 & s2, until the data is 16-byte aligned.
|
||||
*/
|
||||
if ((uintptr_t)buf & 15) {
|
||||
while ((uintptr_t)buf & 15) {
|
||||
s2 += (s1 += *buf++);
|
||||
--len;
|
||||
}
|
||||
|
||||
if (s1 >= BASE)
|
||||
s1 -= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Process the data in blocks.
|
||||
*/
|
||||
const unsigned BLOCK_SIZE = 1 << 5;
|
||||
|
||||
z_size_t blocks = len / BLOCK_SIZE;
|
||||
len -= blocks * BLOCK_SIZE;
|
||||
|
||||
while (blocks)
|
||||
{
|
||||
unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */
|
||||
if (n > blocks)
|
||||
n = (unsigned) blocks;
|
||||
blocks -= n;
|
||||
|
||||
/*
|
||||
* Process n blocks of data. At most NMAX data bytes can be
|
||||
* processed before s2 must be reduced modulo BASE.
|
||||
*/
|
||||
uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, s1 * n };
|
||||
uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 };
|
||||
|
||||
uint16x8_t v_column_sum_1 = vdupq_n_u16(0);
|
||||
uint16x8_t v_column_sum_2 = vdupq_n_u16(0);
|
||||
uint16x8_t v_column_sum_3 = vdupq_n_u16(0);
|
||||
uint16x8_t v_column_sum_4 = vdupq_n_u16(0);
|
||||
|
||||
do {
|
||||
/*
|
||||
* Load 32 input bytes.
|
||||
*/
|
||||
const uint8x16_t bytes1 = vld1q_u8((uint8_t*)(buf));
|
||||
const uint8x16_t bytes2 = vld1q_u8((uint8_t*)(buf + 16));
|
||||
|
||||
/*
|
||||
* Add previous block byte sum to v_s2.
|
||||
*/
|
||||
v_s2 = vaddq_u32(v_s2, v_s1);
|
||||
|
||||
/*
|
||||
* Horizontally add the bytes for s1.
|
||||
*/
|
||||
v_s1 = vpadalq_u16(v_s1, vpadalq_u8(vpaddlq_u8(bytes1), bytes2));
|
||||
|
||||
/*
|
||||
* Vertically add the bytes for s2.
|
||||
*/
|
||||
v_column_sum_1 = vaddw_u8(v_column_sum_1, vget_low_u8 (bytes1));
|
||||
v_column_sum_2 = vaddw_u8(v_column_sum_2, vget_high_u8(bytes1));
|
||||
v_column_sum_3 = vaddw_u8(v_column_sum_3, vget_low_u8 (bytes2));
|
||||
v_column_sum_4 = vaddw_u8(v_column_sum_4, vget_high_u8(bytes2));
|
||||
|
||||
buf += BLOCK_SIZE;
|
||||
|
||||
} while (--n);
|
||||
|
||||
v_s2 = vshlq_n_u32(v_s2, 5);
|
||||
|
||||
/*
|
||||
* Multiply-add bytes by [ 32, 31, 30, ... ] for s2.
|
||||
*/
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_1),
|
||||
(uint16x4_t) { 32, 31, 30, 29 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_1),
|
||||
(uint16x4_t) { 28, 27, 26, 25 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_2),
|
||||
(uint16x4_t) { 24, 23, 22, 21 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_2),
|
||||
(uint16x4_t) { 20, 19, 18, 17 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_3),
|
||||
(uint16x4_t) { 16, 15, 14, 13 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_3),
|
||||
(uint16x4_t) { 12, 11, 10, 9 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_4),
|
||||
(uint16x4_t) { 8, 7, 6, 5 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_4),
|
||||
(uint16x4_t) { 4, 3, 2, 1 });
|
||||
|
||||
/*
|
||||
* Sum epi32 ints v_s1(s2) and accumulate in s1(s2).
|
||||
*/
|
||||
uint32x2_t sum1 = vpadd_u32(vget_low_u32(v_s1), vget_high_u32(v_s1));
|
||||
uint32x2_t sum2 = vpadd_u32(vget_low_u32(v_s2), vget_high_u32(v_s2));
|
||||
uint32x2_t s1s2 = vpadd_u32(sum1, sum2);
|
||||
|
||||
s1 += vget_lane_u32(s1s2, 0);
|
||||
s2 += vget_lane_u32(s1s2, 1);
|
||||
|
||||
/*
|
||||
* Reduce.
|
||||
*/
|
||||
s1 %= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Handle leftover data.
|
||||
*/
|
||||
if (len) {
|
||||
if (len >= 16) {
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
while (len--) {
|
||||
s2 += (s1 += *buf++);
|
||||
}
|
||||
|
||||
if (s1 >= BASE)
|
||||
s1 -= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the recombined sums.
|
||||
*/
|
||||
return s1 | (s2 << 16);
|
||||
}
|
||||
|
||||
#endif /* ADLER32_SIMD_SSSE3 */
|
||||
16
modules/zlib/src/adler32_simd.h
Normal file
16
modules/zlib/src/adler32_simd.h
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
/* adler32_simd.h
|
||||
*
|
||||
* Copyright 2017 The Chromium Authors. All rights reserved.
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the Chromium source repository LICENSE file.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "zconf.h"
|
||||
#include "zutil.h"
|
||||
|
||||
uint32_t ZLIB_INTERNAL adler32_simd_(
|
||||
uint32_t adler,
|
||||
const unsigned char *buf,
|
||||
z_size_t len);
|
||||
425
modules/zlib/src/chunkcopy.h
Normal file
425
modules/zlib/src/chunkcopy.h
Normal file
|
|
@ -0,0 +1,425 @@
|
|||
/* chunkcopy.h -- fast chunk copy and set operations
|
||||
* Copyright (C) 2017 ARM, Inc.
|
||||
* Copyright 2017 The Chromium Authors. All rights reserved.
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the Chromium source repository LICENSE file.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef CHUNKCOPY_H
|
||||
#define CHUNKCOPY_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include "zutil.h"
|
||||
|
||||
#define Z_STATIC_ASSERT(name, assert) typedef char name[(assert) ? 1 : -1]
|
||||
|
||||
#if __STDC_VERSION__ >= 199901L
|
||||
#define Z_RESTRICT restrict
|
||||
#else
|
||||
#define Z_RESTRICT
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) || defined(__GNUC__) || defined(__llvm__)
|
||||
#define Z_BUILTIN_MEMCPY __builtin_memcpy
|
||||
#else
|
||||
#define Z_BUILTIN_MEMCPY zmemcpy
|
||||
#endif
|
||||
|
||||
#if defined(INFLATE_CHUNK_SIMD_NEON)
|
||||
#include <arm_neon.h>
|
||||
typedef uint8x16_t z_vec128i_t;
|
||||
#elif defined(INFLATE_CHUNK_SIMD_SSE2)
|
||||
#include <emmintrin.h>
|
||||
typedef __m128i z_vec128i_t;
|
||||
#else
|
||||
#error chunkcopy.h inflate chunk SIMD is not defined for your build target
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#if _MSC_VER < 1900
|
||||
#define inline
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* chunk copy type: the z_vec128i_t type size should be exactly 128-bits
|
||||
* and equal to CHUNKCOPY_CHUNK_SIZE.
|
||||
*/
|
||||
#define CHUNKCOPY_CHUNK_SIZE sizeof(z_vec128i_t)
|
||||
|
||||
Z_STATIC_ASSERT(vector_128_bits_wide,
|
||||
CHUNKCOPY_CHUNK_SIZE == sizeof(int8_t) * 16);
|
||||
|
||||
/*
|
||||
* Ask the compiler to perform a wide, unaligned load with a machine
|
||||
* instruction appropriate for the z_vec128i_t type.
|
||||
*/
|
||||
static inline z_vec128i_t loadchunk(
|
||||
const unsigned char FAR* s) {
|
||||
z_vec128i_t v;
|
||||
Z_BUILTIN_MEMCPY(&v, s, sizeof(v));
|
||||
return v;
|
||||
}
|
||||
|
||||
/*
|
||||
* Ask the compiler to perform a wide, unaligned store with a machine
|
||||
* instruction appropriate for the z_vec128i_t type.
|
||||
*/
|
||||
static inline void storechunk(
|
||||
unsigned char FAR* d,
|
||||
const z_vec128i_t v) {
|
||||
Z_BUILTIN_MEMCPY(d, &v, sizeof(v));
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform a memcpy-like operation, assuming that length is non-zero and that
|
||||
* it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if
|
||||
* the length is shorter than this.
|
||||
*
|
||||
* It also guarantees that it will properly unroll the data if the distance
|
||||
* between `out` and `from` is at least CHUNKCOPY_CHUNK_SIZE, which we rely on
|
||||
* in chunkcopy_relaxed().
|
||||
*
|
||||
* Aside from better memory bus utilisation, this means that short copies
|
||||
* (CHUNKCOPY_CHUNK_SIZE bytes or fewer) will fall straight through the loop
|
||||
* without iteration, which will hopefully make the branch prediction more
|
||||
* reliable.
|
||||
*/
|
||||
static inline unsigned char FAR* chunkcopy_core(
|
||||
unsigned char FAR* out,
|
||||
const unsigned char FAR* from,
|
||||
unsigned len) {
|
||||
const int bump = (--len % CHUNKCOPY_CHUNK_SIZE) + 1;
|
||||
storechunk(out, loadchunk(from));
|
||||
out += bump;
|
||||
from += bump;
|
||||
len /= CHUNKCOPY_CHUNK_SIZE;
|
||||
while (len-- > 0) {
|
||||
storechunk(out, loadchunk(from));
|
||||
out += CHUNKCOPY_CHUNK_SIZE;
|
||||
from += CHUNKCOPY_CHUNK_SIZE;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/*
|
||||
* Like chunkcopy_core(), but avoid writing beyond of legal output.
|
||||
*
|
||||
* Accepts an additional pointer to the end of safe output. A generic safe
|
||||
* copy would use (out + len), but it's normally the case that the end of the
|
||||
* output buffer is beyond the end of the current copy, and this can still be
|
||||
* exploited.
|
||||
*/
|
||||
static inline unsigned char FAR* chunkcopy_core_safe(
|
||||
unsigned char FAR* out,
|
||||
const unsigned char FAR* from,
|
||||
unsigned len,
|
||||
unsigned char FAR* limit) {
|
||||
Assert(out + len <= limit, "chunk copy exceeds safety limit");
|
||||
if ((limit - out) < (ptrdiff_t)CHUNKCOPY_CHUNK_SIZE) {
|
||||
const unsigned char FAR* Z_RESTRICT rfrom = from;
|
||||
if (len & 8) {
|
||||
Z_BUILTIN_MEMCPY(out, rfrom, 8);
|
||||
out += 8;
|
||||
rfrom += 8;
|
||||
}
|
||||
if (len & 4) {
|
||||
Z_BUILTIN_MEMCPY(out, rfrom, 4);
|
||||
out += 4;
|
||||
rfrom += 4;
|
||||
}
|
||||
if (len & 2) {
|
||||
Z_BUILTIN_MEMCPY(out, rfrom, 2);
|
||||
out += 2;
|
||||
rfrom += 2;
|
||||
}
|
||||
if (len & 1) {
|
||||
*out++ = *rfrom++;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return chunkcopy_core(out, from, len);
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform short copies until distance can be rewritten as being at least
|
||||
* CHUNKCOPY_CHUNK_SIZE.
|
||||
*
|
||||
* Assumes it's OK to overwrite at least the first 2*CHUNKCOPY_CHUNK_SIZE
|
||||
* bytes of output even if the copy is shorter than this. This assumption
|
||||
* holds within zlib inflate_fast(), which starts every iteration with at
|
||||
* least 258 bytes of output space available (258 being the maximum length
|
||||
* output from a single token; see inffast.c).
|
||||
*/
|
||||
static inline unsigned char FAR* chunkunroll_relaxed(
|
||||
unsigned char FAR* out,
|
||||
unsigned FAR* dist,
|
||||
unsigned FAR* len) {
|
||||
const unsigned char FAR* from = out - *dist;
|
||||
while (*dist < *len && *dist < CHUNKCOPY_CHUNK_SIZE) {
|
||||
storechunk(out, loadchunk(from));
|
||||
out += *dist;
|
||||
*len -= *dist;
|
||||
*dist += *dist;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
#if defined(INFLATE_CHUNK_SIMD_NEON)
|
||||
/*
|
||||
* v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in
|
||||
* every 64-bit component of the 128-bit result (64-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load64_dup(const void* src) {
|
||||
return vcombine_u8(vld1_u8(src), vld1_u8(src));
|
||||
}
|
||||
|
||||
/*
|
||||
* v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in
|
||||
* every 32-bit component of the 128-bit result (32-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load32_dup(const void* src) {
|
||||
int32_t i32;
|
||||
Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32));
|
||||
return vreinterpretq_u8_s32(vdupq_n_s32(i32));
|
||||
}
|
||||
|
||||
/*
|
||||
* v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in
|
||||
* every 16-bit component of the 128-bit result (16-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load16_dup(const void* src) {
|
||||
int16_t i16;
|
||||
Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16));
|
||||
return vreinterpretq_u8_s16(vdupq_n_s16(i16));
|
||||
}
|
||||
|
||||
/*
|
||||
* v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit
|
||||
* component of the 128-bit result (8-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load8_dup(const void* src) {
|
||||
return vld1q_dup_u8((const uint8_t*)src);
|
||||
}
|
||||
|
||||
/*
|
||||
* v_store_128(): store the 128-bit vec in a memory destination (that might
|
||||
* not be 16-byte aligned) void* out.
|
||||
*/
|
||||
static inline void v_store_128(void* out, const z_vec128i_t vec) {
|
||||
vst1q_u8(out, vec);
|
||||
}
|
||||
|
||||
#elif defined(INFLATE_CHUNK_SIMD_SSE2)
|
||||
/*
|
||||
* v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in
|
||||
* every 64-bit component of the 128-bit result (64-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load64_dup(const void* src) {
|
||||
int64_t i64;
|
||||
Z_BUILTIN_MEMCPY(&i64, src, sizeof(i64));
|
||||
#if 0//def _WIN64
|
||||
return _mm_set1_epi64x(i64);
|
||||
#else
|
||||
_declspec(align(16)) uint64_t temp[2] = {i64, i64};
|
||||
return _mm_load_si128((const __m128i*)temp);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in
|
||||
* every 32-bit component of the 128-bit result (32-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load32_dup(const void* src) {
|
||||
int32_t i32;
|
||||
Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32));
|
||||
return _mm_set1_epi32(i32);
|
||||
}
|
||||
|
||||
/*
|
||||
* v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in
|
||||
* every 16-bit component of the 128-bit result (16-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load16_dup(const void* src) {
|
||||
int16_t i16;
|
||||
Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16));
|
||||
return _mm_set1_epi16(i16);
|
||||
}
|
||||
|
||||
/*
|
||||
* v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit
|
||||
* component of the 128-bit result (8-bit int splat).
|
||||
*/
|
||||
static inline z_vec128i_t v_load8_dup(const void* src) {
|
||||
return _mm_set1_epi8(*(const char*)src);
|
||||
}
|
||||
|
||||
/*
|
||||
* v_store_128(): store the 128-bit vec in a memory destination (that might
|
||||
* not be 16-byte aligned) void* out.
|
||||
*/
|
||||
static inline void v_store_128(void* out, const z_vec128i_t vec) {
|
||||
_mm_storeu_si128((__m128i*)out, vec);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Perform an overlapping copy which behaves as a memset() operation, but
|
||||
* supporting periods other than one, and assume that length is non-zero and
|
||||
* that it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE*3 bytes of output
|
||||
* even if the length is shorter than this.
|
||||
*/
|
||||
static inline unsigned char FAR* chunkset_core(
|
||||
unsigned char FAR* out,
|
||||
unsigned period,
|
||||
unsigned len) {
|
||||
z_vec128i_t v;
|
||||
const int bump = ((len - 1) % sizeof(v)) + 1;
|
||||
|
||||
switch (period) {
|
||||
case 1:
|
||||
v = v_load8_dup(out - 1);
|
||||
v_store_128(out, v);
|
||||
out += bump;
|
||||
len -= bump;
|
||||
while (len > 0) {
|
||||
v_store_128(out, v);
|
||||
out += sizeof(v);
|
||||
len -= sizeof(v);
|
||||
}
|
||||
return out;
|
||||
case 2:
|
||||
v = v_load16_dup(out - 2);
|
||||
v_store_128(out, v);
|
||||
out += bump;
|
||||
len -= bump;
|
||||
if (len > 0) {
|
||||
v = v_load16_dup(out - 2);
|
||||
do {
|
||||
v_store_128(out, v);
|
||||
out += sizeof(v);
|
||||
len -= sizeof(v);
|
||||
} while (len > 0);
|
||||
}
|
||||
return out;
|
||||
case 4:
|
||||
v = v_load32_dup(out - 4);
|
||||
v_store_128(out, v);
|
||||
out += bump;
|
||||
len -= bump;
|
||||
if (len > 0) {
|
||||
v = v_load32_dup(out - 4);
|
||||
do {
|
||||
v_store_128(out, v);
|
||||
out += sizeof(v);
|
||||
len -= sizeof(v);
|
||||
} while (len > 0);
|
||||
}
|
||||
return out;
|
||||
case 8:
|
||||
v = v_load64_dup(out - 8);
|
||||
v_store_128(out, v);
|
||||
out += bump;
|
||||
len -= bump;
|
||||
if (len > 0) {
|
||||
v = v_load64_dup(out - 8);
|
||||
do {
|
||||
v_store_128(out, v);
|
||||
out += sizeof(v);
|
||||
len -= sizeof(v);
|
||||
} while (len > 0);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
out = chunkunroll_relaxed(out, &period, &len);
|
||||
return chunkcopy_core(out, out - period, len);
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform a memcpy-like operation, but assume that length is non-zero and that
|
||||
* it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if
|
||||
* the length is shorter than this.
|
||||
*
|
||||
* Unlike chunkcopy_core() above, no guarantee is made regarding the behaviour
|
||||
* of overlapping buffers, regardless of the distance between the pointers.
|
||||
* This is reflected in the `restrict`-qualified pointers, allowing the
|
||||
* compiler to re-order loads and stores.
|
||||
*/
|
||||
static inline unsigned char FAR* chunkcopy_relaxed(
|
||||
unsigned char FAR* Z_RESTRICT out,
|
||||
const unsigned char FAR* Z_RESTRICT from,
|
||||
unsigned len) {
|
||||
return chunkcopy_core(out, from, len);
|
||||
}
|
||||
|
||||
/*
|
||||
* Like chunkcopy_relaxed(), but avoid writing beyond of legal output.
|
||||
*
|
||||
* Unlike chunkcopy_core_safe() above, no guarantee is made regarding the
|
||||
* behaviour of overlapping buffers, regardless of the distance between the
|
||||
* pointers. This is reflected in the `restrict`-qualified pointers, allowing
|
||||
* the compiler to re-order loads and stores.
|
||||
*
|
||||
* Accepts an additional pointer to the end of safe output. A generic safe
|
||||
* copy would use (out + len), but it's normally the case that the end of the
|
||||
* output buffer is beyond the end of the current copy, and this can still be
|
||||
* exploited.
|
||||
*/
|
||||
static inline unsigned char FAR* chunkcopy_safe(
|
||||
unsigned char FAR* out,
|
||||
const unsigned char FAR* Z_RESTRICT from,
|
||||
unsigned len,
|
||||
unsigned char FAR* limit) {
|
||||
Assert(out + len <= limit, "chunk copy exceeds safety limit");
|
||||
return chunkcopy_core_safe(out, from, len, limit);
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform chunky copy within the same buffer, where the source and destination
|
||||
* may potentially overlap.
|
||||
*
|
||||
* Assumes that len > 0 on entry, and that it's safe to write at least
|
||||
* CHUNKCOPY_CHUNK_SIZE*3 bytes to the output.
|
||||
*/
|
||||
static inline unsigned char FAR* chunkcopy_lapped_relaxed(
|
||||
unsigned char FAR* out,
|
||||
unsigned dist,
|
||||
unsigned len) {
|
||||
if (dist < len && dist < CHUNKCOPY_CHUNK_SIZE) {
|
||||
return chunkset_core(out, dist, len);
|
||||
}
|
||||
return chunkcopy_core(out, out - dist, len);
|
||||
}
|
||||
|
||||
/*
|
||||
* Behave like chunkcopy_lapped_relaxed(), but avoid writing beyond of legal
|
||||
* output.
|
||||
*
|
||||
* Accepts an additional pointer to the end of safe output. A generic safe
|
||||
* copy would use (out + len), but it's normally the case that the end of the
|
||||
* output buffer is beyond the end of the current copy, and this can still be
|
||||
* exploited.
|
||||
*/
|
||||
static inline unsigned char FAR* chunkcopy_lapped_safe(
|
||||
unsigned char FAR* out,
|
||||
unsigned dist,
|
||||
unsigned len,
|
||||
unsigned char FAR* limit) {
|
||||
Assert(out + len <= limit, "chunk copy exceeds safety limit");
|
||||
if ((limit - out) < (ptrdiff_t)(3 * CHUNKCOPY_CHUNK_SIZE)) {
|
||||
/* TODO(cavalcantii): try harder to optimise this */
|
||||
while (len-- > 0) {
|
||||
*out = *(out - dist);
|
||||
out++;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return chunkcopy_lapped_relaxed(out, dist, len);
|
||||
}
|
||||
|
||||
#undef Z_STATIC_ASSERT
|
||||
#undef Z_RESTRICT
|
||||
#undef Z_BUILTIN_MEMCPY
|
||||
|
||||
#endif /* CHUNKCOPY_H */
|
||||
|
|
@ -28,6 +28,9 @@
|
|||
# endif /* !DYNAMIC_CRC_TABLE */
|
||||
#endif /* MAKECRCH */
|
||||
|
||||
#include "deflate.h"
|
||||
#include "x86.h"
|
||||
#include "crc32_simd.h"
|
||||
#include "zutil.h" /* for STDC and FAR definitions */
|
||||
|
||||
/* Definitions for doing the crc four data bytes at a time. */
|
||||
|
|
@ -204,7 +207,33 @@ unsigned long ZEXPORT crc32_z(crc, buf, len)
|
|||
const unsigned char FAR *buf;
|
||||
z_size_t len;
|
||||
{
|
||||
if (buf == Z_NULL) return 0UL;
|
||||
#if defined(CRC32_SIMD_SSE42_PCLMUL)
|
||||
/*
|
||||
* Use x86 sse4.2+pclmul SIMD to compute the crc32. Since this
|
||||
* routine can be freely used, check the CPU features here.
|
||||
*/
|
||||
if (buf == Z_NULL) {
|
||||
if (!len) /* Assume user is calling crc32(0, NULL, 0); */
|
||||
x86_check_features();
|
||||
return 0UL;
|
||||
}
|
||||
|
||||
if (x86_cpu_enable_simd && len >= Z_CRC32_SSE42_MINIMUM_LENGTH) {
|
||||
/* crc32 16-byte chunks */
|
||||
z_size_t chunk_size = len & ~Z_CRC32_SSE42_CHUNKSIZE_MASK;
|
||||
crc = ~crc32_sse42_simd_(buf, chunk_size, ~(uint32_t)crc);
|
||||
/* check remaining data */
|
||||
len -= chunk_size;
|
||||
if (!len)
|
||||
return crc;
|
||||
/* Fall into the default crc32 for the remaining data. */
|
||||
buf += chunk_size;
|
||||
}
|
||||
#else
|
||||
if (buf == Z_NULL) {
|
||||
return 0UL;
|
||||
}
|
||||
#endif /* CRC32_SIMD_SSE42_PCLMUL */
|
||||
|
||||
#ifdef DYNAMIC_CRC_TABLE
|
||||
if (crc_table_empty)
|
||||
|
|
@ -440,3 +469,28 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2)
|
|||
{
|
||||
return crc32_combine_(crc1, crc2, len2);
|
||||
}
|
||||
|
||||
ZLIB_INTERNAL void crc_reset(deflate_state *const s)
|
||||
{
|
||||
if (x86_cpu_enable_simd) {
|
||||
crc_fold_init(s);
|
||||
return;
|
||||
}
|
||||
s->strm->adler = crc32(0L, Z_NULL, 0);
|
||||
}
|
||||
|
||||
ZLIB_INTERNAL void crc_finalize(deflate_state *const s)
|
||||
{
|
||||
if (x86_cpu_enable_simd)
|
||||
s->strm->adler = crc_fold_512to32(s);
|
||||
}
|
||||
|
||||
ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size)
|
||||
{
|
||||
if (x86_cpu_enable_simd) {
|
||||
crc_fold_copy(strm->state, dst, strm->next_in, size);
|
||||
return;
|
||||
}
|
||||
zmemcpy(dst, strm->next_in, size);
|
||||
strm->adler = crc32(strm->adler, dst, size);
|
||||
}
|
||||
|
|
|
|||
157
modules/zlib/src/crc32_simd.c
Normal file
157
modules/zlib/src/crc32_simd.c
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
/* crc32_simd.c
|
||||
*
|
||||
* Copyright 2017 The Chromium Authors. All rights reserved.
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the Chromium source repository LICENSE file.
|
||||
*/
|
||||
|
||||
#include "crc32_simd.h"
|
||||
|
||||
#if defined(CRC32_SIMD_SSE42_PCLMUL)
|
||||
|
||||
/*
|
||||
* crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer
|
||||
* length must be at least 64, and a multiple of 16. Based on:
|
||||
*
|
||||
* "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
|
||||
* V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0
|
||||
*/
|
||||
|
||||
#include <emmintrin.h>
|
||||
#include <smmintrin.h>
|
||||
#include <wmmintrin.h>
|
||||
|
||||
uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */
|
||||
const unsigned char *buf,
|
||||
z_size_t len,
|
||||
uint32_t crc)
|
||||
{
|
||||
/*
|
||||
* Definitions of the bit-reflected domain constants k1,k2,k3, etc and
|
||||
* the CRC32+Barrett polynomials given at the end of the paper.
|
||||
*/
|
||||
static const uint64_t zalign(16) k1k2[] = { 0x0154442bd4, 0x01c6e41596 };
|
||||
static const uint64_t zalign(16) k3k4[] = { 0x01751997d0, 0x00ccaa009e };
|
||||
static const uint64_t zalign(16) k5k0[] = { 0x0163cd6124, 0x0000000000 };
|
||||
static const uint64_t zalign(16) poly[] = { 0x01db710641, 0x01f7011641 };
|
||||
|
||||
__m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;
|
||||
|
||||
/*
|
||||
* There's at least one block of 64.
|
||||
*/
|
||||
x1 = _mm_loadu_si128((__m128i *)(buf + 0x00));
|
||||
x2 = _mm_loadu_si128((__m128i *)(buf + 0x10));
|
||||
x3 = _mm_loadu_si128((__m128i *)(buf + 0x20));
|
||||
x4 = _mm_loadu_si128((__m128i *)(buf + 0x30));
|
||||
|
||||
x1 = _mm_xor_si128(x1, _mm_cvtsi32_si128(crc));
|
||||
|
||||
x0 = _mm_load_si128((__m128i *)k1k2);
|
||||
|
||||
buf += 64;
|
||||
len -= 64;
|
||||
|
||||
/*
|
||||
* Parallel fold blocks of 64, if any.
|
||||
*/
|
||||
while (len >= 64)
|
||||
{
|
||||
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
||||
x6 = _mm_clmulepi64_si128(x2, x0, 0x00);
|
||||
x7 = _mm_clmulepi64_si128(x3, x0, 0x00);
|
||||
x8 = _mm_clmulepi64_si128(x4, x0, 0x00);
|
||||
|
||||
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
||||
x2 = _mm_clmulepi64_si128(x2, x0, 0x11);
|
||||
x3 = _mm_clmulepi64_si128(x3, x0, 0x11);
|
||||
x4 = _mm_clmulepi64_si128(x4, x0, 0x11);
|
||||
|
||||
y5 = _mm_loadu_si128((__m128i *)(buf + 0x00));
|
||||
y6 = _mm_loadu_si128((__m128i *)(buf + 0x10));
|
||||
y7 = _mm_loadu_si128((__m128i *)(buf + 0x20));
|
||||
y8 = _mm_loadu_si128((__m128i *)(buf + 0x30));
|
||||
|
||||
x1 = _mm_xor_si128(x1, x5);
|
||||
x2 = _mm_xor_si128(x2, x6);
|
||||
x3 = _mm_xor_si128(x3, x7);
|
||||
x4 = _mm_xor_si128(x4, x8);
|
||||
|
||||
x1 = _mm_xor_si128(x1, y5);
|
||||
x2 = _mm_xor_si128(x2, y6);
|
||||
x3 = _mm_xor_si128(x3, y7);
|
||||
x4 = _mm_xor_si128(x4, y8);
|
||||
|
||||
buf += 64;
|
||||
len -= 64;
|
||||
}
|
||||
|
||||
/*
|
||||
* Fold into 128-bits.
|
||||
*/
|
||||
x0 = _mm_load_si128((__m128i *)k3k4);
|
||||
|
||||
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
||||
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
||||
x1 = _mm_xor_si128(x1, x2);
|
||||
x1 = _mm_xor_si128(x1, x5);
|
||||
|
||||
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
||||
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
||||
x1 = _mm_xor_si128(x1, x3);
|
||||
x1 = _mm_xor_si128(x1, x5);
|
||||
|
||||
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
||||
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
||||
x1 = _mm_xor_si128(x1, x4);
|
||||
x1 = _mm_xor_si128(x1, x5);
|
||||
|
||||
/*
|
||||
* Single fold blocks of 16, if any.
|
||||
*/
|
||||
while (len >= 16)
|
||||
{
|
||||
x2 = _mm_loadu_si128((__m128i *)buf);
|
||||
|
||||
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
||||
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
||||
x1 = _mm_xor_si128(x1, x2);
|
||||
x1 = _mm_xor_si128(x1, x5);
|
||||
|
||||
buf += 16;
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
/*
|
||||
* Fold 128-bits to 64-bits.
|
||||
*/
|
||||
x2 = _mm_clmulepi64_si128(x1, x0, 0x10);
|
||||
x3 = _mm_setr_epi32(~0, 0, ~0, 0);
|
||||
x1 = _mm_srli_si128(x1, 8);
|
||||
x1 = _mm_xor_si128(x1, x2);
|
||||
|
||||
x0 = _mm_loadl_epi64((__m128i*)k5k0);
|
||||
|
||||
x2 = _mm_srli_si128(x1, 4);
|
||||
x1 = _mm_and_si128(x1, x3);
|
||||
x1 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
||||
x1 = _mm_xor_si128(x1, x2);
|
||||
|
||||
/*
|
||||
* Barret reduce to 32-bits.
|
||||
*/
|
||||
x0 = _mm_load_si128((__m128i*)poly);
|
||||
|
||||
x2 = _mm_and_si128(x1, x3);
|
||||
x2 = _mm_clmulepi64_si128(x2, x0, 0x10);
|
||||
x2 = _mm_and_si128(x2, x3);
|
||||
x2 = _mm_clmulepi64_si128(x2, x0, 0x00);
|
||||
x1 = _mm_xor_si128(x1, x2);
|
||||
|
||||
/*
|
||||
* Return the crc32.
|
||||
*/
|
||||
return _mm_extract_epi32(x1, 1);
|
||||
}
|
||||
|
||||
#endif /* CRC32_SIMD_SSE42_PCLMUL */
|
||||
27
modules/zlib/src/crc32_simd.h
Normal file
27
modules/zlib/src/crc32_simd.h
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/* crc32_simd.h
|
||||
*
|
||||
* Copyright 2017 The Chromium Authors. All rights reserved.
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the Chromium source repository LICENSE file.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "zconf.h"
|
||||
#include "zutil.h"
|
||||
|
||||
/*
|
||||
* crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer
|
||||
* length must be at least 64, and a multiple of 16.
|
||||
*/
|
||||
uint32_t ZLIB_INTERNAL crc32_sse42_simd_(
|
||||
const unsigned char *buf,
|
||||
z_size_t len,
|
||||
uint32_t crc);
|
||||
|
||||
/*
|
||||
* crc32_sse42_simd_ buffer size constraints: see the use in zlib/crc32.c
|
||||
* for computing the crc32 of an arbitrary length buffer.
|
||||
*/
|
||||
#define Z_CRC32_SSE42_MINIMUM_LENGTH 64
|
||||
#define Z_CRC32_SSE42_CHUNKSIZE_MASK 15
|
||||
493
modules/zlib/src/crc_folding.c
Normal file
493
modules/zlib/src/crc_folding.c
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
/*
|
||||
* Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ
|
||||
* instruction.
|
||||
*
|
||||
* A white paper describing this algorithm can be found at:
|
||||
* http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
|
||||
*
|
||||
* Copyright (C) 2013 Intel Corporation. All rights reserved.
|
||||
* Authors:
|
||||
* Wajdi Feghali <wajdi.k.feghali@intel.com>
|
||||
* Jim Guilford <james.guilford@intel.com>
|
||||
* Vinodh Gopal <vinodh.gopal@intel.com>
|
||||
* Erdinc Ozturk <erdinc.ozturk@intel.com>
|
||||
* Jim Kukunas <james.t.kukunas@linux.intel.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "deflate.h"
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <emmintrin.h>
|
||||
#include <immintrin.h>
|
||||
#include <wmmintrin.h>
|
||||
|
||||
#define CRC_LOAD(s) \
|
||||
do { \
|
||||
__m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\
|
||||
__m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\
|
||||
__m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\
|
||||
__m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\
|
||||
__m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4);
|
||||
|
||||
#define CRC_SAVE(s) \
|
||||
_mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\
|
||||
_mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\
|
||||
_mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\
|
||||
_mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\
|
||||
_mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\
|
||||
} while (0);
|
||||
|
||||
ZLIB_INTERNAL void crc_fold_init(deflate_state *const s)
|
||||
{
|
||||
CRC_LOAD(s)
|
||||
|
||||
xmm_crc0 = _mm_cvtsi32_si128(0x9db42487);
|
||||
xmm_crc1 = _mm_setzero_si128();
|
||||
xmm_crc2 = _mm_setzero_si128();
|
||||
xmm_crc3 = _mm_setzero_si128();
|
||||
|
||||
CRC_SAVE(s)
|
||||
|
||||
s->strm->adler = 0;
|
||||
}
|
||||
|
||||
local void fold_1(deflate_state *const s,
|
||||
__m128i *xmm_crc0, __m128i *xmm_crc1,
|
||||
__m128i *xmm_crc2, __m128i *xmm_crc3)
|
||||
{
|
||||
const __m128i xmm_fold4 = _mm_set_epi32(
|
||||
0x00000001, 0x54442bd4,
|
||||
0x00000001, 0xc6e41596);
|
||||
|
||||
__m128i x_tmp3;
|
||||
__m128 ps_crc0, ps_crc3, ps_res;
|
||||
|
||||
x_tmp3 = *xmm_crc3;
|
||||
|
||||
*xmm_crc3 = *xmm_crc0;
|
||||
*xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
|
||||
*xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10);
|
||||
ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
|
||||
ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
|
||||
ps_res = _mm_xor_ps(ps_crc0, ps_crc3);
|
||||
|
||||
*xmm_crc0 = *xmm_crc1;
|
||||
*xmm_crc1 = *xmm_crc2;
|
||||
*xmm_crc2 = x_tmp3;
|
||||
*xmm_crc3 = _mm_castps_si128(ps_res);
|
||||
}
|
||||
|
||||
local void fold_2(deflate_state *const s,
|
||||
__m128i *xmm_crc0, __m128i *xmm_crc1,
|
||||
__m128i *xmm_crc2, __m128i *xmm_crc3)
|
||||
{
|
||||
const __m128i xmm_fold4 = _mm_set_epi32(
|
||||
0x00000001, 0x54442bd4,
|
||||
0x00000001, 0xc6e41596);
|
||||
|
||||
__m128i x_tmp3, x_tmp2;
|
||||
__m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20;
|
||||
|
||||
x_tmp3 = *xmm_crc3;
|
||||
x_tmp2 = *xmm_crc2;
|
||||
|
||||
*xmm_crc3 = *xmm_crc1;
|
||||
*xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01);
|
||||
*xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10);
|
||||
ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
|
||||
ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
|
||||
ps_res31= _mm_xor_ps(ps_crc3, ps_crc1);
|
||||
|
||||
*xmm_crc2 = *xmm_crc0;
|
||||
*xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
|
||||
*xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10);
|
||||
ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
|
||||
ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
|
||||
ps_res20= _mm_xor_ps(ps_crc0, ps_crc2);
|
||||
|
||||
*xmm_crc0 = x_tmp2;
|
||||
*xmm_crc1 = x_tmp3;
|
||||
*xmm_crc2 = _mm_castps_si128(ps_res20);
|
||||
*xmm_crc3 = _mm_castps_si128(ps_res31);
|
||||
}
|
||||
|
||||
local void fold_3(deflate_state *const s,
|
||||
__m128i *xmm_crc0, __m128i *xmm_crc1,
|
||||
__m128i *xmm_crc2, __m128i *xmm_crc3)
|
||||
{
|
||||
const __m128i xmm_fold4 = _mm_set_epi32(
|
||||
0x00000001, 0x54442bd4,
|
||||
0x00000001, 0xc6e41596);
|
||||
|
||||
__m128i x_tmp3;
|
||||
__m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10;
|
||||
|
||||
x_tmp3 = *xmm_crc3;
|
||||
|
||||
*xmm_crc3 = *xmm_crc2;
|
||||
*xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01);
|
||||
*xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10);
|
||||
ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
|
||||
ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
|
||||
ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3);
|
||||
|
||||
*xmm_crc2 = *xmm_crc1;
|
||||
*xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01);
|
||||
*xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10);
|
||||
ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
|
||||
ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
|
||||
ps_res21= _mm_xor_ps(ps_crc1, ps_crc2);
|
||||
|
||||
*xmm_crc1 = *xmm_crc0;
|
||||
*xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
|
||||
*xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10);
|
||||
ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
|
||||
ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
|
||||
ps_res10= _mm_xor_ps(ps_crc0, ps_crc1);
|
||||
|
||||
*xmm_crc0 = x_tmp3;
|
||||
*xmm_crc1 = _mm_castps_si128(ps_res10);
|
||||
*xmm_crc2 = _mm_castps_si128(ps_res21);
|
||||
*xmm_crc3 = _mm_castps_si128(ps_res32);
|
||||
}
|
||||
|
||||
local void fold_4(deflate_state *const s,
|
||||
__m128i *xmm_crc0, __m128i *xmm_crc1,
|
||||
__m128i *xmm_crc2, __m128i *xmm_crc3)
|
||||
{
|
||||
const __m128i xmm_fold4 = _mm_set_epi32(
|
||||
0x00000001, 0x54442bd4,
|
||||
0x00000001, 0xc6e41596);
|
||||
|
||||
__m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3;
|
||||
__m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3;
|
||||
__m128 ps_t0, ps_t1, ps_t2, ps_t3;
|
||||
__m128 ps_res0, ps_res1, ps_res2, ps_res3;
|
||||
|
||||
x_tmp0 = *xmm_crc0;
|
||||
x_tmp1 = *xmm_crc1;
|
||||
x_tmp2 = *xmm_crc2;
|
||||
x_tmp3 = *xmm_crc3;
|
||||
|
||||
*xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
|
||||
x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10);
|
||||
ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
|
||||
ps_t0 = _mm_castsi128_ps(x_tmp0);
|
||||
ps_res0 = _mm_xor_ps(ps_crc0, ps_t0);
|
||||
|
||||
*xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01);
|
||||
x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10);
|
||||
ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
|
||||
ps_t1 = _mm_castsi128_ps(x_tmp1);
|
||||
ps_res1 = _mm_xor_ps(ps_crc1, ps_t1);
|
||||
|
||||
*xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01);
|
||||
x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10);
|
||||
ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
|
||||
ps_t2 = _mm_castsi128_ps(x_tmp2);
|
||||
ps_res2 = _mm_xor_ps(ps_crc2, ps_t2);
|
||||
|
||||
*xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01);
|
||||
x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10);
|
||||
ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
|
||||
ps_t3 = _mm_castsi128_ps(x_tmp3);
|
||||
ps_res3 = _mm_xor_ps(ps_crc3, ps_t3);
|
||||
|
||||
*xmm_crc0 = _mm_castps_si128(ps_res0);
|
||||
*xmm_crc1 = _mm_castps_si128(ps_res1);
|
||||
*xmm_crc2 = _mm_castps_si128(ps_res2);
|
||||
*xmm_crc3 = _mm_castps_si128(ps_res3);
|
||||
}
|
||||
|
||||
local const unsigned zalign(32) pshufb_shf_table[60] = {
|
||||
0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */
|
||||
0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */
|
||||
0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */
|
||||
0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */
|
||||
0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */
|
||||
0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */
|
||||
0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */
|
||||
0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */
|
||||
0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */
|
||||
0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/
|
||||
0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/
|
||||
0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/
|
||||
0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/
|
||||
0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/
|
||||
0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/
|
||||
};
|
||||
|
||||
local void partial_fold(deflate_state *const s, const size_t len,
|
||||
__m128i *xmm_crc0, __m128i *xmm_crc1,
|
||||
__m128i *xmm_crc2, __m128i *xmm_crc3,
|
||||
__m128i *xmm_crc_part)
|
||||
{
|
||||
|
||||
const __m128i xmm_fold4 = _mm_set_epi32(
|
||||
0x00000001, 0x54442bd4,
|
||||
0x00000001, 0xc6e41596);
|
||||
const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080);
|
||||
|
||||
__m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3;
|
||||
__m128i xmm_a0_0, xmm_a0_1;
|
||||
__m128 ps_crc3, psa0_0, psa0_1, ps_res;
|
||||
|
||||
xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1));
|
||||
xmm_shr = xmm_shl;
|
||||
xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3);
|
||||
|
||||
xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl);
|
||||
|
||||
*xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr);
|
||||
xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl);
|
||||
*xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1);
|
||||
|
||||
*xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr);
|
||||
xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl);
|
||||
*xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2);
|
||||
|
||||
*xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr);
|
||||
xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl);
|
||||
*xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3);
|
||||
|
||||
*xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr);
|
||||
*xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl);
|
||||
*xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part);
|
||||
|
||||
xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10);
|
||||
xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01);
|
||||
|
||||
ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
|
||||
psa0_0 = _mm_castsi128_ps(xmm_a0_0);
|
||||
psa0_1 = _mm_castsi128_ps(xmm_a0_1);
|
||||
|
||||
ps_res = _mm_xor_ps(ps_crc3, psa0_0);
|
||||
ps_res = _mm_xor_ps(ps_res, psa0_1);
|
||||
|
||||
*xmm_crc3 = _mm_castps_si128(ps_res);
|
||||
}
|
||||
|
||||
ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s,
|
||||
unsigned char *dst, const unsigned char *src, long len)
|
||||
{
|
||||
unsigned long algn_diff;
|
||||
__m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3;
|
||||
|
||||
CRC_LOAD(s)
|
||||
|
||||
if (len < 16) {
|
||||
if (len == 0)
|
||||
return;
|
||||
goto partial;
|
||||
}
|
||||
|
||||
algn_diff = 0 - (uintptr_t)src & 0xF;
|
||||
if (algn_diff) {
|
||||
xmm_crc_part = _mm_loadu_si128((__m128i *)src);
|
||||
_mm_storeu_si128((__m128i *)dst, xmm_crc_part);
|
||||
|
||||
dst += algn_diff;
|
||||
src += algn_diff;
|
||||
len -= algn_diff;
|
||||
|
||||
partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3,
|
||||
&xmm_crc_part);
|
||||
}
|
||||
|
||||
while ((len -= 64) >= 0) {
|
||||
xmm_t0 = _mm_load_si128((__m128i *)src);
|
||||
xmm_t1 = _mm_load_si128((__m128i *)src + 1);
|
||||
xmm_t2 = _mm_load_si128((__m128i *)src + 2);
|
||||
xmm_t3 = _mm_load_si128((__m128i *)src + 3);
|
||||
|
||||
fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
|
||||
|
||||
_mm_storeu_si128((__m128i *)dst, xmm_t0);
|
||||
_mm_storeu_si128((__m128i *)dst + 1, xmm_t1);
|
||||
_mm_storeu_si128((__m128i *)dst + 2, xmm_t2);
|
||||
_mm_storeu_si128((__m128i *)dst + 3, xmm_t3);
|
||||
|
||||
xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0);
|
||||
xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1);
|
||||
xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3);
|
||||
|
||||
src += 64;
|
||||
dst += 64;
|
||||
}
|
||||
|
||||
/*
|
||||
* len = num bytes left - 64
|
||||
*/
|
||||
if (len + 16 >= 0) {
|
||||
len += 16;
|
||||
|
||||
xmm_t0 = _mm_load_si128((__m128i *)src);
|
||||
xmm_t1 = _mm_load_si128((__m128i *)src + 1);
|
||||
xmm_t2 = _mm_load_si128((__m128i *)src + 2);
|
||||
|
||||
fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
|
||||
|
||||
_mm_storeu_si128((__m128i *)dst, xmm_t0);
|
||||
_mm_storeu_si128((__m128i *)dst + 1, xmm_t1);
|
||||
_mm_storeu_si128((__m128i *)dst + 2, xmm_t2);
|
||||
|
||||
xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0);
|
||||
xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2);
|
||||
|
||||
if (len == 0)
|
||||
goto done;
|
||||
|
||||
dst += 48;
|
||||
src += 48;
|
||||
} else if (len + 32 >= 0) {
|
||||
len += 32;
|
||||
|
||||
xmm_t0 = _mm_load_si128((__m128i *)src);
|
||||
xmm_t1 = _mm_load_si128((__m128i *)src + 1);
|
||||
|
||||
fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
|
||||
|
||||
_mm_storeu_si128((__m128i *)dst, xmm_t0);
|
||||
_mm_storeu_si128((__m128i *)dst + 1, xmm_t1);
|
||||
|
||||
xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1);
|
||||
|
||||
if (len == 0)
|
||||
goto done;
|
||||
|
||||
dst += 32;
|
||||
src += 32;
|
||||
} else if (len + 48 >= 0) {
|
||||
len += 48;
|
||||
|
||||
xmm_t0 = _mm_load_si128((__m128i *)src);
|
||||
|
||||
fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
|
||||
|
||||
_mm_storeu_si128((__m128i *)dst, xmm_t0);
|
||||
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0);
|
||||
|
||||
if (len == 0)
|
||||
goto done;
|
||||
|
||||
dst += 16;
|
||||
src += 16;
|
||||
} else {
|
||||
len += 64;
|
||||
if (len == 0)
|
||||
goto done;
|
||||
}
|
||||
|
||||
partial:
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
/* VS does not permit the use of _mm_set_epi64x in 32-bit builds */
|
||||
{
|
||||
int32_t parts[4] = {0, 0, 0, 0};
|
||||
memcpy(&parts, src, len);
|
||||
xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]);
|
||||
}
|
||||
#else
|
||||
{
|
||||
int64_t parts[2] = {0, 0};
|
||||
memcpy(&parts, src, len);
|
||||
xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]);
|
||||
}
|
||||
#endif
|
||||
|
||||
_mm_storeu_si128((__m128i *)dst, xmm_crc_part);
|
||||
partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3,
|
||||
&xmm_crc_part);
|
||||
done:
|
||||
CRC_SAVE(s)
|
||||
}
|
||||
|
||||
local const unsigned zalign(16) crc_k[] = {
|
||||
0xccaa009e, 0x00000000, /* rk1 */
|
||||
0x751997d0, 0x00000001, /* rk2 */
|
||||
0xccaa009e, 0x00000000, /* rk5 */
|
||||
0x63cd6124, 0x00000001, /* rk6 */
|
||||
0xf7011640, 0x00000001, /* rk7 */
|
||||
0xdb710640, 0x00000001 /* rk8 */
|
||||
};
|
||||
|
||||
local const unsigned zalign(16) crc_mask[4] = {
|
||||
0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000
|
||||
};
|
||||
|
||||
local const unsigned zalign(16) crc_mask2[4] = {
|
||||
0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF
|
||||
};
|
||||
|
||||
unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s)
|
||||
{
|
||||
const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask);
|
||||
const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2);
|
||||
|
||||
unsigned crc;
|
||||
__m128i x_tmp0, x_tmp1, x_tmp2, crc_fold;
|
||||
|
||||
CRC_LOAD(s)
|
||||
|
||||
/*
|
||||
* k1
|
||||
*/
|
||||
crc_fold = _mm_load_si128((__m128i *)crc_k);
|
||||
|
||||
x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10);
|
||||
xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01);
|
||||
xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0);
|
||||
xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0);
|
||||
|
||||
x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10);
|
||||
xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01);
|
||||
xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1);
|
||||
xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1);
|
||||
|
||||
x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10);
|
||||
xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
|
||||
|
||||
/*
|
||||
* k5
|
||||
*/
|
||||
crc_fold = _mm_load_si128((__m128i *)crc_k + 1);
|
||||
|
||||
xmm_crc0 = xmm_crc3;
|
||||
xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
|
||||
xmm_crc0 = _mm_srli_si128(xmm_crc0, 8);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
|
||||
|
||||
xmm_crc0 = xmm_crc3;
|
||||
xmm_crc3 = _mm_slli_si128(xmm_crc3, 4);
|
||||
xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
|
||||
xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2);
|
||||
|
||||
/*
|
||||
* k7
|
||||
*/
|
||||
xmm_crc1 = xmm_crc3;
|
||||
xmm_crc2 = xmm_crc3;
|
||||
crc_fold = _mm_load_si128((__m128i *)crc_k + 2);
|
||||
|
||||
xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
|
||||
xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask);
|
||||
|
||||
xmm_crc2 = xmm_crc3;
|
||||
xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
|
||||
xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1);
|
||||
|
||||
crc = _mm_extract_epi32(xmm_crc3, 2);
|
||||
return ~crc;
|
||||
CRC_SAVE(s)
|
||||
}
|
||||
|
|
@ -48,8 +48,9 @@
|
|||
*/
|
||||
|
||||
/* @(#) $Id$ */
|
||||
|
||||
#include <assert.h>
|
||||
#include "deflate.h"
|
||||
#include "x86.h"
|
||||
|
||||
const char deflate_copyright[] =
|
||||
" deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler ";
|
||||
|
|
@ -86,7 +87,7 @@ local block_state deflate_huff OF((deflate_state *s, int flush));
|
|||
local void lm_init OF((deflate_state *s));
|
||||
local void putShortMSB OF((deflate_state *s, uInt b));
|
||||
local void flush_pending OF((z_streamp strm));
|
||||
local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size));
|
||||
unsigned ZLIB_INTERNAL deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size));
|
||||
#ifdef ASMV
|
||||
# pragma message("Assembler code may have bugs -- use at your own risk")
|
||||
void match_init OF((void)); /* asm code initialization */
|
||||
|
|
@ -100,6 +101,20 @@ local void check_match OF((deflate_state *s, IPos start, IPos match,
|
|||
int length));
|
||||
#endif
|
||||
|
||||
/* From crc32.c */
|
||||
extern void ZLIB_INTERNAL crc_reset(deflate_state *const s);
|
||||
extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s);
|
||||
extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size);
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define INLINE __inline
|
||||
#else
|
||||
#define INLINE inline
|
||||
#endif
|
||||
|
||||
/* Inline optimisation */
|
||||
local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str);
|
||||
|
||||
/* ===========================================================================
|
||||
* Local data
|
||||
*/
|
||||
|
|
@ -162,7 +177,6 @@ local const config configuration_table[10] = {
|
|||
*/
|
||||
#define UPDATE_HASH(s,h,c) (h = (((h)<<s->hash_shift) ^ (c)) & s->hash_mask)
|
||||
|
||||
|
||||
/* ===========================================================================
|
||||
* Insert string str in the dictionary and set match_head to the previous head
|
||||
* of the hash chain (the most recent string with same hash key). Return
|
||||
|
|
@ -173,17 +187,28 @@ local const config configuration_table[10] = {
|
|||
* characters and the first MIN_MATCH bytes of str are valid (except for
|
||||
* the last MIN_MATCH-1 bytes of the input file).
|
||||
*/
|
||||
local INLINE Pos insert_string_c(deflate_state *const s, const Pos str)
|
||||
{
|
||||
Pos ret;
|
||||
|
||||
UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]);
|
||||
#ifdef FASTEST
|
||||
#define INSERT_STRING(s, str, match_head) \
|
||||
(UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \
|
||||
match_head = s->head[s->ins_h], \
|
||||
s->head[s->ins_h] = (Pos)(str))
|
||||
ret = s->head[s->ins_h];
|
||||
#else
|
||||
#define INSERT_STRING(s, str, match_head) \
|
||||
(UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \
|
||||
match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \
|
||||
s->head[s->ins_h] = (Pos)(str))
|
||||
ret = s->prev[str & s->w_mask] = s->head[s->ins_h];
|
||||
#endif
|
||||
s->head[s->ins_h] = str;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
local INLINE Pos insert_string(deflate_state *const s, const Pos str)
|
||||
{
|
||||
if (x86_cpu_enable_simd)
|
||||
return insert_string_sse(s, str);
|
||||
return insert_string_c(s, str);
|
||||
}
|
||||
|
||||
|
||||
/* ===========================================================================
|
||||
* Initialize the hash table (avoiding 64K overflow for 16 bit systems).
|
||||
|
|
@ -248,6 +273,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
|
|||
const char *version;
|
||||
int stream_size;
|
||||
{
|
||||
unsigned window_padding = 8;
|
||||
deflate_state *s;
|
||||
int wrap = 1;
|
||||
static const char my_version[] = ZLIB_VERSION;
|
||||
|
|
@ -257,6 +283,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
|
|||
* output size for (length,distance) codes is <= 24 bits.
|
||||
*/
|
||||
|
||||
x86_check_features();
|
||||
|
||||
if (version == Z_NULL || version[0] != my_version[0] ||
|
||||
stream_size != sizeof(z_stream)) {
|
||||
return Z_VERSION_ERROR;
|
||||
|
|
@ -313,12 +341,19 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
|
|||
s->w_size = 1 << s->w_bits;
|
||||
s->w_mask = s->w_size - 1;
|
||||
|
||||
s->hash_bits = (uInt)memLevel + 7;
|
||||
if (x86_cpu_enable_simd) {
|
||||
s->hash_bits = 15;
|
||||
} else {
|
||||
s->hash_bits = memLevel + 7;
|
||||
}
|
||||
|
||||
s->hash_size = 1 << s->hash_bits;
|
||||
s->hash_mask = s->hash_size - 1;
|
||||
s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH);
|
||||
|
||||
s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte));
|
||||
s->window = (Bytef *) ZALLOC(strm,
|
||||
s->w_size + window_padding,
|
||||
2*sizeof(Byte));
|
||||
s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos));
|
||||
s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos));
|
||||
|
||||
|
|
@ -394,7 +429,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength)
|
|||
/* when using zlib wrappers, compute Adler-32 for provided dictionary */
|
||||
if (wrap == 1)
|
||||
strm->adler = adler32(strm->adler, dictionary, dictLength);
|
||||
s->wrap = 0; /* avoid computing Adler-32 in read_buf */
|
||||
s->wrap = 0; /* avoid computing Adler-32 in deflate_read_buf */
|
||||
|
||||
/* if dictionary would fill window, just replace the history */
|
||||
if (dictLength >= s->w_size) {
|
||||
|
|
@ -418,11 +453,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength)
|
|||
str = s->strstart;
|
||||
n = s->lookahead - (MIN_MATCH-1);
|
||||
do {
|
||||
UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]);
|
||||
#ifndef FASTEST
|
||||
s->prev[str & s->w_mask] = s->head[s->ins_h];
|
||||
#endif
|
||||
s->head[s->ins_h] = (Pos)str;
|
||||
insert_string(s, str);
|
||||
str++;
|
||||
} while (--n);
|
||||
s->strstart = str;
|
||||
|
|
@ -725,7 +756,7 @@ local void putShortMSB (s, b)
|
|||
* Flush as much pending output as possible. All deflate() output, except for
|
||||
* some deflate_stored() output, goes through this function so some
|
||||
* applications may wish to modify it to avoid allocating a large
|
||||
* strm->next_out buffer and copying into it. (See also read_buf()).
|
||||
* strm->next_out buffer and copying into it. (See also deflate_read_buf()).
|
||||
*/
|
||||
local void flush_pending(strm)
|
||||
z_streamp strm;
|
||||
|
|
@ -848,7 +879,7 @@ int ZEXPORT deflate (strm, flush)
|
|||
#ifdef GZIP
|
||||
if (s->status == GZIP_STATE) {
|
||||
/* gzip header */
|
||||
strm->adler = crc32(0L, Z_NULL, 0);
|
||||
crc_reset(s);
|
||||
put_byte(s, 31);
|
||||
put_byte(s, 139);
|
||||
put_byte(s, 8);
|
||||
|
|
@ -1049,6 +1080,7 @@ int ZEXPORT deflate (strm, flush)
|
|||
/* Write the trailer */
|
||||
#ifdef GZIP
|
||||
if (s->wrap == 2) {
|
||||
crc_finalize(s);
|
||||
put_byte(s, (Byte)(strm->adler & 0xff));
|
||||
put_byte(s, (Byte)((strm->adler >> 8) & 0xff));
|
||||
put_byte(s, (Byte)((strm->adler >> 16) & 0xff));
|
||||
|
|
@ -1161,7 +1193,7 @@ int ZEXPORT deflateCopy (dest, source)
|
|||
* allocating a large strm->next_in buffer and copying from it.
|
||||
* (See also flush_pending()).
|
||||
*/
|
||||
local unsigned read_buf(strm, buf, size)
|
||||
ZLIB_INTERNAL unsigned deflate_read_buf(strm, buf, size)
|
||||
z_streamp strm;
|
||||
Bytef *buf;
|
||||
unsigned size;
|
||||
|
|
@ -1173,15 +1205,16 @@ local unsigned read_buf(strm, buf, size)
|
|||
|
||||
strm->avail_in -= len;
|
||||
|
||||
zmemcpy(buf, strm->next_in, len);
|
||||
if (strm->state->wrap == 1) {
|
||||
strm->adler = adler32(strm->adler, buf, len);
|
||||
}
|
||||
#ifdef GZIP
|
||||
else if (strm->state->wrap == 2) {
|
||||
strm->adler = crc32(strm->adler, buf, len);
|
||||
}
|
||||
if (strm->state->wrap == 2)
|
||||
copy_with_crc(strm, buf, len);
|
||||
else
|
||||
#endif
|
||||
{
|
||||
zmemcpy(buf, strm->next_in, len);
|
||||
if (strm->state->wrap == 1)
|
||||
strm->adler = adler32(strm->adler, buf, len);
|
||||
}
|
||||
strm->next_in += len;
|
||||
strm->total_in += len;
|
||||
|
||||
|
|
@ -1479,7 +1512,19 @@ local void check_match(s, start, match, length)
|
|||
* performed for at least two bytes (required for the zip translate_eol
|
||||
* option -- not supported here).
|
||||
*/
|
||||
local void fill_window(s)
|
||||
local void fill_window_c(deflate_state *s);
|
||||
|
||||
local void fill_window(deflate_state *s)
|
||||
{
|
||||
if (x86_cpu_enable_simd) {
|
||||
fill_window_sse(s);
|
||||
return;
|
||||
}
|
||||
|
||||
fill_window_c(s);
|
||||
}
|
||||
|
||||
local void fill_window_c(s)
|
||||
deflate_state *s;
|
||||
{
|
||||
unsigned n;
|
||||
|
|
@ -1531,7 +1576,7 @@ local void fill_window(s)
|
|||
*/
|
||||
Assert(more >= 2, "more < 2");
|
||||
|
||||
n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more);
|
||||
n = deflate_read_buf(s->strm, s->window + s->strstart + s->lookahead, more);
|
||||
s->lookahead += n;
|
||||
|
||||
/* Initialize the hash value now that we have some input: */
|
||||
|
|
@ -1720,7 +1765,7 @@ local block_state deflate_stored(s, flush)
|
|||
* the check value.
|
||||
*/
|
||||
if (len) {
|
||||
read_buf(s->strm, s->strm->next_out, len);
|
||||
deflate_read_buf(s->strm, s->strm->next_out, len);
|
||||
s->strm->next_out += len;
|
||||
s->strm->avail_out -= len;
|
||||
s->strm->total_out += len;
|
||||
|
|
@ -1783,7 +1828,7 @@ local block_state deflate_stored(s, flush)
|
|||
if (have > s->strm->avail_in)
|
||||
have = s->strm->avail_in;
|
||||
if (have) {
|
||||
read_buf(s->strm, s->window + s->strstart, have);
|
||||
deflate_read_buf(s->strm, s->window + s->strstart, have);
|
||||
s->strstart += have;
|
||||
}
|
||||
if (s->high_water < s->strstart)
|
||||
|
|
@ -1847,7 +1892,7 @@ local block_state deflate_fast(s, flush)
|
|||
*/
|
||||
hash_head = NIL;
|
||||
if (s->lookahead >= MIN_MATCH) {
|
||||
INSERT_STRING(s, s->strstart, hash_head);
|
||||
hash_head = insert_string(s, s->strstart);
|
||||
}
|
||||
|
||||
/* Find the longest match, discarding those <= prev_length.
|
||||
|
|
@ -1878,7 +1923,7 @@ local block_state deflate_fast(s, flush)
|
|||
s->match_length--; /* string at strstart already in table */
|
||||
do {
|
||||
s->strstart++;
|
||||
INSERT_STRING(s, s->strstart, hash_head);
|
||||
hash_head = insert_string(s, s->strstart);
|
||||
/* strstart never exceeds WSIZE-MAX_MATCH, so there are
|
||||
* always MIN_MATCH bytes ahead.
|
||||
*/
|
||||
|
|
@ -1950,7 +1995,7 @@ local block_state deflate_slow(s, flush)
|
|||
*/
|
||||
hash_head = NIL;
|
||||
if (s->lookahead >= MIN_MATCH) {
|
||||
INSERT_STRING(s, s->strstart, hash_head);
|
||||
hash_head = insert_string(s, s->strstart);
|
||||
}
|
||||
|
||||
/* Find the longest match, discarding those <= prev_length.
|
||||
|
|
@ -2001,7 +2046,7 @@ local block_state deflate_slow(s, flush)
|
|||
s->prev_length -= 2;
|
||||
do {
|
||||
if (++s->strstart <= max_insert) {
|
||||
INSERT_STRING(s, s->strstart, hash_head);
|
||||
hash_head = insert_string(s, s->strstart);
|
||||
}
|
||||
} while (--s->prev_length != 0);
|
||||
s->match_available = 0;
|
||||
|
|
@ -2161,3 +2206,37 @@ local block_state deflate_huff(s, flush)
|
|||
FLUSH_BLOCK(s, 0);
|
||||
return block_done;
|
||||
}
|
||||
|
||||
/* Safe to inline this as GCC/clang will use inline asm and Visual Studio will
|
||||
* use intrinsic without extra params
|
||||
*/
|
||||
local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str)
|
||||
{
|
||||
Pos ret;
|
||||
unsigned *ip, val, h = 0;
|
||||
|
||||
ip = (unsigned *)&s->window[str];
|
||||
val = *ip;
|
||||
|
||||
if (s->level >= 6)
|
||||
val &= 0xFFFFFF;
|
||||
|
||||
/* Windows clang should use inline asm */
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
h = _mm_crc32_u32(h, val);
|
||||
#elif defined(__i386__) || defined(__amd64__)
|
||||
__asm__ __volatile__ (
|
||||
"crc32 %1,%0\n\t"
|
||||
: "+r" (h)
|
||||
: "r" (val)
|
||||
);
|
||||
#else
|
||||
/* This should never happen */
|
||||
assert(0);
|
||||
#endif
|
||||
|
||||
ret = s->head[h & s->hash_mask];
|
||||
s->head[h & s->hash_mask] = str;
|
||||
s->prev[str & s->w_mask] = ret;
|
||||
return ret;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ typedef struct internal_state {
|
|||
ulg gzindex; /* where in extra, name, or comment */
|
||||
Byte method; /* can only be DEFLATED */
|
||||
int last_flush; /* value of flush param for previous deflate call */
|
||||
|
||||
unsigned zalign(16) crc0[4 * 5];
|
||||
/* used by deflate.c: */
|
||||
|
||||
uInt w_size; /* LZ77 window size (32K by default) */
|
||||
|
|
@ -346,4 +346,14 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf,
|
|||
flush = _tr_tally(s, distance, length)
|
||||
#endif
|
||||
|
||||
/* Functions that are SIMD optimised on x86 */
|
||||
void ZLIB_INTERNAL crc_fold_init(deflate_state* const s);
|
||||
void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s,
|
||||
unsigned char* dst,
|
||||
const unsigned char* src,
|
||||
long len);
|
||||
unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s);
|
||||
|
||||
void ZLIB_INTERNAL fill_window_sse(deflate_state* s);
|
||||
|
||||
#endif /* DEFLATE_H */
|
||||
|
|
|
|||
177
modules/zlib/src/fill_window_sse.c
Normal file
177
modules/zlib/src/fill_window_sse.c
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
/*
|
||||
* Fill Window with SSE2-optimized hash shifting
|
||||
*
|
||||
* Copyright (C) 2013 Intel Corporation
|
||||
* Authors:
|
||||
* Arjan van de Ven <arjan@linux.intel.com>
|
||||
* Jim Kukunas <james.t.kukunas@linux.intel.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include <immintrin.h>
|
||||
#include "deflate.h"
|
||||
|
||||
#define UPDATE_HASH(s,h,i) \
|
||||
{\
|
||||
if (s->level < 6) { \
|
||||
h = (3483 * (s->window[i]) +\
|
||||
23081* (s->window[i+1]) +\
|
||||
6954 * (s->window[i+2]) +\
|
||||
20947* (s->window[i+3])) & s->hash_mask;\
|
||||
} else {\
|
||||
h = (25881* (s->window[i]) +\
|
||||
24674* (s->window[i+1]) +\
|
||||
25811* (s->window[i+2])) & s->hash_mask;\
|
||||
}\
|
||||
}\
|
||||
|
||||
extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size));
|
||||
|
||||
void fill_window_sse(deflate_state *s)
|
||||
{
|
||||
const __m128i xmm_wsize = _mm_set1_epi16(s->w_size);
|
||||
|
||||
register unsigned n;
|
||||
register Posf *p;
|
||||
unsigned more; /* Amount of free space at the end of the window. */
|
||||
uInt wsize = s->w_size;
|
||||
|
||||
Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead");
|
||||
|
||||
do {
|
||||
more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart);
|
||||
|
||||
/* Deal with !@#$% 64K limit: */
|
||||
if (sizeof(int) <= 2) {
|
||||
if (more == 0 && s->strstart == 0 && s->lookahead == 0) {
|
||||
more = wsize;
|
||||
|
||||
} else if (more == (unsigned)(-1)) {
|
||||
/* Very unlikely, but possible on 16 bit machine if
|
||||
* strstart == 0 && lookahead == 1 (input done a byte at time)
|
||||
*/
|
||||
more--;
|
||||
}
|
||||
}
|
||||
|
||||
/* If the window is almost full and there is insufficient lookahead,
|
||||
* move the upper half to the lower one to make room in the upper half.
|
||||
*/
|
||||
if (s->strstart >= wsize+MAX_DIST(s)) {
|
||||
|
||||
zmemcpy(s->window, s->window+wsize, (unsigned)wsize);
|
||||
s->match_start -= wsize;
|
||||
s->strstart -= wsize; /* we now have strstart >= MAX_DIST */
|
||||
s->block_start -= (long) wsize;
|
||||
|
||||
/* Slide the hash table (could be avoided with 32 bit values
|
||||
at the expense of memory usage). We slide even when level == 0
|
||||
to keep the hash table consistent if we switch back to level > 0
|
||||
later. (Using level 0 permanently is not an optimal usage of
|
||||
zlib, so we don't care about this pathological case.)
|
||||
*/
|
||||
n = s->hash_size;
|
||||
p = &s->head[n];
|
||||
p -= 8;
|
||||
do {
|
||||
__m128i value, result;
|
||||
|
||||
value = _mm_loadu_si128((__m128i *)p);
|
||||
result = _mm_subs_epu16(value, xmm_wsize);
|
||||
_mm_storeu_si128((__m128i *)p, result);
|
||||
|
||||
p -= 8;
|
||||
n -= 8;
|
||||
} while (n > 0);
|
||||
|
||||
n = wsize;
|
||||
#ifndef FASTEST
|
||||
p = &s->prev[n];
|
||||
p -= 8;
|
||||
do {
|
||||
__m128i value, result;
|
||||
|
||||
value = _mm_loadu_si128((__m128i *)p);
|
||||
result = _mm_subs_epu16(value, xmm_wsize);
|
||||
_mm_storeu_si128((__m128i *)p, result);
|
||||
|
||||
p -= 8;
|
||||
n -= 8;
|
||||
} while (n > 0);
|
||||
#endif
|
||||
more += wsize;
|
||||
}
|
||||
if (s->strm->avail_in == 0) break;
|
||||
|
||||
/* If there was no sliding:
|
||||
* strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 &&
|
||||
* more == window_size - lookahead - strstart
|
||||
* => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1)
|
||||
* => more >= window_size - 2*WSIZE + 2
|
||||
* In the BIG_MEM or MMAP case (not yet supported),
|
||||
* window_size == input_size + MIN_LOOKAHEAD &&
|
||||
* strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD.
|
||||
* Otherwise, window_size == 2*WSIZE so more >= 2.
|
||||
* If there was sliding, more >= WSIZE. So in all cases, more >= 2.
|
||||
*/
|
||||
Assert(more >= 2, "more < 2");
|
||||
|
||||
n = deflate_read_buf(s->strm,
|
||||
s->window + s->strstart + s->lookahead,
|
||||
more);
|
||||
s->lookahead += n;
|
||||
|
||||
/* Initialize the hash value now that we have some input: */
|
||||
if (s->lookahead >= MIN_MATCH) {
|
||||
uInt str = s->strstart;
|
||||
s->ins_h = s->window[str];
|
||||
if (str >= 1)
|
||||
UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1));
|
||||
#if MIN_MATCH != 3
|
||||
Call UPDATE_HASH() MIN_MATCH-3 more times
|
||||
#endif
|
||||
}
|
||||
/* If the whole input has less than MIN_MATCH bytes, ins_h is garbage,
|
||||
* but this is not important since only literal bytes will be emitted.
|
||||
*/
|
||||
|
||||
} while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0);
|
||||
|
||||
/* If the WIN_INIT bytes after the end of the current data have never been
|
||||
* written, then zero those bytes in order to avoid memory check reports of
|
||||
* the use of uninitialized (or uninitialised as Julian writes) bytes by
|
||||
* the longest match routines. Update the high water mark for the next
|
||||
* time through here. WIN_INIT is set to MAX_MATCH since the longest match
|
||||
* routines allow scanning to strstart + MAX_MATCH, ignoring lookahead.
|
||||
*/
|
||||
if (s->high_water < s->window_size) {
|
||||
ulg curr = s->strstart + (ulg)(s->lookahead);
|
||||
ulg init;
|
||||
|
||||
if (s->high_water < curr) {
|
||||
/* Previous high water mark below current data -- zero WIN_INIT
|
||||
* bytes or up to end of window, whichever is less.
|
||||
*/
|
||||
init = s->window_size - curr;
|
||||
if (init > WIN_INIT)
|
||||
init = WIN_INIT;
|
||||
zmemzero(s->window + curr, (unsigned)init);
|
||||
s->high_water = curr + init;
|
||||
}
|
||||
else if (s->high_water < (ulg)curr + WIN_INIT) {
|
||||
/* High water mark at or above current data, but below current data
|
||||
* plus WIN_INIT -- zero out to current data plus WIN_INIT, or up
|
||||
* to end of window, whichever is less.
|
||||
*/
|
||||
init = (ulg)curr + WIN_INIT - s->high_water;
|
||||
if (init > s->window_size - s->high_water)
|
||||
init = s->window_size - s->high_water;
|
||||
zmemzero(s->window + s->high_water, (unsigned)init);
|
||||
s->high_water += init;
|
||||
}
|
||||
}
|
||||
|
||||
Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD,
|
||||
"not enough room for search");
|
||||
}
|
||||
|
|
@ -443,7 +443,11 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file)
|
|||
# undef z_gzgetc
|
||||
#else
|
||||
# undef gzgetc
|
||||
# ifdef Z_CR_PREFIX_SET
|
||||
# define gzgetc Cr_z_gzgetc
|
||||
# endif
|
||||
#endif
|
||||
|
||||
int ZEXPORT gzgetc(file)
|
||||
gzFile file;
|
||||
{
|
||||
|
|
|
|||
|
|
@ -480,7 +480,8 @@ void FAR *out_desc;
|
|||
|
||||
case LEN:
|
||||
/* use inflate_fast() if we have enough input and output */
|
||||
if (have >= 6 && left >= 258) {
|
||||
if (have >= INFLATE_FAST_MIN_HAVE &&
|
||||
left >= INFLATE_FAST_MIN_LEFT) {
|
||||
RESTORE();
|
||||
if (state->whave < state->wsize)
|
||||
state->whave = state->wsize - left;
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@
|
|||
Entry assumptions:
|
||||
|
||||
state->mode == LEN
|
||||
strm->avail_in >= 6
|
||||
strm->avail_out >= 258
|
||||
strm->avail_in >= INFLATE_FAST_MIN_HAVE
|
||||
strm->avail_out >= INFLATE_FAST_MIN_LEFT
|
||||
start >= strm->avail_out
|
||||
state->bits < 8
|
||||
|
||||
|
|
@ -80,10 +80,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
|
|||
/* copy state to local variables */
|
||||
state = (struct inflate_state FAR *)strm->state;
|
||||
in = strm->next_in;
|
||||
last = in + (strm->avail_in - 5);
|
||||
last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
|
||||
out = strm->next_out;
|
||||
beg = out - (start - strm->avail_out);
|
||||
end = out + (strm->avail_out - 257);
|
||||
end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
|
||||
#ifdef INFLATE_STRICT
|
||||
dmax = state->dmax;
|
||||
#endif
|
||||
|
|
@ -298,9 +298,12 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
|
|||
/* update state and return */
|
||||
strm->next_in = in;
|
||||
strm->next_out = out;
|
||||
strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last));
|
||||
strm->avail_in = (unsigned)(in < last ?
|
||||
(INFLATE_FAST_MIN_HAVE - 1) + (last - in) :
|
||||
(INFLATE_FAST_MIN_HAVE - 1) - (in - last));
|
||||
strm->avail_out = (unsigned)(out < end ?
|
||||
257 + (end - out) : 257 - (out - end));
|
||||
(INFLATE_FAST_MIN_LEFT - 1) + (end - out) :
|
||||
(INFLATE_FAST_MIN_LEFT - 1) - (out - end));
|
||||
state->hold = hold;
|
||||
state->bits = bits;
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -8,4 +8,18 @@
|
|||
subject to change. Applications should only use zlib.h.
|
||||
*/
|
||||
|
||||
/* INFLATE_FAST_MIN_LEFT is the minimum number of output bytes that are left,
|
||||
so that we can call inflate_fast safely with only one up front bounds check.
|
||||
One length-distance code pair can copy up to 258 bytes.
|
||||
*/
|
||||
#define INFLATE_FAST_MIN_LEFT 258
|
||||
|
||||
/* INFLATE_FAST_MIN_HAVE is the minimum number of input bytes that we have, so
|
||||
that we can call inflate_fast safely with only one up front bounds check.
|
||||
One length-distance code pair (as two Huffman encoded values of up to 15
|
||||
bits each) plus any additional bits (up to 5 for length and 13 for distance)
|
||||
can require reading up to 48 bits, or 6 bytes.
|
||||
*/
|
||||
#define INFLATE_FAST_MIN_HAVE 6
|
||||
|
||||
void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start));
|
||||
|
|
|
|||
318
modules/zlib/src/inffast_chunk.c
Normal file
318
modules/zlib/src/inffast_chunk.c
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
/* inffast_chunk.c -- fast decoding
|
||||
* Copyright (C) 1995-2017 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zutil.h"
|
||||
#include "inftrees.h"
|
||||
#include "inflate.h"
|
||||
#include "inffast_chunk.h"
|
||||
#include "chunkcopy.h"
|
||||
|
||||
#ifdef ASMINF
|
||||
# pragma message("Assembler code may have bugs -- use at your own risk")
|
||||
#else
|
||||
|
||||
/*
|
||||
Decode literal, length, and distance codes and write out the resulting
|
||||
literal and match bytes until either not enough input or output is
|
||||
available, an end-of-block is encountered, or a data error is encountered.
|
||||
When large enough input and output buffers are supplied to inflate(), for
|
||||
example, a 16K input buffer and a 64K output buffer, more than 95% of the
|
||||
inflate() execution time is spent in this routine.
|
||||
|
||||
Entry assumptions:
|
||||
|
||||
state->mode == LEN
|
||||
strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 bytes)
|
||||
strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes)
|
||||
start >= strm->avail_out
|
||||
state->bits < 8
|
||||
strm->next_out[0..strm->avail_out] does not overlap with
|
||||
strm->next_in[0..strm->avail_in]
|
||||
strm->state->window is allocated with an additional
|
||||
CHUNKCOPY_CHUNK_SIZE-1 bytes of padding beyond strm->state->wsize
|
||||
|
||||
On return, state->mode is one of:
|
||||
|
||||
LEN -- ran out of enough output space or enough available input
|
||||
TYPE -- reached end of block code, inflate() to interpret next block
|
||||
BAD -- error in block data
|
||||
|
||||
Notes:
|
||||
|
||||
INFLATE_FAST_MIN_INPUT 6 bytes
|
||||
|
||||
- The maximum input bits used by a length/distance pair is 15 bits for the
|
||||
length code, 5 bits for the length extra, 15 bits for the distance code,
|
||||
and 13 bits for the distance extra. This totals 48 bits, or six bytes.
|
||||
Therefore if strm->avail_in >= 6, then there is enough input to avoid
|
||||
checking for available input while decoding.
|
||||
|
||||
INFLATE_FAST_MIN_OUTPUT 258 bytes
|
||||
|
||||
- The maximum bytes that a single length/distance pair can output is 258
|
||||
bytes, which is the maximum length that can be coded. inflate_fast()
|
||||
requires strm->avail_out >= 258 for each loop to avoid checking for
|
||||
available output space while decoding.
|
||||
*/
|
||||
void ZLIB_INTERNAL inflate_fast_chunk_(strm, start)
|
||||
z_streamp strm;
|
||||
unsigned start; /* inflate()'s starting value for strm->avail_out */
|
||||
{
|
||||
struct inflate_state FAR *state;
|
||||
z_const unsigned char FAR *in; /* local strm->next_in */
|
||||
z_const unsigned char FAR *last; /* have enough input while in < last */
|
||||
unsigned char FAR *out; /* local strm->next_out */
|
||||
unsigned char FAR *beg; /* inflate()'s initial strm->next_out */
|
||||
unsigned char FAR *end; /* while out < end, enough space available */
|
||||
unsigned char FAR *limit; /* safety limit for chunky copies */
|
||||
#ifdef INFLATE_STRICT
|
||||
unsigned dmax; /* maximum distance from zlib header */
|
||||
#endif
|
||||
unsigned wsize; /* window size or zero if not using window */
|
||||
unsigned whave; /* valid bytes in the window */
|
||||
unsigned wnext; /* window write index */
|
||||
unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */
|
||||
unsigned long hold; /* local strm->hold */
|
||||
unsigned bits; /* local strm->bits */
|
||||
code const FAR *lcode; /* local strm->lencode */
|
||||
code const FAR *dcode; /* local strm->distcode */
|
||||
unsigned lmask; /* mask for first level of length codes */
|
||||
unsigned dmask; /* mask for first level of distance codes */
|
||||
code here; /* retrieved table entry */
|
||||
unsigned op; /* code bits, operation, extra bits, or */
|
||||
/* window position, window bytes to copy */
|
||||
unsigned len; /* match length, unused bytes */
|
||||
unsigned dist; /* match distance */
|
||||
unsigned char FAR *from; /* where to copy match from */
|
||||
|
||||
/* copy state to local variables */
|
||||
state = (struct inflate_state FAR *)strm->state;
|
||||
in = strm->next_in;
|
||||
last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1));
|
||||
out = strm->next_out;
|
||||
beg = out - (start - strm->avail_out);
|
||||
end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1));
|
||||
limit = out + strm->avail_out;
|
||||
#ifdef INFLATE_STRICT
|
||||
dmax = state->dmax;
|
||||
#endif
|
||||
wsize = state->wsize;
|
||||
whave = state->whave;
|
||||
wnext = (state->wnext == 0 && whave >= wsize) ? wsize : state->wnext;
|
||||
window = state->window;
|
||||
hold = state->hold;
|
||||
bits = state->bits;
|
||||
lcode = state->lencode;
|
||||
dcode = state->distcode;
|
||||
lmask = (1U << state->lenbits) - 1;
|
||||
dmask = (1U << state->distbits) - 1;
|
||||
|
||||
/* decode literals and length/distances until end-of-block or not enough
|
||||
input data or output space */
|
||||
do {
|
||||
if (bits < 15) {
|
||||
hold += (unsigned long)(*in++) << bits;
|
||||
bits += 8;
|
||||
hold += (unsigned long)(*in++) << bits;
|
||||
bits += 8;
|
||||
}
|
||||
here = lcode[hold & lmask];
|
||||
dolen:
|
||||
op = (unsigned)(here.bits);
|
||||
hold >>= op;
|
||||
bits -= op;
|
||||
op = (unsigned)(here.op);
|
||||
if (op == 0) { /* literal */
|
||||
Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ?
|
||||
"inflate: literal '%c'\n" :
|
||||
"inflate: literal 0x%02x\n", here.val));
|
||||
*out++ = (unsigned char)(here.val);
|
||||
}
|
||||
else if (op & 16) { /* length base */
|
||||
len = (unsigned)(here.val);
|
||||
op &= 15; /* number of extra bits */
|
||||
if (op) {
|
||||
if (bits < op) {
|
||||
hold += (unsigned long)(*in++) << bits;
|
||||
bits += 8;
|
||||
}
|
||||
len += (unsigned)hold & ((1U << op) - 1);
|
||||
hold >>= op;
|
||||
bits -= op;
|
||||
}
|
||||
Tracevv((stderr, "inflate: length %u\n", len));
|
||||
if (bits < 15) {
|
||||
hold += (unsigned long)(*in++) << bits;
|
||||
bits += 8;
|
||||
hold += (unsigned long)(*in++) << bits;
|
||||
bits += 8;
|
||||
}
|
||||
here = dcode[hold & dmask];
|
||||
dodist:
|
||||
op = (unsigned)(here.bits);
|
||||
hold >>= op;
|
||||
bits -= op;
|
||||
op = (unsigned)(here.op);
|
||||
if (op & 16) { /* distance base */
|
||||
dist = (unsigned)(here.val);
|
||||
op &= 15; /* number of extra bits */
|
||||
if (bits < op) {
|
||||
hold += (unsigned long)(*in++) << bits;
|
||||
bits += 8;
|
||||
if (bits < op) {
|
||||
hold += (unsigned long)(*in++) << bits;
|
||||
bits += 8;
|
||||
}
|
||||
}
|
||||
dist += (unsigned)hold & ((1U << op) - 1);
|
||||
#ifdef INFLATE_STRICT
|
||||
if (dist > dmax) {
|
||||
strm->msg = (char *)"invalid distance too far back";
|
||||
state->mode = BAD;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
hold >>= op;
|
||||
bits -= op;
|
||||
Tracevv((stderr, "inflate: distance %u\n", dist));
|
||||
op = (unsigned)(out - beg); /* max distance in output */
|
||||
if (dist > op) { /* see if copy from window */
|
||||
op = dist - op; /* distance back in window */
|
||||
if (op > whave) {
|
||||
if (state->sane) {
|
||||
strm->msg =
|
||||
(char *)"invalid distance too far back";
|
||||
state->mode = BAD;
|
||||
break;
|
||||
}
|
||||
#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
|
||||
if (len <= op - whave) {
|
||||
do {
|
||||
*out++ = 0;
|
||||
} while (--len);
|
||||
continue;
|
||||
}
|
||||
len -= op - whave;
|
||||
do {
|
||||
*out++ = 0;
|
||||
} while (--op > whave);
|
||||
if (op == 0) {
|
||||
from = out - dist;
|
||||
do {
|
||||
*out++ = *from++;
|
||||
} while (--len);
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
from = window;
|
||||
if (wnext >= op) { /* contiguous in window */
|
||||
from += wnext - op;
|
||||
}
|
||||
else { /* wrap around window */
|
||||
op -= wnext;
|
||||
from += wsize - op;
|
||||
if (op < len) { /* some from end of window */
|
||||
len -= op;
|
||||
out = chunkcopy_safe(out, from, op, limit);
|
||||
from = window; /* more from start of window */
|
||||
op = wnext;
|
||||
/* This (rare) case can create a situation where
|
||||
the first chunkcopy below must be checked.
|
||||
*/
|
||||
}
|
||||
}
|
||||
if (op < len) { /* still need some from output */
|
||||
out = chunkcopy_safe(out, from, op, limit);
|
||||
len -= op;
|
||||
/* When dist is small the amount of data that can be
|
||||
copied from the window is also small, and progress
|
||||
towards the dangerous end of the output buffer is
|
||||
also small. This means that for trivial memsets and
|
||||
for chunkunroll_relaxed() a safety check is
|
||||
unnecessary. However, these conditions may not be
|
||||
entered at all, and in that case it's possible that
|
||||
the main copy is near the end.
|
||||
*/
|
||||
out = chunkunroll_relaxed(out, &dist, &len);
|
||||
out = chunkcopy_safe(out, out - dist, len, limit);
|
||||
} else {
|
||||
/* from points to window, so there is no risk of
|
||||
overlapping pointers requiring memset-like behaviour
|
||||
*/
|
||||
out = chunkcopy_safe(out, from, len, limit);
|
||||
}
|
||||
}
|
||||
else {
|
||||
/* Whole reference is in range of current output. No
|
||||
range checks are necessary because we start with room
|
||||
for at least 258 bytes of output, so unroll and roundoff
|
||||
operations can write beyond `out+len` so long as they
|
||||
stay within 258 bytes of `out`.
|
||||
*/
|
||||
out = chunkcopy_lapped_relaxed(out, dist, len);
|
||||
}
|
||||
}
|
||||
else if ((op & 64) == 0) { /* 2nd level distance code */
|
||||
here = dcode[here.val + (hold & ((1U << op) - 1))];
|
||||
goto dodist;
|
||||
}
|
||||
else {
|
||||
strm->msg = (char *)"invalid distance code";
|
||||
state->mode = BAD;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if ((op & 64) == 0) { /* 2nd level length code */
|
||||
here = lcode[here.val + (hold & ((1U << op) - 1))];
|
||||
goto dolen;
|
||||
}
|
||||
else if (op & 32) { /* end-of-block */
|
||||
Tracevv((stderr, "inflate: end of block\n"));
|
||||
state->mode = TYPE;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
strm->msg = (char *)"invalid literal/length code";
|
||||
state->mode = BAD;
|
||||
break;
|
||||
}
|
||||
} while (in < last && out < end);
|
||||
|
||||
/* return unused bytes (on entry, bits < 8, so in won't go too far back) */
|
||||
len = bits >> 3;
|
||||
in -= len;
|
||||
bits -= len << 3;
|
||||
hold &= (1U << bits) - 1;
|
||||
|
||||
/* update state and return */
|
||||
strm->next_in = in;
|
||||
strm->next_out = out;
|
||||
strm->avail_in = (unsigned)(in < last ?
|
||||
(INFLATE_FAST_MIN_INPUT - 1) + (last - in) :
|
||||
(INFLATE_FAST_MIN_INPUT - 1) - (in - last));
|
||||
strm->avail_out = (unsigned)(out < end ?
|
||||
(INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) :
|
||||
(INFLATE_FAST_MIN_OUTPUT - 1) - (out - end));
|
||||
state->hold = hold;
|
||||
state->bits = bits;
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
|
||||
- Using bit fields for code structure
|
||||
- Different op definition to avoid & for extra bits (do & for table bits)
|
||||
- Three separate decoding do-loops for direct, window, and wnext == 0
|
||||
- Special case for distance > 1 copies to do overlapped load and store copy
|
||||
- Explicit branch predictions (based on measured branch probabilities)
|
||||
- Deferring match copy and interspersed it with decoding subsequent codes
|
||||
- Swapping literal/length else
|
||||
- Swapping window/direct else
|
||||
- Larger unrolled copy loops (three is about right)
|
||||
- Moving len -= 3 statement into middle of loop
|
||||
*/
|
||||
|
||||
#endif /* !ASMINF */
|
||||
27
modules/zlib/src/inffast_chunk.h
Normal file
27
modules/zlib/src/inffast_chunk.h
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/* inffast_chunk.h -- header to use inffast_chunk.c
|
||||
* Copyright (C) 1995-2003, 2010 Mark Adler
|
||||
* Copyright (C) 2017 ARM, Inc.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
/* WARNING: this file should *not* be used by applications. It is
|
||||
part of the implementation of the compression library and is
|
||||
subject to change. Applications should only use zlib.h.
|
||||
*/
|
||||
|
||||
/* INFLATE_FAST_MIN_INPUT: the minimum number of input bytes needed so that
|
||||
we can safely call inflate_fast() with only one up-front bounds check. One
|
||||
length/distance code pair (15 bits for the length code, 5 bits for length
|
||||
extra, 15 bits for the distance code, 13 bits for distance extra) requires
|
||||
reading up to 48 input bits (6 bytes).
|
||||
*/
|
||||
#define INFLATE_FAST_MIN_INPUT 6
|
||||
|
||||
/* INFLATE_FAST_MIN_OUTPUT: the minimum number of output bytes needed so that
|
||||
we can safely call inflate_fast() with only one up-front bounds check. One
|
||||
length/distance code pair can output up to 258 bytes, which is the maximum
|
||||
length that can be coded.
|
||||
*/
|
||||
#define INFLATE_FAST_MIN_OUTPUT 258
|
||||
|
||||
void ZLIB_INTERNAL inflate_fast_chunk_ OF((z_streamp strm, unsigned start));
|
||||
|
|
@ -228,6 +228,7 @@ int stream_size;
|
|||
state->strm = strm;
|
||||
state->window = Z_NULL;
|
||||
state->mode = HEAD; /* to pass state test in inflateReset2() */
|
||||
state->check = 1L; /* 1L is the result of adler32() zero length data */
|
||||
ret = inflateReset2(strm, windowBits);
|
||||
if (ret != Z_OK) {
|
||||
ZFREE(strm, state);
|
||||
|
|
@ -1042,7 +1043,8 @@ int flush;
|
|||
case LEN_:
|
||||
state->mode = LEN;
|
||||
case LEN:
|
||||
if (have >= 6 && left >= 258) {
|
||||
if (have >= INFLATE_FAST_MIN_HAVE &&
|
||||
left >= INFLATE_FAST_MIN_LEFT) {
|
||||
RESTORE();
|
||||
inflate_fast(strm, out);
|
||||
LOAD();
|
||||
|
|
|
|||
1583
modules/zlib/src/inflate_simd.c
Normal file
1583
modules/zlib/src/inflate_simd.c
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -10,6 +10,38 @@ EXPORTS += [
|
|||
'zlib.h',
|
||||
]
|
||||
|
||||
if CONFIG['CPU_ARCH'] == 'arm':
|
||||
DEFINES['ADLER32_SIMD_NEON'] = True
|
||||
DEFINES['INFLATE_CHUNK_SIMD_NEON'] = True
|
||||
SOURCES += [
|
||||
'adler32_simd.c',
|
||||
'inffast_chunk.c',
|
||||
'inflate_simd.c',
|
||||
]
|
||||
|
||||
if CONFIG['INTEL_ARCHITECTURE']:
|
||||
DEFINES['ADLER32_SIMD_SSSE3'] = True
|
||||
DEFINES['INFLATE_CHUNK_SIMD_SSE2'] = True
|
||||
DEFINES['CRC32_SIMD_SSE42_PCLMUL'] = True
|
||||
SOURCES += [
|
||||
'adler32_simd.c',
|
||||
'crc32_simd.c',
|
||||
'crc_folding.c',
|
||||
'fill_window_sse.c',
|
||||
'inffast_chunk.c',
|
||||
'inflate_simd.c',
|
||||
'x86.c',
|
||||
]
|
||||
else:
|
||||
SOURCES += [
|
||||
'simd_stub.c',
|
||||
]
|
||||
|
||||
if CONFIG['CPU_ARCH'] != 'arm' and not CONFIG['INTEL_ARCHITECTURE']:
|
||||
SOURCES += [
|
||||
'inflate.c',
|
||||
]
|
||||
|
||||
if CONFIG['ZLIB_IN_MOZGLUE']:
|
||||
FINAL_LIBRARY = 'mozglue'
|
||||
DEFINES['IMPL_MFBT'] = True
|
||||
|
|
@ -28,7 +60,6 @@ SOURCES += [
|
|||
'gzwrite.c',
|
||||
'infback.c',
|
||||
'inffast.c',
|
||||
'inflate.c',
|
||||
'inftrees.c',
|
||||
'trees.c',
|
||||
'uncompr.c',
|
||||
|
|
|
|||
35
modules/zlib/src/simd_stub.c
Normal file
35
modules/zlib/src/simd_stub.c
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/* simd_stub.c -- stub implementations
|
||||
* Copyright (C) 2014 Intel Corporation
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
#include <assert.h>
|
||||
|
||||
#include "deflate.h"
|
||||
#include "x86.h"
|
||||
|
||||
int ZLIB_INTERNAL x86_cpu_enable_simd = 0;
|
||||
|
||||
void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) {
|
||||
assert(0);
|
||||
}
|
||||
|
||||
void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s,
|
||||
unsigned char *dst,
|
||||
const unsigned char *src,
|
||||
long len) {
|
||||
assert(0);
|
||||
}
|
||||
|
||||
unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) {
|
||||
assert(0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ZLIB_INTERNAL fill_window_sse(deflate_state *s)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
void x86_check_features(void)
|
||||
{
|
||||
}
|
||||
122
modules/zlib/src/x86.c
Normal file
122
modules/zlib/src/x86.c
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
/*
|
||||
* x86 feature check
|
||||
*
|
||||
* Copyright (C) 2013 Intel Corporation. All rights reserved.
|
||||
* Author:
|
||||
* Jim Kukunas
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "x86.h"
|
||||
#include "zutil.h"
|
||||
|
||||
int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0;
|
||||
int ZLIB_INTERNAL x86_cpu_enable_simd = 0;
|
||||
|
||||
static void _x86_check_features(void);
|
||||
|
||||
#ifndef _MSC_VER
|
||||
#include <pthread.h>
|
||||
|
||||
pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT;
|
||||
|
||||
void x86_check_features(void)
|
||||
{
|
||||
pthread_once(&cpu_check_inited_once, _x86_check_features);
|
||||
}
|
||||
|
||||
static void _x86_check_features(void)
|
||||
{
|
||||
int x86_cpu_has_sse2;
|
||||
int x86_cpu_has_ssse3;
|
||||
int x86_cpu_has_sse42;
|
||||
int x86_cpu_has_pclmulqdq;
|
||||
unsigned eax, ebx, ecx, edx;
|
||||
|
||||
eax = 1;
|
||||
#ifdef __i386__
|
||||
__asm__ __volatile__ (
|
||||
"xchg %%ebx, %1\n\t"
|
||||
"cpuid\n\t"
|
||||
"xchg %1, %%ebx\n\t"
|
||||
: "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx)
|
||||
);
|
||||
#else
|
||||
__asm__ __volatile__ (
|
||||
"cpuid\n\t"
|
||||
: "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
|
||||
);
|
||||
#endif /* (__i386__) */
|
||||
|
||||
x86_cpu_has_sse2 = edx & 0x4000000;
|
||||
x86_cpu_has_ssse3 = ecx & 0x000200;
|
||||
x86_cpu_has_sse42 = ecx & 0x100000;
|
||||
x86_cpu_has_pclmulqdq = ecx & 0x2;
|
||||
|
||||
x86_cpu_enable_ssse3 = x86_cpu_has_ssse3;
|
||||
|
||||
x86_cpu_enable_simd = x86_cpu_has_sse2 &&
|
||||
x86_cpu_has_sse42 &&
|
||||
x86_cpu_has_pclmulqdq;
|
||||
}
|
||||
#else
|
||||
#include <intrin.h>
|
||||
#include <windows.h>
|
||||
#include <stdint.h>
|
||||
|
||||
static volatile int32_t once_control = 0;
|
||||
static int fake_pthread_once(volatile int32_t *once_control,
|
||||
void (*init_routine)(void));
|
||||
|
||||
void x86_check_features(void)
|
||||
{
|
||||
fake_pthread_once(&once_control, _x86_check_features);
|
||||
}
|
||||
|
||||
/* Copied from "perftools_pthread_once" in tcmalloc */
|
||||
static int fake_pthread_once(volatile int32_t *once_control,
|
||||
void (*init_routine)(void)) {
|
||||
// Try for a fast path first. Note: this should be an acquire semantics read
|
||||
// It is on x86 and x64, where Windows runs.
|
||||
if (*once_control != 1) {
|
||||
while (1) {
|
||||
switch (InterlockedCompareExchange(once_control, 2, 0)) {
|
||||
case 0:
|
||||
init_routine();
|
||||
InterlockedExchange(once_control, 1);
|
||||
return 0;
|
||||
case 1:
|
||||
// The initializer has already been executed
|
||||
return 0;
|
||||
default:
|
||||
// The initializer is being processed by another thread
|
||||
SwitchToThread();
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void _x86_check_features(void)
|
||||
{
|
||||
int x86_cpu_has_sse2;
|
||||
int x86_cpu_has_ssse3;
|
||||
int x86_cpu_has_sse42;
|
||||
int x86_cpu_has_pclmulqdq;
|
||||
int regs[4];
|
||||
|
||||
__cpuid(regs, 1);
|
||||
|
||||
x86_cpu_has_sse2 = regs[3] & 0x4000000;
|
||||
x86_cpu_has_ssse3 = regs[2] & 0x000200;
|
||||
x86_cpu_has_sse42 = regs[2] & 0x100000;
|
||||
x86_cpu_has_pclmulqdq = regs[2] & 0x2;
|
||||
|
||||
x86_cpu_enable_ssse3 = x86_cpu_has_ssse3;
|
||||
|
||||
x86_cpu_enable_simd = x86_cpu_has_sse2 &&
|
||||
x86_cpu_has_sse42 &&
|
||||
x86_cpu_has_pclmulqdq;
|
||||
}
|
||||
#endif /* _MSC_VER */
|
||||
16
modules/zlib/src/x86.h
Normal file
16
modules/zlib/src/x86.h
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
/* x86.h -- check for x86 CPU features
|
||||
* Copyright (C) 2013 Intel Corporation Jim Kukunas
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef X86_H
|
||||
#define X86_H
|
||||
|
||||
#include "zlib.h"
|
||||
|
||||
extern int x86_cpu_enable_ssse3;
|
||||
extern int x86_cpu_enable_simd;
|
||||
|
||||
void x86_check_features(void);
|
||||
|
||||
#endif /* X86_H */
|
||||
|
|
@ -8,7 +8,6 @@
|
|||
#ifndef ZCONF_H
|
||||
#define ZCONF_H
|
||||
|
||||
/* This header does prefixing as below, but with an updated set of names. */
|
||||
#include "mozzconf.h"
|
||||
|
||||
/*
|
||||
|
|
@ -434,7 +433,7 @@ typedef uLong FAR uLongf;
|
|||
typedef unsigned long z_crc_t;
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */
|
||||
#if !defined(_WIN32)
|
||||
# define Z_HAVE_UNISTD_H
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -1824,6 +1824,11 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */
|
|||
# undef z_gzgetc
|
||||
# define z_gzgetc(g) \
|
||||
((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g))
|
||||
#elif defined(Z_CR_PREFIX_SET)
|
||||
# undef gzgetc
|
||||
# define gzgetc(g) \
|
||||
((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) \
|
||||
: (Cr_z_gzgetc)(g))
|
||||
#else
|
||||
# define gzgetc(g) \
|
||||
((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g))
|
||||
|
|
@ -1853,11 +1858,29 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */
|
|||
# define z_adler32_combine z_adler32_combine64
|
||||
# define z_crc32_combine z_crc32_combine64
|
||||
# else
|
||||
# ifdef gzopen
|
||||
# undef gzopen
|
||||
# endif
|
||||
# define gzopen gzopen64
|
||||
# ifdef gzseek
|
||||
# undef gzseek
|
||||
# endif
|
||||
# define gzseek gzseek64
|
||||
# ifdef gztell
|
||||
# undef gztell
|
||||
# endif
|
||||
# define gztell gztell64
|
||||
# ifdef gzoffset
|
||||
# undef gzoffset
|
||||
# endif
|
||||
# define gzoffset gzoffset64
|
||||
# ifdef adler32_combine
|
||||
# undef adler32_combine
|
||||
# endif
|
||||
# define adler32_combine adler32_combine64
|
||||
# ifdef crc32_combine
|
||||
# undef crc32_combine
|
||||
# endif
|
||||
# define crc32_combine crc32_combine64
|
||||
# endif
|
||||
# ifndef Z_LARGE64
|
||||
|
|
|
|||
|
|
@ -28,6 +28,21 @@
|
|||
# include <string.h>
|
||||
# include <stdlib.h>
|
||||
#endif
|
||||
#ifdef NO_ERRNO_H
|
||||
# ifdef _WIN32_WCE
|
||||
/* The Microsoft C Run-Time Library for Windows CE doesn't have
|
||||
* errno. We define it as a global variable to simplify porting.
|
||||
* Its value is always 0 and should not be used. We rename it to
|
||||
* avoid conflict with other libraries that use the same workaround.
|
||||
*/
|
||||
# define errno z_errno
|
||||
# endif
|
||||
extern int errno;
|
||||
#else
|
||||
# ifndef _WIN32_WCE
|
||||
# include <errno.h>
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifdef Z_SOLO
|
||||
typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */
|
||||
|
|
@ -268,4 +283,10 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
|
|||
#define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \
|
||||
(((q) & 0xff00) << 8) + (((q) & 0xff) << 24))
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define zalign(x) __declspec(align(x))
|
||||
#else
|
||||
#define zalign(x) __attribute__((aligned((x))))
|
||||
#endif
|
||||
|
||||
#endif /* ZUTIL_H */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue