mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-08-15 08:53:07 +09:00
revert to rt1's zlib
This commit is contained in:
wuggy
parent
9745f0f1c8
commit
175e0de509
207 changed files with 15023 additions and 33924 deletions
1515
modules/zlib/src/ChangeLog
Normal file
1515
modules/zlib/src/ChangeLog
Normal file
File diff suppressed because it is too large
Load diff
69
modules/zlib/src/ChangeLog.moz
Normal file
69
modules/zlib/src/ChangeLog.moz
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
|
||||
Mozilla ChangeLog file for zlib
|
||||
|
||||
Mozilla.org changes:
|
||||
- 17 June 2004
|
||||
Sync'd with 1.2.1 release
|
||||
see bug #226733
|
||||
|
||||
- 17 June 2004
|
||||
Add mozzconf.h
|
||||
|
||||
- 17 June 2004
|
||||
Turn off ZLIB_DLL to fix static build bustage
|
||||
|
||||
- 24 June 2004
|
||||
do not prototype strerror for VMS in zutil.h
|
||||
|
||||
- 25 June 2004
|
||||
define NO_vsnprintf for VMS in zutil.h
|
||||
|
||||
- 21 September 2004
|
||||
applied patch for DoS vulnerabililty in inflate.c
|
||||
see bug #258009
|
||||
|
||||
- 02 November 2004
|
||||
Sync'ed with 1.2.2 release
|
||||
(keeping '#include "mozzconf.h"' in zconf.h)
|
||||
Our patches of 24, 25 June and 21 Sept are now in the zlib
|
||||
distribution.
|
||||
See bug #248644
|
||||
|
||||
- 24 July 2005
|
||||
Sync'ed with 1.2.3 release
|
||||
(keeping '#include "mozzconf.h"' in zconf.h)
|
||||
See bugs #299445 and #300349
|
||||
|
||||
- 24 July 2005
|
||||
Added treatment of HAVE_VISIBILITY_PRAGMA in mozzconf.h
|
||||
|
||||
- 24 July 2005
|
||||
Updated zlib.def with new symbols in zlib version 1.2.3
|
||||
|
||||
- 13 September 2009
|
||||
Don't enable zlib's debug output when the Mozilla build is in debug mode (bug 431950)
|
||||
|
||||
- 18 September 2010
|
||||
Sync'ed with 1.2.5 release
|
||||
(keeping '#include "mozzconf.h"' in zconf.h)
|
||||
See bug #573137
|
||||
|
||||
- 8 February 2012
|
||||
Sync'ed with 1.2.6 release
|
||||
(keeping '#include "mozzconf.h"' in zconf.h)
|
||||
See bug #722391
|
||||
|
||||
- 4 May 2012
|
||||
Sync'ed with 1.2.7 release
|
||||
(keeping '#include "mozzconf.h"' in zconf.h)
|
||||
See bug #751697
|
||||
|
||||
- 9 May 2013
|
||||
Sync'ed with 1.2.8 release
|
||||
(keeping '#include "mozzconf.h"' in zconf.h)
|
||||
See bug #866964
|
||||
|
||||
- 16 January 2017
|
||||
Sync'ed with 1.2.11 release
|
||||
(keeping '#include "mozzconf.h"' in zconf.h)
|
||||
See bug #1328099
|
||||
368
modules/zlib/src/FAQ
Normal file
368
modules/zlib/src/FAQ
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
|
||||
Frequently Asked Questions about zlib
|
||||
|
||||
|
||||
If your question is not there, please check the zlib home page
|
||||
http://zlib.net/ which may have more recent information.
|
||||
The lastest zlib FAQ is at http://zlib.net/zlib_faq.html
|
||||
|
||||
|
||||
1. Is zlib Y2K-compliant?
|
||||
|
||||
Yes. zlib doesn't handle dates.
|
||||
|
||||
2. Where can I get a Windows DLL version?
|
||||
|
||||
The zlib sources can be compiled without change to produce a DLL. See the
|
||||
file win32/DLL_FAQ.txt in the zlib distribution. Pointers to the
|
||||
precompiled DLL are found in the zlib web site at http://zlib.net/ .
|
||||
|
||||
3. Where can I get a Visual Basic interface to zlib?
|
||||
|
||||
See
|
||||
* http://marknelson.us/1997/01/01/zlib-engine/
|
||||
* win32/DLL_FAQ.txt in the zlib distribution
|
||||
|
||||
4. compress() returns Z_BUF_ERROR.
|
||||
|
||||
Make sure that before the call of compress(), the length of the compressed
|
||||
buffer is equal to the available size of the compressed buffer and not
|
||||
zero. For Visual Basic, check that this parameter is passed by reference
|
||||
("as any"), not by value ("as long").
|
||||
|
||||
5. deflate() or inflate() returns Z_BUF_ERROR.
|
||||
|
||||
Before making the call, make sure that avail_in and avail_out are not zero.
|
||||
When setting the parameter flush equal to Z_FINISH, also make sure that
|
||||
avail_out is big enough to allow processing all pending input. Note that a
|
||||
Z_BUF_ERROR is not fatal--another call to deflate() or inflate() can be
|
||||
made with more input or output space. A Z_BUF_ERROR may in fact be
|
||||
unavoidable depending on how the functions are used, since it is not
|
||||
possible to tell whether or not there is more output pending when
|
||||
strm.avail_out returns with zero. See http://zlib.net/zlib_how.html for a
|
||||
heavily annotated example.
|
||||
|
||||
6. Where's the zlib documentation (man pages, etc.)?
|
||||
|
||||
It's in zlib.h . Examples of zlib usage are in the files test/example.c
|
||||
and test/minigzip.c, with more in examples/ .
|
||||
|
||||
7. Why don't you use GNU autoconf or libtool or ...?
|
||||
|
||||
Because we would like to keep zlib as a very small and simple package.
|
||||
zlib is rather portable and doesn't need much configuration.
|
||||
|
||||
8. I found a bug in zlib.
|
||||
|
||||
Most of the time, such problems are due to an incorrect usage of zlib.
|
||||
Please try to reproduce the problem with a small program and send the
|
||||
corresponding source to us at zlib@gzip.org . Do not send multi-megabyte
|
||||
data files without prior agreement.
|
||||
|
||||
9. Why do I get "undefined reference to gzputc"?
|
||||
|
||||
If "make test" produces something like
|
||||
|
||||
example.o(.text+0x154): undefined reference to `gzputc'
|
||||
|
||||
check that you don't have old files libz.* in /usr/lib, /usr/local/lib or
|
||||
/usr/X11R6/lib. Remove any old versions, then do "make install".
|
||||
|
||||
10. I need a Delphi interface to zlib.
|
||||
|
||||
See the contrib/delphi directory in the zlib distribution.
|
||||
|
||||
11. Can zlib handle .zip archives?
|
||||
|
||||
Not by itself, no. See the directory contrib/minizip in the zlib
|
||||
distribution.
|
||||
|
||||
12. Can zlib handle .Z files?
|
||||
|
||||
No, sorry. You have to spawn an uncompress or gunzip subprocess, or adapt
|
||||
the code of uncompress on your own.
|
||||
|
||||
13. How can I make a Unix shared library?
|
||||
|
||||
By default a shared (and a static) library is built for Unix. So:
|
||||
|
||||
make distclean
|
||||
./configure
|
||||
make
|
||||
|
||||
14. How do I install a shared zlib library on Unix?
|
||||
|
||||
After the above, then:
|
||||
|
||||
make install
|
||||
|
||||
However, many flavors of Unix come with a shared zlib already installed.
|
||||
Before going to the trouble of compiling a shared version of zlib and
|
||||
trying to install it, you may want to check if it's already there! If you
|
||||
can #include <zlib.h>, it's there. The -lz option will probably link to
|
||||
it. You can check the version at the top of zlib.h or with the
|
||||
ZLIB_VERSION symbol defined in zlib.h .
|
||||
|
||||
15. I have a question about OttoPDF.
|
||||
|
||||
We are not the authors of OttoPDF. The real author is on the OttoPDF web
|
||||
site: Joel Hainley, jhainley@myndkryme.com.
|
||||
|
||||
16. Can zlib decode Flate data in an Adobe PDF file?
|
||||
|
||||
Yes. See http://www.pdflib.com/ . To modify PDF forms, see
|
||||
http://sourceforge.net/projects/acroformtool/ .
|
||||
|
||||
17. Why am I getting this "register_frame_info not found" error on Solaris?
|
||||
|
||||
After installing zlib 1.1.4 on Solaris 2.6, running applications using zlib
|
||||
generates an error such as:
|
||||
|
||||
ld.so.1: rpm: fatal: relocation error: file /usr/local/lib/libz.so:
|
||||
symbol __register_frame_info: referenced symbol not found
|
||||
|
||||
The symbol __register_frame_info is not part of zlib, it is generated by
|
||||
the C compiler (cc or gcc). You must recompile applications using zlib
|
||||
which have this problem. This problem is specific to Solaris. See
|
||||
http://www.sunfreeware.com for Solaris versions of zlib and applications
|
||||
using zlib.
|
||||
|
||||
18. Why does gzip give an error on a file I make with compress/deflate?
|
||||
|
||||
The compress and deflate functions produce data in the zlib format, which
|
||||
is different and incompatible with the gzip format. The gz* functions in
|
||||
zlib on the other hand use the gzip format. Both the zlib and gzip formats
|
||||
use the same compressed data format internally, but have different headers
|
||||
and trailers around the compressed data.
|
||||
|
||||
19. Ok, so why are there two different formats?
|
||||
|
||||
The gzip format was designed to retain the directory information about a
|
||||
single file, such as the name and last modification date. The zlib format
|
||||
on the other hand was designed for in-memory and communication channel
|
||||
applications, and has a much more compact header and trailer and uses a
|
||||
faster integrity check than gzip.
|
||||
|
||||
20. Well that's nice, but how do I make a gzip file in memory?
|
||||
|
||||
You can request that deflate write the gzip format instead of the zlib
|
||||
format using deflateInit2(). You can also request that inflate decode the
|
||||
gzip format using inflateInit2(). Read zlib.h for more details.
|
||||
|
||||
21. Is zlib thread-safe?
|
||||
|
||||
Yes. However any library routines that zlib uses and any application-
|
||||
provided memory allocation routines must also be thread-safe. zlib's gz*
|
||||
functions use stdio library routines, and most of zlib's functions use the
|
||||
library memory allocation routines by default. zlib's *Init* functions
|
||||
allow for the application to provide custom memory allocation routines.
|
||||
|
||||
Of course, you should only operate on any given zlib or gzip stream from a
|
||||
single thread at a time.
|
||||
|
||||
22. Can I use zlib in my commercial application?
|
||||
|
||||
Yes. Please read the license in zlib.h.
|
||||
|
||||
23. Is zlib under the GNU license?
|
||||
|
||||
No. Please read the license in zlib.h.
|
||||
|
||||
24. The license says that altered source versions must be "plainly marked". So
|
||||
what exactly do I need to do to meet that requirement?
|
||||
|
||||
You need to change the ZLIB_VERSION and ZLIB_VERNUM #defines in zlib.h. In
|
||||
particular, the final version number needs to be changed to "f", and an
|
||||
identification string should be appended to ZLIB_VERSION. Version numbers
|
||||
x.x.x.f are reserved for modifications to zlib by others than the zlib
|
||||
maintainers. For example, if the version of the base zlib you are altering
|
||||
is "1.2.3.4", then in zlib.h you should change ZLIB_VERNUM to 0x123f, and
|
||||
ZLIB_VERSION to something like "1.2.3.f-zachary-mods-v3". You can also
|
||||
update the version strings in deflate.c and inftrees.c.
|
||||
|
||||
For altered source distributions, you should also note the origin and
|
||||
nature of the changes in zlib.h, as well as in ChangeLog and README, along
|
||||
with the dates of the alterations. The origin should include at least your
|
||||
name (or your company's name), and an email address to contact for help or
|
||||
issues with the library.
|
||||
|
||||
Note that distributing a compiled zlib library along with zlib.h and
|
||||
zconf.h is also a source distribution, and so you should change
|
||||
ZLIB_VERSION and ZLIB_VERNUM and note the origin and nature of the changes
|
||||
in zlib.h as you would for a full source distribution.
|
||||
|
||||
25. Will zlib work on a big-endian or little-endian architecture, and can I
|
||||
exchange compressed data between them?
|
||||
|
||||
Yes and yes.
|
||||
|
||||
26. Will zlib work on a 64-bit machine?
|
||||
|
||||
Yes. It has been tested on 64-bit machines, and has no dependence on any
|
||||
data types being limited to 32-bits in length. If you have any
|
||||
difficulties, please provide a complete problem report to zlib@gzip.org
|
||||
|
||||
27. Will zlib decompress data from the PKWare Data Compression Library?
|
||||
|
||||
No. The PKWare DCL uses a completely different compressed data format than
|
||||
does PKZIP and zlib. However, you can look in zlib's contrib/blast
|
||||
directory for a possible solution to your problem.
|
||||
|
||||
28. Can I access data randomly in a compressed stream?
|
||||
|
||||
No, not without some preparation. If when compressing you periodically use
|
||||
Z_FULL_FLUSH, carefully write all the pending data at those points, and
|
||||
keep an index of those locations, then you can start decompression at those
|
||||
points. You have to be careful to not use Z_FULL_FLUSH too often, since it
|
||||
can significantly degrade compression. Alternatively, you can scan a
|
||||
deflate stream once to generate an index, and then use that index for
|
||||
random access. See examples/zran.c .
|
||||
|
||||
29. Does zlib work on MVS, OS/390, CICS, etc.?
|
||||
|
||||
It has in the past, but we have not heard of any recent evidence. There
|
||||
were working ports of zlib 1.1.4 to MVS, but those links no longer work.
|
||||
If you know of recent, successful applications of zlib on these operating
|
||||
systems, please let us know. Thanks.
|
||||
|
||||
30. Is there some simpler, easier to read version of inflate I can look at to
|
||||
understand the deflate format?
|
||||
|
||||
First off, you should read RFC 1951. Second, yes. Look in zlib's
|
||||
contrib/puff directory.
|
||||
|
||||
31. Does zlib infringe on any patents?
|
||||
|
||||
As far as we know, no. In fact, that was originally the whole point behind
|
||||
zlib. Look here for some more information:
|
||||
|
||||
http://www.gzip.org/#faq11
|
||||
|
||||
32. Can zlib work with greater than 4 GB of data?
|
||||
|
||||
Yes. inflate() and deflate() will process any amount of data correctly.
|
||||
Each call of inflate() or deflate() is limited to input and output chunks
|
||||
of the maximum value that can be stored in the compiler's "unsigned int"
|
||||
type, but there is no limit to the number of chunks. Note however that the
|
||||
strm.total_in and strm_total_out counters may be limited to 4 GB. These
|
||||
counters are provided as a convenience and are not used internally by
|
||||
inflate() or deflate(). The application can easily set up its own counters
|
||||
updated after each call of inflate() or deflate() to count beyond 4 GB.
|
||||
compress() and uncompress() may be limited to 4 GB, since they operate in a
|
||||
single call. gzseek() and gztell() may be limited to 4 GB depending on how
|
||||
zlib is compiled. See the zlibCompileFlags() function in zlib.h.
|
||||
|
||||
The word "may" appears several times above since there is a 4 GB limit only
|
||||
if the compiler's "long" type is 32 bits. If the compiler's "long" type is
|
||||
64 bits, then the limit is 16 exabytes.
|
||||
|
||||
33. Does zlib have any security vulnerabilities?
|
||||
|
||||
The only one that we are aware of is potentially in gzprintf(). If zlib is
|
||||
compiled to use sprintf() or vsprintf(), then there is no protection
|
||||
against a buffer overflow of an 8K string space (or other value as set by
|
||||
gzbuffer()), other than the caller of gzprintf() assuring that the output
|
||||
will not exceed 8K. On the other hand, if zlib is compiled to use
|
||||
snprintf() or vsnprintf(), which should normally be the case, then there is
|
||||
no vulnerability. The ./configure script will display warnings if an
|
||||
insecure variation of sprintf() will be used by gzprintf(). Also the
|
||||
zlibCompileFlags() function will return information on what variant of
|
||||
sprintf() is used by gzprintf().
|
||||
|
||||
If you don't have snprintf() or vsnprintf() and would like one, you can
|
||||
find a portable implementation here:
|
||||
|
||||
http://www.ijs.si/software/snprintf/
|
||||
|
||||
Note that you should be using the most recent version of zlib. Versions
|
||||
1.1.3 and before were subject to a double-free vulnerability, and versions
|
||||
1.2.1 and 1.2.2 were subject to an access exception when decompressing
|
||||
invalid compressed data.
|
||||
|
||||
34. Is there a Java version of zlib?
|
||||
|
||||
Probably what you want is to use zlib in Java. zlib is already included
|
||||
as part of the Java SDK in the java.util.zip package. If you really want
|
||||
a version of zlib written in the Java language, look on the zlib home
|
||||
page for links: http://zlib.net/ .
|
||||
|
||||
35. I get this or that compiler or source-code scanner warning when I crank it
|
||||
up to maximally-pedantic. Can't you guys write proper code?
|
||||
|
||||
Many years ago, we gave up attempting to avoid warnings on every compiler
|
||||
in the universe. It just got to be a waste of time, and some compilers
|
||||
were downright silly as well as contradicted each other. So now, we simply
|
||||
make sure that the code always works.
|
||||
|
||||
36. Valgrind (or some similar memory access checker) says that deflate is
|
||||
performing a conditional jump that depends on an uninitialized value.
|
||||
Isn't that a bug?
|
||||
|
||||
No. That is intentional for performance reasons, and the output of deflate
|
||||
is not affected. This only started showing up recently since zlib 1.2.x
|
||||
uses malloc() by default for allocations, whereas earlier versions used
|
||||
calloc(), which zeros out the allocated memory. Even though the code was
|
||||
correct, versions 1.2.4 and later was changed to not stimulate these
|
||||
checkers.
|
||||
|
||||
37. Will zlib read the (insert any ancient or arcane format here) compressed
|
||||
data format?
|
||||
|
||||
Probably not. Look in the comp.compression FAQ for pointers to various
|
||||
formats and associated software.
|
||||
|
||||
38. How can I encrypt/decrypt zip files with zlib?
|
||||
|
||||
zlib doesn't support encryption. The original PKZIP encryption is very
|
||||
weak and can be broken with freely available programs. To get strong
|
||||
encryption, use GnuPG, http://www.gnupg.org/ , which already includes zlib
|
||||
compression. For PKZIP compatible "encryption", look at
|
||||
http://www.info-zip.org/
|
||||
|
||||
39. What's the difference between the "gzip" and "deflate" HTTP 1.1 encodings?
|
||||
|
||||
"gzip" is the gzip format, and "deflate" is the zlib format. They should
|
||||
probably have called the second one "zlib" instead to avoid confusion with
|
||||
the raw deflate compressed data format. While the HTTP 1.1 RFC 2616
|
||||
correctly points to the zlib specification in RFC 1950 for the "deflate"
|
||||
transfer encoding, there have been reports of servers and browsers that
|
||||
incorrectly produce or expect raw deflate data per the deflate
|
||||
specification in RFC 1951, most notably Microsoft. So even though the
|
||||
"deflate" transfer encoding using the zlib format would be the more
|
||||
efficient approach (and in fact exactly what the zlib format was designed
|
||||
for), using the "gzip" transfer encoding is probably more reliable due to
|
||||
an unfortunate choice of name on the part of the HTTP 1.1 authors.
|
||||
|
||||
Bottom line: use the gzip format for HTTP 1.1 encoding.
|
||||
|
||||
40. Does zlib support the new "Deflate64" format introduced by PKWare?
|
||||
|
||||
No. PKWare has apparently decided to keep that format proprietary, since
|
||||
they have not documented it as they have previous compression formats. In
|
||||
any case, the compression improvements are so modest compared to other more
|
||||
modern approaches, that it's not worth the effort to implement.
|
||||
|
||||
41. I'm having a problem with the zip functions in zlib, can you help?
|
||||
|
||||
There are no zip functions in zlib. You are probably using minizip by
|
||||
Giles Vollant, which is found in the contrib directory of zlib. It is not
|
||||
part of zlib. In fact none of the stuff in contrib is part of zlib. The
|
||||
files in there are not supported by the zlib authors. You need to contact
|
||||
the authors of the respective contribution for help.
|
||||
|
||||
42. The match.asm code in contrib is under the GNU General Public License.
|
||||
Since it's part of zlib, doesn't that mean that all of zlib falls under the
|
||||
GNU GPL?
|
||||
|
||||
No. The files in contrib are not part of zlib. They were contributed by
|
||||
other authors and are provided as a convenience to the user within the zlib
|
||||
distribution. Each item in contrib has its own license.
|
||||
|
||||
43. Is zlib subject to export controls? What is its ECCN?
|
||||
|
||||
zlib is not subject to export controls, and so is classified as EAR99.
|
||||
|
||||
44. Can you please sign these lengthy legal documents and fax them back to us
|
||||
so that we can use your software in our product?
|
||||
|
||||
No. Go away. Shoo.
|
||||
68
modules/zlib/src/INDEX
Normal file
68
modules/zlib/src/INDEX
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
CMakeLists.txt cmake build file
|
||||
ChangeLog history of changes
|
||||
FAQ Frequently Asked Questions about zlib
|
||||
INDEX this file
|
||||
Makefile dummy Makefile that tells you to ./configure
|
||||
Makefile.in template for Unix Makefile
|
||||
README guess what
|
||||
configure configure script for Unix
|
||||
make_vms.com makefile for VMS
|
||||
test/example.c zlib usages examples for build testing
|
||||
test/minigzip.c minimal gzip-like functionality for build testing
|
||||
test/infcover.c inf*.c code coverage for build coverage testing
|
||||
treebuild.xml XML description of source file dependencies
|
||||
zconf.h.cmakein zconf.h template for cmake
|
||||
zconf.h.in zconf.h template for configure
|
||||
zlib.3 Man page for zlib
|
||||
zlib.3.pdf Man page in PDF format
|
||||
zlib.map Linux symbol information
|
||||
zlib.pc.in Template for pkg-config descriptor
|
||||
zlib.pc.cmakein zlib.pc template for cmake
|
||||
zlib2ansi perl script to convert source files for C++ compilation
|
||||
|
||||
amiga/ makefiles for Amiga SAS C
|
||||
as400/ makefiles for AS/400
|
||||
doc/ documentation for formats and algorithms
|
||||
msdos/ makefiles for MSDOS
|
||||
nintendods/ makefile for Nintendo DS
|
||||
old/ makefiles for various architectures and zlib documentation
|
||||
files that have not yet been updated for zlib 1.2.x
|
||||
qnx/ makefiles for QNX
|
||||
watcom/ makefiles for OpenWatcom
|
||||
win32/ makefiles for Windows
|
||||
|
||||
zlib public header files (required for library use):
|
||||
zconf.h
|
||||
zlib.h
|
||||
|
||||
private source files used to build the zlib library:
|
||||
adler32.c
|
||||
compress.c
|
||||
crc32.c
|
||||
crc32.h
|
||||
deflate.c
|
||||
deflate.h
|
||||
gzclose.c
|
||||
gzguts.h
|
||||
gzlib.c
|
||||
gzread.c
|
||||
gzwrite.c
|
||||
infback.c
|
||||
inffast.c
|
||||
inffast.h
|
||||
inffixed.h
|
||||
inflate.c
|
||||
inflate.h
|
||||
inftrees.c
|
||||
inftrees.h
|
||||
trees.c
|
||||
trees.h
|
||||
uncompr.c
|
||||
zutil.c
|
||||
zutil.h
|
||||
|
||||
source files for sample programs
|
||||
See examples/README.examples
|
||||
|
||||
unsupported contributions by third parties
|
||||
See contrib/README.contrib
|
||||
115
modules/zlib/src/README
Normal file
115
modules/zlib/src/README
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
ZLIB DATA COMPRESSION LIBRARY
|
||||
|
||||
zlib 1.2.11 is a general purpose data compression library. All the code is
|
||||
thread safe. The data format used by the zlib library is described by RFCs
|
||||
(Request for Comments) 1950 to 1952 in the files
|
||||
http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and
|
||||
rfc1952 (gzip format).
|
||||
|
||||
All functions of the compression library are documented in the file zlib.h
|
||||
(volunteer to write man pages welcome, contact zlib@gzip.org). A usage example
|
||||
of the library is given in the file test/example.c which also tests that
|
||||
the library is working correctly. Another example is given in the file
|
||||
test/minigzip.c. The compression library itself is composed of all source
|
||||
files in the root directory.
|
||||
|
||||
To compile all files and run the test program, follow the instructions given at
|
||||
the top of Makefile.in. In short "./configure; make test", and if that goes
|
||||
well, "make install" should work for most flavors of Unix. For Windows, use
|
||||
one of the special makefiles in win32/ or contrib/vstudio/ . For VMS, use
|
||||
make_vms.com.
|
||||
|
||||
Questions about zlib should be sent to <zlib@gzip.org>, or to Gilles Vollant
|
||||
<info@winimage.com> for the Windows DLL version. The zlib home page is
|
||||
http://zlib.net/ . Before reporting a problem, please check this site to
|
||||
verify that you have the latest version of zlib; otherwise get the latest
|
||||
version and check whether the problem still exists or not.
|
||||
|
||||
PLEASE read the zlib FAQ http://zlib.net/zlib_faq.html before asking for help.
|
||||
|
||||
Mark Nelson <markn@ieee.org> wrote an article about zlib for the Jan. 1997
|
||||
issue of Dr. Dobb's Journal; a copy of the article is available at
|
||||
http://marknelson.us/1997/01/01/zlib-engine/ .
|
||||
|
||||
The changes made in version 1.2.11 are documented in the file ChangeLog.
|
||||
|
||||
Unsupported third party contributions are provided in directory contrib/ .
|
||||
|
||||
zlib is available in Java using the java.util.zip package, documented at
|
||||
http://java.sun.com/developer/technicalArticles/Programming/compression/ .
|
||||
|
||||
A Perl interface to zlib written by Paul Marquess <pmqs@cpan.org> is available
|
||||
at CPAN (Comprehensive Perl Archive Network) sites, including
|
||||
http://search.cpan.org/~pmqs/IO-Compress-Zlib/ .
|
||||
|
||||
A Python interface to zlib written by A.M. Kuchling <amk@amk.ca> is
|
||||
available in Python 1.5 and later versions, see
|
||||
http://docs.python.org/library/zlib.html .
|
||||
|
||||
zlib is built into tcl: http://wiki.tcl.tk/4610 .
|
||||
|
||||
An experimental package to read and write files in .zip format, written on top
|
||||
of zlib by Gilles Vollant <info@winimage.com>, is available in the
|
||||
contrib/minizip directory of zlib.
|
||||
|
||||
|
||||
Notes for some targets:
|
||||
|
||||
- For Windows DLL versions, please see win32/DLL_FAQ.txt
|
||||
|
||||
- For 64-bit Irix, deflate.c must be compiled without any optimization. With
|
||||
-O, one libpng test fails. The test works in 32 bit mode (with the -n32
|
||||
compiler flag). The compiler bug has been reported to SGI.
|
||||
|
||||
- zlib doesn't work with gcc 2.6.3 on a DEC 3000/300LX under OSF/1 2.1 it works
|
||||
when compiled with cc.
|
||||
|
||||
- On Digital Unix 4.0D (formely OSF/1) on AlphaServer, the cc option -std1 is
|
||||
necessary to get gzprintf working correctly. This is done by configure.
|
||||
|
||||
- zlib doesn't work on HP-UX 9.05 with some versions of /bin/cc. It works with
|
||||
other compilers. Use "make test" to check your compiler.
|
||||
|
||||
- gzdopen is not supported on RISCOS or BEOS.
|
||||
|
||||
- For PalmOs, see http://palmzlib.sourceforge.net/
|
||||
|
||||
|
||||
Acknowledgments:
|
||||
|
||||
The deflate format used by zlib was defined by Phil Katz. The deflate and
|
||||
zlib specifications were written by L. Peter Deutsch. Thanks to all the
|
||||
people who reported problems and suggested various improvements in zlib; they
|
||||
are too numerous to cite here.
|
||||
|
||||
Copyright notice:
|
||||
|
||||
(C) 1995-2017 Jean-loup Gailly and Mark Adler
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any damages
|
||||
arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not
|
||||
claim that you wrote the original software. If you use this software
|
||||
in a product, an acknowledgment in the product documentation would be
|
||||
appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be
|
||||
misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
|
||||
Jean-loup Gailly Mark Adler
|
||||
jloup@gzip.org madler@alumni.caltech.edu
|
||||
|
||||
If you use the zlib library in a product, we would appreciate *not* receiving
|
||||
lengthy legal documents to sign. The sources are provided for free but without
|
||||
warranty of any kind. The library has been entirely written by Jean-loup
|
||||
Gailly and Mark Adler; it does not include third-party code.
|
||||
|
||||
If you redistribute modified sources, we would appreciate that you include in
|
||||
the file ChangeLog history information documenting your changes. Please read
|
||||
the FAQ for more information on the distribution of modified source versions.
|
||||
|
|
@ -3,75 +3,214 @@
|
|||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "functable.h"
|
||||
#include "adler32_p.h"
|
||||
/* @(#) $Id$ */
|
||||
|
||||
#ifdef ZLIB_COMPAT
|
||||
unsigned long Z_EXPORT PREFIX(adler32_z)(unsigned long adler, const unsigned char *buf, size_t len) {
|
||||
if (buf == NULL)
|
||||
return ADLER32_INITIAL_VALUE;
|
||||
return (unsigned long)FUNCTABLE_CALL(adler32)((uint32_t)adler, buf, len);
|
||||
}
|
||||
#include "zutil.h"
|
||||
|
||||
local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2));
|
||||
|
||||
#define BASE 65521U /* largest prime smaller than 65536 */
|
||||
#define NMAX 5552
|
||||
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
|
||||
|
||||
#define DO1(buf,i) {adler += (buf)[i]; sum2 += adler;}
|
||||
#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1);
|
||||
#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2);
|
||||
#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4);
|
||||
#define DO16(buf) DO8(buf,0); DO8(buf,8);
|
||||
|
||||
/* use NO_DIVIDE if your processor does not do division in hardware --
|
||||
try it both ways to see which is faster */
|
||||
#ifdef NO_DIVIDE
|
||||
/* note that this assumes BASE is 65521, where 65536 % 65521 == 15
|
||||
(thank you to John Reiser for pointing this out) */
|
||||
# define CHOP(a) \
|
||||
do { \
|
||||
unsigned long tmp = a >> 16; \
|
||||
a &= 0xffffUL; \
|
||||
a += (tmp << 4) - tmp; \
|
||||
} while (0)
|
||||
# define MOD28(a) \
|
||||
do { \
|
||||
CHOP(a); \
|
||||
if (a >= BASE) a -= BASE; \
|
||||
} while (0)
|
||||
# define MOD(a) \
|
||||
do { \
|
||||
CHOP(a); \
|
||||
MOD28(a); \
|
||||
} while (0)
|
||||
# define MOD63(a) \
|
||||
do { /* this assumes a is not negative */ \
|
||||
z_off64_t tmp = a >> 32; \
|
||||
a &= 0xffffffffL; \
|
||||
a += (tmp << 8) - (tmp << 5) + tmp; \
|
||||
tmp = a >> 16; \
|
||||
a &= 0xffffL; \
|
||||
a += (tmp << 4) - tmp; \
|
||||
tmp = a >> 16; \
|
||||
a &= 0xffffL; \
|
||||
a += (tmp << 4) - tmp; \
|
||||
if (a >= BASE) a -= BASE; \
|
||||
} while (0)
|
||||
#else
|
||||
uint32_t Z_EXPORT PREFIX(adler32_z)(uint32_t adler, const unsigned char *buf, size_t len) {
|
||||
if (buf == NULL)
|
||||
return ADLER32_INITIAL_VALUE;
|
||||
return FUNCTABLE_CALL(adler32)(adler, buf, len);
|
||||
}
|
||||
# define MOD(a) a %= BASE
|
||||
# define MOD28(a) a %= BASE
|
||||
# define MOD63(a) a %= BASE
|
||||
#endif
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
#include "adler32_simd.h"
|
||||
#include "x86.h"
|
||||
#elif defined(ADLER32_SIMD_NEON)
|
||||
#include "adler32_simd.h"
|
||||
#endif
|
||||
|
||||
/* ========================================================================= */
|
||||
#ifdef ZLIB_COMPAT
|
||||
unsigned long Z_EXPORT PREFIX(adler32)(unsigned long adler, const unsigned char *buf, unsigned int len) {
|
||||
if (buf == NULL)
|
||||
return ADLER32_INITIAL_VALUE;
|
||||
return (unsigned long)FUNCTABLE_CALL(adler32)((uint32_t)adler, buf, len);
|
||||
}
|
||||
#else
|
||||
uint32_t Z_EXPORT PREFIX(adler32)(uint32_t adler, const unsigned char *buf, uint32_t len) {
|
||||
if (buf == NULL)
|
||||
return ADLER32_INITIAL_VALUE;
|
||||
return FUNCTABLE_CALL(adler32)(adler, buf, len);
|
||||
}
|
||||
uLong ZEXPORT adler32_z(adler, buf, len)
|
||||
uLong adler;
|
||||
const Bytef *buf;
|
||||
z_size_t len;
|
||||
{
|
||||
unsigned long sum2;
|
||||
unsigned n;
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
if (x86_cpu_enable_ssse3 && buf && len >= 64)
|
||||
return adler32_simd_(adler, buf, len);
|
||||
#elif defined(ADLER32_SIMD_NEON)
|
||||
if (buf && len >= 64)
|
||||
return adler32_simd_(adler, buf, len);
|
||||
#endif
|
||||
|
||||
/* split Adler-32 into component sums */
|
||||
sum2 = (adler >> 16) & 0xffff;
|
||||
adler &= 0xffff;
|
||||
|
||||
/* in case user likes doing a byte at a time, keep it fast */
|
||||
if (len == 1) {
|
||||
adler += buf[0];
|
||||
if (adler >= BASE)
|
||||
adler -= BASE;
|
||||
sum2 += adler;
|
||||
if (sum2 >= BASE)
|
||||
sum2 -= BASE;
|
||||
return adler | (sum2 << 16);
|
||||
}
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
/*
|
||||
* Use SSSE3 to compute the adler32. Since this routine can be
|
||||
* freely used, check CPU features here. zlib convention is to
|
||||
* call adler32(0, NULL, 0), before making calls to adler32().
|
||||
* So this is a good early (and infrequent) place to cache CPU
|
||||
* features for those later, more interesting adler32() calls.
|
||||
*/
|
||||
if (buf == Z_NULL) {
|
||||
if (!len) /* Assume user is calling adler32(0, NULL, 0); */
|
||||
x86_check_features();
|
||||
return 1L;
|
||||
}
|
||||
#else
|
||||
/* initial Adler-32 value (deferred check for len == 1 speed) */
|
||||
if (buf == Z_NULL)
|
||||
return 1L;
|
||||
#endif
|
||||
|
||||
/* in case short lengths are provided, keep it somewhat fast */
|
||||
if (len < 16) {
|
||||
while (len--) {
|
||||
adler += *buf++;
|
||||
sum2 += adler;
|
||||
}
|
||||
if (adler >= BASE)
|
||||
adler -= BASE;
|
||||
MOD28(sum2); /* only added so many BASE's */
|
||||
return adler | (sum2 << 16);
|
||||
}
|
||||
|
||||
/* do length NMAX blocks -- requires just one modulo operation */
|
||||
while (len >= NMAX) {
|
||||
len -= NMAX;
|
||||
n = NMAX / 16; /* NMAX is divisible by 16 */
|
||||
do {
|
||||
DO16(buf); /* 16 sums unrolled */
|
||||
buf += 16;
|
||||
} while (--n);
|
||||
MOD(adler);
|
||||
MOD(sum2);
|
||||
}
|
||||
|
||||
/* do remaining bytes (less than NMAX, still just one modulo) */
|
||||
if (len) { /* avoid modulos if none remaining */
|
||||
while (len >= 16) {
|
||||
len -= 16;
|
||||
DO16(buf);
|
||||
buf += 16;
|
||||
}
|
||||
while (len--) {
|
||||
adler += *buf++;
|
||||
sum2 += adler;
|
||||
}
|
||||
MOD(adler);
|
||||
MOD(sum2);
|
||||
}
|
||||
|
||||
/* return recombined sums */
|
||||
return adler | (sum2 << 16);
|
||||
}
|
||||
|
||||
/* ========================================================================= */
|
||||
static uint32_t adler32_combine_(uint32_t adler1, uint32_t adler2, z_off64_t len2) {
|
||||
uint32_t sum1;
|
||||
uint32_t sum2;
|
||||
uLong ZEXPORT adler32(adler, buf, len)
|
||||
uLong adler;
|
||||
const Bytef *buf;
|
||||
uInt len;
|
||||
{
|
||||
return adler32_z(adler, buf, len);
|
||||
}
|
||||
|
||||
/* ========================================================================= */
|
||||
local uLong adler32_combine_(adler1, adler2, len2)
|
||||
uLong adler1;
|
||||
uLong adler2;
|
||||
z_off64_t len2;
|
||||
{
|
||||
unsigned long sum1;
|
||||
unsigned long sum2;
|
||||
unsigned rem;
|
||||
|
||||
/* for negative len, return invalid adler32 as a clue for debugging */
|
||||
if (len2 < 0)
|
||||
return 0xffffffff;
|
||||
return 0xffffffffUL;
|
||||
|
||||
/* the derivation of this formula is left as an exercise for the reader */
|
||||
len2 %= BASE; /* assumes len2 >= 0 */
|
||||
MOD63(len2); /* assumes len2 >= 0 */
|
||||
rem = (unsigned)len2;
|
||||
sum1 = adler1 & 0xffff;
|
||||
sum2 = rem * sum1;
|
||||
sum2 %= BASE;
|
||||
MOD(sum2);
|
||||
sum1 += (adler2 & 0xffff) + BASE - 1;
|
||||
sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem;
|
||||
if (sum1 >= BASE) sum1 -= BASE;
|
||||
if (sum1 >= BASE) sum1 -= BASE;
|
||||
if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1);
|
||||
if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1);
|
||||
if (sum2 >= BASE) sum2 -= BASE;
|
||||
return sum1 | (sum2 << 16);
|
||||
}
|
||||
|
||||
/* ========================================================================= */
|
||||
#ifdef ZLIB_COMPAT
|
||||
unsigned long Z_EXPORT PREFIX(adler32_combine)(unsigned long adler1, unsigned long adler2, z_off_t len2) {
|
||||
return (unsigned long)adler32_combine_((uint32_t)adler1, (uint32_t)adler2, len2);
|
||||
}
|
||||
|
||||
unsigned long Z_EXPORT PREFIX4(adler32_combine)(unsigned long adler1, unsigned long adler2, z_off64_t len2) {
|
||||
return (unsigned long)adler32_combine_((uint32_t)adler1, (uint32_t)adler2, len2);
|
||||
}
|
||||
#else
|
||||
uint32_t Z_EXPORT PREFIX4(adler32_combine)(uint32_t adler1, uint32_t adler2, z_off64_t len2) {
|
||||
uLong ZEXPORT adler32_combine(adler1, adler2, len2)
|
||||
uLong adler1;
|
||||
uLong adler2;
|
||||
z_off_t len2;
|
||||
{
|
||||
return adler32_combine_(adler1, adler2, len2);
|
||||
}
|
||||
|
||||
uLong ZEXPORT adler32_combine64(adler1, adler2, len2)
|
||||
uLong adler1;
|
||||
uLong adler2;
|
||||
z_off64_t len2;
|
||||
{
|
||||
return adler32_combine_(adler1, adler2, len2);
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
/* adler32_p.h -- Private inline functions and macros shared with
|
||||
* different computation of the Adler-32 checksum
|
||||
* of a data stream.
|
||||
* Copyright (C) 1995-2011, 2016 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef ADLER32_P_H
|
||||
#define ADLER32_P_H
|
||||
|
||||
#define BASE 65521U /* largest prime smaller than 65536 */
|
||||
#define NMAX 5552
|
||||
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
|
||||
#define NMAX_ALIGNED32 (NMAX & ~31)
|
||||
/* NMAX rounded down to a multiple of 32 is 5536 */
|
||||
|
||||
#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);}
|
||||
#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);}
|
||||
#define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);}
|
||||
#define ADLER_DO8(sum1, sum2, buf, i) {ADLER_DO4(sum1, sum2, buf, i); ADLER_DO4(sum1, sum2, buf, i+4);}
|
||||
#define ADLER_DO16(sum1, sum2, buf) {ADLER_DO8(sum1, sum2, buf, 0); ADLER_DO8(sum1, sum2, buf, 8);}
|
||||
|
||||
Z_FORCEINLINE static void adler32_copy_align(uint32_t *Z_RESTRICT adler, uint8_t *dst, const uint8_t *buf, size_t len,
|
||||
uint32_t *Z_RESTRICT sum2, const int MAX_LEN, const int COPY) {
|
||||
Z_UNUSED(MAX_LEN);
|
||||
if (len & 1) {
|
||||
if (COPY) {
|
||||
*dst = *buf;
|
||||
dst += 1;
|
||||
}
|
||||
ADLER_DO1(*adler, *sum2, buf, 0);
|
||||
buf += 1;
|
||||
}
|
||||
if (len & 2) {
|
||||
if (COPY) {
|
||||
memcpy(dst, buf, 2);
|
||||
dst += 2;
|
||||
}
|
||||
ADLER_DO2(*adler, *sum2, buf, 0);
|
||||
buf += 2;
|
||||
}
|
||||
while (len >= 4) {
|
||||
if (COPY) {
|
||||
memcpy(dst, buf, 4);
|
||||
dst += 4;
|
||||
}
|
||||
len -= 4;
|
||||
ADLER_DO4(*adler, *sum2, buf, 0);
|
||||
buf += 4;
|
||||
}
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static uint32_t adler32_copy_tail(uint32_t adler, uint8_t *dst, const uint8_t *buf, size_t len,
|
||||
uint32_t sum2, const int REBASE, const int MAX_LEN, const int COPY) {
|
||||
if (len) {
|
||||
/* DO16 loop for large remainders only (scalar, risc-v). */
|
||||
if (MAX_LEN >= 32) {
|
||||
while (len >= 16) {
|
||||
if (COPY) {
|
||||
memcpy(dst, buf, 16);
|
||||
dst += 16;
|
||||
}
|
||||
len -= 16;
|
||||
ADLER_DO16(adler, sum2, buf);
|
||||
buf += 16;
|
||||
}
|
||||
}
|
||||
/* DO4 loop avoids GCC x86 register pressure from hoisted DO8/DO16 loads. */
|
||||
while (len >= 4) {
|
||||
if (COPY) {
|
||||
memcpy(dst, buf, 4);
|
||||
dst += 4;
|
||||
}
|
||||
len -= 4;
|
||||
ADLER_DO4(adler, sum2, buf, 0);
|
||||
buf += 4;
|
||||
}
|
||||
if (len & 2) {
|
||||
if (COPY) {
|
||||
memcpy(dst, buf, 2);
|
||||
dst += 2;
|
||||
}
|
||||
ADLER_DO2(adler, sum2, buf, 0);
|
||||
buf += 2;
|
||||
}
|
||||
if (len & 1) {
|
||||
if (COPY)
|
||||
*dst = *buf;
|
||||
ADLER_DO1(adler, sum2, buf, 0);
|
||||
}
|
||||
}
|
||||
if (REBASE) {
|
||||
adler %= BASE;
|
||||
sum2 %= BASE;
|
||||
}
|
||||
/* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */
|
||||
return adler | (sum2 << 16);
|
||||
}
|
||||
|
||||
#endif /* ADLER32_P_H */
|
||||
366
modules/zlib/src/adler32_simd.c
Normal file
366
modules/zlib/src/adler32_simd.c
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
/* adler32_simd.c
|
||||
*
|
||||
* Copyright 2017 The Chromium Authors. All rights reserved.
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the Chromium source repository LICENSE file.
|
||||
*
|
||||
* Per http://en.wikipedia.org/wiki/Adler-32 the adler32 A value (aka s1) is
|
||||
* the sum of N input data bytes D1 ... DN,
|
||||
*
|
||||
* A = A0 + D1 + D2 + ... + DN
|
||||
*
|
||||
* where A0 is the initial value.
|
||||
*
|
||||
* SSE2 _mm_sad_epu8() can be used for byte sums (see http://bit.ly/2wpUOeD,
|
||||
* for example) and accumulating the byte sums can use SSE shuffle-adds (see
|
||||
* the "Integer" section of http://bit.ly/2erPT8t for details). Arm NEON has
|
||||
* similar instructions.
|
||||
*
|
||||
* The adler32 B value (aka s2) sums the A values from each step:
|
||||
*
|
||||
* B0 + (A0 + D1) + (A0 + D1 + D2) + ... + (A0 + D1 + D2 + ... + DN) or
|
||||
*
|
||||
* B0 + N.A0 + N.D1 + (N-1).D2 + (N-2).D3 + ... + (N-(N-1)).DN
|
||||
*
|
||||
* B0 being the initial value. For 32 bytes (ideal for garden-variety SIMD):
|
||||
*
|
||||
* B = B0 + 32.A0 + [D1 D2 D3 ... D32] x [32 31 30 ... 1].
|
||||
*
|
||||
* Adjacent blocks of 32 input bytes can be iterated with the expressions to
|
||||
* compute the adler32 s1 s2 of M >> 32 input bytes [1].
|
||||
*
|
||||
* As M grows, the s1 s2 sums grow. If left unchecked, they would eventually
|
||||
* overflow the precision of their integer representation (bad). However, s1
|
||||
* and s2 also need to be computed modulo the adler BASE value (reduced). If
|
||||
* at most NMAX bytes are processed before a reduce, s1 s2 _cannot_ overflow
|
||||
* a uint32_t type (the NMAX constraint) [2].
|
||||
*
|
||||
* [1] the iterative equations for s2 contain constant factors; these can be
|
||||
* hoisted from the n-blocks do loop of the SIMD code.
|
||||
*
|
||||
* [2] zlib adler32_z() uses this fact to implement NMAX-block-based updates
|
||||
* of the adler s1 s2 of uint32_t type (see adler32.c).
|
||||
*/
|
||||
|
||||
#include "adler32_simd.h"
|
||||
|
||||
/* Definitions from adler32.c: largest prime smaller than 65536 */
|
||||
#define BASE 65521U
|
||||
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
|
||||
#define NMAX 5552
|
||||
|
||||
#if defined(ADLER32_SIMD_SSSE3)
|
||||
|
||||
#include <tmmintrin.h>
|
||||
|
||||
uint32_t ZLIB_INTERNAL adler32_simd_( /* SSSE3 */
|
||||
uint32_t adler,
|
||||
const unsigned char *buf,
|
||||
z_size_t len)
|
||||
{
|
||||
/*
|
||||
* Split Adler-32 into component sums.
|
||||
*/
|
||||
uint32_t s1 = adler & 0xffff;
|
||||
uint32_t s2 = adler >> 16;
|
||||
|
||||
/*
|
||||
* Process the data in blocks.
|
||||
*/
|
||||
const unsigned BLOCK_SIZE = 1 << 5;
|
||||
|
||||
z_size_t blocks = len / BLOCK_SIZE;
|
||||
len -= blocks * BLOCK_SIZE;
|
||||
|
||||
while (blocks)
|
||||
{
|
||||
unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */
|
||||
if (n > blocks)
|
||||
n = (unsigned) blocks;
|
||||
blocks -= n;
|
||||
|
||||
const __m128i tap1 =
|
||||
_mm_setr_epi8(32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17);
|
||||
const __m128i tap2 =
|
||||
_mm_setr_epi8(16,15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1);
|
||||
const __m128i zero =
|
||||
_mm_setr_epi8( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
|
||||
const __m128i ones =
|
||||
_mm_set_epi16( 1, 1, 1, 1, 1, 1, 1, 1);
|
||||
|
||||
/*
|
||||
* Process n blocks of data. At most NMAX data bytes can be
|
||||
* processed before s2 must be reduced modulo BASE.
|
||||
*/
|
||||
__m128i v_ps = _mm_set_epi32(0, 0, 0, s1 * n);
|
||||
__m128i v_s2 = _mm_set_epi32(0, 0, 0, s2);
|
||||
__m128i v_s1 = _mm_set_epi32(0, 0, 0, 0);
|
||||
|
||||
do {
|
||||
/*
|
||||
* Load 32 input bytes.
|
||||
*/
|
||||
const __m128i bytes1 = _mm_loadu_si128((__m128i*)(buf));
|
||||
const __m128i bytes2 = _mm_loadu_si128((__m128i*)(buf + 16));
|
||||
|
||||
/*
|
||||
* Add previous block byte sum to v_ps.
|
||||
*/
|
||||
v_ps = _mm_add_epi32(v_ps, v_s1);
|
||||
|
||||
/*
|
||||
* Horizontally add the bytes for s1, multiply-adds the
|
||||
* bytes by [ 32, 31, 30, ... ] for s2.
|
||||
*/
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes1, zero));
|
||||
const __m128i mad1 = _mm_maddubs_epi16(bytes1, tap1);
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad1, ones));
|
||||
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes2, zero));
|
||||
const __m128i mad2 = _mm_maddubs_epi16(bytes2, tap2);
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad2, ones));
|
||||
|
||||
buf += BLOCK_SIZE;
|
||||
|
||||
} while (--n);
|
||||
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_slli_epi32(v_ps, 5));
|
||||
|
||||
/*
|
||||
* Sum epi32 ints v_s1(s2) and accumulate in s1(s2).
|
||||
*/
|
||||
|
||||
#define S23O1 _MM_SHUFFLE(2,3,0,1) /* A B C D -> B A D C */
|
||||
#define S1O32 _MM_SHUFFLE(1,0,3,2) /* A B C D -> C D A B */
|
||||
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S23O1));
|
||||
v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S1O32));
|
||||
|
||||
s1 += _mm_cvtsi128_si32(v_s1);
|
||||
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S23O1));
|
||||
v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S1O32));
|
||||
|
||||
s2 = _mm_cvtsi128_si32(v_s2);
|
||||
|
||||
#undef S23O1
|
||||
#undef S1O32
|
||||
|
||||
/*
|
||||
* Reduce.
|
||||
*/
|
||||
s1 %= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Handle leftover data.
|
||||
*/
|
||||
if (len) {
|
||||
if (len >= 16) {
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
while (len--) {
|
||||
s2 += (s1 += *buf++);
|
||||
}
|
||||
|
||||
if (s1 >= BASE)
|
||||
s1 -= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the recombined sums.
|
||||
*/
|
||||
return s1 | (s2 << 16);
|
||||
}
|
||||
|
||||
#elif defined(ADLER32_SIMD_NEON)
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
uint32_t ZLIB_INTERNAL adler32_simd_( /* NEON */
|
||||
uint32_t adler,
|
||||
const unsigned char *buf,
|
||||
z_size_t len)
|
||||
{
|
||||
/*
|
||||
* Split Adler-32 into component sums.
|
||||
*/
|
||||
uint32_t s1 = adler & 0xffff;
|
||||
uint32_t s2 = adler >> 16;
|
||||
|
||||
/*
|
||||
* Serially compute s1 & s2, until the data is 16-byte aligned.
|
||||
*/
|
||||
if ((uintptr_t)buf & 15) {
|
||||
while ((uintptr_t)buf & 15) {
|
||||
s2 += (s1 += *buf++);
|
||||
--len;
|
||||
}
|
||||
|
||||
if (s1 >= BASE)
|
||||
s1 -= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Process the data in blocks.
|
||||
*/
|
||||
const unsigned BLOCK_SIZE = 1 << 5;
|
||||
|
||||
z_size_t blocks = len / BLOCK_SIZE;
|
||||
len -= blocks * BLOCK_SIZE;
|
||||
|
||||
while (blocks)
|
||||
{
|
||||
unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */
|
||||
if (n > blocks)
|
||||
n = (unsigned) blocks;
|
||||
blocks -= n;
|
||||
|
||||
/*
|
||||
* Process n blocks of data. At most NMAX data bytes can be
|
||||
* processed before s2 must be reduced modulo BASE.
|
||||
*/
|
||||
uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, s1 * n };
|
||||
uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 };
|
||||
|
||||
uint16x8_t v_column_sum_1 = vdupq_n_u16(0);
|
||||
uint16x8_t v_column_sum_2 = vdupq_n_u16(0);
|
||||
uint16x8_t v_column_sum_3 = vdupq_n_u16(0);
|
||||
uint16x8_t v_column_sum_4 = vdupq_n_u16(0);
|
||||
|
||||
do {
|
||||
/*
|
||||
* Load 32 input bytes.
|
||||
*/
|
||||
const uint8x16_t bytes1 = vld1q_u8((uint8_t*)(buf));
|
||||
const uint8x16_t bytes2 = vld1q_u8((uint8_t*)(buf + 16));
|
||||
|
||||
/*
|
||||
* Add previous block byte sum to v_s2.
|
||||
*/
|
||||
v_s2 = vaddq_u32(v_s2, v_s1);
|
||||
|
||||
/*
|
||||
* Horizontally add the bytes for s1.
|
||||
*/
|
||||
v_s1 = vpadalq_u16(v_s1, vpadalq_u8(vpaddlq_u8(bytes1), bytes2));
|
||||
|
||||
/*
|
||||
* Vertically add the bytes for s2.
|
||||
*/
|
||||
v_column_sum_1 = vaddw_u8(v_column_sum_1, vget_low_u8 (bytes1));
|
||||
v_column_sum_2 = vaddw_u8(v_column_sum_2, vget_high_u8(bytes1));
|
||||
v_column_sum_3 = vaddw_u8(v_column_sum_3, vget_low_u8 (bytes2));
|
||||
v_column_sum_4 = vaddw_u8(v_column_sum_4, vget_high_u8(bytes2));
|
||||
|
||||
buf += BLOCK_SIZE;
|
||||
|
||||
} while (--n);
|
||||
|
||||
v_s2 = vshlq_n_u32(v_s2, 5);
|
||||
|
||||
/*
|
||||
* Multiply-add bytes by [ 32, 31, 30, ... ] for s2.
|
||||
*/
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_1),
|
||||
(uint16x4_t) { 32, 31, 30, 29 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_1),
|
||||
(uint16x4_t) { 28, 27, 26, 25 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_2),
|
||||
(uint16x4_t) { 24, 23, 22, 21 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_2),
|
||||
(uint16x4_t) { 20, 19, 18, 17 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_3),
|
||||
(uint16x4_t) { 16, 15, 14, 13 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_3),
|
||||
(uint16x4_t) { 12, 11, 10, 9 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_4),
|
||||
(uint16x4_t) { 8, 7, 6, 5 });
|
||||
v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_4),
|
||||
(uint16x4_t) { 4, 3, 2, 1 });
|
||||
|
||||
/*
|
||||
* Sum epi32 ints v_s1(s2) and accumulate in s1(s2).
|
||||
*/
|
||||
uint32x2_t sum1 = vpadd_u32(vget_low_u32(v_s1), vget_high_u32(v_s1));
|
||||
uint32x2_t sum2 = vpadd_u32(vget_low_u32(v_s2), vget_high_u32(v_s2));
|
||||
uint32x2_t s1s2 = vpadd_u32(sum1, sum2);
|
||||
|
||||
s1 += vget_lane_u32(s1s2, 0);
|
||||
s2 += vget_lane_u32(s1s2, 1);
|
||||
|
||||
/*
|
||||
* Reduce.
|
||||
*/
|
||||
s1 %= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Handle leftover data.
|
||||
*/
|
||||
if (len) {
|
||||
if (len >= 16) {
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
s2 += (s1 += *buf++);
|
||||
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
while (len--) {
|
||||
s2 += (s1 += *buf++);
|
||||
}
|
||||
|
||||
if (s1 >= BASE)
|
||||
s1 -= BASE;
|
||||
s2 %= BASE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the recombined sums.
|
||||
*/
|
||||
return s1 | (s2 << 16);
|
||||
}
|
||||
|
||||
#endif /* ADLER32_SIMD_SSSE3 */
|
||||
16
modules/zlib/src/adler32_simd.h
Normal file
16
modules/zlib/src/adler32_simd.h
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
/* adler32_simd.h
|
||||
*
|
||||
* Copyright 2017 The Chromium Authors. All rights reserved.
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the Chromium source repository LICENSE file.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "zconf.h"
|
||||
#include "zutil.h"
|
||||
|
||||
uint32_t ZLIB_INTERNAL adler32_simd_(
|
||||
uint32_t adler,
|
||||
const unsigned char *buf,
|
||||
z_size_t len);
|
||||
2
modules/zlib/src/arch/.gitignore
vendored
2
modules/zlib/src/arch/.gitignore
vendored
|
|
@ -1,2 +0,0 @@
|
|||
# ignore Makefiles; they're all automatically generated
|
||||
Makefile
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
# Makefile for zlib
|
||||
# Copyright (C) 1995-2013 Jean-loup Gailly, Mark Adler
|
||||
# For conditions of distribution and use, see copyright notice in zlib.h
|
||||
|
||||
CC=
|
||||
CFLAGS=
|
||||
SFLAGS=
|
||||
INCLUDES=
|
||||
SUFFIX=
|
||||
|
||||
ARMV8FLAG=
|
||||
PMULLEOR3FLAG=
|
||||
NEONFLAG=
|
||||
ARMV6FLAG=
|
||||
NOLTOFLAG=
|
||||
|
||||
SRCDIR=.
|
||||
SRCTOP=../..
|
||||
TOPDIR=$(SRCTOP)
|
||||
|
||||
all: \
|
||||
adler32_neon.o adler32_neon.lo \
|
||||
arm_features.o arm_features.lo \
|
||||
chunkset_neon.o chunkset_neon.lo \
|
||||
compare256_neon.o compare256_neon.lo \
|
||||
crc32_armv8.o crc32_armv8.lo \
|
||||
crc32_armv8_pmull_eor3.o crc32_armv8_pmull_eor3.lo \
|
||||
slide_hash_neon.o slide_hash_neon.lo \
|
||||
slide_hash_armv6.o slide_hash_armv6.lo \
|
||||
|
||||
adler32_neon.o:
|
||||
$(CC) $(CFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon.c
|
||||
|
||||
adler32_neon.lo:
|
||||
$(CC) $(SFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_neon.c
|
||||
|
||||
arm_features.o:
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/arm_features.c
|
||||
|
||||
arm_features.lo:
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/arm_features.c
|
||||
|
||||
chunkset_neon.o:
|
||||
$(CC) $(CFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_neon.c
|
||||
|
||||
chunkset_neon.lo:
|
||||
$(CC) $(SFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_neon.c
|
||||
|
||||
compare256_neon.o:
|
||||
$(CC) $(CFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_neon.c
|
||||
|
||||
compare256_neon.lo:
|
||||
$(CC) $(SFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_neon.c
|
||||
|
||||
crc32_armv8.o:
|
||||
$(CC) $(CFLAGS) $(ARMV8FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_armv8.c
|
||||
|
||||
crc32_armv8.lo:
|
||||
$(CC) $(SFLAGS) $(ARMV8FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_armv8.c
|
||||
|
||||
crc32_armv8_pmull_eor3.o:
|
||||
$(CC) $(CFLAGS) $(PMULLEOR3FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_armv8_pmull_eor3.c
|
||||
|
||||
crc32_armv8_pmull_eor3.lo:
|
||||
$(CC) $(SFLAGS) $(PMULLEOR3FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_armv8_pmull_eor3.c
|
||||
|
||||
slide_hash_neon.o:
|
||||
$(CC) $(CFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_neon.c
|
||||
|
||||
slide_hash_neon.lo:
|
||||
$(CC) $(SFLAGS) $(NEONFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_neon.c
|
||||
|
||||
slide_hash_armv6.o:
|
||||
$(CC) $(CFLAGS) $(ARMV6FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_armv6.c
|
||||
|
||||
slide_hash_armv6.lo:
|
||||
$(CC) $(SFLAGS) $(ARMV6FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_armv6.c
|
||||
|
||||
mostlyclean: clean
|
||||
clean:
|
||||
rm -f *.o *.lo *~
|
||||
rm -rf objs
|
||||
rm -f *.gcda *.gcno *.gcov
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
#ifndef ARM_ACLE_INTRINS_H
|
||||
#define ARM_ACLE_INTRINS_H
|
||||
|
||||
#include <stdint.h>
|
||||
#ifdef _MSC_VER
|
||||
# include <intrin.h>
|
||||
#elif defined(HAVE_ARM_ACLE_H)
|
||||
# include <arm_acle.h>
|
||||
#endif
|
||||
|
||||
#ifdef ARM_CRC32
|
||||
#if defined(ARCH_ARM) && defined(ARCH_64BIT)
|
||||
# define Z_TARGET_CRC Z_TARGET("+crc")
|
||||
#else
|
||||
# define Z_TARGET_CRC
|
||||
#endif
|
||||
#ifdef ARM_PMULL_EOR3
|
||||
# define Z_TARGET_PMULL_EOR3 Z_TARGET("+crc+crypto+sha3")
|
||||
#else
|
||||
# define Z_TARGET_PMULL_EOR3
|
||||
#endif
|
||||
|
||||
#if !defined(ARM_CRC32_INTRIN) && !defined(_MSC_VER)
|
||||
#if defined(ARCH_ARM) && defined(ARCH_64BIT)
|
||||
static inline uint32_t __crc32b(uint32_t __a, uint8_t __b) {
|
||||
uint32_t __c;
|
||||
__asm__("crc32b %w0, %w1, %w2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
|
||||
static inline uint32_t __crc32h(uint32_t __a, uint16_t __b) {
|
||||
uint32_t __c;
|
||||
__asm__("crc32h %w0, %w1, %w2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
|
||||
static inline uint32_t __crc32w(uint32_t __a, uint32_t __b) {
|
||||
uint32_t __c;
|
||||
__asm__("crc32w %w0, %w1, %w2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
|
||||
static inline uint32_t __crc32d(uint32_t __a, uint64_t __b) {
|
||||
uint32_t __c;
|
||||
__asm__("crc32x %w0, %w1, %x2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
#else
|
||||
static inline uint32_t __crc32b(uint32_t __a, uint8_t __b) {
|
||||
uint32_t __c;
|
||||
__asm__("crc32b %0, %1, %2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
|
||||
static inline uint32_t __crc32h(uint32_t __a, uint16_t __b) {
|
||||
uint32_t __c;
|
||||
__asm__("crc32h %0, %1, %2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
|
||||
static inline uint32_t __crc32w(uint32_t __a, uint32_t __b) {
|
||||
uint32_t __c;
|
||||
__asm__("crc32w %0, %1, %2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
|
||||
static inline uint32_t __crc32d(uint32_t __a, uint64_t __b) {
|
||||
return __crc32w (__crc32w (__a, __b & 0xffffffffULL), __b >> 32);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef ARM_SIMD
|
||||
#ifdef _MSC_VER
|
||||
typedef uint32_t uint16x2_t;
|
||||
|
||||
#define __uqsub16 _arm_uqsub16
|
||||
#elif !defined(ARM_SIMD_INTRIN)
|
||||
typedef uint32_t uint16x2_t;
|
||||
|
||||
static inline uint16x2_t __uqsub16(uint16x2_t __a, uint16x2_t __b) {
|
||||
uint16x2_t __c;
|
||||
__asm__("uqsub16 %0, %1, %2" : "=r" (__c) : "r"(__a), "r"(__b));
|
||||
return __c;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#endif // include guard ARM_ACLE_INTRINS_H
|
||||
|
|
@ -1,346 +0,0 @@
|
|||
/* Copyright (C) 1995-2011, 2016 Mark Adler
|
||||
* Copyright (C) 2017 ARM Holdings Inc.
|
||||
* Authors:
|
||||
* Adenilson Cavalcanti <adenilson.cavalcanti@arm.com>
|
||||
* Adam Stylinski <kungfujesus06@gmail.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef ARM_NEON
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "neon_intrins.h"
|
||||
#include "adler32_p.h"
|
||||
|
||||
static const uint16_t ALIGNED_(64) taps[64] = {
|
||||
64, 63, 62, 61, 60, 59, 58, 57,
|
||||
56, 55, 54, 53, 52, 51, 50, 49,
|
||||
48, 47, 46, 45, 44, 43, 42, 41,
|
||||
40, 39, 38, 37, 36, 35, 34, 33,
|
||||
32, 31, 30, 29, 28, 27, 26, 25,
|
||||
24, 23, 22, 21, 20, 19, 18, 17,
|
||||
16, 15, 14, 13, 12, 11, 10, 9,
|
||||
8, 7, 6, 5, 4, 3, 2, 1 };
|
||||
|
||||
Z_FORCEINLINE static void NEON_accum32_copy(uint32_t *s, uint8_t *dst, const uint8_t *buf, size_t len) {
|
||||
uint32x4_t adacc = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc_0 = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc_1 = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc_2 = vdupq_n_u32(0);
|
||||
|
||||
adacc = vsetq_lane_u32(s[0], adacc, 0);
|
||||
s2acc = vsetq_lane_u32(s[1], s2acc, 0);
|
||||
|
||||
uint32x4_t s3acc = vdupq_n_u32(0);
|
||||
uint32x4_t adacc_prev = adacc;
|
||||
|
||||
uint16x8_t s2_0, s2_1, s2_2, s2_3;
|
||||
s2_0 = s2_1 = s2_2 = s2_3 = vdupq_n_u16(0);
|
||||
|
||||
uint16x8_t s2_4, s2_5, s2_6, s2_7;
|
||||
s2_4 = s2_5 = s2_6 = s2_7 = vdupq_n_u16(0);
|
||||
|
||||
size_t num_iter = len >> 2;
|
||||
int rem = len & 3;
|
||||
|
||||
for (size_t i = 0; i < num_iter; ++i) {
|
||||
uint8x16_t d0 = vld1q_u8_ex(buf, 128);
|
||||
uint8x16_t d1 = vld1q_u8_ex(buf + 16, 128);
|
||||
uint8x16_t d2 = vld1q_u8_ex(buf + 32, 128);
|
||||
uint8x16_t d3 = vld1q_u8_ex(buf + 48, 128);
|
||||
|
||||
vst1q_u8(dst, d0);
|
||||
vst1q_u8(dst + 16, d1);
|
||||
vst1q_u8(dst + 32, d2);
|
||||
vst1q_u8(dst + 48, d3);
|
||||
dst += 64;
|
||||
|
||||
/* Unfortunately it doesn't look like there's a direct sum 8 bit to 32
|
||||
* bit instruction, we'll have to make due summing to 16 bits first */
|
||||
uint16x8x2_t hsum, hsum_fold;
|
||||
hsum.val[0] = vpaddlq_u8(d0);
|
||||
hsum.val[1] = vpaddlq_u8(d1);
|
||||
|
||||
hsum_fold.val[0] = vpadalq_u8(hsum.val[0], d2);
|
||||
hsum_fold.val[1] = vpadalq_u8(hsum.val[1], d3);
|
||||
|
||||
adacc = vpadalq_u16(adacc, hsum_fold.val[0]);
|
||||
s3acc = vaddq_u32(s3acc, adacc_prev);
|
||||
adacc = vpadalq_u16(adacc, hsum_fold.val[1]);
|
||||
|
||||
/* If we do straight widening additions to the 16 bit values, we don't incur
|
||||
* the usual penalties of a pairwise add. We can defer the multiplications
|
||||
* until the very end. These will not overflow because we are incurring at
|
||||
* most 408 loop iterations (NMAX / 64), and a given lane is only going to be
|
||||
* summed into once. This means for the maximum input size, the largest value
|
||||
* we will see is 255 * 102 = 26010, safely under uint16 max */
|
||||
s2_0 = vaddw_u8(s2_0, vget_low_u8(d0));
|
||||
s2_1 = vaddw_high_u8(s2_1, d0);
|
||||
s2_2 = vaddw_u8(s2_2, vget_low_u8(d1));
|
||||
s2_3 = vaddw_high_u8(s2_3, d1);
|
||||
s2_4 = vaddw_u8(s2_4, vget_low_u8(d2));
|
||||
s2_5 = vaddw_high_u8(s2_5, d2);
|
||||
s2_6 = vaddw_u8(s2_6, vget_low_u8(d3));
|
||||
s2_7 = vaddw_high_u8(s2_7, d3);
|
||||
|
||||
adacc_prev = adacc;
|
||||
buf += 64;
|
||||
}
|
||||
|
||||
s3acc = vshlq_n_u32(s3acc, 6);
|
||||
|
||||
if (rem) {
|
||||
uint32x4_t s3acc_0 = vdupq_n_u32(0);
|
||||
while (rem--) {
|
||||
uint8x16_t d0 = vld1q_u8_ex(buf, 128);
|
||||
vst1q_u8(dst, d0);
|
||||
dst += 16;
|
||||
uint16x8_t adler;
|
||||
adler = vpaddlq_u8(d0);
|
||||
s2_6 = vaddw_u8(s2_6, vget_low_u8(d0));
|
||||
s2_7 = vaddw_high_u8(s2_7, d0);
|
||||
adacc = vpadalq_u16(adacc, adler);
|
||||
s3acc_0 = vaddq_u32(s3acc_0, adacc_prev);
|
||||
adacc_prev = adacc;
|
||||
buf += 16;
|
||||
}
|
||||
|
||||
s3acc_0 = vshlq_n_u32(s3acc_0, 4);
|
||||
s3acc = vaddq_u32(s3acc_0, s3acc);
|
||||
}
|
||||
|
||||
uint16x8x4_t t0_t3 = vld1q_u16_x4_ex(taps, 256);
|
||||
uint16x8x4_t t4_t7 = vld1q_u16_x4_ex(taps + 32, 256);
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t0_t3.val[0], s2_0);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t0_t3.val[0]), vget_low_u16(s2_0));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t0_t3.val[1], s2_1);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t0_t3.val[1]), vget_low_u16(s2_1));
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t0_t3.val[2], s2_2);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t0_t3.val[2]), vget_low_u16(s2_2));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t0_t3.val[3], s2_3);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t0_t3.val[3]), vget_low_u16(s2_3));
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t4_t7.val[0], s2_4);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t4_t7.val[0]), vget_low_u16(s2_4));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t4_t7.val[1], s2_5);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t4_t7.val[1]), vget_low_u16(s2_5));
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t4_t7.val[2], s2_6);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t4_t7.val[2]), vget_low_u16(s2_6));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t4_t7.val[3], s2_7);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t4_t7.val[3]), vget_low_u16(s2_7));
|
||||
|
||||
s2acc = vaddq_u32(s2acc_0, s2acc);
|
||||
s2acc_2 = vaddq_u32(s2acc_1, s2acc_2);
|
||||
s2acc = vaddq_u32(s2acc, s2acc_2);
|
||||
|
||||
uint32x2_t adacc2, s2acc2, as;
|
||||
s2acc = vaddq_u32(s2acc, s3acc);
|
||||
adacc2 = vpadd_u32(vget_low_u32(adacc), vget_high_u32(adacc));
|
||||
s2acc2 = vpadd_u32(vget_low_u32(s2acc), vget_high_u32(s2acc));
|
||||
as = vpadd_u32(adacc2, s2acc2);
|
||||
s[0] = vget_lane_u32(as, 0);
|
||||
s[1] = vget_lane_u32(as, 1);
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static void NEON_accum32(uint32_t *s, const uint8_t *buf, size_t len) {
|
||||
uint32x4_t adacc = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc_0 = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc_1 = vdupq_n_u32(0);
|
||||
uint32x4_t s2acc_2 = vdupq_n_u32(0);
|
||||
|
||||
adacc = vsetq_lane_u32(s[0], adacc, 0);
|
||||
s2acc = vsetq_lane_u32(s[1], s2acc, 0);
|
||||
|
||||
uint32x4_t s3acc = vdupq_n_u32(0);
|
||||
uint32x4_t adacc_prev = adacc;
|
||||
|
||||
uint16x8_t s2_0, s2_1, s2_2, s2_3;
|
||||
s2_0 = s2_1 = s2_2 = s2_3 = vdupq_n_u16(0);
|
||||
|
||||
uint16x8_t s2_4, s2_5, s2_6, s2_7;
|
||||
s2_4 = s2_5 = s2_6 = s2_7 = vdupq_n_u16(0);
|
||||
|
||||
size_t num_iter = len >> 2;
|
||||
int rem = len & 3;
|
||||
|
||||
for (size_t i = 0; i < num_iter; ++i) {
|
||||
uint8x16x4_t d0_d3 = vld1q_u8_x4_ex(buf, 256);
|
||||
|
||||
/* Unfortunately it doesn't look like there's a direct sum 8 bit to 32
|
||||
* bit instruction, we'll have to make due summing to 16 bits first */
|
||||
uint16x8x2_t hsum, hsum_fold;
|
||||
hsum.val[0] = vpaddlq_u8(d0_d3.val[0]);
|
||||
hsum.val[1] = vpaddlq_u8(d0_d3.val[1]);
|
||||
|
||||
hsum_fold.val[0] = vpadalq_u8(hsum.val[0], d0_d3.val[2]);
|
||||
hsum_fold.val[1] = vpadalq_u8(hsum.val[1], d0_d3.val[3]);
|
||||
|
||||
adacc = vpadalq_u16(adacc, hsum_fold.val[0]);
|
||||
s3acc = vaddq_u32(s3acc, adacc_prev);
|
||||
adacc = vpadalq_u16(adacc, hsum_fold.val[1]);
|
||||
|
||||
/* If we do straight widening additions to the 16 bit values, we don't incur
|
||||
* the usual penalties of a pairwise add. We can defer the multiplications
|
||||
* until the very end. These will not overflow because we are incurring at
|
||||
* most 408 loop iterations (NMAX / 64), and a given lane is only going to be
|
||||
* summed into once. This means for the maximum input size, the largest value
|
||||
* we will see is 255 * 102 = 26010, safely under uint16 max */
|
||||
s2_0 = vaddw_u8(s2_0, vget_low_u8(d0_d3.val[0]));
|
||||
s2_1 = vaddw_high_u8(s2_1, d0_d3.val[0]);
|
||||
s2_2 = vaddw_u8(s2_2, vget_low_u8(d0_d3.val[1]));
|
||||
s2_3 = vaddw_high_u8(s2_3, d0_d3.val[1]);
|
||||
s2_4 = vaddw_u8(s2_4, vget_low_u8(d0_d3.val[2]));
|
||||
s2_5 = vaddw_high_u8(s2_5, d0_d3.val[2]);
|
||||
s2_6 = vaddw_u8(s2_6, vget_low_u8(d0_d3.val[3]));
|
||||
s2_7 = vaddw_high_u8(s2_7, d0_d3.val[3]);
|
||||
|
||||
adacc_prev = adacc;
|
||||
buf += 64;
|
||||
}
|
||||
|
||||
s3acc = vshlq_n_u32(s3acc, 6);
|
||||
|
||||
if (rem) {
|
||||
uint32x4_t s3acc_0 = vdupq_n_u32(0);
|
||||
while (rem--) {
|
||||
uint8x16_t d0 = vld1q_u8_ex(buf, 128);
|
||||
uint16x8_t adler;
|
||||
adler = vpaddlq_u8(d0);
|
||||
s2_6 = vaddw_u8(s2_6, vget_low_u8(d0));
|
||||
s2_7 = vaddw_high_u8(s2_7, d0);
|
||||
adacc = vpadalq_u16(adacc, adler);
|
||||
s3acc_0 = vaddq_u32(s3acc_0, adacc_prev);
|
||||
adacc_prev = adacc;
|
||||
buf += 16;
|
||||
}
|
||||
|
||||
s3acc_0 = vshlq_n_u32(s3acc_0, 4);
|
||||
s3acc = vaddq_u32(s3acc_0, s3acc);
|
||||
}
|
||||
|
||||
uint16x8x4_t t0_t3 = vld1q_u16_x4_ex(taps, 256);
|
||||
uint16x8x4_t t4_t7 = vld1q_u16_x4_ex(taps + 32, 256);
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t0_t3.val[0], s2_0);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t0_t3.val[0]), vget_low_u16(s2_0));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t0_t3.val[1], s2_1);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t0_t3.val[1]), vget_low_u16(s2_1));
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t0_t3.val[2], s2_2);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t0_t3.val[2]), vget_low_u16(s2_2));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t0_t3.val[3], s2_3);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t0_t3.val[3]), vget_low_u16(s2_3));
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t4_t7.val[0], s2_4);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t4_t7.val[0]), vget_low_u16(s2_4));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t4_t7.val[1], s2_5);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t4_t7.val[1]), vget_low_u16(s2_5));
|
||||
|
||||
s2acc = vmlal_high_u16(s2acc, t4_t7.val[2], s2_6);
|
||||
s2acc_0 = vmlal_u16(s2acc_0, vget_low_u16(t4_t7.val[2]), vget_low_u16(s2_6));
|
||||
s2acc_1 = vmlal_high_u16(s2acc_1, t4_t7.val[3], s2_7);
|
||||
s2acc_2 = vmlal_u16(s2acc_2, vget_low_u16(t4_t7.val[3]), vget_low_u16(s2_7));
|
||||
|
||||
s2acc = vaddq_u32(s2acc_0, s2acc);
|
||||
s2acc_2 = vaddq_u32(s2acc_1, s2acc_2);
|
||||
s2acc = vaddq_u32(s2acc, s2acc_2);
|
||||
|
||||
uint32x2_t adacc2, s2acc2, as;
|
||||
s2acc = vaddq_u32(s2acc, s3acc);
|
||||
adacc2 = vpadd_u32(vget_low_u32(adacc), vget_high_u32(adacc));
|
||||
s2acc2 = vpadd_u32(vget_low_u32(s2acc), vget_high_u32(s2acc));
|
||||
as = vpadd_u32(adacc2, s2acc2);
|
||||
s[0] = vget_lane_u32(as, 0);
|
||||
s[1] = vget_lane_u32(as, 1);
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len, const int COPY) {
|
||||
/* split Adler-32 into component sums */
|
||||
uint32_t sum2 = (adler >> 16) & 0xffff;
|
||||
adler &= 0xffff;
|
||||
|
||||
/* in case user likes doing a byte at a time, keep it fast */
|
||||
if (UNLIKELY(len == 1))
|
||||
return adler32_copy_tail(adler, dst, src, 1, sum2, 1, 1, COPY);
|
||||
|
||||
/* in case short lengths are provided, keep it somewhat fast */
|
||||
if (UNLIKELY(len < 16))
|
||||
return adler32_copy_tail(adler, dst, src, len, sum2, 1, 15, COPY);
|
||||
|
||||
uint32_t pair[2];
|
||||
|
||||
/* Split Adler-32 into component sums, it can be supplied by
|
||||
* the caller sites (e.g. in a PNG file).
|
||||
*/
|
||||
pair[0] = adler;
|
||||
pair[1] = sum2;
|
||||
|
||||
/* If memory is not SIMD aligned, do scalar sums to an aligned
|
||||
* offset, provided that doing so doesn't completely eliminate
|
||||
* SIMD operation. Aligned loads are still faster on ARM, even
|
||||
* when there's no explicit aligned load instruction. Note:
|
||||
* the code currently emits an alignment hint in the instruction
|
||||
* for exactly 256 bits when supported by the compiler. Several ARM
|
||||
* SIPs have small penalties for cacheline crossing loads as well (so
|
||||
* really 512 bits is the optimal alignment of the buffer). 32 bytes
|
||||
* should strike a balance, though. The Cortex-A8 and Cortex-A9
|
||||
* processors are documented to benefit from 128 bit and 64 bit
|
||||
* alignment, but it's unclear which other SIPs will benefit from it.
|
||||
* In the copying variant we use fallback to 4x loads and 4x stores,
|
||||
* as ld1x4 seems to block ILP when stores are in the mix */
|
||||
size_t align_diff = MIN(ALIGN_DIFF(src, 32), len);
|
||||
size_t n = NMAX_ALIGNED32;
|
||||
if (align_diff) {
|
||||
adler32_copy_align(&pair[0], dst, src, align_diff, &pair[1], 31, COPY);
|
||||
|
||||
if (COPY)
|
||||
dst += align_diff;
|
||||
src += align_diff;
|
||||
len -= align_diff;
|
||||
n = ALIGN_DOWN(n - align_diff, 32);
|
||||
}
|
||||
|
||||
while (len >= 16) {
|
||||
n = MIN(len, n);
|
||||
|
||||
if (COPY)
|
||||
NEON_accum32_copy(pair, dst, src, n >> 4);
|
||||
else
|
||||
NEON_accum32(pair, src, n >> 4);
|
||||
|
||||
pair[0] %= BASE;
|
||||
pair[1] %= BASE;
|
||||
|
||||
size_t k = (n >> 4) << 4;
|
||||
src += k;
|
||||
if (COPY)
|
||||
dst += k;
|
||||
len -= k;
|
||||
n = NMAX_ALIGNED32;
|
||||
}
|
||||
|
||||
/* Process tail (len < 16). */
|
||||
return adler32_copy_tail(pair[0], dst, src, len, pair[1], len != 0 || align_diff, 15, COPY);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_neon(uint32_t adler, const uint8_t *src, size_t len) {
|
||||
return adler32_copy_impl(adler, NULL, src, len, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_copy_neon(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
#if OPTIMAL_CMP >= 32
|
||||
return adler32_copy_impl(adler, dst, src, len, 1);
|
||||
#else
|
||||
/* Without unaligned access, interleaved stores get decomposed into byte ops */
|
||||
adler = adler32_neon(adler, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return adler;
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,334 +0,0 @@
|
|||
#ifdef ARM_FEATURES
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arm_features.h"
|
||||
|
||||
#if defined(HAVE_SYS_AUXV_H)
|
||||
# include <sys/auxv.h>
|
||||
# ifdef ARM_ASM_HWCAP
|
||||
# include <asm/hwcap.h>
|
||||
# endif
|
||||
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
|
||||
# include <machine/armreg.h>
|
||||
# ifndef ID_AA64ISAR0_CRC32_VAL
|
||||
# define ID_AA64ISAR0_CRC32_VAL ID_AA64ISAR0_CRC32
|
||||
# endif
|
||||
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
|
||||
# include <machine/armreg.h>
|
||||
# include <machine/cpu.h>
|
||||
# include <sys/sysctl.h>
|
||||
# include <sys/types.h>
|
||||
#elif defined(__APPLE__)
|
||||
# if !defined(_DARWIN_C_SOURCE)
|
||||
# define _DARWIN_C_SOURCE /* enable types aliases (eg u_int) */
|
||||
# endif
|
||||
# include <sys/sysctl.h>
|
||||
#elif defined(_WIN32)
|
||||
# include <windows.h>
|
||||
#endif
|
||||
|
||||
static int arm_has_crc32(void) {
|
||||
int has_crc32 = 0;
|
||||
#if defined(__ARM_FEATURE_CRC32)
|
||||
/* Compile-time check */
|
||||
has_crc32 = 1;
|
||||
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_CRC32
|
||||
has_crc32 = (getauxval(AT_HWCAP) & HWCAP_CRC32) != 0;
|
||||
# elif defined(HWCAP2_CRC32)
|
||||
has_crc32 = (getauxval(AT_HWCAP2) & HWCAP2_CRC32) != 0;
|
||||
# endif
|
||||
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_CRC32
|
||||
unsigned long hwcap = 0;
|
||||
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
||||
has_crc32 = (hwcap & HWCAP_CRC32) != 0;
|
||||
# elif defined(HWCAP2_CRC32)
|
||||
unsigned long hwcap2 = 0;
|
||||
elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2));
|
||||
has_crc32 = (hwcap2 & HWCAP2_CRC32) != 0;
|
||||
# endif
|
||||
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
|
||||
has_crc32 = getenv("QEMU_EMULATING") == NULL
|
||||
&& ID_AA64ISAR0_CRC32_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_CRC32_BASE;
|
||||
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
|
||||
int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
|
||||
uint64_t isar0 = 0;
|
||||
size_t len = sizeof(isar0);
|
||||
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
|
||||
has_crc32 = ID_AA64ISAR0_CRC32(isar0) >= ID_AA64ISAR0_CRC32_BASE;
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
int has_feat = 0;
|
||||
size_t size = sizeof(has_feat);
|
||||
has_crc32 = sysctlbyname("hw.optional.armv8_crc32", &has_feat, &size, NULL, 0) == 0
|
||||
&& has_feat == 1;
|
||||
#elif defined(_WIN32)
|
||||
has_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE);
|
||||
#endif
|
||||
return has_crc32;
|
||||
}
|
||||
|
||||
static int arm_has_pmull(void) {
|
||||
int has_pmull = 0;
|
||||
#if defined(__ARM_FEATURE_CRYPTO) || defined(__ARM_FEATURE_AES)
|
||||
/* Compile-time check */
|
||||
has_pmull = 1;
|
||||
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_PMULL
|
||||
has_pmull = (getauxval(AT_HWCAP) & HWCAP_PMULL) != 0;
|
||||
# elif defined(HWCAP_AES)
|
||||
/* PMULL is part of crypto extension, check for AES as proxy */
|
||||
has_pmull = (getauxval(AT_HWCAP) & HWCAP_AES) != 0;
|
||||
# endif
|
||||
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_PMULL
|
||||
unsigned long hwcap = 0;
|
||||
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
||||
has_pmull = (hwcap & HWCAP_PMULL) != 0;
|
||||
# elif defined(HWCAP_AES)
|
||||
/* PMULL is part of crypto extension, check for AES as proxy */
|
||||
unsigned long hwcap = 0;
|
||||
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
||||
has_pmull = (hwcap & HWCAP_AES) != 0;
|
||||
# endif
|
||||
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
|
||||
/* Check for AES feature as PMULL is part of crypto extension */
|
||||
has_pmull = getenv("QEMU_EMULATING") == NULL
|
||||
&& ID_AA64ISAR0_AES_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_AES_BASE;
|
||||
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
|
||||
int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
|
||||
uint64_t isar0 = 0;
|
||||
size_t len = sizeof(isar0);
|
||||
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
|
||||
has_pmull = ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_BASE;
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
int has_feat = 0;
|
||||
size_t size = sizeof(has_feat);
|
||||
has_pmull = sysctlbyname("hw.optional.arm.FEAT_PMULL", &has_feat, &size, NULL, 0) == 0
|
||||
&& has_feat == 1;
|
||||
#elif defined(_WIN32)
|
||||
/* Windows checks for crypto/AES support */
|
||||
# ifdef PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE
|
||||
has_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE);
|
||||
# endif
|
||||
#endif
|
||||
return has_pmull;
|
||||
}
|
||||
|
||||
static int arm_has_eor3(void) {
|
||||
int has_eor3 = 0;
|
||||
#if defined(__ARM_FEATURE_SHA3)
|
||||
/* Compile-time check */
|
||||
has_eor3 = 1;
|
||||
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
/* EOR3 is part of SHA3 extension, check HWCAP2_SHA3 */
|
||||
# ifdef HWCAP2_SHA3
|
||||
has_eor3 = (getauxval(AT_HWCAP2) & HWCAP2_SHA3) != 0;
|
||||
# elif defined(HWCAP_SHA3)
|
||||
has_eor3 = (getauxval(AT_HWCAP) & HWCAP_SHA3) != 0;
|
||||
# endif
|
||||
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP2_SHA3
|
||||
unsigned long hwcap2 = 0;
|
||||
elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2));
|
||||
has_eor3 = (hwcap2 & HWCAP2_SHA3) != 0;
|
||||
# elif defined(HWCAP_SHA3)
|
||||
unsigned long hwcap = 0;
|
||||
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
||||
has_eor3 = (hwcap & HWCAP_SHA3) != 0;
|
||||
# endif
|
||||
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
|
||||
/* FreeBSD: check for SHA3 in id_aa64isar0_el1 */
|
||||
# ifdef ID_AA64ISAR0_SHA3_VAL
|
||||
has_eor3 = getenv("QEMU_EMULATING") == NULL
|
||||
&& ID_AA64ISAR0_SHA3_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_SHA3_BASE;
|
||||
# endif
|
||||
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
|
||||
# ifdef ID_AA64ISAR0_SHA3
|
||||
int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
|
||||
uint64_t isar0 = 0;
|
||||
size_t len = sizeof(isar0);
|
||||
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
|
||||
has_eor3 = ID_AA64ISAR0_SHA3(isar0) >= ID_AA64ISAR0_SHA3_IMPL;
|
||||
}
|
||||
# endif
|
||||
#elif defined(__APPLE__)
|
||||
/* All Apple Silicon (M1+) has SHA3/EOR3 support */
|
||||
int has_feat = 0;
|
||||
size_t size = sizeof(has_feat);
|
||||
has_eor3 = sysctlbyname("hw.optional.arm.FEAT_SHA3", &has_feat, &size, NULL, 0) == 0
|
||||
&& has_feat == 1;
|
||||
/* Fallback to legacy name for older macOS versions */
|
||||
if (!has_eor3) {
|
||||
size = sizeof(has_feat);
|
||||
has_eor3 = sysctlbyname("hw.optional.armv8_2_sha3", &has_feat, &size, NULL, 0) == 0
|
||||
&& has_feat == 1;
|
||||
}
|
||||
#elif defined(_WIN32)
|
||||
# ifdef PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE
|
||||
has_eor3 = IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE);
|
||||
# endif
|
||||
#endif
|
||||
return has_eor3;
|
||||
}
|
||||
|
||||
/* AArch64 has neon. */
|
||||
#ifdef ARCH_32BIT
|
||||
static inline int arm_has_neon(void) {
|
||||
int has_neon = 0;
|
||||
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
|
||||
/* Compile-time check */
|
||||
has_neon = 1;
|
||||
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_ARM_NEON
|
||||
has_neon = (getauxval(AT_HWCAP) & HWCAP_ARM_NEON) != 0;
|
||||
# elif defined(HWCAP_NEON)
|
||||
has_neon = (getauxval(AT_HWCAP) & HWCAP_NEON) != 0;
|
||||
# endif
|
||||
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_NEON
|
||||
unsigned long hwcap = 0;
|
||||
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
||||
has_neon = (hwcap & HWCAP_NEON) != 0;
|
||||
# endif
|
||||
#elif defined(__APPLE__)
|
||||
int has_feat = 0;
|
||||
size_t size = sizeof(has_feat);
|
||||
has_neon = sysctlbyname("hw.optional.neon", &has_feat, &size, NULL, 0) == 0
|
||||
&& has_feat == 1;
|
||||
#elif defined(_M_ARM) && defined(WINAPI_FAMILY_PARTITION)
|
||||
# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP)
|
||||
has_neon = 1; /* Always supported */
|
||||
# endif
|
||||
#endif
|
||||
return has_neon;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* AArch64 does not have ARMv6 SIMD. */
|
||||
#ifdef ARCH_32BIT
|
||||
static inline int arm_has_simd(void) {
|
||||
int has_simd = 0;
|
||||
#if defined(__ARM_FEATURE_SIMD32)
|
||||
/* Compile-time check for ARMv6 SIMD */
|
||||
has_simd = 1;
|
||||
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
const char *platform = (const char *)getauxval(AT_PLATFORM);
|
||||
has_simd = platform
|
||||
&& (strncmp(platform, "v6l", 3) == 0
|
||||
|| strncmp(platform, "v7l", 3) == 0
|
||||
|| strncmp(platform, "v8l", 3) == 0);
|
||||
#endif
|
||||
return has_simd;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(ARCH_64BIT) && !defined(__APPLE__) && !defined(_WIN32)
|
||||
/* MIDR_EL1 bit field definitions */
|
||||
#define MIDR_IMPLEMENTOR(midr) (((midr) & (0xffU << 24)) >> 24)
|
||||
#define MIDR_PARTNUM(midr) (((midr) & (0xfffU << 4)) >> 4)
|
||||
|
||||
/* ARM CPU Implementer IDs */
|
||||
#define ARM_IMPLEMENTER_ARM 0x41
|
||||
#define ARM_IMPLEMENTER_QUALCOMM 0x51
|
||||
#define ARM_IMPLEMENTER_APPLE 0x61
|
||||
|
||||
/* ARM CPU Part Numbers */
|
||||
|
||||
/* Cortex-X series - Multiple PMULL lanes */
|
||||
#define ARM_PART_CORTEX_X1 0xd44
|
||||
#define ARM_PART_CORTEX_X1C 0xd4c
|
||||
#define ARM_PART_CORTEX_X2 0xd48
|
||||
#define ARM_PART_CORTEX_X3 0xd4e
|
||||
#define ARM_PART_CORTEX_X4 0xd82
|
||||
#define ARM_PART_CORTEX_X925 0xd85
|
||||
|
||||
/* Neoverse V/N2 series - Multiple PMULL lanes */
|
||||
#define ARM_PART_NEOVERSE_N2 0xd49
|
||||
#define ARM_PART_NEOVERSE_V1 0xd40
|
||||
#define ARM_PART_NEOVERSE_V2 0xd4f
|
||||
#define ARM_PART_NEOVERSE_V3 0xd8e
|
||||
|
||||
/* Snapdragon X Elite/Plus - Custom core */
|
||||
#define QUALCOMM_PART_ORYON 0x001
|
||||
|
||||
static inline int arm_has_cpuid(void) {
|
||||
int has_cpuid = 0;
|
||||
#if defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_CPUID
|
||||
has_cpuid = (getauxval(AT_HWCAP) & HWCAP_CPUID) != 0;
|
||||
# elif defined(HWCAP2_CPUID)
|
||||
has_cpuid = (getauxval(AT_HWCAP2) & HWCAP2_CPUID) != 0;
|
||||
# endif
|
||||
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
|
||||
# ifdef HWCAP_CPUID
|
||||
unsigned long hwcap = 0;
|
||||
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
||||
has_cpuid = (hwcap & HWCAP_CPUID) != 0;
|
||||
# endif
|
||||
#endif
|
||||
return has_cpuid;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Determine if CPU has fast PMULL (multiple execution units) */
|
||||
static inline int arm_cpu_has_fast_pmull(void) {
|
||||
int has_fast_pmull = 0;
|
||||
#if defined(__APPLE__)
|
||||
/* On macOS, all Apple Silicon has fast PMULL */
|
||||
has_fast_pmull = 1;
|
||||
#elif defined(ARCH_64BIT) && !defined(_WIN32)
|
||||
/* We need CPUID feature to read MIDR register */
|
||||
if (!arm_has_cpuid())
|
||||
return has_fast_pmull;
|
||||
|
||||
uint64_t midr;
|
||||
__asm__ ("mrs %0, midr_el1" : "=r" (midr));
|
||||
|
||||
uint32_t implementer = MIDR_IMPLEMENTOR(midr);
|
||||
uint32_t part = MIDR_PARTNUM(midr);
|
||||
|
||||
if (implementer == ARM_IMPLEMENTER_APPLE) {
|
||||
/* All Apple Silicon (M1+) have fast PMULL */
|
||||
has_fast_pmull = 1;
|
||||
} else if (implementer == ARM_IMPLEMENTER_ARM) {
|
||||
/* ARM Cortex-X and Neoverse V/N2 series have multi-lane PMULL */
|
||||
switch (part) {
|
||||
case ARM_PART_CORTEX_X1:
|
||||
case ARM_PART_CORTEX_X1C:
|
||||
case ARM_PART_CORTEX_X2:
|
||||
case ARM_PART_CORTEX_X3:
|
||||
case ARM_PART_CORTEX_X4:
|
||||
case ARM_PART_CORTEX_X925:
|
||||
case ARM_PART_NEOVERSE_N2:
|
||||
case ARM_PART_NEOVERSE_V1:
|
||||
case ARM_PART_NEOVERSE_V2:
|
||||
case ARM_PART_NEOVERSE_V3:
|
||||
has_fast_pmull = 1;
|
||||
}
|
||||
} else if (implementer == ARM_IMPLEMENTER_QUALCOMM) {
|
||||
/* Qualcomm Oryon (Snapdragon X Elite/Plus) has fast PMULL */
|
||||
if (part == QUALCOMM_PART_ORYON)
|
||||
has_fast_pmull = 1;
|
||||
}
|
||||
#endif
|
||||
return has_fast_pmull;
|
||||
}
|
||||
|
||||
void Z_INTERNAL arm_check_features(struct arm_cpu_features *features) {
|
||||
#ifdef ARCH_64BIT
|
||||
features->has_simd = 0; /* never available */
|
||||
features->has_neon = 1; /* always available */
|
||||
#else
|
||||
features->has_simd = arm_has_simd();
|
||||
features->has_neon = arm_has_neon();
|
||||
#endif
|
||||
features->has_crc32 = arm_has_crc32();
|
||||
features->has_pmull = arm_has_pmull();
|
||||
features->has_eor3 = arm_has_eor3();
|
||||
features->has_fast_pmull = features->has_pmull && arm_cpu_has_fast_pmull();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
/* arm_features.h -- check for ARM features.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef ARM_FEATURES_H_
|
||||
#define ARM_FEATURES_H_
|
||||
|
||||
struct arm_cpu_features {
|
||||
int has_simd;
|
||||
int has_neon;
|
||||
int has_crc32;
|
||||
int has_pmull;
|
||||
int has_eor3;
|
||||
int has_fast_pmull;
|
||||
};
|
||||
|
||||
void Z_INTERNAL arm_check_features(struct arm_cpu_features *features);
|
||||
|
||||
#endif /* ARM_FEATURES_H_ */
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
/* arm_functions.h -- ARM implementations for arch-specific functions.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef ARM_FUNCTIONS_H_
|
||||
#define ARM_FUNCTIONS_H_
|
||||
|
||||
#include "arm_natives.h"
|
||||
|
||||
#ifdef ARM_NEON
|
||||
uint32_t adler32_neon(uint32_t adler, const uint8_t *buf, size_t len);
|
||||
uint32_t adler32_copy_neon(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
uint8_t* chunkmemset_safe_neon(uint8_t *out, uint8_t *from, size_t len, size_t left);
|
||||
uint32_t compare256_neon(const uint8_t *src0, const uint8_t *src1);
|
||||
void inflate_fast_neon(PREFIX3(stream) *strm, uint32_t start);
|
||||
uint32_t longest_match_neon(deflate_state *const s, uint32_t cur_match);
|
||||
uint32_t longest_match_roll_neon(deflate_state *const s, uint32_t cur_match);
|
||||
void slide_hash_neon(deflate_state *s);
|
||||
#endif
|
||||
|
||||
#ifndef ARM_NEON_NATIVE
|
||||
# define ADLER32_FALLBACK
|
||||
# define CHUNKSET_FALLBACK
|
||||
# define COMPARE256_FALLBACK
|
||||
# ifndef ARM_SIMD_NATIVE
|
||||
# define SLIDE_HASH_FALLBACK
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifdef ARM_CRC32
|
||||
uint32_t crc32_armv8(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_armv8(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
#endif
|
||||
|
||||
#ifdef ARM_PMULL_EOR3
|
||||
uint32_t crc32_armv8_pmull_eor3(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_armv8_pmull_eor3(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
#endif
|
||||
|
||||
#if !defined(ARM_CRC32_NATIVE) && !defined(ARM_PMULL_EOR3_NATIVE)
|
||||
# define CRC32_BRAID_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef ARM_SIMD
|
||||
void slide_hash_armv6(deflate_state *s);
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_RUNTIME_CPU_DETECTION
|
||||
// ARM - SIMD
|
||||
# ifdef ARM_SIMD_NATIVE
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_armv6
|
||||
# endif
|
||||
// ARM - NEON
|
||||
# ifdef ARM_NEON_NATIVE
|
||||
# undef native_adler32
|
||||
# define native_adler32 adler32_neon
|
||||
# undef native_adler32_copy
|
||||
# define native_adler32_copy adler32_copy_neon
|
||||
# undef native_chunkmemset_safe
|
||||
# define native_chunkmemset_safe chunkmemset_safe_neon
|
||||
# undef native_compare256
|
||||
# define native_compare256 compare256_neon
|
||||
# undef native_inflate_fast
|
||||
# define native_inflate_fast inflate_fast_neon
|
||||
# undef native_longest_match
|
||||
# define native_longest_match longest_match_neon
|
||||
# undef native_longest_match_roll
|
||||
# define native_longest_match_roll longest_match_roll_neon
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_neon
|
||||
# endif
|
||||
// ARM - CRC32
|
||||
# ifdef ARM_CRC32_NATIVE
|
||||
# undef native_crc32
|
||||
# define native_crc32 crc32_armv8
|
||||
# undef native_crc32_copy
|
||||
# define native_crc32_copy crc32_copy_armv8
|
||||
# endif
|
||||
// ARM - PMULL EOR3
|
||||
# ifdef ARM_PMULL_EOR3_NATIVE
|
||||
# undef native_crc32
|
||||
# define native_crc32 crc32_armv8_pmull_eor3
|
||||
# undef native_crc32_copy
|
||||
# define native_crc32_copy crc32_copy_armv8_pmull_eor3
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* ARM_FUNCTIONS_H_ */
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
/* arm_natives.h -- ARM compile-time feature detection macros.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef ARM_NATIVES_H_
|
||||
#define ARM_NATIVES_H_
|
||||
|
||||
#if defined(__ARM_FEATURE_SIMD32)
|
||||
# ifdef ARM_SIMD
|
||||
# define ARM_SIMD_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
/* NEON is guaranteed on ARM64 (like SSE2 on x86-64) */
|
||||
#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(ARCH_64BIT)
|
||||
# ifdef ARM_NEON
|
||||
# define ARM_NEON_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
/* CRC32 is optional in ARMv8.0, mandatory in ARMv8.1+ */
|
||||
#if defined(__ARM_FEATURE_CRC32) || (defined(__ARM_ARCH) && __ARM_ARCH >= 801)
|
||||
# ifdef ARM_CRC32
|
||||
# define ARM_CRC32_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
#if defined(__ARM_FEATURE_CRC32) && defined(__ARM_FEATURE_CRYPTO) && defined(__ARM_FEATURE_SHA3)
|
||||
# ifdef ARM_PMULL_EOR3
|
||||
# define ARM_PMULL_EOR3_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* ARM_NATIVES_H_ */
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
/* chunkset_neon.c -- NEON inline functions to copy small data chunks.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef ARM_NEON
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zsanitizer.h"
|
||||
#include "zmemory.h"
|
||||
#include "neon_intrins.h"
|
||||
#include "arch/shared/chunk_128bit_perm_idx_lut.h"
|
||||
|
||||
typedef uint8x16_t chunk_t;
|
||||
|
||||
#define HAVE_CHUNKMEMSET_2
|
||||
#define HAVE_CHUNKMEMSET_4
|
||||
#define HAVE_CHUNKMEMSET_8
|
||||
#define HAVE_CHUNK_MAG
|
||||
|
||||
|
||||
static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = vreinterpretq_u8_u16(vdupq_n_u16(zng_memread_2(from)));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = vreinterpretq_u8_u32(vdupq_n_u32(zng_memread_4(from)));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = vreinterpretq_u8_u64(vdupq_n_u64(zng_memread_8(from)));
|
||||
}
|
||||
|
||||
#define CHUNKSIZE chunksize_neon
|
||||
#define CHUNKCOPY chunkcopy_neon
|
||||
#define CHUNKUNROLL chunkunroll_neon
|
||||
#define CHUNKMEMSET chunkmemset_neon
|
||||
#define CHUNKMEMSET_SAFE chunkmemset_safe_neon
|
||||
|
||||
static inline void loadchunk(uint8_t const *s, chunk_t *chunk) {
|
||||
*chunk = vld1q_u8(s);
|
||||
}
|
||||
|
||||
static inline void storechunk(uint8_t *out, chunk_t *chunk) {
|
||||
vst1q_u8(out, *chunk);
|
||||
}
|
||||
|
||||
static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, size_t *chunk_rem, size_t dist) {
|
||||
lut_rem_pair lut_rem = perm_idx_lut[dist - 3];
|
||||
*chunk_rem = lut_rem.remval;
|
||||
|
||||
/* See note in chunkset_ssse3.c for why this is ok */
|
||||
__msan_unpoison(buf + dist, 16 - dist);
|
||||
|
||||
/* This version of table is only available on aarch64 */
|
||||
#if defined(ARCH_ARM) && defined(ARCH_64BIT)
|
||||
uint8x16_t ret_vec = vld1q_u8(buf);
|
||||
|
||||
uint8x16_t perm_vec = vld1q_u8_ex(permute_table + lut_rem.idx, 128);
|
||||
return vqtbl1q_u8(ret_vec, perm_vec);
|
||||
#else
|
||||
uint8x8_t ret0, ret1, a, b, perm_vec0, perm_vec1;
|
||||
perm_vec0 = vld1_u8_ex(permute_table + lut_rem.idx, 64);
|
||||
perm_vec1 = vld1_u8_ex(permute_table + lut_rem.idx + 8, 64);
|
||||
a = vld1_u8(buf);
|
||||
b = vld1_u8(buf + 8);
|
||||
ret0 = vtbl1_u8(a, perm_vec0);
|
||||
uint8x8x2_t ab;
|
||||
ab.val[0] = a;
|
||||
ab.val[1] = b;
|
||||
ret1 = vtbl2_u8(ab, perm_vec1);
|
||||
return vcombine_u8(ret0, ret1);
|
||||
#endif
|
||||
}
|
||||
|
||||
#include "chunkset_tpl.h"
|
||||
|
||||
#define INFLATE_FAST inflate_fast_neon
|
||||
|
||||
#include "inffast_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
/* compare256_neon.c - NEON version of compare256
|
||||
* Copyright (C) 2022 Nathan Moinvaziri
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zmemory.h"
|
||||
#include "deflate.h"
|
||||
#include "fallback_builtins.h"
|
||||
|
||||
#if defined(ARM_NEON)
|
||||
#include "neon_intrins.h"
|
||||
|
||||
static inline uint32_t compare256_neon_static(const uint8_t *src0, const uint8_t *src1) {
|
||||
uint32_t len = 0;
|
||||
|
||||
do {
|
||||
uint8x16_t a, b, cmp;
|
||||
uint64_t lane;
|
||||
|
||||
a = vld1q_u8(src0);
|
||||
b = vld1q_u8(src1);
|
||||
|
||||
cmp = veorq_u8(a, b);
|
||||
|
||||
lane = vgetq_lane_u64(vreinterpretq_u64_u8(cmp), 0);
|
||||
if (lane)
|
||||
return len + zng_first_diff_byte64(lane);
|
||||
len += 8;
|
||||
lane = vgetq_lane_u64(vreinterpretq_u64_u8(cmp), 1);
|
||||
if (lane)
|
||||
return len + zng_first_diff_byte64(lane);
|
||||
len += 8;
|
||||
|
||||
src0 += 16, src1 += 16;
|
||||
} while (len < 256);
|
||||
|
||||
return 256;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t compare256_neon(const uint8_t *src0, const uint8_t *src1) {
|
||||
return compare256_neon_static(src0, src1);
|
||||
}
|
||||
|
||||
#define LONGEST_MATCH longest_match_neon
|
||||
#define COMPARE256 compare256_neon_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#define LONGEST_MATCH_ROLL
|
||||
#define LONGEST_MATCH longest_match_roll_neon
|
||||
#define COMPARE256 compare256_neon_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
/* crc32_armv8.c -- compute the CRC-32 of a data stream
|
||||
* Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler
|
||||
* Copyright (C) 2016 Yang Zhang
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef ARM_CRC32
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "acle_intrins.h"
|
||||
#include "crc32_armv8_p.h"
|
||||
|
||||
#include "arch/shared/crc32_hw_copy_impl_tpl.h"
|
||||
|
||||
|
||||
Z_INTERNAL Z_TARGET_CRC uint32_t crc32_armv8(uint32_t crc, const uint8_t *buf, size_t len) {
|
||||
return crc32_hw_copy_impl(crc, NULL, buf, len, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL Z_TARGET_CRC uint32_t crc32_copy_armv8(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
#if OPTIMAL_CMP >= 32
|
||||
return crc32_hw_copy_impl(crc, dst, src, len, 1);
|
||||
#else
|
||||
/* Without unaligned access, interleaved stores get decomposed into byte ops */
|
||||
crc = crc32_armv8(crc, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return crc;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
/* crc32_armv8_p.h -- Private shared inline ARMv8 CRC32 functions
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef CRC32_ARMV8_P_H
|
||||
#define CRC32_ARMV8_P_H
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "acle_intrins.h"
|
||||
|
||||
#define CRC32B(crc, val) __crc32b((crc), (val))
|
||||
#define CRC32H(crc, val) __crc32h((crc), (val))
|
||||
#define CRC32W(crc, val) __crc32w((crc), (val))
|
||||
#define CRC32D(crc, val) __crc32d((crc), (val))
|
||||
|
||||
#include "arch/shared/crc32_hw_common_tpl.h"
|
||||
|
||||
#endif /* CRC32_ARMV8_P_H */
|
||||
|
|
@ -1,366 +0,0 @@
|
|||
/* crc32_armv8_pmull_eor3.c -- ARMv8 CRC32 using PMULL + EOR3 (SHA3 extension)
|
||||
* Copyright (C) 2025 Peter Cawley
|
||||
* https://github.com/corsix/fast-crc32
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*
|
||||
* This uses EOR3 (3-way XOR) from ARMv8.2-A SHA3 extension to save instructions.
|
||||
* Uses 3-way parallel scalar CRC + 9 PMULL vector lanes, processing 192 bytes/iter.
|
||||
*/
|
||||
|
||||
#ifdef ARM_PMULL_EOR3
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zutil.h"
|
||||
#include "acle_intrins.h"
|
||||
#include "neon_intrins.h"
|
||||
#include "crc32_armv8_p.h"
|
||||
|
||||
/* Carryless multiply low 64 bits: a[0] * b[0] */
|
||||
static inline uint64x2_t clmul_lo(uint64x2_t a, uint64x2_t b) {
|
||||
#ifdef _MSC_VER
|
||||
return vreinterpretq_u64_p128(vmull_p64(
|
||||
vget_low_p64(vreinterpret_p64_u64(a)),
|
||||
vget_low_p64(vreinterpret_p64_u64(b))));
|
||||
#else
|
||||
return vreinterpretq_u64_p128(vmull_p64(
|
||||
vget_lane_p64(vreinterpret_p64_u64(vget_low_u64(a)), 0),
|
||||
vget_lane_p64(vreinterpret_p64_u64(vget_low_u64(b)), 0)));
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Carryless multiply high 64 bits: a[1] * b[1] */
|
||||
static inline uint64x2_t clmul_hi(uint64x2_t a, uint64x2_t b) {
|
||||
return vreinterpretq_u64_p128(vmull_high_p64(vreinterpretq_p64_u64(a), vreinterpretq_p64_u64(b)));
|
||||
}
|
||||
|
||||
/* Carryless multiply of two 32-bit scalars: a * b (returns 64-bit result in 128-bit vector) */
|
||||
static inline uint64x2_t clmul_scalar(uint32_t a, uint32_t b) {
|
||||
#ifdef _MSC_VER
|
||||
return vreinterpretq_u64_p128(vmull_p64(vdup_n_p64((poly64_t)a), vdup_n_p64((poly64_t)b)));
|
||||
#else
|
||||
return vreinterpretq_u64_p128(vmull_p64((poly64_t)a, (poly64_t)b));
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Compute x^n mod P (CRC-32 polynomial) in log(n) time, where P = 0x104c11db7 */
|
||||
static uint32_t xnmodp(uint64_t n) {
|
||||
uint64_t stack = ~(uint64_t)1;
|
||||
uint32_t acc, low;
|
||||
for (; n > 191; n = (n >> 1) - 16) {
|
||||
stack = (stack << 1) + (n & 1);
|
||||
}
|
||||
stack = ~stack;
|
||||
acc = ((uint32_t)0x80000000) >> (n & 31);
|
||||
for (n >>= 5; n; --n) {
|
||||
acc = __crc32w(acc, 0);
|
||||
}
|
||||
while ((low = stack & 1), stack >>= 1) {
|
||||
poly8x8_t x = vreinterpret_p8_u64(vmov_n_u64(acc));
|
||||
uint64_t y = vgetq_lane_u64(vreinterpretq_u64_p16(vmull_p8(x, x)), 0);
|
||||
acc = __crc32d(0, y << low);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
/* Shift CRC forward by nbytes: equivalent to appending nbytes of zeros to the data stream */
|
||||
static inline uint64x2_t crc_shift(uint32_t crc, size_t nbytes) {
|
||||
Assert(nbytes >= 5, "crc_shift requires nbytes >= 5");
|
||||
return clmul_scalar(crc, xnmodp(nbytes * 8 - 33));
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static Z_TARGET_PMULL_EOR3 uint32_t crc32_copy_impl(uint32_t crc, uint8_t *dst, const uint8_t *src,
|
||||
size_t len, const int COPY) {
|
||||
uint32_t crc0 = ~crc;
|
||||
|
||||
if (UNLIKELY(len == 1)) {
|
||||
if (COPY)
|
||||
*dst = *src;
|
||||
crc0 = __crc32b(crc0, *src);
|
||||
return ~crc0;
|
||||
}
|
||||
|
||||
/* Align to 16-byte boundary for vector path */
|
||||
uintptr_t align_diff = ALIGN_DIFF(src, 16);
|
||||
if (align_diff)
|
||||
crc0 = crc32_hw_align(crc0, &dst, &src, &len, align_diff, COPY);
|
||||
|
||||
/* 3-way scalar CRC + 9-way PMULL folding (192 bytes/iter) */
|
||||
if (len >= 192) {
|
||||
size_t blk = len / 192; /* Number of 192-byte blocks */
|
||||
size_t klen = blk * 16; /* Scalar stride per CRC lane */
|
||||
const uint8_t *end = src + len;
|
||||
const uint8_t *src0 = src;
|
||||
const uint8_t *src1 = src + klen;
|
||||
const uint8_t *src2 = src + klen * 2;
|
||||
const uint8_t *srcv = src + klen * 3; /* Vector data starts after scalar lanes */
|
||||
uint32_t crc1 = 0, crc2 = 0;
|
||||
uint64x2_t vc0, vc1, vc2;
|
||||
uint64_t vc;
|
||||
|
||||
/* Load first 9 vector chunks (144 bytes) */
|
||||
uint64x2_t x0 = vld1q_u64_ex((const uint64_t*)srcv, 128), y0;
|
||||
uint64x2_t x1 = vld1q_u64_ex((const uint64_t*)(srcv + 16), 128), y1;
|
||||
uint64x2_t x2 = vld1q_u64_ex((const uint64_t*)(srcv + 32), 128), y2;
|
||||
uint64x2_t x3 = vld1q_u64_ex((const uint64_t*)(srcv + 48), 128), y3;
|
||||
uint64x2_t x4 = vld1q_u64_ex((const uint64_t*)(srcv + 64), 128), y4;
|
||||
uint64x2_t x5 = vld1q_u64_ex((const uint64_t*)(srcv + 80), 128), y5;
|
||||
uint64x2_t x6 = vld1q_u64_ex((const uint64_t*)(srcv + 96), 128), y6;
|
||||
uint64x2_t x7 = vld1q_u64_ex((const uint64_t*)(srcv + 112), 128), y7;
|
||||
uint64x2_t x8 = vld1q_u64_ex((const uint64_t*)(srcv + 128), 128), y8;
|
||||
uint64x2_t k;
|
||||
/* k = {x^144 mod P, x^144+64 mod P} for 144-byte fold */
|
||||
{ static const uint64_t ALIGNED_(16) k_[] = {0x26b70c3d, 0x3f41287a}; k = vld1q_u64_ex(k_, 128); }
|
||||
|
||||
/* Per-region dst pointers */
|
||||
uint8_t *dst0 = dst;
|
||||
uint8_t *dst1 = NULL;
|
||||
uint8_t *dst2 = NULL;
|
||||
uint8_t *dst_v = NULL;
|
||||
|
||||
if (COPY) {
|
||||
dst1 = dst + klen;
|
||||
dst2 = dst + klen * 2;
|
||||
dst_v = dst + klen * 3;
|
||||
vst1q_u8(dst_v, vreinterpretq_u8_u64(x0));
|
||||
vst1q_u8(dst_v + 16, vreinterpretq_u8_u64(x1));
|
||||
vst1q_u8(dst_v + 32, vreinterpretq_u8_u64(x2));
|
||||
vst1q_u8(dst_v + 48, vreinterpretq_u8_u64(x3));
|
||||
vst1q_u8(dst_v + 64, vreinterpretq_u8_u64(x4));
|
||||
vst1q_u8(dst_v + 80, vreinterpretq_u8_u64(x5));
|
||||
vst1q_u8(dst_v + 96, vreinterpretq_u8_u64(x6));
|
||||
vst1q_u8(dst_v + 112, vreinterpretq_u8_u64(x7));
|
||||
vst1q_u8(dst_v + 128, vreinterpretq_u8_u64(x8));
|
||||
dst_v += 144;
|
||||
}
|
||||
srcv += 144;
|
||||
|
||||
/* Fold 9 vectors + 3-way parallel scalar CRC */
|
||||
if (blk > 1) {
|
||||
/* Only form a limit pointer when we have at least 2 blocks. */
|
||||
const uint8_t *limit = src0 + klen - 32;
|
||||
while (src0 <= limit) {
|
||||
/* Fold all 9 vector lanes using PMULL */
|
||||
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
|
||||
y1 = clmul_lo(x1, k), x1 = clmul_hi(x1, k);
|
||||
y2 = clmul_lo(x2, k), x2 = clmul_hi(x2, k);
|
||||
y3 = clmul_lo(x3, k), x3 = clmul_hi(x3, k);
|
||||
y4 = clmul_lo(x4, k), x4 = clmul_hi(x4, k);
|
||||
y5 = clmul_lo(x5, k), x5 = clmul_hi(x5, k);
|
||||
y6 = clmul_lo(x6, k), x6 = clmul_hi(x6, k);
|
||||
y7 = clmul_lo(x7, k), x7 = clmul_hi(x7, k);
|
||||
y8 = clmul_lo(x8, k), x8 = clmul_hi(x8, k);
|
||||
|
||||
/* EOR3: combine hi*k, lo*k, and new data in one instruction */
|
||||
{
|
||||
uint64x2_t d0 = vld1q_u64_ex((const uint64_t*)srcv, 128);
|
||||
uint64x2_t d1 = vld1q_u64_ex((const uint64_t*)(srcv + 16), 128);
|
||||
uint64x2_t d2 = vld1q_u64_ex((const uint64_t*)(srcv + 32), 128);
|
||||
uint64x2_t d3 = vld1q_u64_ex((const uint64_t*)(srcv + 48), 128);
|
||||
uint64x2_t d4 = vld1q_u64_ex((const uint64_t*)(srcv + 64), 128);
|
||||
uint64x2_t d5 = vld1q_u64_ex((const uint64_t*)(srcv + 80), 128);
|
||||
uint64x2_t d6 = vld1q_u64_ex((const uint64_t*)(srcv + 96), 128);
|
||||
uint64x2_t d7 = vld1q_u64_ex((const uint64_t*)(srcv + 112), 128);
|
||||
uint64x2_t d8 = vld1q_u64_ex((const uint64_t*)(srcv + 128), 128);
|
||||
if (COPY) {
|
||||
vst1q_u8(dst_v, vreinterpretq_u8_u64(d0));
|
||||
vst1q_u8(dst_v + 16, vreinterpretq_u8_u64(d1));
|
||||
vst1q_u8(dst_v + 32, vreinterpretq_u8_u64(d2));
|
||||
vst1q_u8(dst_v + 48, vreinterpretq_u8_u64(d3));
|
||||
vst1q_u8(dst_v + 64, vreinterpretq_u8_u64(d4));
|
||||
vst1q_u8(dst_v + 80, vreinterpretq_u8_u64(d5));
|
||||
vst1q_u8(dst_v + 96, vreinterpretq_u8_u64(d6));
|
||||
vst1q_u8(dst_v + 112, vreinterpretq_u8_u64(d7));
|
||||
vst1q_u8(dst_v + 128, vreinterpretq_u8_u64(d8));
|
||||
dst_v += 144;
|
||||
}
|
||||
x0 = veor3q_u64(x0, y0, d0);
|
||||
x1 = veor3q_u64(x1, y1, d1);
|
||||
x2 = veor3q_u64(x2, y2, d2);
|
||||
x3 = veor3q_u64(x3, y3, d3);
|
||||
x4 = veor3q_u64(x4, y4, d4);
|
||||
x5 = veor3q_u64(x5, y5, d5);
|
||||
x6 = veor3q_u64(x6, y6, d6);
|
||||
x7 = veor3q_u64(x7, y7, d7);
|
||||
x8 = veor3q_u64(x8, y8, d8);
|
||||
}
|
||||
|
||||
/* 3-way parallel scalar CRC (16 bytes each) */
|
||||
{
|
||||
uint64_t s0a = *(const uint64_t*)src0;
|
||||
uint64_t s0b = *(const uint64_t*)(src0 + 8);
|
||||
uint64_t s1a = *(const uint64_t*)src1;
|
||||
uint64_t s1b = *(const uint64_t*)(src1 + 8);
|
||||
uint64_t s2a = *(const uint64_t*)src2;
|
||||
uint64_t s2b = *(const uint64_t*)(src2 + 8);
|
||||
if (COPY) {
|
||||
memcpy(dst0, &s0a, 8);
|
||||
memcpy(dst0 + 8, &s0b, 8);
|
||||
dst0 += 16;
|
||||
memcpy(dst1, &s1a, 8);
|
||||
memcpy(dst1 + 8, &s1b, 8);
|
||||
dst1 += 16;
|
||||
memcpy(dst2, &s2a, 8);
|
||||
memcpy(dst2 + 8, &s2b, 8);
|
||||
dst2 += 16;
|
||||
}
|
||||
crc0 = __crc32d(crc0, s0a);
|
||||
crc0 = __crc32d(crc0, s0b);
|
||||
crc1 = __crc32d(crc1, s1a);
|
||||
crc1 = __crc32d(crc1, s1b);
|
||||
crc2 = __crc32d(crc2, s2a);
|
||||
crc2 = __crc32d(crc2, s2b);
|
||||
}
|
||||
src0 += 16;
|
||||
src1 += 16;
|
||||
src2 += 16;
|
||||
srcv += 144;
|
||||
}
|
||||
}
|
||||
|
||||
/* Reduce 9 vectors to 1 using tree reduction */
|
||||
/* Step 1: x0 = fold(x0, x1), shift x2..x8 down */
|
||||
{ static const uint64_t ALIGNED_(16) k_[] = {0xae689191, 0xccaa009e}; k = vld1q_u64_ex(k_, 128); }
|
||||
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
|
||||
x0 = veor3q_u64(x0, y0, x1);
|
||||
x1 = x2, x2 = x3, x3 = x4, x4 = x5, x5 = x6, x6 = x7, x7 = x8;
|
||||
|
||||
/* Step 2: fold pairs (x0,x1), (x2,x3), (x4,x5), (x6,x7) */
|
||||
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
|
||||
y2 = clmul_lo(x2, k), x2 = clmul_hi(x2, k);
|
||||
y4 = clmul_lo(x4, k), x4 = clmul_hi(x4, k);
|
||||
y6 = clmul_lo(x6, k), x6 = clmul_hi(x6, k);
|
||||
x0 = veor3q_u64(x0, y0, x1);
|
||||
x2 = veor3q_u64(x2, y2, x3);
|
||||
x4 = veor3q_u64(x4, y4, x5);
|
||||
x6 = veor3q_u64(x6, y6, x7);
|
||||
|
||||
/* Step 3: fold pairs (x0,x2), (x4,x6) */
|
||||
{ static const uint64_t ALIGNED_(16) k_[] = {0xf1da05aa, 0x81256527}; k = vld1q_u64_ex(k_, 128); }
|
||||
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
|
||||
y4 = clmul_lo(x4, k), x4 = clmul_hi(x4, k);
|
||||
x0 = veor3q_u64(x0, y0, x2);
|
||||
x4 = veor3q_u64(x4, y4, x6);
|
||||
|
||||
/* Step 4: final fold (x0, x4) -> x0 */
|
||||
{ static const uint64_t ALIGNED_(16) k_[] = {0x8f352d95, 0x1d9513d7}; k = vld1q_u64_ex(k_, 128); }
|
||||
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
|
||||
x0 = veor3q_u64(x0, y0, x4);
|
||||
|
||||
/* Process final scalar chunk */
|
||||
{
|
||||
uint64_t s0a = *(const uint64_t*)src0;
|
||||
uint64_t s0b = *(const uint64_t*)(src0 + 8);
|
||||
uint64_t s1a = *(const uint64_t*)src1;
|
||||
uint64_t s1b = *(const uint64_t*)(src1 + 8);
|
||||
uint64_t s2a = *(const uint64_t*)src2;
|
||||
uint64_t s2b = *(const uint64_t*)(src2 + 8);
|
||||
if (COPY) {
|
||||
memcpy(dst0, &s0a, 8);
|
||||
memcpy(dst0 + 8, &s0b, 8);
|
||||
memcpy(dst1, &s1a, 8);
|
||||
memcpy(dst1 + 8, &s1b, 8);
|
||||
memcpy(dst2, &s2a, 8);
|
||||
memcpy(dst2 + 8, &s2b, 8);
|
||||
}
|
||||
crc0 = __crc32d(crc0, s0a);
|
||||
crc0 = __crc32d(crc0, s0b);
|
||||
crc1 = __crc32d(crc1, s1a);
|
||||
crc1 = __crc32d(crc1, s1b);
|
||||
crc2 = __crc32d(crc2, s2a);
|
||||
crc2 = __crc32d(crc2, s2b);
|
||||
}
|
||||
|
||||
/* Shift and combine 3 scalar CRCs */
|
||||
vc0 = crc_shift(crc0, klen * 2 + blk * 144);
|
||||
vc1 = crc_shift(crc1, klen + blk * 144);
|
||||
vc2 = crc_shift(crc2, blk * 144);
|
||||
vc = vgetq_lane_u64(veor3q_u64(vc0, vc1, vc2), 0);
|
||||
|
||||
/* Final reduction: 128-bit vector + scalar CRCs -> 32-bit */
|
||||
crc0 = __crc32d(0, vgetq_lane_u64(x0, 0));
|
||||
crc0 = __crc32d(crc0, vc ^ vgetq_lane_u64(x0, 1));
|
||||
if (COPY)
|
||||
dst += blk * 192;
|
||||
src = srcv;
|
||||
len = end - srcv;
|
||||
}
|
||||
|
||||
/* 3-way scalar CRC (24 bytes/iter) */
|
||||
if (len >= 80) {
|
||||
size_t klen = ((len - 8) / 24) * 8; /* Stride for 3-way parallel */
|
||||
const uint8_t *buf0 = src;
|
||||
const uint8_t *buf1 = src + klen;
|
||||
const uint8_t *buf2 = src + klen * 2;
|
||||
uint32_t crc1 = 0, crc2 = 0;
|
||||
uint64x2_t vc0, vc1;
|
||||
uint64_t vc;
|
||||
|
||||
/* Per-lane dst pointers */
|
||||
uint8_t *dst0 = dst;
|
||||
uint8_t *dst1 = NULL;
|
||||
uint8_t *dst2 = NULL;
|
||||
if (COPY) {
|
||||
dst1 = dst + klen;
|
||||
dst2 = dst + klen * 2;
|
||||
}
|
||||
|
||||
/* 3-way parallel scalar CRC */
|
||||
do {
|
||||
uint64_t v0 = *(const uint64_t*)buf0;
|
||||
uint64_t v1 = *(const uint64_t*)buf1;
|
||||
uint64_t v2 = *(const uint64_t*)buf2;
|
||||
if (COPY) {
|
||||
memcpy(dst0, &v0, 8);
|
||||
dst0 += 8;
|
||||
memcpy(dst1, &v1, 8);
|
||||
dst1 += 8;
|
||||
memcpy(dst2, &v2, 8);
|
||||
dst2 += 8;
|
||||
}
|
||||
crc0 = __crc32d(crc0, v0);
|
||||
crc1 = __crc32d(crc1, v1);
|
||||
crc2 = __crc32d(crc2, v2);
|
||||
buf0 += 8;
|
||||
buf1 += 8;
|
||||
buf2 += 8;
|
||||
len -= 24;
|
||||
} while (len >= 32);
|
||||
|
||||
/* Combine the 3 CRCs */
|
||||
vc0 = crc_shift(crc0, klen * 2 + 8);
|
||||
vc1 = crc_shift(crc1, klen + 8);
|
||||
vc = vgetq_lane_u64(veorq_u64(vc0, vc1), 0);
|
||||
|
||||
/* Process final 8 bytes with combined CRC */
|
||||
crc0 = crc2;
|
||||
{
|
||||
uint64_t vf = *(const uint64_t*)buf2;
|
||||
if (COPY)
|
||||
memcpy(dst2, &vf, 8);
|
||||
crc0 = __crc32d(crc0, vf ^ vc);
|
||||
}
|
||||
src = buf2 + 8;
|
||||
len -= 8;
|
||||
if (COPY)
|
||||
dst = dst2 + 8;
|
||||
}
|
||||
|
||||
/* Process remaining bytes */
|
||||
return crc32_hw_tail(crc0, dst, src, len, COPY);
|
||||
}
|
||||
|
||||
Z_INTERNAL Z_TARGET_PMULL_EOR3 uint32_t crc32_armv8_pmull_eor3(uint32_t crc, const uint8_t *buf, size_t len) {
|
||||
return crc32_copy_impl(crc, NULL, buf, len, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL Z_TARGET_PMULL_EOR3 uint32_t crc32_copy_armv8_pmull_eor3(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
#if OPTIMAL_CMP >= 32
|
||||
return crc32_copy_impl(crc, dst, src, len, 1);
|
||||
#else
|
||||
/* Without unaligned access, interleaved stores get decomposed into byte ops */
|
||||
crc = crc32_armv8_pmull_eor3(crc, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return crc;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
#ifndef ARM_NEON_INTRINS_H
|
||||
#define ARM_NEON_INTRINS_H
|
||||
|
||||
#if defined(_MSC_VER) && defined(ARCH_ARM) && defined(ARCH_64BIT)
|
||||
/* arm64_neon.h is MSVC specific */
|
||||
# include <arm64_neon.h>
|
||||
#else
|
||||
# include <arm_neon.h>
|
||||
#endif
|
||||
|
||||
#if defined(ARM_NEON) && defined(ARCH_ARM) && defined(ARCH_32BIT)
|
||||
/* Compatibility shim for the _high family of functions */
|
||||
#define vmull_high_u8(a, b) vmull_u8(vget_high_u8(a), vget_high_u8(b))
|
||||
#define vmlal_high_u8(a, b, c) vmlal_u8(a, vget_high_u8(b), vget_high_u8(c))
|
||||
#define vmlal_high_u16(a, b, c) vmlal_u16(a, vget_high_u16(b), vget_high_u16(c))
|
||||
#define vaddw_high_u8(a, b) vaddw_u8(a, vget_high_u8(b))
|
||||
#endif
|
||||
|
||||
#ifdef ARM_NEON
|
||||
|
||||
#define vqsubq_u16_x4_x1(out, a, b) do { \
|
||||
out.val[0] = vqsubq_u16(a.val[0], b); \
|
||||
out.val[1] = vqsubq_u16(a.val[1], b); \
|
||||
out.val[2] = vqsubq_u16(a.val[2], b); \
|
||||
out.val[3] = vqsubq_u16(a.val[3], b); \
|
||||
} while (0)
|
||||
|
||||
# if defined(ARCH_ARM) && defined(ARCH_32BIT) && defined(__clang__) && \
|
||||
(!defined(__clang_major__) || __clang_major__ < 20)
|
||||
/* Clang versions before 20 have too strict of an
|
||||
* alignment requirement (:256) for x4 NEON intrinsics */
|
||||
# undef ARM_NEON_HASLD4
|
||||
# undef vld1q_u16_x4
|
||||
# undef vld1q_u8_x4
|
||||
# undef vst1q_u16_x4
|
||||
# endif
|
||||
|
||||
# ifndef ARM_NEON_HASLD4
|
||||
|
||||
static inline uint16x8x4_t vld1q_u16_x4(uint16_t const *a) {
|
||||
uint16x8x4_t ret;
|
||||
ret.val[0] = vld1q_u16(a);
|
||||
ret.val[1] = vld1q_u16(a+8);
|
||||
ret.val[2] = vld1q_u16(a+16);
|
||||
ret.val[3] = vld1q_u16(a+24);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline uint8x16x4_t vld1q_u8_x4(uint8_t const *a) {
|
||||
uint8x16x4_t ret;
|
||||
ret.val[0] = vld1q_u8(a);
|
||||
ret.val[1] = vld1q_u8(a+16);
|
||||
ret.val[2] = vld1q_u8(a+32);
|
||||
ret.val[3] = vld1q_u8(a+48);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline void vst1q_u16_x4(uint16_t *p, uint16x8x4_t a) {
|
||||
vst1q_u16(p, a.val[0]);
|
||||
vst1q_u16(p + 8, a.val[1]);
|
||||
vst1q_u16(p + 16, a.val[2]);
|
||||
vst1q_u16(p + 24, a.val[3]);
|
||||
}
|
||||
# endif // HASLD4 check
|
||||
|
||||
# ifndef _MSC_VER
|
||||
# define vld1_u8_ex(p, align) vld1_u8(HINT_ALIGNED((p), (align)/8))
|
||||
# define vld1q_u8_ex(p, align) vld1q_u8(HINT_ALIGNED((p), (align)/8))
|
||||
# define vld1q_u64_ex(p, align) vld1q_u64(HINT_ALIGNED((p), (align)/8))
|
||||
# endif
|
||||
# if !defined(_MSC_VER) || !defined(ARM_NEON_HASLD4)
|
||||
# define vld1q_u8_x4_ex(p, align) vld1q_u8_x4(HINT_ALIGNED((p), (align)/8))
|
||||
# define vld1q_u16_x4_ex(p, align) vld1q_u16_x4(HINT_ALIGNED((p), (align)/8))
|
||||
# define vst1q_u16_x4_ex(p, a, align) vst1q_u16_x4(HINT_ALIGNED((p), (align)/8), a)
|
||||
# endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif // include guard ARM_NEON_INTRINS_H
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
/* slide_hash_armv6.c -- Optimized hash table shifting for ARMv6 with support for SIMD instructions
|
||||
* Copyright (C) 2023 Cameron Cawley
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef ARM_SIMD
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "acle_intrins.h"
|
||||
#include "deflate.h"
|
||||
|
||||
/* SIMD version of hash_chain rebase */
|
||||
static inline void slide_hash_chain(Pos *table, uint32_t entries, uint16_t wsize) {
|
||||
Z_REGISTER uint16x2_t v;
|
||||
uint16x2_t p0, p1, p2, p3;
|
||||
Z_REGISTER size_t n;
|
||||
|
||||
size_t size = entries*sizeof(table[0]);
|
||||
Assert((size % (sizeof(uint16x2_t) * 4) == 0), "hash table size err");
|
||||
|
||||
Assert(sizeof(Pos) == 2, "Wrong Pos size");
|
||||
v = wsize | (wsize << 16);
|
||||
|
||||
n = size / (sizeof(uint16x2_t) * 4);
|
||||
do {
|
||||
p0 = *((const uint16x2_t *)(table));
|
||||
p1 = *((const uint16x2_t *)(table+2));
|
||||
p2 = *((const uint16x2_t *)(table+4));
|
||||
p3 = *((const uint16x2_t *)(table+6));
|
||||
p0 = __uqsub16(p0, v);
|
||||
p1 = __uqsub16(p1, v);
|
||||
p2 = __uqsub16(p2, v);
|
||||
p3 = __uqsub16(p3, v);
|
||||
*((uint16x2_t *)(table)) = p0;
|
||||
*((uint16x2_t *)(table+2)) = p1;
|
||||
*((uint16x2_t *)(table+4)) = p2;
|
||||
*((uint16x2_t *)(table+6)) = p3;
|
||||
table += 8;
|
||||
} while (--n);
|
||||
}
|
||||
|
||||
Z_INTERNAL void slide_hash_armv6(deflate_state *s) {
|
||||
Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
|
||||
slide_hash_chain(s->head, HASH_SIZE, wsize);
|
||||
slide_hash_chain(s->prev, wsize, wsize);
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
/* slide_hash_neon.c -- Optimized hash table shifting for ARM with support for NEON instructions
|
||||
* Copyright (C) 2017-2020 Mika T. Lindqvist
|
||||
*
|
||||
* Authors:
|
||||
* Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* Jun He <jun.he@arm.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef ARM_NEON
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "neon_intrins.h"
|
||||
#include "deflate.h"
|
||||
|
||||
/* SIMD version of hash_chain rebase */
|
||||
static inline void slide_hash_chain(Pos *table, uint32_t entries, uint16_t wsize) {
|
||||
Z_REGISTER uint16x8_t v;
|
||||
uint16x8x4_t p0, p1;
|
||||
Z_REGISTER size_t n;
|
||||
|
||||
size_t size = entries*sizeof(table[0]);
|
||||
Assert((size % sizeof(uint16x8_t) * 8 == 0), "hash table size err");
|
||||
|
||||
Assert(sizeof(Pos) == 2, "Wrong Pos size");
|
||||
v = vdupq_n_u16(wsize);
|
||||
|
||||
n = size / (sizeof(uint16x8_t) * 8);
|
||||
do {
|
||||
p0 = vld1q_u16_x4_ex(table, 256);
|
||||
p1 = vld1q_u16_x4_ex(table+32, 256);
|
||||
vqsubq_u16_x4_x1(p0, p0, v);
|
||||
vqsubq_u16_x4_x1(p1, p1, v);
|
||||
vst1q_u16_x4_ex(table, p0, 256);
|
||||
vst1q_u16_x4_ex(table+32, p1, 256);
|
||||
table += 64;
|
||||
} while (--n);
|
||||
}
|
||||
|
||||
Z_INTERNAL void slide_hash_neon(deflate_state *s) {
|
||||
Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
|
||||
slide_hash_chain(s->head, HASH_SIZE, wsize);
|
||||
slide_hash_chain(s->prev, wsize, wsize);
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
# Makefile for zlib-ng
|
||||
# Copyright (C) 1995-2013 Jean-loup Gailly, Mark Adler
|
||||
# Copyright (C) 2024 Hans Kristian Rosbach
|
||||
# For conditions of distribution and use, see copyright notice in zlib.h
|
||||
|
||||
CC=
|
||||
CFLAGS=
|
||||
SFLAGS=
|
||||
INCLUDES=
|
||||
|
||||
SRCDIR=.
|
||||
SRCTOP=../..
|
||||
TOPDIR=$(SRCTOP)
|
||||
|
||||
all: \
|
||||
adler32_c.o adler32_c.lo \
|
||||
chunkset_c.o chunkset_c.lo \
|
||||
compare256_c.o compare256_c.lo \
|
||||
crc32_braid_c.o crc32_braid_c.lo \
|
||||
crc32_chorba_c.o crc32_chorba_c.lo \
|
||||
slide_hash_c.o slide_hash_c.lo
|
||||
|
||||
|
||||
adler32_c.o: $(SRCDIR)/adler32_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/adler32_p.h
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_c.c
|
||||
|
||||
adler32_c.lo: $(SRCDIR)/adler32_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/adler32_p.h
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_c.c
|
||||
|
||||
chunkset_c.o: $(SRCDIR)/chunkset_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/chunkset_tpl.h $(SRCTOP)/inffast_tpl.h
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_c.c
|
||||
|
||||
chunkset_c.lo: $(SRCDIR)/chunkset_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/chunkset_tpl.h $(SRCTOP)/inffast_tpl.h
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_c.c
|
||||
|
||||
compare256_c.o: $(SRCDIR)/compare256_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/zendian.h $(SRCTOP)/zmemory.h $(SRCTOP)/deflate.h $(SRCTOP)/fallback_builtins.h
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_c.c
|
||||
|
||||
compare256_c.lo: $(SRCDIR)/compare256_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/zendian.h $(SRCTOP)/zmemory.h $(SRCTOP)/deflate.h $(SRCTOP)/fallback_builtins.h
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_c.c
|
||||
|
||||
crc32_braid_c.o: $(SRCDIR)/crc32_braid_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/crc32_braid_p.h $(SRCTOP)/crc32_braid_tbl.h
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_braid_c.c
|
||||
|
||||
crc32_braid_c.lo: $(SRCDIR)/crc32_braid_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/crc32_braid_p.h $(SRCTOP)/crc32_braid_tbl.h
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_braid_c.c
|
||||
|
||||
crc32_chorba_c.o: $(SRCDIR)/crc32_chorba_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/crc32_braid_p.h $(SRCTOP)/crc32_braid_tbl.h
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_chorba_c.c
|
||||
|
||||
crc32_chorba_c.lo: $(SRCDIR)/crc32_chorba_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/crc32_braid_p.h $(SRCTOP)/crc32_braid_tbl.h
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_chorba_c.c
|
||||
|
||||
slide_hash_c.o: $(SRCDIR)/slide_hash_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/deflate.h
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_c.c
|
||||
|
||||
slide_hash_c.lo: $(SRCDIR)/slide_hash_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/deflate.h
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_c.c
|
||||
|
||||
|
||||
mostlyclean: clean
|
||||
clean:
|
||||
rm -f *.o *.lo *~
|
||||
rm -rf objs
|
||||
rm -f *.gcda *.gcno *.gcov
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
/* adler32.c -- compute the Adler-32 checksum of a data stream
|
||||
* Copyright (C) 1995-2011, 2016 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arch_functions.h"
|
||||
|
||||
#ifdef ADLER32_FALLBACK
|
||||
|
||||
#include "functable.h"
|
||||
#include "adler32_p.h"
|
||||
|
||||
Z_INTERNAL uint32_t adler32_c(uint32_t adler, const uint8_t *buf, size_t len) {
|
||||
uint32_t sum2;
|
||||
unsigned n;
|
||||
|
||||
/* split Adler-32 into component sums */
|
||||
sum2 = (adler >> 16) & 0xffff;
|
||||
adler &= 0xffff;
|
||||
|
||||
/* in case user likes doing a byte at a time, keep it fast */
|
||||
if (UNLIKELY(len == 1))
|
||||
return adler32_copy_tail(adler, NULL, buf, 1, sum2, 1, 1, 0);
|
||||
|
||||
/* in case short lengths are provided, keep it somewhat fast */
|
||||
if (UNLIKELY(len < 16))
|
||||
return adler32_copy_tail(adler, NULL, buf, len, sum2, 1, 15, 0);
|
||||
|
||||
/* do length NMAX blocks -- requires just one modulo operation */
|
||||
while (len >= NMAX) {
|
||||
len -= NMAX;
|
||||
#ifdef UNROLL_MORE
|
||||
n = NMAX / 16; /* NMAX is divisible by 16 */
|
||||
#else
|
||||
n = NMAX / 8; /* NMAX is divisible by 8 */
|
||||
#endif
|
||||
do {
|
||||
#ifdef UNROLL_MORE
|
||||
ADLER_DO16(adler, sum2, buf); /* 16 sums unrolled */
|
||||
buf += 16;
|
||||
#else
|
||||
ADLER_DO8(adler, sum2, buf, 0); /* 8 sums unrolled */
|
||||
buf += 8;
|
||||
#endif
|
||||
} while (--n);
|
||||
adler %= BASE;
|
||||
sum2 %= BASE;
|
||||
}
|
||||
|
||||
/* do remaining bytes (less than NMAX, still just one modulo) */
|
||||
return adler32_copy_tail(adler, NULL, buf, len, sum2, len != 0, NMAX - 1, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_copy_c(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
adler = FUNCTABLE_CALL(adler32)(adler, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return adler;
|
||||
}
|
||||
|
||||
#endif /* ADLER32_FALLBACK */
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
/* chunkset.c -- inline functions to copy small data chunks.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arch_functions.h"
|
||||
|
||||
#ifdef CHUNKSET_FALLBACK
|
||||
|
||||
#include "zmemory.h"
|
||||
|
||||
typedef uint64_t chunk_t;
|
||||
|
||||
#define HAVE_CHUNKMEMSET_4
|
||||
#define HAVE_CHUNKMEMSET_8
|
||||
|
||||
static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) {
|
||||
uint32_t tmp = zng_memread_4(from);
|
||||
*chunk = tmp | ((chunk_t)tmp << 32);
|
||||
}
|
||||
|
||||
static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = zng_memread_8(from);
|
||||
}
|
||||
|
||||
static inline void loadchunk(uint8_t const *s, chunk_t *chunk) {
|
||||
*chunk = zng_memread_8(s);
|
||||
}
|
||||
|
||||
static inline void storechunk(uint8_t *out, chunk_t *chunk) {
|
||||
zng_memwrite_8(out, *chunk);
|
||||
}
|
||||
|
||||
#define CHUNKSIZE chunksize_c
|
||||
#define CHUNKCOPY chunkcopy_c
|
||||
#define CHUNKUNROLL chunkunroll_c
|
||||
#define CHUNKMEMSET chunkmemset_c
|
||||
#define CHUNKMEMSET_SAFE chunkmemset_safe_c
|
||||
|
||||
#include "chunkset_tpl.h"
|
||||
|
||||
#define INFLATE_FAST inflate_fast_c
|
||||
|
||||
#include "inffast_tpl.h"
|
||||
|
||||
#endif /* CHUNKSET_FALLBACK */
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
/* compare256.c -- 256 byte memory comparison with match length return
|
||||
* Copyright (C) 2020 Nathan Moinvaziri
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arch_functions.h"
|
||||
|
||||
#ifdef COMPARE256_FALLBACK
|
||||
|
||||
#include "zendian.h"
|
||||
#include "deflate.h"
|
||||
#include "fallback_builtins.h"
|
||||
|
||||
/* 8-bit integer comparison for hardware without unaligned loads */
|
||||
static inline uint32_t compare256_8_static(const uint8_t *src0, const uint8_t *src1) {
|
||||
uint32_t len = 0;
|
||||
|
||||
do {
|
||||
if (src0[0] != src1[0])
|
||||
return len;
|
||||
if (src0[1] != src1[1])
|
||||
return len + 1;
|
||||
if (src0[2] != src1[2])
|
||||
return len + 2;
|
||||
if (src0[3] != src1[3])
|
||||
return len + 3;
|
||||
if (src0[4] != src1[4])
|
||||
return len + 4;
|
||||
if (src0[5] != src1[5])
|
||||
return len + 5;
|
||||
if (src0[6] != src1[6])
|
||||
return len + 6;
|
||||
if (src0[7] != src1[7])
|
||||
return len + 7;
|
||||
src0 += 8, src1 += 8, len += 8;
|
||||
} while (len < 256);
|
||||
|
||||
return 256;
|
||||
}
|
||||
|
||||
/* 64-bit integer comparison for hardware with unaligned loads */
|
||||
static inline uint32_t compare256_64_static(const uint8_t *src0, const uint8_t *src1) {
|
||||
uint32_t len = 0;
|
||||
|
||||
do {
|
||||
uint64_t sv = zng_memread_8(src0);
|
||||
uint64_t mv = zng_memread_8(src1);
|
||||
uint64_t diff = sv ^ mv;
|
||||
if (diff)
|
||||
return len + zng_first_diff_byte64(diff);
|
||||
src0 += 8, src1 += 8, len += 8;
|
||||
|
||||
sv = zng_memread_8(src0);
|
||||
mv = zng_memread_8(src1);
|
||||
diff = sv ^ mv;
|
||||
if (diff)
|
||||
return len + zng_first_diff_byte64(diff);
|
||||
src0 += 8, src1 += 8, len += 8;
|
||||
} while (len < 256);
|
||||
|
||||
return 256;
|
||||
}
|
||||
|
||||
#if OPTIMAL_CMP == 8
|
||||
# define COMPARE256 compare256_8_static
|
||||
#else
|
||||
# define COMPARE256 compare256_64_static
|
||||
#endif
|
||||
|
||||
Z_INTERNAL uint32_t compare256_8(const uint8_t *src0, const uint8_t *src1) {
|
||||
return compare256_8_static(src0, src1);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t compare256_64(const uint8_t *src0, const uint8_t *src1) {
|
||||
return compare256_64_static(src0, src1);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t compare256_c(const uint8_t *src0, const uint8_t *src1) {
|
||||
return COMPARE256(src0, src1);
|
||||
}
|
||||
|
||||
// Generate longest_match_c
|
||||
#define LONGEST_MATCH longest_match_c
|
||||
#include "match_tpl.h"
|
||||
|
||||
// Generate longest_match_roll_c
|
||||
#define LONGEST_MATCH_ROLL
|
||||
#define LONGEST_MATCH longest_match_roll_c
|
||||
#include "match_tpl.h"
|
||||
|
||||
#endif /* COMPARE256_FALLBACK */
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
/* crc32_braid.c -- compute the CRC-32 of a data stream
|
||||
* Copyright (C) 1995-2022 Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*
|
||||
* This interleaved implementation of a CRC makes use of pipelined multiple
|
||||
* arithmetic-logic units, commonly found in modern CPU cores. It is due to
|
||||
* Kadatch and Jenkins (2010). See doc/crc-doc.1.0.pdf in this distribution.
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arch_functions.h"
|
||||
|
||||
/* Used by chorba fallback and by arch-specific implementations (s390 vx, risc-v zbc). */
|
||||
#ifdef CRC32_BRAID_FALLBACK
|
||||
|
||||
#include "crc32_braid_p.h"
|
||||
#include "crc32_braid_tbl.h"
|
||||
#include "crc32_p.h"
|
||||
|
||||
/*
|
||||
A CRC of a message is computed on BRAID_N braids of words in the message, where
|
||||
each word consists of BRAID_W bytes (4 or 8). If BRAID_N is 3, for example, then
|
||||
three running sparse CRCs are calculated respectively on each braid, at these
|
||||
indices in the array of words: 0, 3, 6, ..., 1, 4, 7, ..., and 2, 5, 8, ...
|
||||
This is done starting at a word boundary, and continues until as many blocks of
|
||||
BRAID_N * BRAID_W bytes as are available have been processed. The results are
|
||||
combined into a single CRC at the end. For this code, BRAID_N must be in the
|
||||
range 1..6 and BRAID_W must be 4 or 8. The upper limit on BRAID_N can be increased
|
||||
if desired by adding more #if blocks, extending the patterns apparent in the code.
|
||||
In addition, crc32 tables would need to be regenerated, if the maximum BRAID_N
|
||||
value is increased.
|
||||
|
||||
BRAID_N and BRAID_W are chosen empirically by benchmarking the execution time
|
||||
on a given processor. The choices for BRAID_N and BRAID_W below were based on
|
||||
testing on Intel Kaby Lake i7, AMD Ryzen 7, ARM Cortex-A57, Sparc64-VII, PowerPC
|
||||
POWER9, and MIPS64 Octeon II processors.
|
||||
The Intel, AMD, and ARM processors were all fastest with BRAID_N=5, BRAID_W=8.
|
||||
The Sparc, PowerPC, and MIPS64 were all fastest at BRAID_N=5, BRAID_W=4.
|
||||
They were all tested with either gcc or clang, all using the -O3 optimization
|
||||
level. Your mileage may vary.
|
||||
*/
|
||||
|
||||
/* ========================================================================= */
|
||||
#ifdef BRAID_W
|
||||
/*
|
||||
Return the CRC of the BRAID_W bytes in the word_t data, taking the
|
||||
least-significant byte of the word as the first byte of data, without any pre
|
||||
or post conditioning. This is used to combine the CRCs of each braid.
|
||||
*/
|
||||
# if BYTE_ORDER == LITTLE_ENDIAN
|
||||
static uint32_t crc_word(z_word_t data) {
|
||||
int k;
|
||||
for (k = 0; k < BRAID_W; k++)
|
||||
data = (data >> 8) ^ crc_table[data & 0xff];
|
||||
return (uint32_t)data;
|
||||
}
|
||||
# elif BYTE_ORDER == BIG_ENDIAN
|
||||
static z_word_t crc_word(z_word_t data) {
|
||||
int k;
|
||||
for (k = 0; k < BRAID_W; k++)
|
||||
data = (data << 8) ^
|
||||
crc_big_table[(data >> ((BRAID_W - 1) << 3)) & 0xff];
|
||||
return data;
|
||||
}
|
||||
# endif /* BYTE_ORDER */
|
||||
#endif /* BRAID_W */
|
||||
|
||||
/* ========================================================================= */
|
||||
Z_INTERNAL uint32_t crc32_braid(uint32_t crc, const uint8_t *buf, size_t len) {
|
||||
crc = ~crc;
|
||||
|
||||
#ifdef BRAID_W
|
||||
/* If provided enough bytes, do a braided CRC calculation. */
|
||||
if (len >= BRAID_N * BRAID_W + BRAID_W - 1) {
|
||||
size_t blks;
|
||||
z_word_t const *words;
|
||||
int k;
|
||||
|
||||
/* Compute the CRC up to a z_word_t boundary. */
|
||||
size_t align_diff = (size_t)MIN(ALIGN_DIFF(buf, BRAID_W), len);
|
||||
if (align_diff) {
|
||||
crc = crc32_copy_small(crc, NULL, buf, align_diff, BRAID_W - 1, 0);
|
||||
len -= align_diff;
|
||||
buf += align_diff;
|
||||
}
|
||||
|
||||
/* Compute the CRC on as many BRAID_N z_word_t blocks as are available. */
|
||||
blks = len / (BRAID_N * BRAID_W);
|
||||
len -= blks * BRAID_N * BRAID_W;
|
||||
words = (z_word_t const *)buf;
|
||||
|
||||
z_word_t crc0, word0, comb;
|
||||
#if BRAID_N > 1
|
||||
z_word_t crc1, word1;
|
||||
#if BRAID_N > 2
|
||||
z_word_t crc2, word2;
|
||||
#if BRAID_N > 3
|
||||
z_word_t crc3, word3;
|
||||
#if BRAID_N > 4
|
||||
z_word_t crc4, word4;
|
||||
#if BRAID_N > 5
|
||||
z_word_t crc5, word5;
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
/* Initialize the CRC for each braid. */
|
||||
crc0 = Z_WORD_FROM_LE(crc);
|
||||
#if BRAID_N > 1
|
||||
crc1 = 0;
|
||||
#if BRAID_N > 2
|
||||
crc2 = 0;
|
||||
#if BRAID_N > 3
|
||||
crc3 = 0;
|
||||
#if BRAID_N > 4
|
||||
crc4 = 0;
|
||||
#if BRAID_N > 5
|
||||
crc5 = 0;
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
/* Process the first blks-1 blocks, computing the CRCs on each braid independently. */
|
||||
while (--blks) {
|
||||
/* Load the word for each braid into registers. */
|
||||
word0 = crc0 ^ words[0];
|
||||
#if BRAID_N > 1
|
||||
word1 = crc1 ^ words[1];
|
||||
#if BRAID_N > 2
|
||||
word2 = crc2 ^ words[2];
|
||||
#if BRAID_N > 3
|
||||
word3 = crc3 ^ words[3];
|
||||
#if BRAID_N > 4
|
||||
word4 = crc4 ^ words[4];
|
||||
#if BRAID_N > 5
|
||||
word5 = crc5 ^ words[5];
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
words += BRAID_N;
|
||||
|
||||
/* Compute and update the CRC for each word. The loop should get unrolled. */
|
||||
crc0 = BRAID_TABLE[0][word0 & 0xff];
|
||||
#if BRAID_N > 1
|
||||
crc1 = BRAID_TABLE[0][word1 & 0xff];
|
||||
#if BRAID_N > 2
|
||||
crc2 = BRAID_TABLE[0][word2 & 0xff];
|
||||
#if BRAID_N > 3
|
||||
crc3 = BRAID_TABLE[0][word3 & 0xff];
|
||||
#if BRAID_N > 4
|
||||
crc4 = BRAID_TABLE[0][word4 & 0xff];
|
||||
#if BRAID_N > 5
|
||||
crc5 = BRAID_TABLE[0][word5 & 0xff];
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
for (k = 1; k < BRAID_W; k++) {
|
||||
crc0 ^= BRAID_TABLE[k][(word0 >> (k << 3)) & 0xff];
|
||||
#if BRAID_N > 1
|
||||
crc1 ^= BRAID_TABLE[k][(word1 >> (k << 3)) & 0xff];
|
||||
#if BRAID_N > 2
|
||||
crc2 ^= BRAID_TABLE[k][(word2 >> (k << 3)) & 0xff];
|
||||
#if BRAID_N > 3
|
||||
crc3 ^= BRAID_TABLE[k][(word3 >> (k << 3)) & 0xff];
|
||||
#if BRAID_N > 4
|
||||
crc4 ^= BRAID_TABLE[k][(word4 >> (k << 3)) & 0xff];
|
||||
#if BRAID_N > 5
|
||||
crc5 ^= BRAID_TABLE[k][(word5 >> (k << 3)) & 0xff];
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
/* Process the last block, combining the CRCs of the BRAID_N braids at the same time. */
|
||||
comb = crc_word(crc0 ^ words[0]);
|
||||
#if BRAID_N > 1
|
||||
comb = crc_word(crc1 ^ words[1] ^ comb);
|
||||
#if BRAID_N > 2
|
||||
comb = crc_word(crc2 ^ words[2] ^ comb);
|
||||
#if BRAID_N > 3
|
||||
comb = crc_word(crc3 ^ words[3] ^ comb);
|
||||
#if BRAID_N > 4
|
||||
comb = crc_word(crc4 ^ words[4] ^ comb);
|
||||
#if BRAID_N > 5
|
||||
comb = crc_word(crc5 ^ words[5] ^ comb);
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
words += BRAID_N;
|
||||
Assert(comb <= UINT32_MAX, "comb should fit in uint32_t");
|
||||
crc = (uint32_t)Z_WORD_FROM_LE(comb);
|
||||
|
||||
/* Update the pointer to the remaining bytes to process. */
|
||||
buf = (const unsigned char *)words;
|
||||
}
|
||||
|
||||
#endif /* BRAID_W */
|
||||
|
||||
/* Complete the computation of the CRC on any remaining bytes. */
|
||||
return ~crc32_copy_small(crc, NULL, buf, len, (BRAID_N * BRAID_W) - 1, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t crc32_copy_braid(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
crc = crc32_braid(crc, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return crc;
|
||||
}
|
||||
|
||||
#endif /* CRC32_BRAID_FALLBACK */
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,80 +0,0 @@
|
|||
/* generic_functions.h -- generic C implementations for arch-specific functions.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef GENERIC_FUNCTIONS_H_
|
||||
#define GENERIC_FUNCTIONS_H_
|
||||
|
||||
typedef uint32_t (*adler32_func)(uint32_t adler, const uint8_t *buf, size_t len);
|
||||
typedef uint32_t (*adler32_copy_func)(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
typedef uint32_t (*compare256_func)(const uint8_t *src0, const uint8_t *src1);
|
||||
typedef uint32_t (*crc32_func)(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
typedef uint32_t (*crc32_copy_func)(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
typedef void (*slide_hash_func)(deflate_state *s);
|
||||
|
||||
#ifdef ADLER32_FALLBACK
|
||||
uint32_t adler32_c(uint32_t adler, const uint8_t *buf, size_t len);
|
||||
uint32_t adler32_copy_c(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
#endif
|
||||
#ifdef CHUNKSET_FALLBACK
|
||||
uint8_t* chunkmemset_safe_c(uint8_t *out, uint8_t *from, size_t len, size_t left);
|
||||
#endif
|
||||
#ifdef COMPARE256_FALLBACK
|
||||
uint32_t compare256_8(const uint8_t *src0, const uint8_t *src1);
|
||||
uint32_t compare256_64(const uint8_t *src0, const uint8_t *src1);
|
||||
uint32_t compare256_c(const uint8_t *src0, const uint8_t *src1);
|
||||
#endif
|
||||
|
||||
#ifdef CRC32_BRAID_FALLBACK
|
||||
uint32_t crc32_braid(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_braid(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
#endif
|
||||
|
||||
/* Chorba is available whenever braid is needed as a fallback and hasn't been disabled. */
|
||||
#if defined(CRC32_BRAID_FALLBACK) && !defined(WITHOUT_CHORBA)
|
||||
# define CRC32_CHORBA_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef CRC32_CHORBA_FALLBACK
|
||||
uint32_t crc32_chorba(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_chorba(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
#endif
|
||||
#ifdef CHUNKSET_FALLBACK
|
||||
void inflate_fast_c(PREFIX3(stream) *strm, uint32_t start);
|
||||
#endif
|
||||
#ifdef COMPARE256_FALLBACK
|
||||
uint32_t longest_match_c(deflate_state *const s, uint32_t cur_match);
|
||||
uint32_t longest_match_roll_c(deflate_state *const s, uint32_t cur_match);
|
||||
#endif
|
||||
#ifdef SLIDE_HASH_FALLBACK
|
||||
void slide_hash_c(deflate_state *s);
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_RUNTIME_CPU_DETECTION
|
||||
// Generic fallbacks when no native implementation exists
|
||||
# ifdef ADLER32_FALLBACK
|
||||
# define native_adler32 adler32_c
|
||||
# define native_adler32_copy adler32_copy_c
|
||||
# endif
|
||||
# ifdef CHUNKSET_FALLBACK
|
||||
# define native_chunkmemset_safe chunkmemset_safe_c
|
||||
# define native_inflate_fast inflate_fast_c
|
||||
# endif
|
||||
# ifdef COMPARE256_FALLBACK
|
||||
# define native_compare256 compare256_c
|
||||
# define native_longest_match longest_match_c
|
||||
# define native_longest_match_roll longest_match_roll_c
|
||||
# endif
|
||||
# ifdef CRC32_CHORBA_FALLBACK
|
||||
# define native_crc32 crc32_chorba
|
||||
# define native_crc32_copy crc32_copy_chorba
|
||||
# elif defined(CRC32_BRAID_FALLBACK)
|
||||
# define native_crc32 crc32_braid
|
||||
# define native_crc32_copy crc32_copy_braid
|
||||
# endif
|
||||
# ifdef SLIDE_HASH_FALLBACK
|
||||
# define native_slide_hash slide_hash_c
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
/* slide_hash.c -- slide hash table C implementation
|
||||
*
|
||||
* Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arch_functions.h"
|
||||
|
||||
#ifdef SLIDE_HASH_FALLBACK
|
||||
|
||||
#include "deflate.h"
|
||||
|
||||
/* ===========================================================================
|
||||
* Slide the hash table when sliding the window down (could be avoided with 32
|
||||
* bit values at the expense of memory usage). We slide even when level == 0 to
|
||||
* keep the hash table consistent if we switch back to level > 0 later.
|
||||
*/
|
||||
static inline void slide_hash_c_chain(Pos *table, uint32_t entries, uint16_t wsize) {
|
||||
#ifdef NOT_TWEAK_COMPILER
|
||||
table += entries;
|
||||
do {
|
||||
unsigned m;
|
||||
m = *--table;
|
||||
*table = (Pos)(m >= wsize ? m-wsize : 0);
|
||||
/* If entries is not on any hash chain, prev[entries] is garbage but
|
||||
* its value will never be used.
|
||||
*/
|
||||
} while (--entries);
|
||||
#else
|
||||
{
|
||||
/* As of I make this change, gcc (4.8.*) isn't able to vectorize
|
||||
* this hot loop using saturated-subtraction on x86-64 architecture.
|
||||
* To avoid this defect, we can change the loop such that
|
||||
* o. the pointer advance forward, and
|
||||
* o. demote the variable 'm' to be local to the loop, and
|
||||
* choose type "Pos" (instead of 'unsigned int') for the
|
||||
* variable to avoid unnecessary zero-extension.
|
||||
*/
|
||||
unsigned int i;
|
||||
Pos *q = table;
|
||||
for (i = 0; i < entries; i++) {
|
||||
Pos m = *q;
|
||||
Pos t = (Pos)wsize;
|
||||
*q++ = (Pos)(m >= t ? m-t: 0);
|
||||
}
|
||||
}
|
||||
#endif /* NOT_TWEAK_COMPILER */
|
||||
}
|
||||
|
||||
Z_INTERNAL void slide_hash_c(deflate_state *s) {
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
|
||||
slide_hash_c_chain(s->head, HASH_SIZE, wsize);
|
||||
slide_hash_c_chain(s->prev, wsize, wsize);
|
||||
}
|
||||
|
||||
#endif /* SLIDE_HASH_FALLBACK */
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
# Makefile for zlib-ng
|
||||
# Copyright (C) 1995-2013 Jean-loup Gailly, Mark Adler
|
||||
# Copyright (C) 2024 Hans Kristian Rosbach
|
||||
# Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
# For conditions of distribution and use, see copyright notice in zlib.h
|
||||
|
||||
CC=
|
||||
CFLAGS=
|
||||
SFLAGS=
|
||||
INCLUDES=
|
||||
SUFFIX=
|
||||
|
||||
LSXFLAG=-mlsx
|
||||
LASXFLAG=-mlasx
|
||||
|
||||
SRCDIR=.
|
||||
SRCTOP=../..
|
||||
TOPDIR=$(SRCTOP)
|
||||
|
||||
all: \
|
||||
loongarch_features.o loongarch_features.lo \
|
||||
crc32_la.o crc32_la.lo \
|
||||
adler32_lasx.o adler32_lasx.lo \
|
||||
adler32_lsx.o adler32_lsx.lo \
|
||||
chunkset_lasx.o chunkset_lasx.lo \
|
||||
chunkset_lsx.o chunkset_lsx.lo \
|
||||
compare256_lasx.o compare256_lasx.lo \
|
||||
compare256_lsx.o compare256_lsx.lo \
|
||||
slide_hash_lasx.o slide_hash_lasx.lo \
|
||||
slide_hash_lsx.o slide_hash_lsx.lo
|
||||
|
||||
loongarch_features.o: $(SRCDIR)/loongarch_features.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/loongarch_features.c
|
||||
|
||||
loongarch_features.lo: $(SRCDIR)/loongarch_features.c
|
||||
$(CC) $(SFLAGS) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/loongarch_features.c
|
||||
|
||||
crc32_la.o: $(SRCDIR)/crc32_la.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_la.c
|
||||
|
||||
crc32_la.lo: $(SRCDIR)/crc32_la.c
|
||||
$(CC) $(SFLAGS) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_la.c
|
||||
|
||||
adler32_lasx.o:
|
||||
$(CC) $(CFLAGS) $(LASXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_lasx.c
|
||||
|
||||
adler32_lasx.lo:
|
||||
$(CC) $(SFLAGS) $(LASXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_lasx.c
|
||||
|
||||
adler32_lsx.o:
|
||||
$(CC) $(CFLAGS) $(LSXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_lsx.c
|
||||
|
||||
adler32_lsx.lo:
|
||||
$(CC) $(SFLAGS) $(LSXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_lsx.c
|
||||
|
||||
chunkset_lasx.o:
|
||||
$(CC) $(CFLAGS) $(LASXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_lasx.c
|
||||
|
||||
chunkset_lasx.lo:
|
||||
$(CC) $(SFLAGS) $(LASXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_lasx.c
|
||||
|
||||
chunkset_lsx.o:
|
||||
$(CC) $(CFLAGS) $(LSXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_lsx.c
|
||||
|
||||
chunkset_lsx.lo:
|
||||
$(CC) $(SFLAGS) $(LSXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_lsx.c
|
||||
|
||||
compare256_lasx.o:
|
||||
$(CC) $(CFLAGS) $(LASXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_lasx.c
|
||||
|
||||
compare256_lasx.lo:
|
||||
$(CC) $(SFLAGS) $(LASXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_lasx.c
|
||||
|
||||
compare256_lsx.o:
|
||||
$(CC) $(CFLAGS) $(LSXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_lsx.c
|
||||
|
||||
compare256_lsx.lo:
|
||||
$(CC) $(SFLAGS) $(LSXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_lsx.c
|
||||
|
||||
slide_hash_lasx.o:
|
||||
$(CC) $(CFLAGS) $(LASXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_lasx.c
|
||||
|
||||
slide_hash_lasx.lo:
|
||||
$(CC) $(SFLAGS) $(LASXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_lasx.c
|
||||
|
||||
slide_hash_lsx.o:
|
||||
$(CC) $(CFLAGS) $(LSXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_lsx.c
|
||||
|
||||
slide_hash_lsx.lo:
|
||||
$(CC) $(SFLAGS) $(LSXFLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_lsx.c
|
||||
|
||||
mostlyclean: clean
|
||||
clean:
|
||||
rm -f *.o *.lo *~
|
||||
rm -rf objs
|
||||
rm -f *.gcda *.gcno *.gcov
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
|
@ -1,154 +0,0 @@
|
|||
/* adler32_lasx.c -- compute the Adler-32 checksum of a data stream, based on Intel AVX2 implementation
|
||||
* Copyright (C) 1995-2011 Mark Adler
|
||||
* Copyright (C) 2022 Adam Stylinski
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* Authors:
|
||||
* Brian Bockelman <bockelman@gmail.com>
|
||||
* Adam Stylinski <kungfujesus06@gmail.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_LASX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "adler32_p.h"
|
||||
|
||||
#include <lasxintrin.h>
|
||||
#include "lasxintrin_ext.h"
|
||||
|
||||
|
||||
/* 32 bit horizontal sum */
|
||||
static inline uint32_t hsum256(__m256i x) {
|
||||
__m256i sum1 = __lasx_xvadd_w(x, __lasx_xvbsrl_v(x, 8));
|
||||
__m256i sum2 = __lasx_xvadd_w(sum1, __lasx_xvpermi_d(sum1, 0x2));
|
||||
__m256i sum3 = __lasx_xvadd_w(sum2, __lasx_xvbsrl_v(sum2, 4));
|
||||
return (uint32_t)__lasx_xvpickve2gr_wu(sum3, 0);
|
||||
}
|
||||
|
||||
static inline uint32_t partial_hsum256(__m256i x) {
|
||||
__m256i sum1 = __lasx_xvadd_w(x, __lasx_xvbsrl_v(x, 8));
|
||||
__m256i sum2 = __lasx_xvadd_w(sum1, __lasx_xvpermi_d(sum1, 0x2));
|
||||
return (uint32_t)__lasx_xvpickve2gr_wu(sum2, 0);
|
||||
}
|
||||
|
||||
extern uint32_t adler32_copy_lsx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
extern uint32_t adler32_lsx(uint32_t adler, const uint8_t *src, size_t len);
|
||||
|
||||
Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len, const int COPY) {
|
||||
uint32_t adler0, adler1;
|
||||
adler1 = (adler >> 16) & 0xffff;
|
||||
adler0 = adler & 0xffff;
|
||||
|
||||
rem_peel:
|
||||
if (len < 16) {
|
||||
return adler32_copy_tail(adler0, dst, src, len, adler1, 1, 15, COPY);
|
||||
} else if (len < 32) {
|
||||
if (COPY) {
|
||||
return adler32_copy_lsx(adler, dst, src, len);
|
||||
} else {
|
||||
return adler32_lsx(adler, src, len);
|
||||
}
|
||||
}
|
||||
|
||||
__m256i vs1, vs2, vs2_0;
|
||||
|
||||
const __m256i dot2v = (__m256i)((v32i8){ 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47,
|
||||
46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33 });
|
||||
const __m256i dot2v_0 = (__m256i)((v32i8){ 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15,
|
||||
14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 });
|
||||
const __m256i dot3v = __lasx_xvreplgr2vr_h(1);
|
||||
const __m256i zero = __lasx_xvldi(0);
|
||||
|
||||
while (len >= 32) {
|
||||
vs1 = __lasx_xvinsgr2vr_w(zero, adler0, 0);
|
||||
vs2 = __lasx_xvinsgr2vr_w(zero, adler1, 0);
|
||||
|
||||
__m256i vs1_0 = vs1;
|
||||
__m256i vs3 = __lasx_xvldi(0);
|
||||
vs2_0 = vs3;
|
||||
|
||||
size_t k = ALIGN_DOWN(MIN(len, NMAX), 32);
|
||||
len -= k;
|
||||
|
||||
while (k >= 64) {
|
||||
__m256i vbuf = __lasx_xvld(src, 0);
|
||||
__m256i vbuf_0 = __lasx_xvld(src, 32);
|
||||
src += 64;
|
||||
k -= 64;
|
||||
|
||||
__m256i vs1_sad = lasx_sad_bu(vbuf, zero);
|
||||
__m256i vs1_sad2 = lasx_sad_bu(vbuf_0, zero);
|
||||
|
||||
if (COPY) {
|
||||
__lasx_xvst(vbuf, dst, 0);
|
||||
__lasx_xvst(vbuf_0, dst, 32);
|
||||
dst += 64;
|
||||
}
|
||||
|
||||
vs1 = __lasx_xvadd_w(vs1, vs1_sad);
|
||||
vs3 = __lasx_xvadd_w(vs3, vs1_0);
|
||||
__m256i v_short_sum2 = lasx_maddubs_w_h(vbuf, dot2v); // sum 32 uint8s to 16 shorts
|
||||
__m256i v_short_sum2_0 = lasx_maddubs_w_h(vbuf_0, dot2v_0); // sum 32 uint8s to 16 shorts
|
||||
__m256i vsum2 = lasx_madd_w_h(v_short_sum2, dot3v); // sum 16 shorts to 8 uint32s
|
||||
__m256i vsum2_0 = lasx_madd_w_h(v_short_sum2_0, dot3v); // sum 16 shorts to 8 uint32s
|
||||
vs1 = __lasx_xvadd_w(vs1_sad2, vs1);
|
||||
vs2 = __lasx_xvadd_w(vsum2, vs2);
|
||||
vs2_0 = __lasx_xvadd_w(vsum2_0, vs2_0);
|
||||
vs1_0 = vs1;
|
||||
}
|
||||
|
||||
vs2 = __lasx_xvadd_w(vs2_0, vs2);
|
||||
vs3 = __lasx_xvslli_w(vs3, 6);
|
||||
vs2 = __lasx_xvadd_w(vs3, vs2);
|
||||
vs3 = __lasx_xvldi(0);
|
||||
|
||||
while (k >= 32) {
|
||||
/*
|
||||
vs1 = adler + sum(c[i])
|
||||
vs2 = sum2 + 32 vs1 + sum( (32-i+1) c[i] )
|
||||
*/
|
||||
__m256i vbuf = __lasx_xvld(src, 0);
|
||||
src += 32;
|
||||
k -= 32;
|
||||
|
||||
__m256i vs1_sad = lasx_sad_bu(vbuf, zero); // Sum of abs diff, resulting in 2 x int32's
|
||||
|
||||
if (COPY) {
|
||||
__lasx_xvst(vbuf, dst, 0);
|
||||
dst += 32;
|
||||
}
|
||||
|
||||
vs1 = __lasx_xvadd_w(vs1, vs1_sad);
|
||||
vs3 = __lasx_xvadd_w(vs3, vs1_0);
|
||||
__m256i v_short_sum2 = lasx_maddubs_w_h(vbuf, dot2v_0); // sum 32 uint8s to 16 shorts
|
||||
__m256i vsum2 = lasx_madd_w_h(v_short_sum2, dot3v); // sum 16 shorts to 8 uint32s
|
||||
vs2 = __lasx_xvadd_w(vsum2, vs2);
|
||||
vs1_0 = vs1;
|
||||
}
|
||||
|
||||
/* Defer the multiplication with 32 to outside of the loop */
|
||||
vs3 = __lasx_xvslli_w(vs3, 5);
|
||||
vs2 = __lasx_xvadd_w(vs2, vs3);
|
||||
|
||||
adler0 = partial_hsum256(vs1) % BASE;
|
||||
adler1 = hsum256(vs2) % BASE;
|
||||
}
|
||||
|
||||
adler = adler0 | (adler1 << 16);
|
||||
|
||||
if (len) {
|
||||
goto rem_peel;
|
||||
}
|
||||
|
||||
return adler;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_lasx(uint32_t adler, const uint8_t *src, size_t len) {
|
||||
return adler32_copy_impl(adler, NULL, src, len, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_copy_lasx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
return adler32_copy_impl(adler, dst, src, len, 1);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,147 +0,0 @@
|
|||
/* adler32_lsx.c -- compute the Adler-32 checksum of a data stream, based on Intel SSE4.2 implementation
|
||||
* Copyright (C) 1995-2011 Mark Adler
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* Authors:
|
||||
* Adam Stylinski <kungfujesus06@gmail.com>
|
||||
* Brian Bockelman <bockelman@gmail.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_LSX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "adler32_p.h"
|
||||
|
||||
#include <lsxintrin.h>
|
||||
#include "lsxintrin_ext.h"
|
||||
|
||||
static inline uint32_t partial_hsum(__m128i x) {
|
||||
__m128i second_int = __lsx_vbsrl_v(x, 8);
|
||||
__m128i sum = __lsx_vadd_w(x, second_int);
|
||||
return __lsx_vpickve2gr_w(sum, 0);
|
||||
}
|
||||
|
||||
static inline uint32_t hsum(__m128i x) {
|
||||
__m128i sum1 = __lsx_vilvh_d(x, x);
|
||||
__m128i sum2 = __lsx_vadd_w(x, sum1);
|
||||
__m128i sum3 = __lsx_vshuf4i_w(sum2, 0x01);
|
||||
__m128i sum4 = __lsx_vadd_w(sum2, sum3);
|
||||
return __lsx_vpickve2gr_w(sum4, 0);
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len, const int COPY) {
|
||||
uint32_t adler0, adler1;
|
||||
adler1 = (adler >> 16) & 0xffff;
|
||||
adler0 = adler & 0xffff;
|
||||
|
||||
rem_peel:
|
||||
if (len < 16)
|
||||
return adler32_copy_tail(adler0, dst, src, len, adler1, 1, 15, COPY);
|
||||
|
||||
__m128i vbuf, vbuf_0;
|
||||
__m128i vs1_0, vs3, vs1, vs2, vs2_0, v_sad_sum1, v_short_sum2, v_short_sum2_0,
|
||||
v_sad_sum2, vsum2, vsum2_0;
|
||||
__m128i zero = __lsx_vldi(0);
|
||||
const __m128i dot2v = (__m128i)((v16i8){ 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17 });
|
||||
const __m128i dot2v_0 = (__m128i)((v16i8){ 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 });
|
||||
const __m128i dot3v = __lsx_vreplgr2vr_h(1);
|
||||
size_t k;
|
||||
|
||||
while (len >= 16) {
|
||||
|
||||
k = ALIGN_DOWN(MIN(len, NMAX), 16);
|
||||
len -= k;
|
||||
|
||||
vs1 = __lsx_vinsgr2vr_w(zero, adler0, 0);
|
||||
vs2 = __lsx_vinsgr2vr_w(zero, adler1, 0);
|
||||
|
||||
vs3 = __lsx_vldi(0);
|
||||
vs2_0 = __lsx_vldi(0);
|
||||
vs1_0 = vs1;
|
||||
|
||||
while (k >= 32) {
|
||||
/*
|
||||
vs1 = adler + sum(c[i])
|
||||
vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] )
|
||||
*/
|
||||
vbuf = __lsx_vld(src, 0);
|
||||
vbuf_0 = __lsx_vld(src, 16);
|
||||
src += 32;
|
||||
k -= 32;
|
||||
|
||||
v_sad_sum1 = lsx_sad_bu(vbuf, zero);
|
||||
v_sad_sum2 = lsx_sad_bu(vbuf_0, zero);
|
||||
|
||||
if (COPY) {
|
||||
__lsx_vst(vbuf, dst, 0);
|
||||
__lsx_vst(vbuf_0, dst, 16);
|
||||
dst += 32;
|
||||
}
|
||||
|
||||
v_short_sum2 = __lsx_vsadd_h(__lsx_vmulwev_h_bu_b(vbuf, dot2v), __lsx_vmulwod_h_bu_b(vbuf, dot2v));
|
||||
v_short_sum2_0 = __lsx_vsadd_h(__lsx_vmulwev_h_bu_b(vbuf_0, dot2v_0), __lsx_vmulwod_h_bu_b(vbuf_0, dot2v_0));
|
||||
|
||||
vs1 = __lsx_vadd_w(v_sad_sum1, vs1);
|
||||
vs3 = __lsx_vadd_w(vs1_0, vs3);
|
||||
|
||||
vsum2 = __lsx_vmaddwod_w_h(__lsx_vmulwev_w_h(v_short_sum2, dot3v), v_short_sum2, dot3v);
|
||||
vsum2_0 = __lsx_vmaddwod_w_h(__lsx_vmulwev_w_h(v_short_sum2_0, dot3v), v_short_sum2_0, dot3v);
|
||||
vs1 = __lsx_vadd_w(v_sad_sum2, vs1);
|
||||
vs2 = __lsx_vadd_w(vsum2, vs2);
|
||||
vs2_0 = __lsx_vadd_w(vsum2_0, vs2_0);
|
||||
vs1_0 = vs1;
|
||||
}
|
||||
|
||||
vs2 = __lsx_vadd_w(vs2_0, vs2);
|
||||
vs3 = __lsx_vslli_w(vs3, 5);
|
||||
vs2 = __lsx_vadd_w(vs3, vs2);
|
||||
vs3 = __lsx_vldi(0);
|
||||
|
||||
while (k >= 16) {
|
||||
/*
|
||||
vs1 = adler + sum(c[i])
|
||||
vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] )
|
||||
*/
|
||||
vbuf = __lsx_vld(src, 0);
|
||||
src += 16;
|
||||
k -= 16;
|
||||
|
||||
v_sad_sum1 = lsx_sad_bu(vbuf, zero);
|
||||
v_short_sum2 = __lsx_vsadd_h(__lsx_vmulwev_h_bu_b(vbuf, dot2v_0), __lsx_vmulwod_h_bu_b(vbuf, dot2v_0));
|
||||
|
||||
vs1 = __lsx_vadd_w(v_sad_sum1, vs1);
|
||||
vs3 = __lsx_vadd_w(vs1_0, vs3);
|
||||
vsum2 = __lsx_vmaddwod_w_h(__lsx_vmulwev_w_h(v_short_sum2, dot3v), v_short_sum2, dot3v);
|
||||
vs2 = __lsx_vadd_w(vsum2, vs2);
|
||||
vs1_0 = vs1;
|
||||
|
||||
if (COPY) {
|
||||
__lsx_vst(vbuf, dst, 0);
|
||||
dst += 16;
|
||||
}
|
||||
}
|
||||
|
||||
vs3 = __lsx_vslli_w(vs3, 4);
|
||||
vs2 = __lsx_vadd_w(vs2, vs3);
|
||||
|
||||
adler0 = partial_hsum(vs1) % BASE;
|
||||
adler1 = hsum(vs2) % BASE;
|
||||
}
|
||||
|
||||
/* If this is true, there's fewer than 16 elements remaining */
|
||||
if (len) {
|
||||
goto rem_peel;
|
||||
}
|
||||
|
||||
return adler0 | (adler1 << 16);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_lsx(uint32_t adler, const uint8_t *src, size_t len) {
|
||||
return adler32_copy_impl(adler, NULL, src, len, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_copy_lsx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
return adler32_copy_impl(adler, dst, src, len, 1);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,126 +0,0 @@
|
|||
/* chunkset_lasx.c -- LASX inline functions to copy small data chunks, based on Intel AVX2 implementation
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_LASX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zsanitizer.h"
|
||||
#include "zmemory.h"
|
||||
|
||||
#include <lasxintrin.h>
|
||||
#include "lasxintrin_ext.h"
|
||||
#include "lsxintrin_ext.h"
|
||||
|
||||
#include "arch/shared/chunk_256bit_perm_idx_lut.h"
|
||||
|
||||
typedef __m256i chunk_t;
|
||||
typedef __m128i halfchunk_t;
|
||||
|
||||
#define HAVE_CHUNKMEMSET_2
|
||||
#define HAVE_CHUNKMEMSET_4
|
||||
#define HAVE_CHUNKMEMSET_8
|
||||
#define HAVE_CHUNKMEMSET_16
|
||||
#define HAVE_CHUNK_MAG
|
||||
#define HAVE_HALF_CHUNK
|
||||
|
||||
static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = __lasx_xvreplgr2vr_h(zng_memread_2(from));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = __lasx_xvreplgr2vr_w(zng_memread_4(from));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = __lasx_xvreplgr2vr_d(zng_memread_8(from));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_16(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = lasx_broadcast_128(__lsx_vld(from, 0));
|
||||
}
|
||||
|
||||
static inline void loadchunk(uint8_t const *s, chunk_t *chunk) {
|
||||
*chunk = __lasx_xvld(s, 0);
|
||||
}
|
||||
|
||||
static inline void storechunk(uint8_t *out, chunk_t *chunk) {
|
||||
__lasx_xvst(*chunk, out, 0);
|
||||
}
|
||||
|
||||
static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, size_t *chunk_rem, size_t dist) {
|
||||
lut_rem_pair lut_rem = perm_idx_lut[dist - 3];
|
||||
__m256i ret_vec;
|
||||
/* While technically we only need to read 4 or 8 bytes into this vector register for a lot of cases, GCC is
|
||||
* compiling this to a shared load for all branches, preferring the simpler code. Given that the buf value isn't in
|
||||
* GPRs to begin with the 256 bit load is _probably_ just as inexpensive */
|
||||
*chunk_rem = lut_rem.remval;
|
||||
|
||||
/* See note in chunkset_ssse3.c for why this is ok */
|
||||
__msan_unpoison(buf + dist, 32 - dist);
|
||||
|
||||
if (dist < 16) {
|
||||
/* This simpler case still requires us to shuffle in 128 bit lanes, so we must apply a static offset after
|
||||
* broadcasting the first vector register to both halves. This is _marginally_ faster than doing two separate
|
||||
* shuffles and combining the halves later */
|
||||
__m256i perm_vec = __lasx_xvld(permute_table+lut_rem.idx, 0);
|
||||
__m128i ret_vec0 = __lsx_vld(buf, 0);
|
||||
ret_vec = __lasx_concat_128(ret_vec0, ret_vec0);
|
||||
ret_vec = lasx_shuffle_b(ret_vec, perm_vec);
|
||||
} else {
|
||||
__m128i ret_vec0 = __lsx_vld(buf, 0);
|
||||
__m128i ret_vec1 = __lsx_vld(buf, 16);
|
||||
/* Take advantage of the fact that only the latter half of the 256 bit vector will actually differ */
|
||||
__m128i perm_vec1 = __lsx_vld(permute_table + lut_rem.idx, 0);
|
||||
__m128i xlane_permutes = __lsx_vslt_b(perm_vec1, __lsx_vreplgr2vr_b(16));
|
||||
__m128i xlane_res = lsx_shuffle_b(ret_vec0, perm_vec1);
|
||||
/* Since we can't wrap twice, we can simply keep the later half exactly how it is instead of having to _also_
|
||||
* shuffle those values */
|
||||
__m128i latter_half = __lsx_vbitsel_v(ret_vec1, xlane_res, xlane_permutes);
|
||||
ret_vec = __lasx_concat_128(ret_vec0, latter_half);
|
||||
}
|
||||
|
||||
return ret_vec;
|
||||
}
|
||||
|
||||
static inline void loadhalfchunk(uint8_t const *s, halfchunk_t *chunk) {
|
||||
*chunk = __lsx_vld(s, 0);
|
||||
}
|
||||
|
||||
static inline void storehalfchunk(uint8_t *out, halfchunk_t *chunk) {
|
||||
__lsx_vst(*chunk, out, 0);
|
||||
}
|
||||
|
||||
static inline chunk_t halfchunk2whole(halfchunk_t *chunk) {
|
||||
/* We zero extend mostly to appease some memory sanitizers. These bytes are ultimately
|
||||
* unlikely to be actually written or read from */
|
||||
return lasx_zext_128(*chunk);
|
||||
}
|
||||
|
||||
static inline halfchunk_t GET_HALFCHUNK_MAG(uint8_t *buf, size_t *chunk_rem, size_t dist) {
|
||||
lut_rem_pair lut_rem = perm_idx_lut[dist - 3];
|
||||
__m128i perm_vec, ret_vec;
|
||||
__msan_unpoison(buf + dist, 16 - dist);
|
||||
ret_vec = __lsx_vld(buf, 0);
|
||||
*chunk_rem = half_rem_vals[dist - 3];
|
||||
|
||||
perm_vec = __lsx_vld(permute_table + lut_rem.idx, 0);
|
||||
ret_vec = lsx_shuffle_b(ret_vec, perm_vec);
|
||||
|
||||
return ret_vec;
|
||||
}
|
||||
|
||||
#define CHUNKSIZE chunksize_lasx
|
||||
#define CHUNKCOPY chunkcopy_lasx
|
||||
#define CHUNKUNROLL chunkunroll_lasx
|
||||
#define CHUNKMEMSET chunkmemset_lasx
|
||||
#define CHUNKMEMSET_SAFE chunkmemset_safe_lasx
|
||||
|
||||
#include "chunkset_tpl.h"
|
||||
|
||||
#define INFLATE_FAST inflate_fast_lasx
|
||||
|
||||
#include "inffast_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
/* chunkset_lsx.c -- LSX inline functions to copy small data chunks, based on Intel SSSE3 implementation
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_LSX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zsanitizer.h"
|
||||
#include "zmemory.h"
|
||||
|
||||
#include <lsxintrin.h>
|
||||
#include "lsxintrin_ext.h"
|
||||
#include "arch/shared/chunk_128bit_perm_idx_lut.h"
|
||||
|
||||
typedef __m128i chunk_t;
|
||||
|
||||
#define HAVE_CHUNKMEMSET_2
|
||||
#define HAVE_CHUNKMEMSET_4
|
||||
#define HAVE_CHUNKMEMSET_8
|
||||
#define HAVE_CHUNK_MAG
|
||||
|
||||
|
||||
static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = __lsx_vreplgr2vr_h(zng_memread_2(from));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = __lsx_vreplgr2vr_w(zng_memread_4(from));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = __lsx_vreplgr2vr_d(zng_memread_8(from));
|
||||
}
|
||||
|
||||
static inline void loadchunk(uint8_t const *s, chunk_t *chunk) {
|
||||
*chunk = __lsx_vld(s, 0);
|
||||
}
|
||||
|
||||
static inline void storechunk(uint8_t *out, chunk_t *chunk) {
|
||||
__lsx_vst(*chunk, out, 0);
|
||||
}
|
||||
|
||||
static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, size_t *chunk_rem, size_t dist) {
|
||||
lut_rem_pair lut_rem = perm_idx_lut[dist - 3];
|
||||
__m128i perm_vec, ret_vec;
|
||||
/* Important to note:
|
||||
* This is _not_ to subvert the memory sanitizer but to instead unpoison some
|
||||
* bytes we willingly and purposefully load uninitialized that we swizzle over
|
||||
* in a vector register, anyway. If what we assume is wrong about what is used,
|
||||
* the memory sanitizer will still usefully flag it */
|
||||
__msan_unpoison(buf + dist, 16 - dist);
|
||||
ret_vec = __lsx_vld(buf, 0);
|
||||
*chunk_rem = lut_rem.remval;
|
||||
|
||||
perm_vec = __lsx_vld(permute_table + lut_rem.idx, 0);
|
||||
ret_vec = lsx_shuffle_b(ret_vec, perm_vec);
|
||||
|
||||
return ret_vec;
|
||||
}
|
||||
|
||||
#define CHUNKSIZE chunksize_lsx
|
||||
#define CHUNKMEMSET chunkmemset_lsx
|
||||
#define CHUNKMEMSET_SAFE chunkmemset_safe_lsx
|
||||
#define CHUNKCOPY chunkcopy_lsx
|
||||
#define CHUNKUNROLL chunkunroll_lsx
|
||||
|
||||
#include "chunkset_tpl.h"
|
||||
|
||||
#define INFLATE_FAST inflate_fast_lsx
|
||||
|
||||
#include "inffast_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
/* compare256_lasx.c -- LASX version of compare256, based on Intel AVX2 implementation
|
||||
* Copyright Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zendian.h"
|
||||
#include "zmemory.h"
|
||||
#include "deflate.h"
|
||||
#include "fallback_builtins.h"
|
||||
|
||||
#ifdef LOONGARCH_LASX
|
||||
|
||||
#include <lasxintrin.h>
|
||||
#include "lasxintrin_ext.h"
|
||||
|
||||
static inline uint32_t compare256_lasx_static(const uint8_t *src0, const uint8_t *src1) {
|
||||
uint32_t len = 0;
|
||||
|
||||
do {
|
||||
__m256i ymm_src0, ymm_src1, ymm_cmp;
|
||||
ymm_src0 = __lasx_xvld(src0, 0);
|
||||
ymm_src1 = __lasx_xvld(src1, 0);
|
||||
ymm_cmp = __lasx_xvseq_b(ymm_src0, ymm_src1); /* non-identical bytes = 00, identical bytes = FF */
|
||||
unsigned mask = (unsigned)lasx_movemask_b(ymm_cmp);
|
||||
if (mask != 0xFFFFFFFF)
|
||||
return len + zng_ctz32(~mask); /* Invert bits so identical = 0 */
|
||||
|
||||
src0 += 32, src1 += 32, len += 32;
|
||||
|
||||
ymm_src0 = __lasx_xvld(src0, 0);
|
||||
ymm_src1 = __lasx_xvld(src1, 0);
|
||||
ymm_cmp = __lasx_xvseq_b(ymm_src0, ymm_src1);
|
||||
mask = (unsigned)lasx_movemask_b(ymm_cmp);
|
||||
if (mask != 0xFFFFFFFF)
|
||||
return len + zng_ctz32(~mask);
|
||||
|
||||
src0 += 32, src1 += 32, len += 32;
|
||||
} while (len < 256);
|
||||
|
||||
return 256;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t compare256_lasx(const uint8_t *src0, const uint8_t *src1) {
|
||||
return compare256_lasx_static(src0, src1);
|
||||
}
|
||||
|
||||
#define LONGEST_MATCH longest_match_lasx
|
||||
#define COMPARE256 compare256_lasx_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#define LONGEST_MATCH_ROLL
|
||||
#define LONGEST_MATCH longest_match_roll_lasx
|
||||
#define COMPARE256 compare256_lasx_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
/* compare256_lsx.c -- LSX version of compare256, based on Intel SSE implementation
|
||||
* Copyright Adam Stylinski <kungfujesus06@gmail.com>
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zendian.h"
|
||||
#include "zmemory.h"
|
||||
#include "deflate.h"
|
||||
#include "fallback_builtins.h"
|
||||
|
||||
#ifdef LOONGARCH_LSX
|
||||
|
||||
#include <lsxintrin.h>
|
||||
#include "lsxintrin_ext.h"
|
||||
|
||||
static inline uint32_t compare256_lsx_static(const uint8_t *src0, const uint8_t *src1) {
|
||||
__m128i xmm_src0, xmm_src1, xmm_cmp;
|
||||
|
||||
/* Do the first load unaligned, than all subsequent ones we have at least
|
||||
* one aligned load. Sadly aligning both loads is probably unrealistic */
|
||||
xmm_src0 = __lsx_vld(src0, 0);
|
||||
xmm_src1 = __lsx_vld(src1, 0);
|
||||
xmm_cmp = __lsx_vseq_b(xmm_src0, xmm_src1);
|
||||
|
||||
unsigned mask = (unsigned)lsx_movemask_b(xmm_cmp);
|
||||
|
||||
/* Compiler _may_ turn this branch into a ptest + movemask,
|
||||
* since a lot of those uops are shared and fused */
|
||||
if (mask != 0xFFFF)
|
||||
return zng_ctz32(~mask);
|
||||
|
||||
const uint8_t *last0 = src0 + 240;
|
||||
const uint8_t *last1 = src1 + 240;
|
||||
|
||||
int align_offset = ((uintptr_t)src0) & 15;
|
||||
int align_adv = 16 - align_offset;
|
||||
uint32_t len = align_adv;
|
||||
|
||||
src0 += align_adv;
|
||||
src1 += align_adv;
|
||||
|
||||
for (int i = 0; i < 15; i++) {
|
||||
xmm_src0 = __lsx_vld(src0, 0);
|
||||
xmm_src1 = __lsx_vld(src1, 0);
|
||||
xmm_cmp = __lsx_vseq_b(xmm_src0, xmm_src1);
|
||||
|
||||
mask = (unsigned)lsx_movemask_b(xmm_cmp);
|
||||
|
||||
/* Compiler _may_ turn this branch into a ptest + movemask,
|
||||
* since a lot of those uops are shared and fused */
|
||||
if (mask != 0xFFFF)
|
||||
return len + zng_ctz32(~mask);
|
||||
|
||||
len += 16, src0 += 16, src1 += 16;
|
||||
}
|
||||
|
||||
if (align_offset) {
|
||||
xmm_src0 = __lsx_vld(last0, 0);
|
||||
xmm_src1 = __lsx_vld(last1, 0);
|
||||
xmm_cmp = __lsx_vseq_b(xmm_src0, xmm_src1);
|
||||
|
||||
mask = (unsigned)lsx_movemask_b(xmm_cmp);
|
||||
|
||||
if (mask != 0xFFFF)
|
||||
return 240 + zng_ctz32(~mask);
|
||||
}
|
||||
|
||||
return 256;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t compare256_lsx(const uint8_t *src0, const uint8_t *src1) {
|
||||
return compare256_lsx_static(src0, src1);
|
||||
}
|
||||
|
||||
#define LONGEST_MATCH longest_match_lsx
|
||||
#define COMPARE256 compare256_lsx_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#define LONGEST_MATCH_ROLL
|
||||
#define LONGEST_MATCH longest_match_roll_lsx
|
||||
#define COMPARE256 compare256_lsx_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
/* crc32_la.c - LoongArch version of crc32
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_CRC
|
||||
|
||||
#include "zbuild.h"
|
||||
|
||||
#include <larchintrin.h>
|
||||
|
||||
#define Z_TARGET_CRC
|
||||
#define CRC32B(crc, val) (uint32_t)__crc_w_b_w((char)(val), (int)(crc))
|
||||
#define CRC32H(crc, val) (uint32_t)__crc_w_h_w((short)(val), (int)(crc))
|
||||
#define CRC32W(crc, val) (uint32_t)__crc_w_w_w((int)(val), (int)(crc))
|
||||
#define CRC32D(crc, val) (uint32_t)__crc_w_d_w((long int)(val), (int)(crc))
|
||||
|
||||
#include "arch/shared/crc32_hw_common_tpl.h"
|
||||
|
||||
#include "arch/shared/crc32_hw_copy_impl_tpl.h"
|
||||
|
||||
|
||||
Z_INTERNAL uint32_t crc32_loongarch64(uint32_t crc, const uint8_t *buf, size_t len) {
|
||||
return crc32_hw_copy_impl(crc, NULL, buf, len, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t crc32_copy_loongarch64(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
return crc32_hw_copy_impl(crc, dst, src, len, 1);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
/* lasxintrin_ext.h
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
#ifndef LASXINTRIN_EXT_H
|
||||
#define LASXINTRIN_EXT_H
|
||||
|
||||
#include <lsxintrin.h>
|
||||
#include <lasxintrin.h>
|
||||
|
||||
|
||||
static inline __m256i lasx_zext_128(__m128i src) {
|
||||
#ifdef __loongarch_asx_sx_conv
|
||||
return __lasx_insert_128_lo(__lasx_xvldi(0), src);
|
||||
#else
|
||||
__m256i dest = __lasx_xvldi(0);
|
||||
__asm__ volatile ("xvpermi.q %u0,%u2,0x30\n" : "=f"(dest) : "0"(dest), "f"(src));
|
||||
return dest;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef __loongarch_asx_sx_conv
|
||||
static inline __m256i __lasx_concat_128(__m128i lo, __m128i hi) {
|
||||
__m256i dest;
|
||||
__asm__ volatile ("xvpermi.q %u0,%u2,0x02\n" : "=f"(dest) : "0"(lo), "f"(hi));
|
||||
return dest;
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline __m256i lasx_broadcast_128(__m128i in) {
|
||||
return __lasx_concat_128(in, in);
|
||||
}
|
||||
|
||||
static inline __m256i lasx_sad_bu(__m256i a, __m256i b) {
|
||||
__m256i tmp = __lasx_xvabsd_bu(a, b);
|
||||
tmp = __lasx_xvhaddw_hu_bu(tmp, tmp);
|
||||
tmp = __lasx_xvhaddw_wu_hu(tmp, tmp);
|
||||
return __lasx_xvhaddw_du_wu(tmp, tmp);
|
||||
}
|
||||
|
||||
static inline __m256i lasx_maddubs_w_h(__m256i a, __m256i b) {
|
||||
return __lasx_xvsadd_h(__lasx_xvmulwod_h_bu_b(a, b), __lasx_xvmulwev_h_bu_b(a, b));
|
||||
}
|
||||
|
||||
static inline __m256i lasx_madd_w_h(__m256i a, __m256i b) {
|
||||
return __lasx_xvmaddwod_w_h(__lasx_xvmulwev_w_h(a, b), a, b);
|
||||
}
|
||||
|
||||
static inline int lasx_movemask_b(__m256i v) {
|
||||
v = __lasx_xvmskltz_b(v);
|
||||
return __lasx_xvpickve2gr_w(v, 0) | (__lasx_xvpickve2gr_w(v, 4) << 16);
|
||||
}
|
||||
|
||||
/* See: lsx_shuffle_b */
|
||||
static inline __m256i lasx_shuffle_b(__m256i a, __m256i b) {
|
||||
__m256i msb_mask = __lasx_xvslti_b(b, 0);
|
||||
__m256i dst = __lasx_xvshuf_b(a, a, __lasx_xvandi_b(b, 0xF));
|
||||
return __lasx_xvand_v(dst, __lasx_xvnor_v(msb_mask, msb_mask));
|
||||
}
|
||||
|
||||
#endif // include guard LASXINTRIN_EXT_H
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
/* loongarch_features.c -- check for LoongArch features.
|
||||
*
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_FEATURES
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "loongarch_features.h"
|
||||
|
||||
#include <larchintrin.h>
|
||||
|
||||
/*
|
||||
* https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html
|
||||
*
|
||||
* Word number Bit number Annotation Implication
|
||||
* 0x1 25 CRC 1 indicates support for CRC instruction
|
||||
* 0x1 6 LSX 1 indicates support for 128-bit vector extension
|
||||
* 0x1 7 LASX 1 indicates support for 256-bit vector expansion
|
||||
*/
|
||||
|
||||
void Z_INTERNAL loongarch_check_features(struct loongarch_cpu_features *features) {
|
||||
unsigned int w1 = __cpucfg(0x1);
|
||||
features->has_crc = w1 & 0x2000000;
|
||||
features->has_lsx = w1 & 0x40;
|
||||
features->has_lasx = w1 & 0x80;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
/* loongarch_features.h -- check for LoongArch features.
|
||||
*
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef LOONGARCH_FEATURES_H_
|
||||
#define LOONGARCH_FEATURES_H_
|
||||
|
||||
struct loongarch_cpu_features {
|
||||
int has_crc;
|
||||
int has_lsx;
|
||||
int has_lasx;
|
||||
};
|
||||
|
||||
void Z_INTERNAL loongarch_check_features(struct loongarch_cpu_features *features);
|
||||
|
||||
#endif /* LOONGARCH_FEATURES_H_ */
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
/* loongarch_functions.h -- LoongArch implementations for arch-specific functions.
|
||||
*
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef LOONGARCH_FUNCTIONS_H_
|
||||
#define LOONGARCH_FUNCTIONS_H_
|
||||
|
||||
#include "loongarch_natives.h"
|
||||
|
||||
#ifdef LOONGARCH_CRC
|
||||
uint32_t crc32_loongarch64(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_loongarch64(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
#endif
|
||||
|
||||
#ifndef LOONGARCH_CRC_NATIVE
|
||||
# define CRC32_BRAID_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef LOONGARCH_LSX
|
||||
uint32_t adler32_lsx(uint32_t adler, const uint8_t *src, size_t len);
|
||||
uint32_t adler32_copy_lsx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
uint8_t* chunkmemset_safe_lsx(uint8_t *out, uint8_t *from, size_t len, size_t left);
|
||||
uint32_t compare256_lsx(const uint8_t *src0, const uint8_t *src1);
|
||||
void inflate_fast_lsx(PREFIX3(stream) *strm, uint32_t start);
|
||||
uint32_t longest_match_lsx(deflate_state *const s, uint32_t cur_match);
|
||||
uint32_t longest_match_roll_lsx(deflate_state *const s, uint32_t cur_match);
|
||||
void slide_hash_lsx(deflate_state *s);
|
||||
#endif
|
||||
|
||||
#ifndef LOONGARCH_LSX_NATIVE
|
||||
# define ADLER32_FALLBACK
|
||||
# define CHUNKSET_FALLBACK
|
||||
# define COMPARE256_FALLBACK
|
||||
# define SLIDE_HASH_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef LOONGARCH_LASX
|
||||
uint32_t adler32_lasx(uint32_t adler, const uint8_t *src, size_t len);
|
||||
uint32_t adler32_copy_lasx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
uint8_t* chunkmemset_safe_lasx(uint8_t *out, uint8_t *from, size_t len, size_t left);
|
||||
uint32_t compare256_lasx(const uint8_t *src0, const uint8_t *src1);
|
||||
void inflate_fast_lasx(PREFIX3(stream) *strm, uint32_t start);
|
||||
uint32_t longest_match_lasx(deflate_state *const s, uint32_t cur_match);
|
||||
uint32_t longest_match_roll_lasx(deflate_state *const s, uint32_t cur_match);
|
||||
void slide_hash_lasx(deflate_state *s);
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_RUNTIME_CPU_DETECTION
|
||||
// LOONGARCH - CRC32
|
||||
# ifdef LOONGARCH_CRC_NATIVE
|
||||
# undef native_crc32
|
||||
# define native_crc32 crc32_loongarch64
|
||||
# undef native_crc32_copy
|
||||
# define native_crc32_copy crc32_copy_loongarch64
|
||||
# endif
|
||||
# ifdef LOONGARCH_LSX_NATIVE
|
||||
# undef native_adler32
|
||||
# define native_adler32 adler32_lsx
|
||||
# undef native_adler32_copy
|
||||
# define native_adler32_copy adler32_copy_lsx
|
||||
# undef native_chunkmemset_safe
|
||||
# define native_chunkmemset_safe chunkmemset_safe_lsx
|
||||
# undef native_compare256
|
||||
# define native_compare256 compare256_lsx
|
||||
# undef native_inflate_fast
|
||||
# define native_inflate_fast inflate_fast_lsx
|
||||
# undef native_longest_match
|
||||
# define native_longest_match longest_match_lsx
|
||||
# undef native_longest_match_roll
|
||||
# define native_longest_match_roll longest_match_roll_lsx
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_lsx
|
||||
# endif
|
||||
# ifdef LOONGARCH_LASX_NATIVE
|
||||
# undef native_adler32
|
||||
# define native_adler32 adler32_lasx
|
||||
# undef native_adler32_copy
|
||||
# define native_adler32_copy adler32_copy_lasx
|
||||
# undef native_chunkmemset_safe
|
||||
# define native_chunkmemset_safe chunkmemset_safe_lasx
|
||||
# undef native_compare256
|
||||
# define native_compare256 compare256_lasx
|
||||
# undef native_inflate_fast
|
||||
# define native_inflate_fast inflate_fast_lasx
|
||||
# undef native_longest_match
|
||||
# define native_longest_match longest_match_lasx
|
||||
# undef native_longest_match_roll
|
||||
# define native_longest_match_roll longest_match_roll_lasx
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_lasx
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* LOONGARCH_FUNCTIONS_H_ */
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
/* loongarch_natives.h -- LoongArch compile-time feature detection macros.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef LOONGARCH_NATIVES_H_
|
||||
#define LOONGARCH_NATIVES_H_
|
||||
|
||||
#if defined(__loongarch__)
|
||||
// All known CPUs have crc instructions
|
||||
# ifdef LOONGARCH_CRC
|
||||
# define LOONGARCH_CRC_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
#if defined(__loongarch_sx)
|
||||
# ifdef LOONGARCH_LSX
|
||||
# define LOONGARCH_LSX_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
#if defined(__loongarch_asx)
|
||||
# ifdef LOONGARCH_LASX
|
||||
# define LOONGARCH_LASX_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* LOONGARCH_NATIVES_H_ */
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
/* lsxintrin_ext.h
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
#ifndef LSXINTRIN_EXT_H
|
||||
#define LSXINTRIN_EXT_H
|
||||
|
||||
#include <lsxintrin.h>
|
||||
|
||||
|
||||
static inline __m128i lsx_sad_bu(__m128i a, __m128i b) {
|
||||
__m128i tmp = __lsx_vabsd_bu(a, b);
|
||||
tmp = __lsx_vhaddw_hu_bu(tmp, tmp);
|
||||
tmp = __lsx_vhaddw_wu_hu(tmp, tmp);
|
||||
return __lsx_vhaddw_du_wu(tmp, tmp);
|
||||
}
|
||||
|
||||
static inline int lsx_movemask_b(__m128i v) {
|
||||
return __lsx_vpickve2gr_w(__lsx_vmskltz_b(v), 0);
|
||||
}
|
||||
|
||||
static inline __m128i lsx_shuffle_b(__m128i a, __m128i b) {
|
||||
/* most significant bit is set - negative 8-bit integer */
|
||||
__m128i msb_mask = __lsx_vslti_b(b, 0);
|
||||
|
||||
/* shuffle, clear msb in indices vector b */
|
||||
__m128i dst = __lsx_vshuf_b(a, a, __lsx_vandi_b(b, 0xF));
|
||||
|
||||
/* invert and apply mask - clear dst-element if b-msb is set */
|
||||
return __lsx_vand_v(dst, __lsx_vnor_v(msb_mask, msb_mask));
|
||||
}
|
||||
|
||||
#endif // include guard LSXINTRIN_EXT_H
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
/*
|
||||
* LASX optimized hash slide, based on Intel AVX2 implementation
|
||||
*
|
||||
* Copyright (C) 2017 Intel Corporation
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* Authors:
|
||||
* Arjan van de Ven <arjan@linux.intel.com>
|
||||
* Jim Kukunas <james.t.kukunas@linux.intel.com>
|
||||
* Mika T. Lindqvist <postmaster@raasu.org>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_LASX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "deflate.h"
|
||||
|
||||
#include <lasxintrin.h>
|
||||
|
||||
static inline void slide_hash_chain(Pos *table, uint32_t entries, const __m256i wsize) {
|
||||
table += entries;
|
||||
table -= 32;
|
||||
|
||||
do {
|
||||
__m256i value1, value2, result1, result2;
|
||||
|
||||
value1 = __lasx_xvld(table, 0);
|
||||
value2 = __lasx_xvld(table, 32);
|
||||
result1 = __lasx_xvssub_hu(value1, wsize);
|
||||
result2 = __lasx_xvssub_hu(value2, wsize);
|
||||
__lasx_xvst(result1, table, 0);
|
||||
__lasx_xvst(result2, table, 32);
|
||||
|
||||
table -= 32;
|
||||
entries -= 32;
|
||||
} while (entries > 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL void slide_hash_lasx(deflate_state *s) {
|
||||
Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
const __m256i ymm_wsize = __lasx_xvreplgr2vr_h((short)wsize);
|
||||
|
||||
slide_hash_chain(s->head, HASH_SIZE, ymm_wsize);
|
||||
slide_hash_chain(s->prev, wsize, ymm_wsize);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
/*
|
||||
* LSX optimized hash slide, based on Intel SSE implementation
|
||||
*
|
||||
* Copyright (C) 2017 Intel Corporation
|
||||
* Copyright (C) 2025 Vladislav Shchapov <vladislav@shchapov.ru>
|
||||
* Authors:
|
||||
* Arjan van de Ven <arjan@linux.intel.com>
|
||||
* Jim Kukunas <james.t.kukunas@linux.intel.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef LOONGARCH_LSX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "deflate.h"
|
||||
|
||||
#include <lsxintrin.h>
|
||||
#include <assert.h>
|
||||
|
||||
static inline void slide_hash_chain(Pos *table, uint32_t entries, const __m128i wsize) {
|
||||
table += entries;
|
||||
table -= 16;
|
||||
|
||||
/* ZALLOC allocates this pointer unless the user chose a custom allocator.
|
||||
* Our alloc function is aligned to 64 byte boundaries */
|
||||
do {
|
||||
__m128i value0, value1, result0, result1;
|
||||
|
||||
value0 = __lsx_vld(table, 0);
|
||||
value1 = __lsx_vld(table, 16);
|
||||
result0 = __lsx_vssub_hu(value0, wsize);
|
||||
result1 = __lsx_vssub_hu(value1, wsize);
|
||||
__lsx_vst(result0, table, 0);
|
||||
__lsx_vst(result1, table, 16);
|
||||
|
||||
table -= 16;
|
||||
entries -= 16;
|
||||
} while (entries > 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL void slide_hash_lsx(deflate_state *s) {
|
||||
Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
const __m128i xmm_wsize = __lsx_vreplgr2vr_h((short)wsize);
|
||||
|
||||
assert(((uintptr_t)s->head & 15) == 0);
|
||||
assert(((uintptr_t)s->prev & 15) == 0);
|
||||
|
||||
slide_hash_chain(s->head, HASH_SIZE, xmm_wsize);
|
||||
slide_hash_chain(s->prev, wsize, xmm_wsize);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
# Makefile for POWER-specific files
|
||||
# Copyright (C) 2020 Matheus Castanho <msc@linux.ibm.com>, IBM
|
||||
# Copyright (C) 2021 Mika T. Lindqvist <postmaster@raasu.org>
|
||||
# For conditions of distribution and use, see copyright notice in zlib.h
|
||||
|
||||
CC=
|
||||
CFLAGS=
|
||||
SFLAGS=
|
||||
INCLUDES=
|
||||
SUFFIX=
|
||||
|
||||
P8FLAGS=-mcpu=power8
|
||||
P9FLAGS=-mcpu=power9
|
||||
PPCFLAGS=-maltivec
|
||||
NOLTOFLAG=
|
||||
|
||||
SRCDIR=.
|
||||
SRCTOP=../..
|
||||
TOPDIR=$(SRCTOP)
|
||||
|
||||
all: power_features.o \
|
||||
power_features.lo \
|
||||
adler32_power8.o \
|
||||
adler32_power8.lo \
|
||||
adler32_vmx.o \
|
||||
adler32_vmx.lo \
|
||||
chunkset_power8.o \
|
||||
chunkset_power8.lo \
|
||||
compare256_power9.o \
|
||||
compare256_power9.lo \
|
||||
crc32_power8.o \
|
||||
crc32_power8.lo \
|
||||
slide_hash_power8.o \
|
||||
slide_hash_power8.lo \
|
||||
slide_hash_vmx.o \
|
||||
slide_hash_vmx.lo
|
||||
|
||||
power_features.o:
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/power_features.c
|
||||
|
||||
power_features.lo:
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/power_features.c
|
||||
|
||||
adler32_power8.o:
|
||||
$(CC) $(CFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_power8.c
|
||||
|
||||
adler32_power8.lo:
|
||||
$(CC) $(SFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_power8.c
|
||||
|
||||
adler32_vmx.o:
|
||||
$(CC) $(CFLAGS) $(PPCFLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_vmx.c
|
||||
|
||||
adler32_vmx.lo:
|
||||
$(CC) $(SFLAGS) $(PPCFLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_vmx.c
|
||||
|
||||
chunkset_power8.o:
|
||||
$(CC) $(CFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_power8.c
|
||||
|
||||
chunkset_power8.lo:
|
||||
$(CC) $(SFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_power8.c
|
||||
|
||||
compare256_power9.o:
|
||||
$(CC) $(CFLAGS) $(P9FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_power9.c
|
||||
|
||||
compare256_power9.lo:
|
||||
$(CC) $(SFLAGS) $(P9FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_power9.c
|
||||
|
||||
crc32_power8.o:
|
||||
$(CC) $(CFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_power8.c
|
||||
|
||||
crc32_power8.lo:
|
||||
$(CC) $(SFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_power8.c
|
||||
|
||||
slide_hash_power8.o:
|
||||
$(CC) $(CFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_power8.c
|
||||
|
||||
slide_hash_power8.lo:
|
||||
$(CC) $(SFLAGS) $(P8FLAGS) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_power8.c
|
||||
|
||||
slide_hash_vmx.o:
|
||||
$(CC) $(CFLAGS) ${PPCFLAGS} $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_vmx.c
|
||||
|
||||
slide_hash_vmx.lo:
|
||||
$(CC) $(SFLAGS) ${PPCFLAGS} $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_vmx.c
|
||||
|
||||
mostlyclean: clean
|
||||
clean:
|
||||
rm -f *.o *.lo *~
|
||||
rm -rf objs
|
||||
rm -f *.gcda *.gcno *.gcov
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
/* Adler32 for POWER8 using VSX instructions.
|
||||
* Copyright (C) 2020 IBM Corporation
|
||||
* Author: Rogerio Alves <rcardoso@linux.ibm.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*
|
||||
* Calculate adler32 checksum for 16 bytes at once using POWER8+ VSX (vector)
|
||||
* instructions.
|
||||
*
|
||||
* If adler32 do 1 byte at time on the first iteration s1 is s1_0 (_n means
|
||||
* iteration n) is the initial value of adler - at start _0 is 1 unless
|
||||
* adler initial value is different than 1. So s1_1 = s1_0 + c[0] after
|
||||
* the first calculation. For the iteration s1_2 = s1_1 + c[1] and so on.
|
||||
* Hence, for iteration N, s1_N = s1_(N-1) + c[N] is the value of s1 on
|
||||
* after iteration N.
|
||||
*
|
||||
* Therefore, for s2 and iteration N, s2_N = s2_0 + N*s1_N + N*c[0] +
|
||||
* N-1*c[1] + ... + c[N]
|
||||
*
|
||||
* In a more general way:
|
||||
*
|
||||
* s1_N = s1_0 + sum(i=1 to N)c[i]
|
||||
* s2_N = s2_0 + N*s1 + sum (i=1 to N)(N-i+1)*c[i]
|
||||
*
|
||||
* Where s1_N, s2_N are the values for s1, s2 after N iterations. So if we
|
||||
* can process N-bit at time we can do this at once.
|
||||
*
|
||||
* Since VSX can support 16-bit vector instructions, we can process
|
||||
* 16-bit at time using N = 16 we have:
|
||||
*
|
||||
* s1 = s1_16 = s1_(16-1) + c[16] = s1_0 + sum(i=1 to 16)c[i]
|
||||
* s2 = s2_16 = s2_0 + 16*s1 + sum(i=1 to 16)(16-i+1)*c[i]
|
||||
*
|
||||
* After the first iteration we calculate the adler32 checksum for 16 bytes.
|
||||
*
|
||||
* For more background about adler32 please check the RFC:
|
||||
* https://www.ietf.org/rfc/rfc1950.txt
|
||||
*/
|
||||
|
||||
#ifdef POWER8_VSX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "adler32_p.h"
|
||||
|
||||
#include <altivec.h>
|
||||
|
||||
/* Vector across sum unsigned int (saturate). */
|
||||
static inline vector unsigned int vec_sumsu(vector unsigned int __a, vector unsigned int __b) {
|
||||
__b = vec_sld(__a, __a, 8);
|
||||
__b = vec_add(__b, __a);
|
||||
__a = vec_sld(__b, __b, 4);
|
||||
__a = vec_add(__a, __b);
|
||||
|
||||
return __a;
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static uint32_t adler32_impl(uint32_t adler, const uint8_t *buf, size_t len) {
|
||||
uint32_t s1 = adler & 0xffff;
|
||||
uint32_t s2 = (adler >> 16) & 0xffff;
|
||||
|
||||
/* in case user likes doing a byte at a time, keep it fast */
|
||||
if (UNLIKELY(len == 1))
|
||||
return adler32_copy_tail(s1, NULL, buf, 1, s2, 1, 1, 0);
|
||||
|
||||
/* This is faster than VSX code for len < 64. */
|
||||
if (len < 64)
|
||||
return adler32_copy_tail(s1, NULL, buf, len, s2, 1, 63, 0);
|
||||
|
||||
/* Use POWER VSX instructions for len >= 64. */
|
||||
const vector unsigned int v_zeros = { 0 };
|
||||
const vector unsigned char v_mul = {16, 15, 14, 13, 12, 11, 10, 9, 8, 7,
|
||||
6, 5, 4, 3, 2, 1};
|
||||
const vector unsigned char vsh = vec_splat_u8(4);
|
||||
const vector unsigned int vmask = {0xffffffff, 0x0, 0x0, 0x0};
|
||||
vector unsigned int vs1 = { 0 };
|
||||
vector unsigned int vs2 = { 0 };
|
||||
vector unsigned int vs1_save = { 0 };
|
||||
vector unsigned int vsum1, vsum2;
|
||||
vector unsigned char vbuf;
|
||||
int n;
|
||||
|
||||
vs1[0] = s1;
|
||||
vs2[0] = s2;
|
||||
|
||||
/* Do length bigger than NMAX in blocks of NMAX size. */
|
||||
while (len >= NMAX) {
|
||||
len -= NMAX;
|
||||
n = NMAX / 16;
|
||||
do {
|
||||
vbuf = vec_xl(0, (unsigned char *) buf);
|
||||
vsum1 = vec_sum4s(vbuf, v_zeros); /* sum(i=1 to 16) buf[i]. */
|
||||
/* sum(i=1 to 16) buf[i]*(16-i+1). */
|
||||
vsum2 = vec_msum(vbuf, v_mul, v_zeros);
|
||||
/* Save vs1. */
|
||||
vs1_save = vec_add(vs1_save, vs1);
|
||||
/* Accumulate the sums. */
|
||||
vs1 = vec_add(vsum1, vs1);
|
||||
vs2 = vec_add(vsum2, vs2);
|
||||
|
||||
buf += 16;
|
||||
} while (--n);
|
||||
/* Once each block of NMAX size. */
|
||||
vs1 = vec_sumsu(vs1, vsum1);
|
||||
vs1_save = vec_sll(vs1_save, vsh); /* 16*vs1_save. */
|
||||
vs2 = vec_add(vs1_save, vs2);
|
||||
vs2 = vec_sumsu(vs2, vsum2);
|
||||
|
||||
/* vs1[0] = (s1_i + sum(i=1 to 16)buf[i]) mod 65521. */
|
||||
vs1[0] = vs1[0] % BASE;
|
||||
/* vs2[0] = s2_i + 16*s1_save +
|
||||
sum(i=1 to 16)(16-i+1)*buf[i] mod 65521. */
|
||||
vs2[0] = vs2[0] % BASE;
|
||||
|
||||
vs1 = vec_and(vs1, vmask);
|
||||
vs2 = vec_and(vs2, vmask);
|
||||
vs1_save = v_zeros;
|
||||
}
|
||||
|
||||
/* len is less than NMAX one modulo is needed. */
|
||||
if (len >= 16) {
|
||||
while (len >= 16) {
|
||||
len -= 16;
|
||||
|
||||
vbuf = vec_xl(0, (unsigned char *) buf);
|
||||
|
||||
vsum1 = vec_sum4s(vbuf, v_zeros); /* sum(i=1 to 16) buf[i]. */
|
||||
/* sum(i=1 to 16) buf[i]*(16-i+1). */
|
||||
vsum2 = vec_msum(vbuf, v_mul, v_zeros);
|
||||
/* Save vs1. */
|
||||
vs1_save = vec_add(vs1_save, vs1);
|
||||
/* Accumulate the sums. */
|
||||
vs1 = vec_add(vsum1, vs1);
|
||||
vs2 = vec_add(vsum2, vs2);
|
||||
|
||||
buf += 16;
|
||||
}
|
||||
/* Since the size will be always less than NMAX we do this once. */
|
||||
vs1 = vec_sumsu(vs1, vsum1);
|
||||
vs1_save = vec_sll(vs1_save, vsh); /* 16*vs1_save. */
|
||||
vs2 = vec_add(vs1_save, vs2);
|
||||
vs2 = vec_sumsu(vs2, vsum2);
|
||||
}
|
||||
/* Copy result back to s1, s2 (mod 65521). */
|
||||
s1 = vs1[0] % BASE;
|
||||
s2 = vs2[0] % BASE;
|
||||
|
||||
/* Process tail (len < 16). */
|
||||
return adler32_copy_tail(s1, NULL, buf, len, s2, len != 0, 15, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_power8(uint32_t adler, const uint8_t *buf, size_t len) {
|
||||
return adler32_impl(adler, buf, len);
|
||||
}
|
||||
|
||||
/* VSX/VMX stores can have higher latency than optimized memcpy on POWER8+ */
|
||||
Z_INTERNAL uint32_t adler32_copy_power8(uint32_t adler, uint8_t *dst, const uint8_t *buf, size_t len) {
|
||||
adler = adler32_impl(adler, buf, len);
|
||||
memcpy(dst, buf, len);
|
||||
return adler;
|
||||
}
|
||||
#endif /* POWER8_VSX */
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
/* adler32_vmx.c -- compute the Adler-32 checksum of a data stream
|
||||
* Copyright (C) 1995-2011 Mark Adler
|
||||
* Copyright (C) 2017-2023 Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* Copyright (C) 2021 Adam Stylinski <kungfujesus06@gmail.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef PPC_VMX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zendian.h"
|
||||
#include "adler32_p.h"
|
||||
|
||||
#include <altivec.h>
|
||||
|
||||
#define vmx_zero() (vec_splat_u32(0))
|
||||
|
||||
static void vmx_accum32(uint32_t *s, const uint8_t *buf, size_t len) {
|
||||
/* Different taps for the separable components of sums */
|
||||
const vector unsigned char t0 = {64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49};
|
||||
const vector unsigned char t1 = {48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33};
|
||||
const vector unsigned char t2 = {32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17};
|
||||
const vector unsigned char t3 = {16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
|
||||
/* As silly and inefficient as it seems, creating 1 permutation vector to permute
|
||||
* a 2 element vector from a single load + a subsequent shift is just barely faster
|
||||
* than doing 2 indexed insertions into zero initialized vectors from unaligned memory. */
|
||||
const vector unsigned char s0_perm = {0, 1, 2, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};
|
||||
const vector unsigned char shift_vec = vec_sl(vec_splat_u8(8), vec_splat_u8(2));
|
||||
vector unsigned int adacc, s2acc;
|
||||
vector unsigned int pair_vec = vec_ld(0, s);
|
||||
adacc = vec_perm(pair_vec, pair_vec, s0_perm);
|
||||
#if BYTE_ORDER == LITTLE_ENDIAN
|
||||
s2acc = vec_sro(pair_vec, shift_vec);
|
||||
#else
|
||||
s2acc = vec_slo(pair_vec, shift_vec);
|
||||
#endif
|
||||
|
||||
vector unsigned int zero = vmx_zero();
|
||||
vector unsigned int s3acc = zero;
|
||||
vector unsigned int s3acc_0 = zero;
|
||||
vector unsigned int adacc_prev = adacc;
|
||||
vector unsigned int adacc_prev_0 = zero;
|
||||
|
||||
vector unsigned int s2acc_0 = zero;
|
||||
vector unsigned int s2acc_1 = zero;
|
||||
vector unsigned int s2acc_2 = zero;
|
||||
|
||||
/* Maintain a running sum of a second half, this might help use break yet another
|
||||
* data dependency bubble in the sum */
|
||||
vector unsigned int adacc_0 = zero;
|
||||
|
||||
int num_iter = len / 4;
|
||||
int rem = len & 3;
|
||||
|
||||
for (int i = 0; i < num_iter; ++i) {
|
||||
vector unsigned char d0 = vec_ld(0, buf);
|
||||
vector unsigned char d1 = vec_ld(16, buf);
|
||||
vector unsigned char d2 = vec_ld(32, buf);
|
||||
vector unsigned char d3 = vec_ld(48, buf);
|
||||
|
||||
/* The core operation of the loop, basically
|
||||
* what is being unrolled below */
|
||||
adacc = vec_sum4s(d0, adacc);
|
||||
s3acc = vec_add(s3acc, adacc_prev);
|
||||
s3acc_0 = vec_add(s3acc_0, adacc_prev_0);
|
||||
s2acc = vec_msum(t0, d0, s2acc);
|
||||
|
||||
/* interleave dependent sums in here */
|
||||
adacc_0 = vec_sum4s(d1, adacc_0);
|
||||
s2acc_0 = vec_msum(t1, d1, s2acc_0);
|
||||
adacc = vec_sum4s(d2, adacc);
|
||||
s2acc_1 = vec_msum(t2, d2, s2acc_1);
|
||||
s2acc_2 = vec_msum(t3, d3, s2acc_2);
|
||||
adacc_0 = vec_sum4s(d3, adacc_0);
|
||||
|
||||
adacc_prev = adacc;
|
||||
adacc_prev_0 = adacc_0;
|
||||
buf += 64;
|
||||
}
|
||||
|
||||
adacc = vec_add(adacc, adacc_0);
|
||||
s3acc = vec_add(s3acc, s3acc_0);
|
||||
s3acc = vec_sl(s3acc, vec_splat_u32(6));
|
||||
|
||||
if (rem) {
|
||||
adacc_prev = vec_add(adacc_prev_0, adacc_prev);
|
||||
adacc_prev = vec_sl(adacc_prev, vec_splat_u32(4));
|
||||
while (rem--) {
|
||||
vector unsigned char d0 = vec_ld(0, buf);
|
||||
adacc = vec_sum4s(d0, adacc);
|
||||
s3acc = vec_add(s3acc, adacc_prev);
|
||||
s2acc = vec_msum(t3, d0, s2acc);
|
||||
adacc_prev = vec_sl(adacc, vec_splat_u32(4));
|
||||
buf += 16;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* Sum up independent second sums */
|
||||
s2acc = vec_add(s2acc, s2acc_0);
|
||||
s2acc_2 = vec_add(s2acc_1, s2acc_2);
|
||||
s2acc = vec_add(s2acc, s2acc_2);
|
||||
|
||||
s2acc = vec_add(s2acc, s3acc);
|
||||
|
||||
adacc = vec_add(adacc, vec_sld(adacc, adacc, 8));
|
||||
s2acc = vec_add(s2acc, vec_sld(s2acc, s2acc, 8));
|
||||
adacc = vec_add(adacc, vec_sld(adacc, adacc, 4));
|
||||
s2acc = vec_add(s2acc, vec_sld(s2acc, s2acc, 4));
|
||||
|
||||
vec_ste(adacc, 0, s);
|
||||
vec_ste(s2acc, 0, s+1);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_vmx(uint32_t adler, const uint8_t *buf, size_t len) {
|
||||
/* Split Adler-32 into component sums */
|
||||
uint32_t sum2 = (adler >> 16) & 0xffff;
|
||||
adler &= 0xffff;
|
||||
|
||||
/* in case user likes doing a byte at a time, keep it fast */
|
||||
if (UNLIKELY(len == 1))
|
||||
return adler32_copy_tail(adler, NULL, buf, 1, sum2, 1, 1, 0);
|
||||
|
||||
/* in case short lengths are provided, keep it somewhat fast */
|
||||
if (UNLIKELY(len < 16))
|
||||
return adler32_copy_tail(adler, NULL, buf, len, sum2, 1, 15, 0);
|
||||
|
||||
uint32_t pair[4] ALIGNED_(16);
|
||||
pair[0] = adler;
|
||||
pair[1] = sum2;
|
||||
pair[2] = 0;
|
||||
pair[3] = 0;
|
||||
|
||||
// Align buffer
|
||||
size_t align_diff = MIN(ALIGN_DIFF(buf, 16), len);
|
||||
size_t n = NMAX;
|
||||
if (align_diff) {
|
||||
adler32_copy_align(&pair[0], NULL, buf, align_diff, &pair[1], 15, 0);
|
||||
|
||||
buf += align_diff;
|
||||
len -= align_diff;
|
||||
n -= align_diff;
|
||||
}
|
||||
|
||||
while (len >= 16) {
|
||||
n = MIN(len, n);
|
||||
|
||||
vmx_accum32(pair, buf, n / 16);
|
||||
pair[0] %= BASE;
|
||||
pair[1] %= BASE;
|
||||
|
||||
size_t k = (n / 16) * 16;
|
||||
buf += k;
|
||||
len -= k;
|
||||
n = NMAX;
|
||||
}
|
||||
|
||||
/* Process tail (len < 16). */
|
||||
return adler32_copy_tail(pair[0], NULL, buf, len, pair[1], len != 0 || align_diff, 15, 0);
|
||||
}
|
||||
|
||||
/* VMX stores can have higher latency than optimized memcpy */
|
||||
Z_INTERNAL uint32_t adler32_copy_vmx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
adler = adler32_vmx(adler, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return adler;
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
/* chunkset_power8.c -- VSX inline functions to copy small data chunks.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef POWER8_VSX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zmemory.h"
|
||||
|
||||
#include <altivec.h>
|
||||
|
||||
typedef vector unsigned char chunk_t;
|
||||
|
||||
#define HAVE_CHUNKMEMSET_2
|
||||
#define HAVE_CHUNKMEMSET_4
|
||||
#define HAVE_CHUNKMEMSET_8
|
||||
|
||||
static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = (vector unsigned char)vec_splats(zng_memread_2(from));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = (vector unsigned char)vec_splats(zng_memread_4(from));
|
||||
}
|
||||
|
||||
static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) {
|
||||
*chunk = (vector unsigned char)vec_splats((unsigned long long)zng_memread_8(from));
|
||||
}
|
||||
|
||||
static inline void loadchunk(uint8_t const *s, chunk_t *chunk) {
|
||||
*chunk = vec_xl(0, s);
|
||||
}
|
||||
|
||||
static inline void storechunk(uint8_t *out, chunk_t *chunk) {
|
||||
vec_xst(*chunk, 0, out);
|
||||
}
|
||||
|
||||
#define CHUNKSIZE chunksize_power8
|
||||
#define CHUNKCOPY chunkcopy_power8
|
||||
#define CHUNKUNROLL chunkunroll_power8
|
||||
#define CHUNKMEMSET chunkmemset_power8
|
||||
#define CHUNKMEMSET_SAFE chunkmemset_safe_power8
|
||||
|
||||
#include "chunkset_tpl.h"
|
||||
|
||||
#define INFLATE_FAST inflate_fast_power8
|
||||
|
||||
#include "inffast_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
/* compare256_power9.c - Power9 version of compare256
|
||||
* Copyright (C) 2019 Matheus Castanho <msc@linux.ibm.com>, IBM
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef POWER9
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zmemory.h"
|
||||
#include "deflate.h"
|
||||
#include "zendian.h"
|
||||
|
||||
#include <altivec.h>
|
||||
|
||||
/* Older versions of GCC misimplemented semantics for these bit counting builtins.
|
||||
* https://gcc.gnu.org/git/gitweb.cgi?p=gcc.git;h=3f30f2d1dbb3228b8468b26239fe60c2974ce2ac */
|
||||
#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 12)
|
||||
#if BYTE_ORDER == LITTLE_ENDIAN
|
||||
# define zng_vec_vctzlsbb(vc, len) len = __builtin_vec_vctzlsbb(vc)
|
||||
#else
|
||||
# define zng_vec_vctzlsbb(vc, len) len = __builtin_vec_vclzlsbb(vc)
|
||||
#endif
|
||||
#else
|
||||
# define zng_vec_vctzlsbb(vc, len) len = vec_cntlz_lsbb(vc)
|
||||
#endif
|
||||
|
||||
static inline uint32_t compare256_power9_static(const uint8_t *src0, const uint8_t *src1) {
|
||||
uint32_t len = 0, cmplen;
|
||||
|
||||
do {
|
||||
vector unsigned char vsrc0, vsrc1, vc;
|
||||
|
||||
vsrc0 = *((vector unsigned char *)src0);
|
||||
vsrc1 = *((vector unsigned char *)src1);
|
||||
|
||||
/* Compare 16 bytes at a time. Each byte of vc will be either
|
||||
* all ones or all zeroes, depending on the result of the comparison. */
|
||||
vc = (vector unsigned char)vec_cmpne(vsrc0, vsrc1);
|
||||
|
||||
/* Since the index of matching bytes will contain only zeroes
|
||||
* on vc (since we used cmpne), counting the number of consecutive
|
||||
* bytes where LSB == 0 is the same as counting the length of the match. */
|
||||
zng_vec_vctzlsbb(vc, cmplen);
|
||||
if (cmplen != 16)
|
||||
return len + cmplen;
|
||||
|
||||
src0 += 16, src1 += 16, len += 16;
|
||||
} while (len < 256);
|
||||
|
||||
return 256;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t compare256_power9(const uint8_t *src0, const uint8_t *src1) {
|
||||
return compare256_power9_static(src0, src1);
|
||||
}
|
||||
|
||||
#define LONGEST_MATCH longest_match_power9
|
||||
#define COMPARE256 compare256_power9_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#define LONGEST_MATCH_ROLL
|
||||
#define LONGEST_MATCH longest_match_roll_power9
|
||||
#define COMPARE256 compare256_power9_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,593 +0,0 @@
|
|||
/* crc32 for POWER8 using VSX instructions
|
||||
* Copyright (C) 2021 IBM Corporation
|
||||
*
|
||||
* Author: Rogerio Alves <rogealve@br.ibm.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*
|
||||
* Calculate the checksum of data that is 16 byte aligned and a multiple of
|
||||
* 16 bytes.
|
||||
*
|
||||
* The first step is to reduce it to 1024 bits. We do this in 8 parallel
|
||||
* chunks in order to mask the latency of the vpmsum instructions. If we
|
||||
* have more than 32 kB of data to checksum we repeat this step multiple
|
||||
* times, passing in the previous 1024 bits.
|
||||
*
|
||||
* The next step is to reduce the 1024 bits to 64 bits. This step adds
|
||||
* 32 bits of 0s to the end - this matches what a CRC does. We just
|
||||
* calculate constants that land the data in this 32 bits.
|
||||
*
|
||||
* We then use fixed point Barrett reduction to compute a mod n over GF(2)
|
||||
* for n = CRC using POWER8 instructions. We use x = 32.
|
||||
*
|
||||
* http://en.wikipedia.org/wiki/Barrett_reduction
|
||||
*
|
||||
* This code uses gcc vector builtins instead using assembly directly.
|
||||
*/
|
||||
|
||||
#ifdef POWER8_VSX_CRC32
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zendian.h"
|
||||
|
||||
#include "crc32_constants.h"
|
||||
#include "crc32_braid_tbl.h"
|
||||
|
||||
#include "power_intrins.h"
|
||||
|
||||
#define MAX_SIZE 32768
|
||||
#define VMX_ALIGN 16
|
||||
#define VMX_ALIGN_MASK (VMX_ALIGN-1)
|
||||
|
||||
static unsigned int crc32_align(unsigned int crc, const unsigned char *p, unsigned long len) {
|
||||
while (len--)
|
||||
crc = crc_table[(crc ^ *p++) & 0xff] ^ (crc >> 8);
|
||||
return crc;
|
||||
}
|
||||
|
||||
static unsigned int ALIGNED_(32) __crc32_vpmsum(unsigned int crc, const void* p, unsigned long len);
|
||||
|
||||
Z_INTERNAL uint32_t crc32_power8(uint32_t crc, const unsigned char *p, size_t _len) {
|
||||
unsigned int prealign;
|
||||
unsigned int tail;
|
||||
|
||||
unsigned long len = (unsigned long) _len;
|
||||
|
||||
crc ^= 0xffffffff;
|
||||
|
||||
if (len < VMX_ALIGN + VMX_ALIGN_MASK) {
|
||||
crc = crc32_align(crc, p, len);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if ((unsigned long)p & VMX_ALIGN_MASK) {
|
||||
prealign = (unsigned int)ALIGN_DIFF(p, VMX_ALIGN);
|
||||
crc = crc32_align(crc, p, prealign);
|
||||
len -= prealign;
|
||||
p += prealign;
|
||||
}
|
||||
|
||||
crc = __crc32_vpmsum(crc, p, ALIGN_DOWN(len, VMX_ALIGN));
|
||||
|
||||
tail = len & VMX_ALIGN_MASK;
|
||||
if (tail) {
|
||||
p += ALIGN_DOWN(len, VMX_ALIGN);
|
||||
crc = crc32_align(crc, p, tail);
|
||||
}
|
||||
|
||||
out:
|
||||
crc ^= 0xffffffff;
|
||||
|
||||
return crc;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t crc32_copy_power8(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
crc = crc32_power8(crc, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return crc;
|
||||
}
|
||||
|
||||
/* When we have a load-store in a single-dispatch group and address overlap
|
||||
* such that forward is not allowed (load-hit-store) the group must be flushed.
|
||||
* A group ending NOP prevents the flush.
|
||||
*/
|
||||
#define GROUP_ENDING_NOP __asm__("ori 2,2,0" ::: "memory")
|
||||
|
||||
#if BYTE_ORDER == BIG_ENDIAN
|
||||
#define BYTESWAP_DATA
|
||||
#endif
|
||||
|
||||
#ifdef BYTESWAP_DATA
|
||||
#define VEC_PERM(vr, va, vb, vc) vr = vec_perm(va, vb, (__vector unsigned char) vc)
|
||||
#if BYTE_ORDER == LITTLE_ENDIAN
|
||||
/* Byte reverse permute constant LE. */
|
||||
static const __vector unsigned long long vperm_const ALIGNED_(16) = { 0x08090A0B0C0D0E0FUL, 0x0001020304050607UL };
|
||||
#else
|
||||
static const __vector unsigned long long vperm_const ALIGNED_(16) = { 0x0F0E0D0C0B0A0908UL, 0X0706050403020100UL };
|
||||
#endif
|
||||
#else
|
||||
#define VEC_PERM(vr, va, vb, vc)
|
||||
#endif
|
||||
|
||||
static unsigned int ALIGNED_(32) __crc32_vpmsum(unsigned int crc, const void* p, unsigned long len) {
|
||||
|
||||
const __vector unsigned long long vzero = {0,0};
|
||||
const __vector unsigned long long vones = {0xffffffffffffffffUL, 0xffffffffffffffffUL};
|
||||
|
||||
const __vector unsigned long long vmask_32bit =
|
||||
(__vector unsigned long long)vec_sld((__vector unsigned char)vzero, (__vector unsigned char)vones, 4);
|
||||
|
||||
const __vector unsigned long long vmask_64bit =
|
||||
(__vector unsigned long long)vec_sld((__vector unsigned char)vzero, (__vector unsigned char)vones, 8);
|
||||
|
||||
__vector unsigned long long vcrc;
|
||||
|
||||
__vector unsigned long long vconst1, vconst2;
|
||||
|
||||
/* vdata0-vdata7 will contain our data (p). */
|
||||
__vector unsigned long long vdata0, vdata1, vdata2, vdata3, vdata4, vdata5, vdata6, vdata7;
|
||||
|
||||
/* v0-v7 will contain our checksums */
|
||||
__vector unsigned long long v0 = {0,0};
|
||||
__vector unsigned long long v1 = {0,0};
|
||||
__vector unsigned long long v2 = {0,0};
|
||||
__vector unsigned long long v3 = {0,0};
|
||||
__vector unsigned long long v4 = {0,0};
|
||||
__vector unsigned long long v5 = {0,0};
|
||||
__vector unsigned long long v6 = {0,0};
|
||||
__vector unsigned long long v7 = {0,0};
|
||||
|
||||
|
||||
/* Vector auxiliary variables. */
|
||||
__vector unsigned long long va0, va1, va2, va3, va4, va5, va6, va7;
|
||||
|
||||
unsigned int offset; /* Constant table offset. */
|
||||
|
||||
unsigned long i; /* Counter. */
|
||||
unsigned long chunks;
|
||||
|
||||
unsigned long block_size;
|
||||
int next_block = 0;
|
||||
|
||||
/* Align by 128 bits. The last 128 bit block will be processed at end. */
|
||||
unsigned long length = len & 0xFFFFFFFFFFFFFF80UL;
|
||||
|
||||
vcrc = (__vector unsigned long long)__builtin_pack_vector_int128(0UL, crc);
|
||||
|
||||
/* Short version. */
|
||||
if (len < 256) {
|
||||
/* Calculate where in the constant table we need to start. */
|
||||
offset = 256 - len;
|
||||
|
||||
vconst1 = vec_ld(offset, vcrc_short_const);
|
||||
vdata0 = vec_ld(0, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata0, vdata0, vconst1, vperm_const);
|
||||
|
||||
/* xor initial value */
|
||||
vdata0 = vec_xor(vdata0, vcrc);
|
||||
|
||||
vdata0 = (__vector unsigned long long) __builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata0, (__vector unsigned int)vconst1);
|
||||
v0 = vec_xor(v0, vdata0);
|
||||
|
||||
for (i = 16; i < len; i += 16) {
|
||||
vconst1 = vec_ld(offset + i, vcrc_short_const);
|
||||
vdata0 = vec_ld(i, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata0, vdata0, vconst1, vperm_const);
|
||||
vdata0 = (__vector unsigned long long) __builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata0, (__vector unsigned int)vconst1);
|
||||
v0 = vec_xor(v0, vdata0);
|
||||
}
|
||||
} else {
|
||||
|
||||
/* Load initial values. */
|
||||
vdata0 = vec_ld(0, (__vector unsigned long long*) p);
|
||||
vdata1 = vec_ld(16, (__vector unsigned long long*) p);
|
||||
|
||||
VEC_PERM(vdata0, vdata0, vdata0, vperm_const);
|
||||
VEC_PERM(vdata1, vdata1, vdata1, vperm_const);
|
||||
|
||||
vdata2 = vec_ld(32, (__vector unsigned long long*) p);
|
||||
vdata3 = vec_ld(48, (__vector unsigned long long*) p);
|
||||
|
||||
VEC_PERM(vdata2, vdata2, vdata2, vperm_const);
|
||||
VEC_PERM(vdata3, vdata3, vdata3, vperm_const);
|
||||
|
||||
vdata4 = vec_ld(64, (__vector unsigned long long*) p);
|
||||
vdata5 = vec_ld(80, (__vector unsigned long long*) p);
|
||||
|
||||
VEC_PERM(vdata4, vdata4, vdata4, vperm_const);
|
||||
VEC_PERM(vdata5, vdata5, vdata5, vperm_const);
|
||||
|
||||
vdata6 = vec_ld(96, (__vector unsigned long long*) p);
|
||||
vdata7 = vec_ld(112, (__vector unsigned long long*) p);
|
||||
|
||||
VEC_PERM(vdata6, vdata6, vdata6, vperm_const);
|
||||
VEC_PERM(vdata7, vdata7, vdata7, vperm_const);
|
||||
|
||||
/* xor in initial value */
|
||||
vdata0 = vec_xor(vdata0, vcrc);
|
||||
|
||||
p = (char *)p + 128;
|
||||
|
||||
do {
|
||||
/* Checksum in blocks of MAX_SIZE. */
|
||||
block_size = length;
|
||||
if (block_size > MAX_SIZE) {
|
||||
block_size = MAX_SIZE;
|
||||
}
|
||||
|
||||
length = length - block_size;
|
||||
|
||||
/*
|
||||
* Work out the offset into the constants table to start at. Each
|
||||
* constant is 16 bytes, and it is used against 128 bytes of input
|
||||
* data - 128 / 16 = 8
|
||||
*/
|
||||
offset = (MAX_SIZE/8) - (block_size/8);
|
||||
/* We reduce our final 128 bytes in a separate step */
|
||||
chunks = (block_size/128)-1;
|
||||
|
||||
vconst1 = vec_ld(offset, vcrc_const);
|
||||
|
||||
va0 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata0,
|
||||
(__vector unsigned long long)vconst1);
|
||||
va1 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata1,
|
||||
(__vector unsigned long long)vconst1);
|
||||
va2 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata2,
|
||||
(__vector unsigned long long)vconst1);
|
||||
va3 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata3,
|
||||
(__vector unsigned long long)vconst1);
|
||||
va4 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata4,
|
||||
(__vector unsigned long long)vconst1);
|
||||
va5 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata5,
|
||||
(__vector unsigned long long)vconst1);
|
||||
va6 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata6,
|
||||
(__vector unsigned long long)vconst1);
|
||||
va7 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata7,
|
||||
(__vector unsigned long long)vconst1);
|
||||
|
||||
if (chunks > 1) {
|
||||
offset += 16;
|
||||
vconst2 = vec_ld(offset, vcrc_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
vdata0 = vec_ld(0, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata0, vdata0, vdata0, vperm_const);
|
||||
|
||||
vdata1 = vec_ld(16, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata1, vdata1, vdata1, vperm_const);
|
||||
|
||||
vdata2 = vec_ld(32, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata2, vdata2, vdata2, vperm_const);
|
||||
|
||||
vdata3 = vec_ld(48, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata3, vdata3, vdata3, vperm_const);
|
||||
|
||||
vdata4 = vec_ld(64, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata4, vdata4, vdata4, vperm_const);
|
||||
|
||||
vdata5 = vec_ld(80, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata5, vdata5, vdata5, vperm_const);
|
||||
|
||||
vdata6 = vec_ld(96, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata6, vdata6, vdata6, vperm_const);
|
||||
|
||||
vdata7 = vec_ld(112, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata7, vdata7, vdata7, vperm_const);
|
||||
|
||||
p = (char *)p + 128;
|
||||
|
||||
/*
|
||||
* main loop. Each iteration calculates the CRC for a 128-byte
|
||||
* block.
|
||||
*/
|
||||
for (i = 0; i < chunks-2; i++) {
|
||||
vconst1 = vec_ld(offset, vcrc_const);
|
||||
offset += 16;
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v0 = vec_xor(v0, va0);
|
||||
va0 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata0,
|
||||
(__vector unsigned long long)vconst2);
|
||||
vdata0 = vec_ld(0, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata0, vdata0, vdata0, vperm_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v1 = vec_xor(v1, va1);
|
||||
va1 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata1,
|
||||
(__vector unsigned long long)vconst2);
|
||||
vdata1 = vec_ld(16, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata1, vdata1, vdata1, vperm_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v2 = vec_xor(v2, va2);
|
||||
va2 = __builtin_crypto_vpmsumd((__vector unsigned long long)
|
||||
vdata2, (__vector unsigned long long)vconst2);
|
||||
vdata2 = vec_ld(32, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata2, vdata2, vdata2, vperm_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v3 = vec_xor(v3, va3);
|
||||
va3 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata3,
|
||||
(__vector unsigned long long)vconst2);
|
||||
vdata3 = vec_ld(48, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata3, vdata3, vdata3, vperm_const);
|
||||
|
||||
vconst2 = vec_ld(offset, vcrc_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v4 = vec_xor(v4, va4);
|
||||
va4 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata4,
|
||||
(__vector unsigned long long)vconst1);
|
||||
vdata4 = vec_ld(64, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata4, vdata4, vdata4, vperm_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v5 = vec_xor(v5, va5);
|
||||
va5 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata5,
|
||||
(__vector unsigned long long)vconst1);
|
||||
vdata5 = vec_ld(80, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata5, vdata5, vdata5, vperm_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v6 = vec_xor(v6, va6);
|
||||
va6 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata6,
|
||||
(__vector unsigned long long)vconst1);
|
||||
vdata6 = vec_ld(96, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata6, vdata6, vdata6, vperm_const);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v7 = vec_xor(v7, va7);
|
||||
va7 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata7,
|
||||
(__vector unsigned long long)vconst1);
|
||||
vdata7 = vec_ld(112, (__vector unsigned long long*) p);
|
||||
VEC_PERM(vdata7, vdata7, vdata7, vperm_const);
|
||||
|
||||
p = (char *)p + 128;
|
||||
}
|
||||
|
||||
/* First cool down */
|
||||
vconst1 = vec_ld(offset, vcrc_const);
|
||||
offset += 16;
|
||||
|
||||
v0 = vec_xor(v0, va0);
|
||||
va0 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata0,
|
||||
(__vector unsigned long long)vconst1);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v1 = vec_xor(v1, va1);
|
||||
va1 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata1,
|
||||
(__vector unsigned long long)vconst1);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v2 = vec_xor(v2, va2);
|
||||
va2 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata2,
|
||||
(__vector unsigned long long)vconst1);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v3 = vec_xor(v3, va3);
|
||||
va3 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata3,
|
||||
(__vector unsigned long long)vconst1);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v4 = vec_xor(v4, va4);
|
||||
va4 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata4,
|
||||
(__vector unsigned long long)vconst1);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v5 = vec_xor(v5, va5);
|
||||
va5 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata5,
|
||||
(__vector unsigned long long)vconst1);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v6 = vec_xor(v6, va6);
|
||||
va6 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata6,
|
||||
(__vector unsigned long long)vconst1);
|
||||
GROUP_ENDING_NOP;
|
||||
|
||||
v7 = vec_xor(v7, va7);
|
||||
va7 = __builtin_crypto_vpmsumd((__vector unsigned long long)vdata7,
|
||||
(__vector unsigned long long)vconst1);
|
||||
}/* else */
|
||||
|
||||
/* Second cool down. */
|
||||
v0 = vec_xor(v0, va0);
|
||||
v1 = vec_xor(v1, va1);
|
||||
v2 = vec_xor(v2, va2);
|
||||
v3 = vec_xor(v3, va3);
|
||||
v4 = vec_xor(v4, va4);
|
||||
v5 = vec_xor(v5, va5);
|
||||
v6 = vec_xor(v6, va6);
|
||||
v7 = vec_xor(v7, va7);
|
||||
|
||||
/*
|
||||
* vpmsumd produces a 96 bit result in the least significant bits
|
||||
* of the register. Since we are bit reflected we have to shift it
|
||||
* left 32 bits so it occupies the least significant bits in the
|
||||
* bit reflected domain.
|
||||
*/
|
||||
v0 = (__vector unsigned long long)vec_sld((__vector unsigned char)v0,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
v1 = (__vector unsigned long long)vec_sld((__vector unsigned char)v1,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
v2 = (__vector unsigned long long)vec_sld((__vector unsigned char)v2,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
v3 = (__vector unsigned long long)vec_sld((__vector unsigned char)v3,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
v4 = (__vector unsigned long long)vec_sld((__vector unsigned char)v4,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
v5 = (__vector unsigned long long)vec_sld((__vector unsigned char)v5,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
v6 = (__vector unsigned long long)vec_sld((__vector unsigned char)v6,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
v7 = (__vector unsigned long long)vec_sld((__vector unsigned char)v7,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
|
||||
/* xor with the last 1024 bits. */
|
||||
va0 = vec_ld(0, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va0, va0, va0, vperm_const);
|
||||
|
||||
va1 = vec_ld(16, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va1, va1, va1, vperm_const);
|
||||
|
||||
va2 = vec_ld(32, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va2, va2, va2, vperm_const);
|
||||
|
||||
va3 = vec_ld(48, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va3, va3, va3, vperm_const);
|
||||
|
||||
va4 = vec_ld(64, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va4, va4, va4, vperm_const);
|
||||
|
||||
va5 = vec_ld(80, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va5, va5, va5, vperm_const);
|
||||
|
||||
va6 = vec_ld(96, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va6, va6, va6, vperm_const);
|
||||
|
||||
va7 = vec_ld(112, (__vector unsigned long long*) p);
|
||||
VEC_PERM(va7, va7, va7, vperm_const);
|
||||
|
||||
p = (char *)p + 128;
|
||||
|
||||
vdata0 = vec_xor(v0, va0);
|
||||
vdata1 = vec_xor(v1, va1);
|
||||
vdata2 = vec_xor(v2, va2);
|
||||
vdata3 = vec_xor(v3, va3);
|
||||
vdata4 = vec_xor(v4, va4);
|
||||
vdata5 = vec_xor(v5, va5);
|
||||
vdata6 = vec_xor(v6, va6);
|
||||
vdata7 = vec_xor(v7, va7);
|
||||
|
||||
/* Check if we have more blocks to process */
|
||||
next_block = 0;
|
||||
if (length != 0) {
|
||||
next_block = 1;
|
||||
|
||||
/* zero v0-v7 */
|
||||
v0 = vec_xor(v0, v0);
|
||||
v1 = vec_xor(v1, v1);
|
||||
v2 = vec_xor(v2, v2);
|
||||
v3 = vec_xor(v3, v3);
|
||||
v4 = vec_xor(v4, v4);
|
||||
v5 = vec_xor(v5, v5);
|
||||
v6 = vec_xor(v6, v6);
|
||||
v7 = vec_xor(v7, v7);
|
||||
}
|
||||
length = length + 128;
|
||||
|
||||
} while (next_block);
|
||||
|
||||
/* Calculate how many bytes we have left. */
|
||||
length = (len & 127);
|
||||
|
||||
/* Calculate where in (short) constant table we need to start. */
|
||||
offset = 128 - length;
|
||||
|
||||
v0 = vec_ld(offset, vcrc_short_const);
|
||||
v1 = vec_ld(offset + 16, vcrc_short_const);
|
||||
v2 = vec_ld(offset + 32, vcrc_short_const);
|
||||
v3 = vec_ld(offset + 48, vcrc_short_const);
|
||||
v4 = vec_ld(offset + 64, vcrc_short_const);
|
||||
v5 = vec_ld(offset + 80, vcrc_short_const);
|
||||
v6 = vec_ld(offset + 96, vcrc_short_const);
|
||||
v7 = vec_ld(offset + 112, vcrc_short_const);
|
||||
|
||||
offset += 128;
|
||||
|
||||
v0 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata0, (__vector unsigned int)v0);
|
||||
v1 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata1, (__vector unsigned int)v1);
|
||||
v2 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata2, (__vector unsigned int)v2);
|
||||
v3 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata3, (__vector unsigned int)v3);
|
||||
v4 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata4, (__vector unsigned int)v4);
|
||||
v5 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata5, (__vector unsigned int)v5);
|
||||
v6 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata6, (__vector unsigned int)v6);
|
||||
v7 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata7, (__vector unsigned int)v7);
|
||||
|
||||
/* Now reduce the tail (0-112 bytes). */
|
||||
for (i = 0; i < length; i+=16) {
|
||||
vdata0 = vec_ld(i,(__vector unsigned long long*)p);
|
||||
VEC_PERM(vdata0, vdata0, vdata0, vperm_const);
|
||||
va0 = vec_ld(offset + i,vcrc_short_const);
|
||||
va0 = (__vector unsigned long long)__builtin_crypto_vpmsumw(
|
||||
(__vector unsigned int)vdata0, (__vector unsigned int)va0);
|
||||
v0 = vec_xor(v0, va0);
|
||||
}
|
||||
|
||||
/* xor all parallel chunks together. */
|
||||
v0 = vec_xor(v0, v1);
|
||||
v2 = vec_xor(v2, v3);
|
||||
v4 = vec_xor(v4, v5);
|
||||
v6 = vec_xor(v6, v7);
|
||||
|
||||
v0 = vec_xor(v0, v2);
|
||||
v4 = vec_xor(v4, v6);
|
||||
|
||||
v0 = vec_xor(v0, v4);
|
||||
}
|
||||
|
||||
/* Barrett Reduction */
|
||||
vconst1 = vec_ld(0, v_Barrett_const);
|
||||
vconst2 = vec_ld(16, v_Barrett_const);
|
||||
|
||||
v1 = (__vector unsigned long long)vec_sld((__vector unsigned char)v0,
|
||||
(__vector unsigned char)v0, 8);
|
||||
v0 = vec_xor(v1,v0);
|
||||
|
||||
/* shift left one bit */
|
||||
__vector unsigned char vsht_splat = vec_splat_u8 (1);
|
||||
v0 = (__vector unsigned long long)vec_sll((__vector unsigned char)v0, vsht_splat);
|
||||
|
||||
v0 = vec_and(v0, vmask_64bit);
|
||||
|
||||
/*
|
||||
* The reflected version of Barrett reduction. Instead of bit
|
||||
* reflecting our data (which is expensive to do), we bit reflect our
|
||||
* constants and our algorithm, which means the intermediate data in
|
||||
* our vector registers goes from 0-63 instead of 63-0. We can reflect
|
||||
* the algorithm because we don't carry in mod 2 arithmetic.
|
||||
*/
|
||||
|
||||
/* bottom 32 bits of a */
|
||||
v1 = vec_and(v0, vmask_32bit);
|
||||
|
||||
/* ma */
|
||||
v1 = __builtin_crypto_vpmsumd((__vector unsigned long long)v1,
|
||||
(__vector unsigned long long)vconst1);
|
||||
|
||||
/* bottom 32bits of ma */
|
||||
v1 = vec_and(v1, vmask_32bit);
|
||||
/* qn */
|
||||
v1 = __builtin_crypto_vpmsumd((__vector unsigned long long)v1,
|
||||
(__vector unsigned long long)vconst2);
|
||||
/* a - qn, subtraction is xor in GF(2) */
|
||||
v0 = vec_xor (v0, v1);
|
||||
|
||||
/*
|
||||
* Since we are bit reflected, the result (ie the low 32 bits) is in
|
||||
* the high 32 bits. We just need to shift it left 4 bytes
|
||||
* V0 [ 0 1 X 3 ]
|
||||
* V0 [ 0 X 2 3 ]
|
||||
*/
|
||||
|
||||
/* shift result into top 64 bits of */
|
||||
v0 = (__vector unsigned long long)vec_sld((__vector unsigned char)v0,
|
||||
(__vector unsigned char)vzero, 4);
|
||||
|
||||
#if BYTE_ORDER == BIG_ENDIAN
|
||||
return v0[0];
|
||||
#else
|
||||
return v0[1];
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
/* power_features.c - POWER feature check
|
||||
* Copyright (C) 2020 Matheus Castanho <msc@linux.ibm.com>, IBM
|
||||
* Copyright (C) 2021-2024 Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#if defined(PPC_FEATURES) || defined(POWER_FEATURES)
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "power_features.h"
|
||||
|
||||
#ifdef HAVE_SYS_AUXV_H
|
||||
# include <sys/auxv.h>
|
||||
#endif
|
||||
#ifdef POWER_NEED_AUXVEC_H
|
||||
# include <linux/auxvec.h>
|
||||
#endif
|
||||
#ifdef __FreeBSD__
|
||||
# include <machine/cpu.h>
|
||||
#endif
|
||||
|
||||
void Z_INTERNAL power_check_features(struct power_cpu_features *features) {
|
||||
#ifdef PPC_FEATURES
|
||||
unsigned long hwcap;
|
||||
#if defined(__FreeBSD__) || defined(__OpenBSD__)
|
||||
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
|
||||
#else
|
||||
hwcap = getauxval(AT_HWCAP);
|
||||
#endif
|
||||
|
||||
if (hwcap & PPC_FEATURE_HAS_ALTIVEC)
|
||||
features->has_altivec = 1;
|
||||
#endif
|
||||
|
||||
#ifdef POWER_FEATURES
|
||||
unsigned long hwcap2;
|
||||
#if defined(__FreeBSD__) || defined(__OpenBSD__)
|
||||
elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2));
|
||||
#else
|
||||
hwcap2 = getauxval(AT_HWCAP2);
|
||||
#endif
|
||||
|
||||
#ifdef POWER8_VSX
|
||||
if (hwcap2 & PPC_FEATURE2_ARCH_2_07)
|
||||
features->has_arch_2_07 = 1;
|
||||
#endif
|
||||
#ifdef POWER9
|
||||
if (hwcap2 & PPC_FEATURE2_ARCH_3_00)
|
||||
features->has_arch_3_00 = 1;
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
/* power_features.h -- check for POWER CPU features
|
||||
* Copyright (C) 2020 Matheus Castanho <msc@linux.ibm.com>, IBM
|
||||
* Copyright (C) 2021 Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef POWER_FEATURES_H_
|
||||
#define POWER_FEATURES_H_
|
||||
|
||||
struct power_cpu_features {
|
||||
int has_altivec;
|
||||
int has_arch_2_07;
|
||||
int has_arch_3_00;
|
||||
};
|
||||
|
||||
void Z_INTERNAL power_check_features(struct power_cpu_features *features);
|
||||
|
||||
#endif /* POWER_FEATURES_H_ */
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
/* power_functions.h -- POWER implementations for arch-specific functions.
|
||||
* Copyright (C) 2020 Matheus Castanho <msc@linux.ibm.com>, IBM
|
||||
* Copyright (C) 2021 Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef POWER_FUNCTIONS_H_
|
||||
#define POWER_FUNCTIONS_H_
|
||||
|
||||
#include "power_natives.h"
|
||||
|
||||
#ifdef PPC_VMX
|
||||
uint32_t adler32_vmx(uint32_t adler, const uint8_t *buf, size_t len);
|
||||
uint32_t adler32_copy_vmx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
void slide_hash_vmx(deflate_state *s);
|
||||
#endif
|
||||
|
||||
#ifdef POWER8_VSX
|
||||
uint32_t adler32_power8(uint32_t adler, const uint8_t *buf, size_t len);
|
||||
uint32_t adler32_copy_power8(uint32_t adler, uint8_t *dst, const uint8_t *buf, size_t len);
|
||||
uint8_t* chunkmemset_safe_power8(uint8_t *out, uint8_t *from, size_t len, size_t left);
|
||||
uint32_t crc32_power8(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_power8(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
void slide_hash_power8(deflate_state *s);
|
||||
void inflate_fast_power8(PREFIX3(stream) *strm, uint32_t start);
|
||||
#endif
|
||||
|
||||
#if !defined(PPC_VMX_NATIVE) && !defined(POWER8_VSX_NATIVE)
|
||||
# define ADLER32_FALLBACK
|
||||
# define SLIDE_HASH_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifndef POWER8_VSX_NATIVE
|
||||
# define CHUNKSET_FALLBACK
|
||||
#endif
|
||||
#ifndef POWER8_VSX_CRC32_NATIVE
|
||||
# define CRC32_BRAID_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef POWER9
|
||||
uint32_t compare256_power9(const uint8_t *src0, const uint8_t *src1);
|
||||
uint32_t longest_match_power9(deflate_state *const s, uint32_t cur_match);
|
||||
uint32_t longest_match_roll_power9(deflate_state *const s, uint32_t cur_match);
|
||||
#endif
|
||||
|
||||
#ifndef POWER9_NATIVE
|
||||
# define COMPARE256_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_RUNTIME_CPU_DETECTION
|
||||
// Power - VMX
|
||||
# ifdef PPC_VMX_NATIVE
|
||||
# undef native_adler32
|
||||
# define native_adler32 adler32_vmx
|
||||
# undef native_adler32_copy
|
||||
# define native_adler32_copy adler32_copy_vmx
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_vmx
|
||||
# endif
|
||||
// Power8 - VSX
|
||||
# ifdef POWER8_VSX_NATIVE
|
||||
# undef native_adler32
|
||||
# define native_adler32 adler32_power8
|
||||
# undef native_adler32_copy
|
||||
# define native_adler32_copy adler32_copy_power8
|
||||
# undef native_chunkmemset_safe
|
||||
# define native_chunkmemset_safe chunkmemset_safe_power8
|
||||
# undef native_inflate_fast
|
||||
# define native_inflate_fast inflate_fast_power8
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_power8
|
||||
# endif
|
||||
# ifdef POWER8_VSX_CRC32_NATIVE
|
||||
# undef native_crc32
|
||||
# define native_crc32 crc32_power8
|
||||
# undef native_crc32_copy
|
||||
# define native_crc32_copy crc32_copy_power8
|
||||
# endif
|
||||
// Power9
|
||||
# ifdef POWER9_NATIVE
|
||||
# undef native_compare256
|
||||
# define native_compare256 compare256_power9
|
||||
# undef native_longest_match
|
||||
# define native_longest_match longest_match_power9
|
||||
# undef native_longest_match_roll
|
||||
# define native_longest_match_roll longest_match_roll_power9
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* POWER_FUNCTIONS_H_ */
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
/* Helper functions to work around issues with clang builtins
|
||||
* Copyright (C) 2021 IBM Corporation
|
||||
*
|
||||
* Authors:
|
||||
* Daniel Black <daniel@linux.vnet.ibm.com>
|
||||
* Rogerio Alves <rogealve@br.ibm.com>
|
||||
* Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef POWER_INTRINS_H
|
||||
#define POWER_INTRINS_H
|
||||
|
||||
#include <altivec.h>
|
||||
|
||||
#if defined (__clang__)
|
||||
/*
|
||||
* These stubs fix clang incompatibilities with GCC builtins.
|
||||
*/
|
||||
|
||||
#ifndef __builtin_crypto_vpmsumw
|
||||
#define __builtin_crypto_vpmsumw __builtin_crypto_vpmsumb
|
||||
#endif
|
||||
#ifndef __builtin_crypto_vpmsumd
|
||||
#define __builtin_crypto_vpmsumd __builtin_crypto_vpmsumb
|
||||
#endif
|
||||
|
||||
#ifdef __VSX__
|
||||
static inline __vector unsigned long long __attribute__((overloadable))
|
||||
vec_ld(int __a, const __vector unsigned long long* __b) {
|
||||
return (__vector unsigned long long)__builtin_altivec_lvx(__a, __b);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
/* There's no version of this that operates over unsigned and if casted, it does
|
||||
* sign extension. Let's write an endian independent version and hope the compiler
|
||||
* eliminates creating another zero idiom for the zero value if one exists locally */
|
||||
static inline vector unsigned short vec_unpackl(vector unsigned char a) {
|
||||
vector unsigned char zero = vec_splat_u8(0);
|
||||
|
||||
#if BYTE_ORDER == BIG_ENDIAN
|
||||
return (vector unsigned short)vec_mergel(zero, a);
|
||||
#else
|
||||
return (vector unsigned short)vec_mergel(a, zero);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline vector unsigned short vec_unpackh(vector unsigned char a) {
|
||||
vector unsigned char zero = vec_splat_u8(0);
|
||||
|
||||
#if BYTE_ORDER == BIG_ENDIAN
|
||||
return (vector unsigned short)vec_mergeh(zero, a);
|
||||
#else
|
||||
return (vector unsigned short)vec_mergeh(a, zero);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
/* power_natives.h -- POWER compile-time feature detection macros.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef POWER_NATIVES_H_
|
||||
#define POWER_NATIVES_H_
|
||||
|
||||
#if defined(__ALTIVEC__)
|
||||
# ifdef PPC_VMX
|
||||
# define PPC_VMX_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
#if defined(_ARCH_PWR8) && defined(__VSX__)
|
||||
# ifdef POWER8_VSX
|
||||
# define POWER8_VSX_NATIVE
|
||||
# endif
|
||||
# ifdef POWER8_VSX_CRC32
|
||||
# define POWER8_VSX_CRC32_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
#if defined(_ARCH_PWR9)
|
||||
# ifdef POWER9
|
||||
# define POWER9_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* POWER_NATIVES_H_ */
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
/* Optimized slide_hash for POWER processors
|
||||
* Copyright (C) 2019-2020 IBM Corporation
|
||||
* Author: Matheus Castanho <msc@linux.ibm.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef POWER8_VSX
|
||||
|
||||
#define SLIDE_PPC slide_hash_power8
|
||||
#include "slide_ppc_tpl.h"
|
||||
|
||||
#endif /* POWER8_VSX */
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
/* Optimized slide_hash for PowerPC processors with VMX instructions
|
||||
* Copyright (C) 2017-2021 Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
#ifdef PPC_VMX
|
||||
|
||||
#define SLIDE_PPC slide_hash_vmx
|
||||
#include "slide_ppc_tpl.h"
|
||||
|
||||
#endif /* PPC_VMX */
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
/* Optimized slide_hash for PowerPC processors
|
||||
* Copyright (C) 2017-2021 Mika T. Lindqvist <postmaster@raasu.org>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include <altivec.h>
|
||||
#include "zbuild.h"
|
||||
#include "deflate.h"
|
||||
|
||||
static inline void slide_hash_chain(Pos *table, uint32_t entries, uint16_t wsize) {
|
||||
const vector unsigned short vmx_wsize = vec_splats(wsize);
|
||||
Pos *p = table;
|
||||
|
||||
do {
|
||||
/* Do the pointer arithmetic early to hopefully overlap the vector unit */
|
||||
Pos *q = p;
|
||||
p += 32;
|
||||
vector unsigned short value0, value1, value2, value3;
|
||||
vector unsigned short result0, result1, result2, result3;
|
||||
|
||||
value0 = vec_ld(0, q);
|
||||
value1 = vec_ld(16, q);
|
||||
value2 = vec_ld(32, q);
|
||||
value3 = vec_ld(48, q);
|
||||
result0 = vec_subs(value0, vmx_wsize);
|
||||
result1 = vec_subs(value1, vmx_wsize);
|
||||
result2 = vec_subs(value2, vmx_wsize);
|
||||
result3 = vec_subs(value3, vmx_wsize);
|
||||
vec_st(result0, 0, q);
|
||||
vec_st(result1, 16, q);
|
||||
vec_st(result2, 32, q);
|
||||
vec_st(result3, 48, q);
|
||||
|
||||
entries -= 32;
|
||||
} while (entries);
|
||||
}
|
||||
|
||||
void Z_INTERNAL SLIDE_PPC(deflate_state *s) {
|
||||
Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
|
||||
slide_hash_chain(s->head, HASH_SIZE, wsize);
|
||||
slide_hash_chain(s->prev, wsize, wsize);
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
# Makefile for zlib-ng
|
||||
# Copyright (C) 1995-2013 Jean-loup Gailly, Mark Adler
|
||||
# Copyright (C) 2024 Hans Kristian Rosbach
|
||||
# Copyright (C) 2025 Yin Tong <yintong.ustc@bytedance.com>, ByteDance
|
||||
# For conditions of distribution and use, see copyright notice in zlib.h
|
||||
|
||||
CC=
|
||||
CFLAGS=
|
||||
SFLAGS=
|
||||
INCLUDES=
|
||||
SUFFIX=
|
||||
|
||||
SRCDIR=.
|
||||
SRCTOP=../..
|
||||
TOPDIR=$(SRCTOP)
|
||||
|
||||
RVVFLAG=
|
||||
RVVZBCFLAG=
|
||||
ZBCFLAG=
|
||||
|
||||
all: \
|
||||
riscv_features.o riscv_features.lo \
|
||||
adler32_rvv.o adler32_rvv.lo \
|
||||
chunkset_rvv.o chunkset_rvv.lo \
|
||||
compare256_rvv.o compare256_rvv.lo \
|
||||
slide_hash_rvv.o slide_hash_rvv.lo \
|
||||
crc32_zbc.o crc32_zbc.lo
|
||||
|
||||
riscv_features.o: $(SRCDIR)/riscv_features.c
|
||||
$(CC) $(CFLAGS) $(RVVZBCFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/riscv_features.c
|
||||
|
||||
riscv_features.lo: $(SRCDIR)/riscv_features.c
|
||||
$(CC) $(SFLAGS) $(RVVZBCFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/riscv_features.c
|
||||
|
||||
adler32_rvv.o: $(SRCDIR)/adler32_rvv.c
|
||||
$(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_rvv.c
|
||||
|
||||
adler32_rvv.lo: $(SRCDIR)/adler32_rvv.c
|
||||
$(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_rvv.c
|
||||
|
||||
chunkset_rvv.o: $(SRCDIR)/chunkset_rvv.c
|
||||
$(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_rvv.c
|
||||
|
||||
chunkset_rvv.lo: $(SRCDIR)/chunkset_rvv.c
|
||||
$(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_rvv.c
|
||||
|
||||
compare256_rvv.o: $(SRCDIR)/compare256_rvv.c
|
||||
$(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_rvv.c
|
||||
|
||||
compare256_rvv.lo: $(SRCDIR)/compare256_rvv.c
|
||||
$(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_rvv.c
|
||||
|
||||
slide_hash_rvv.o: $(SRCDIR)/slide_hash_rvv.c
|
||||
$(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_rvv.c
|
||||
|
||||
slide_hash_rvv.lo: $(SRCDIR)/slide_hash_rvv.c
|
||||
$(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_rvv.c
|
||||
|
||||
crc32_zbc.o: $(SRCDIR)/crc32_zbc.c
|
||||
$(CC) $(CFLAGS) $(ZBCFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_zbc.c
|
||||
|
||||
crc32_zbc.lo: $(SRCDIR)/crc32_zbc.c
|
||||
$(CC) $(SFLAGS) $(ZBCFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_zbc.c
|
||||
|
||||
mostlyclean: clean
|
||||
clean:
|
||||
rm -f *.o *.lo *~
|
||||
rm -rf objs
|
||||
rm -f *.gcda *.gcno *.gcov
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
# Building RISC-V Target with Cmake #
|
||||
|
||||
> **Warning**
|
||||
> Runtime rvv detection (using `hwcap`) requires linux kernel 6.5 or newer.
|
||||
>
|
||||
> When running on older kernels, we fall back to compile-time detection, potentially this can cause crashes if rvv is enabled at compile but not supported by the target cpu.
|
||||
> Therefore if older kernel support is needed, rvv should be disabled if the target cpu does not support it.
|
||||
## Prerequisite: Build RISC-V Clang Toolchain and QEMU ##
|
||||
|
||||
If you don't have prebuilt clang and riscv64 qemu, you can refer to the [script](https://github.com/sifive/prepare-riscv-toolchain-qemu/blob/main/prepare_riscv_toolchain_qemu.sh) to get the source. Copy the script to the zlib-ng root directory, and run it to download the source and build them. Modify the content according to your conditions (e.g., toolchain version).
|
||||
|
||||
```bash
|
||||
./prepare_riscv_toolchain_qemu.sh
|
||||
```
|
||||
|
||||
After running script, clang & qemu are built in `build-toolchain-qemu/riscv-clang/` & `build-toolchain-qemu/riscv-qemu/`.
|
||||
|
||||
`build-toolchain-qemu/riscv-clang/` is your `TOOLCHAIN_PATH`.
|
||||
`build-toolchain-qemu/riscv-qemu/bin/qemu-riscv64` is your `QEMU_PATH`.
|
||||
|
||||
You can also download the prebuilt toolchain & qemu from [the release page](https://github.com/sifive/prepare-riscv-toolchain-qemu/releases), and enjoy using them.
|
||||
|
||||
## Cross-Compile for RISC-V Target ##
|
||||
|
||||
```bash
|
||||
cmake -G Ninja -B ./build-riscv \
|
||||
-D CMAKE_TOOLCHAIN_FILE=./cmake/toolchain-riscv.cmake \
|
||||
-D CMAKE_INSTALL_PREFIX=./build-riscv/install \
|
||||
-D TOOLCHAIN_PATH={TOOLCHAIN_PATH} \
|
||||
-D QEMU_PATH={QEMU_PATH} \
|
||||
.
|
||||
|
||||
cmake --build ./build-riscv
|
||||
```
|
||||
|
||||
Disable the option if there is no RVV support:
|
||||
```
|
||||
-D WITH_RVV=OFF
|
||||
```
|
||||
|
||||
## Run Unittests on User Mode QEMU ##
|
||||
|
||||
```bash
|
||||
cd ./build-riscv && ctest --verbose
|
||||
```
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
/* adler32_rvv.c - RVV version of adler32
|
||||
* Copyright (C) 2023 SiFive, Inc. All rights reserved.
|
||||
* Contributed by Alex Chiang <alex.chiang@sifive.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef RISCV_RVV
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "adler32_p.h"
|
||||
|
||||
#include <riscv_vector.h>
|
||||
|
||||
Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t* restrict dst, const uint8_t *src, size_t len, int COPY) {
|
||||
/* split Adler-32 into component sums */
|
||||
uint32_t sum2 = (adler >> 16) & 0xffff;
|
||||
adler &= 0xffff;
|
||||
|
||||
/* in case user likes doing a byte at a time, keep it fast */
|
||||
if (UNLIKELY(len == 1))
|
||||
return adler32_copy_tail(adler, dst, src, 1, sum2, 1, 1, COPY);
|
||||
|
||||
/* in case short lengths are provided, keep it somewhat fast */
|
||||
if (UNLIKELY(len < 16))
|
||||
return adler32_copy_tail(adler, dst, src, len, sum2, 1, 15, COPY);
|
||||
|
||||
size_t left = len;
|
||||
size_t vl = __riscv_vsetvlmax_e8m1();
|
||||
vl = MIN(vl, 256);
|
||||
vuint32m4_t v_buf32_accu = __riscv_vmv_v_x_u32m4(0, vl);
|
||||
vuint32m4_t v_adler32_prev_accu = __riscv_vmv_v_x_u32m4(0, vl);
|
||||
vuint16m2_t v_buf16_accu;
|
||||
|
||||
/*
|
||||
* We accumulate 8-bit data, and to prevent overflow, we have to use a 32-bit accumulator.
|
||||
* However, adding 8-bit data into a 32-bit accumulator isn't efficient. We use 16-bit & 32-bit
|
||||
* accumulators to boost performance.
|
||||
*
|
||||
* The block_size is the largest multiple of vl that <= 256, because overflow would occur when
|
||||
* vl > 256 (255 * 256 <= UINT16_MAX).
|
||||
*
|
||||
* We accumulate 8-bit data into a 16-bit accumulator and then
|
||||
* move the data into the 32-bit accumulator at the last iteration.
|
||||
*/
|
||||
size_t block_size = (256 / vl) * vl;
|
||||
size_t nmax_limit = (NMAX / block_size);
|
||||
size_t cnt = 0;
|
||||
while (left >= block_size) {
|
||||
v_buf16_accu = __riscv_vmv_v_x_u16m2(0, vl);
|
||||
size_t subprob = block_size;
|
||||
while (subprob > 0) {
|
||||
vuint8m1_t v_buf8 = __riscv_vle8_v_u8m1(src, vl);
|
||||
if (COPY) __riscv_vse8_v_u8m1(dst, v_buf8, vl);
|
||||
v_adler32_prev_accu = __riscv_vwaddu_wv_u32m4(v_adler32_prev_accu, v_buf16_accu, vl);
|
||||
v_buf16_accu = __riscv_vwaddu_wv_u16m2(v_buf16_accu, v_buf8, vl);
|
||||
src += vl;
|
||||
if (COPY) dst += vl;
|
||||
subprob -= vl;
|
||||
}
|
||||
v_adler32_prev_accu = __riscv_vmacc_vx_u32m4(v_adler32_prev_accu, block_size / vl, v_buf32_accu, vl);
|
||||
v_buf32_accu = __riscv_vwaddu_wv_u32m4(v_buf32_accu, v_buf16_accu, vl);
|
||||
left -= block_size;
|
||||
/* do modulo once each block of NMAX size */
|
||||
if (++cnt >= nmax_limit) {
|
||||
v_adler32_prev_accu = __riscv_vremu_vx_u32m4(v_adler32_prev_accu, BASE, vl);
|
||||
v_buf32_accu = __riscv_vremu_vx_u32m4(v_buf32_accu, BASE, vl);
|
||||
cnt = 0;
|
||||
}
|
||||
}
|
||||
/* the left len <= 256 now, we can use 16-bit accum safely */
|
||||
v_buf16_accu = __riscv_vmv_v_x_u16m2(0, vl);
|
||||
size_t res = left;
|
||||
while (left >= vl) {
|
||||
vuint8m1_t v_buf8 = __riscv_vle8_v_u8m1(src, vl);
|
||||
if (COPY) __riscv_vse8_v_u8m1(dst, v_buf8, vl);
|
||||
v_adler32_prev_accu = __riscv_vwaddu_wv_u32m4(v_adler32_prev_accu, v_buf16_accu, vl);
|
||||
v_buf16_accu = __riscv_vwaddu_wv_u16m2(v_buf16_accu, v_buf8, vl);
|
||||
src += vl;
|
||||
if (COPY) dst += vl;
|
||||
left -= vl;
|
||||
}
|
||||
v_adler32_prev_accu = __riscv_vmacc_vx_u32m4(v_adler32_prev_accu, res / vl, v_buf32_accu, vl);
|
||||
v_adler32_prev_accu = __riscv_vremu_vx_u32m4(v_adler32_prev_accu, BASE, vl);
|
||||
v_buf32_accu = __riscv_vwaddu_wv_u32m4(v_buf32_accu, v_buf16_accu, vl);
|
||||
|
||||
vuint32m4_t v_seq = __riscv_vid_v_u32m4(vl);
|
||||
vuint32m4_t v_rev_seq = __riscv_vrsub_vx_u32m4(v_seq, vl, vl);
|
||||
vuint32m4_t v_sum32_accu = __riscv_vmul_vv_u32m4(v_buf32_accu, v_rev_seq, vl);
|
||||
|
||||
v_sum32_accu = __riscv_vadd_vv_u32m4(v_sum32_accu, __riscv_vmul_vx_u32m4(v_adler32_prev_accu, vl, vl), vl);
|
||||
|
||||
vuint32m1_t v_sum2_sum = __riscv_vmv_s_x_u32m1(0, vl);
|
||||
v_sum2_sum = __riscv_vredsum_vs_u32m4_u32m1(v_sum32_accu, v_sum2_sum, vl);
|
||||
uint32_t sum2_sum = __riscv_vmv_x_s_u32m1_u32(v_sum2_sum) % BASE;
|
||||
|
||||
sum2 += (sum2_sum + adler * ((len - left) % BASE));
|
||||
|
||||
vuint32m1_t v_adler_sum = __riscv_vmv_s_x_u32m1(0, vl);
|
||||
v_adler_sum = __riscv_vredsum_vs_u32m4_u32m1(v_buf32_accu, v_adler_sum, vl);
|
||||
uint32_t adler_sum = __riscv_vmv_x_s_u32m1_u32(v_adler_sum) % BASE;
|
||||
|
||||
adler += adler_sum;
|
||||
|
||||
sum2 %= BASE;
|
||||
adler %= BASE;
|
||||
|
||||
/* Process tail (left < 256). */
|
||||
return adler32_copy_tail(adler, dst, src, left, sum2, left != 0, 255, COPY);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_rvv(uint32_t adler, const uint8_t *buf, size_t len) {
|
||||
return adler32_copy_impl(adler, NULL, buf, len, 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t adler32_copy_rvv(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
return adler32_copy_impl(adler, dst, src, len, 1);
|
||||
}
|
||||
|
||||
#endif // RISCV_RVV
|
||||
|
|
@ -1,126 +0,0 @@
|
|||
/* chunkset_rvv.c - RVV version of chunkset
|
||||
* Copyright (C) 2023 SiFive, Inc. All rights reserved.
|
||||
* Contributed by Alex Chiang <alex.chiang@sifive.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef RISCV_RVV
|
||||
|
||||
#include "zbuild.h"
|
||||
|
||||
#include <riscv_vector.h>
|
||||
|
||||
/*
|
||||
* RISC-V glibc would enable RVV optimized memcpy at runtime by IFUNC,
|
||||
* so we prefer using large size chunk and copy memory as much as possible.
|
||||
*/
|
||||
#define HAVE_CHUNKMEMSET_2
|
||||
#define HAVE_CHUNKMEMSET_4
|
||||
#define HAVE_CHUNKMEMSET_8
|
||||
|
||||
#define CHUNK_MEMSET_RVV_IMPL(from, chunk, elen) \
|
||||
do { \
|
||||
size_t vl, len = sizeof(*chunk) / sizeof(uint##elen##_t); \
|
||||
uint##elen##_t val = *(uint##elen##_t*)from; \
|
||||
uint##elen##_t* chunk_p = (uint##elen##_t*)chunk; \
|
||||
do { \
|
||||
vl = __riscv_vsetvl_e##elen##m4(len); \
|
||||
vuint##elen##m4_t v_val = __riscv_vmv_v_x_u##elen##m4(val, vl); \
|
||||
__riscv_vse##elen##_v_u##elen##m4(chunk_p, v_val, vl); \
|
||||
len -= vl; chunk_p += vl; \
|
||||
} while (len > 0); \
|
||||
} while (0)
|
||||
|
||||
/* We don't have a 32-byte datatype for RISC-V arch. */
|
||||
typedef struct chunk_s {
|
||||
uint64_t data[4];
|
||||
} chunk_t;
|
||||
|
||||
static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) {
|
||||
CHUNK_MEMSET_RVV_IMPL(from, chunk, 16);
|
||||
}
|
||||
|
||||
static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) {
|
||||
CHUNK_MEMSET_RVV_IMPL(from, chunk, 32);
|
||||
}
|
||||
|
||||
static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) {
|
||||
CHUNK_MEMSET_RVV_IMPL(from, chunk, 64);
|
||||
}
|
||||
|
||||
static inline void loadchunk(uint8_t const *s, chunk_t *chunk) {
|
||||
memcpy(chunk->data, (uint8_t *)s, sizeof(*chunk));
|
||||
}
|
||||
|
||||
static inline void storechunk(uint8_t *out, chunk_t *chunk) {
|
||||
memcpy(out, chunk->data, sizeof(*chunk));
|
||||
}
|
||||
|
||||
#define CHUNKSIZE chunksize_rvv
|
||||
#define CHUNKCOPY chunkcopy_rvv
|
||||
#define CHUNKUNROLL chunkunroll_rvv
|
||||
#define CHUNKMEMSET chunkmemset_rvv
|
||||
#define CHUNKMEMSET_SAFE chunkmemset_safe_rvv
|
||||
|
||||
#define HAVE_CHUNKCOPY
|
||||
|
||||
/*
|
||||
* Assuming that the length is non-zero, and that `from` lags `out` by at least
|
||||
* sizeof chunk_t bytes, please see the comments in chunkset_tpl.h.
|
||||
*
|
||||
* We load/store a single chunk once in the `CHUNKCOPY`.
|
||||
* However, RISC-V glibc would enable RVV optimized memcpy at runtime by IFUNC,
|
||||
* such that, we prefer copy large memory size once to make good use of the the RVV advance.
|
||||
*
|
||||
* To be aligned to the other platforms, we didn't modify `CHUNKCOPY` method a lot,
|
||||
* but we still copy as much memory as possible for some conditions.
|
||||
*
|
||||
* case 1: out - from >= len (no overlap)
|
||||
* We can use memcpy to copy `len` size once
|
||||
* because the memory layout would be the same.
|
||||
*
|
||||
* case 2: overlap
|
||||
* We copy N chunks using memcpy at once, aiming to achieve our goal:
|
||||
* to copy as much memory as possible.
|
||||
*
|
||||
* After using a single memcpy to copy N chunks, we have to use series of
|
||||
* loadchunk and storechunk to ensure the result is correct.
|
||||
*/
|
||||
static inline uint8_t* CHUNKCOPY(uint8_t *out, uint8_t const *from, size_t len) {
|
||||
Assert(len > 0, "chunkcopy should never have a length 0");
|
||||
size_t dist = out - from;
|
||||
if (out < from || dist >= len) {
|
||||
memcpy(out, from, len);
|
||||
out += len;
|
||||
from += len;
|
||||
return out;
|
||||
}
|
||||
|
||||
size_t align = ((len - 1) % sizeof(chunk_t)) + 1;
|
||||
memcpy(out, from, sizeof(chunk_t));
|
||||
out += align;
|
||||
from += align;
|
||||
len -= align;
|
||||
|
||||
size_t vl = (dist / sizeof(chunk_t)) * sizeof(chunk_t);
|
||||
while (len > dist) {
|
||||
memcpy(out, from, vl);
|
||||
out += vl;
|
||||
from += vl;
|
||||
len -= vl;
|
||||
}
|
||||
|
||||
if (len > 0) {
|
||||
memcpy(out, from, len);
|
||||
out += len;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
#include "chunkset_tpl.h"
|
||||
|
||||
#define INFLATE_FAST inflate_fast_rvv
|
||||
|
||||
#include "inffast_tpl.h"
|
||||
|
||||
#endif
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
/* compare256_rvv.c - RVV version of compare256
|
||||
* Copyright (C) 2023 SiFive, Inc. All rights reserved.
|
||||
* Contributed by Alex Chiang <alex.chiang@sifive.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef RISCV_RVV
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zmemory.h"
|
||||
#include "deflate.h"
|
||||
|
||||
#include <riscv_vector.h>
|
||||
|
||||
static inline uint32_t compare256_rvv_static(const uint8_t *src0, const uint8_t *src1) {
|
||||
uint32_t len = 0;
|
||||
size_t vl;
|
||||
long found_diff;
|
||||
do {
|
||||
vl = __riscv_vsetvl_e8m4(256 - len);
|
||||
vuint8m4_t v_src0 = __riscv_vle8_v_u8m4(src0, vl);
|
||||
vuint8m4_t v_src1 = __riscv_vle8_v_u8m4(src1, vl);
|
||||
vbool2_t v_mask = __riscv_vmsne_vv_u8m4_b2(v_src0, v_src1, vl);
|
||||
found_diff = __riscv_vfirst_m_b2(v_mask, vl);
|
||||
if (found_diff >= 0)
|
||||
return len + (uint32_t)found_diff;
|
||||
src0 += vl, src1 += vl, len += vl;
|
||||
} while (len < 256);
|
||||
|
||||
return 256;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t compare256_rvv(const uint8_t *src0, const uint8_t *src1) {
|
||||
return compare256_rvv_static(src0, src1);
|
||||
}
|
||||
|
||||
#define LONGEST_MATCH longest_match_rvv
|
||||
#define COMPARE256 compare256_rvv_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#define LONGEST_MATCH_ROLL
|
||||
#define LONGEST_MATCH longest_match_roll_rvv
|
||||
#define COMPARE256 compare256_rvv_static
|
||||
|
||||
#include "match_tpl.h"
|
||||
|
||||
#endif // RISCV_RVV
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
/* crc32_zbc.c - RISCV Zbc version of crc32
|
||||
* Copyright (C) 2025 ByteDance. All rights reserved.
|
||||
* Contributed by Yin Tong <yintong.ustc@bytedance.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef RISCV_CRC32_ZBC
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arch_functions.h"
|
||||
|
||||
#define CLMUL_MIN_LEN 16 // Minimum size of buffer for _crc32_clmul
|
||||
#define CLMUL_CHUNK_LEN 16 // Length of chunk for clmul
|
||||
|
||||
#define CONSTANT_R3 0x1751997d0ULL
|
||||
#define CONSTANT_R4 0x0ccaa009eULL
|
||||
#define CONSTANT_R5 0x163cd6124ULL
|
||||
#define MASK32 0xFFFFFFFF
|
||||
#define CRCPOLY_TRUE_LE_FULL 0x1DB710641ULL
|
||||
#define CONSTANT_RU 0x1F7011641ULL
|
||||
|
||||
static inline uint64_t clmul(uint64_t a, uint64_t b) {
|
||||
uint64_t res;
|
||||
__asm__ volatile("clmul %0, %1, %2" : "=r"(res) : "r"(a), "r"(b));
|
||||
return res;
|
||||
}
|
||||
|
||||
static inline uint64_t clmulh(uint64_t a, uint64_t b) {
|
||||
uint64_t res;
|
||||
__asm__ volatile("clmulh %0, %1, %2" : "=r"(res) : "r"(a), "r"(b));
|
||||
return res;
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static uint32_t crc32_clmul_impl(uint64_t crc, const unsigned char *buf, uint64_t len) {
|
||||
const uint64_t *buf64 = (const uint64_t *)buf;
|
||||
uint64_t low = buf64[0] ^ crc;
|
||||
uint64_t high = buf64[1];
|
||||
|
||||
if (len < 16)
|
||||
goto finish_fold;
|
||||
len -= 16;
|
||||
buf64 += 2;
|
||||
|
||||
// process each 16-byte block
|
||||
while (len >= 16) {
|
||||
uint64_t t2 = clmul(CONSTANT_R4, high);
|
||||
uint64_t t3 = clmulh(CONSTANT_R4, high);
|
||||
|
||||
uint64_t t0_new = clmul(CONSTANT_R3, low);
|
||||
uint64_t t1_new = clmulh(CONSTANT_R3, low);
|
||||
|
||||
// Combine the results and XOR with new data
|
||||
low = t0_new ^ t2;
|
||||
high = t1_new ^ t3;
|
||||
low ^= buf64[0];
|
||||
high ^= buf64[1];
|
||||
|
||||
buf64 += 2;
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
finish_fold:
|
||||
// Fold the 128-bit result into 64 bits
|
||||
uint64_t fold_t3 = clmulh(low, CONSTANT_R4);
|
||||
uint64_t fold_t2 = clmul(low, CONSTANT_R4);
|
||||
low = high ^ fold_t2;
|
||||
high = fold_t3;
|
||||
|
||||
// Combine the low and high parts and perform polynomial reduction
|
||||
uint64_t combined = (low >> 32) | ((high & MASK32) << 32);
|
||||
uint64_t reduced_low = clmul(low & MASK32, CONSTANT_R5) ^ combined;
|
||||
|
||||
// Barrett reduction step
|
||||
uint64_t barrett = clmul(reduced_low & MASK32, CONSTANT_RU) & MASK32;
|
||||
barrett = clmul(barrett, CRCPOLY_TRUE_LE_FULL);
|
||||
uint64_t final = barrett ^ reduced_low;
|
||||
|
||||
// Return the high 32 bits as the final CRC
|
||||
return (uint32_t)(final >> 32);
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t crc32_riscv64_zbc(uint32_t crc, const uint8_t *buf, size_t len) {
|
||||
if (len < CLMUL_MIN_LEN) {
|
||||
return crc32_braid(crc, buf, len);
|
||||
}
|
||||
|
||||
uint64_t unaligned_length = len % CLMUL_CHUNK_LEN;
|
||||
if (unaligned_length) {
|
||||
crc = crc32_braid(crc, buf, unaligned_length);
|
||||
buf += unaligned_length;
|
||||
len -= unaligned_length;
|
||||
}
|
||||
|
||||
crc = crc32_clmul_impl(~crc, buf, len);
|
||||
return ~crc;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t crc32_copy_riscv64_zbc(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
crc = crc32_riscv64_zbc(crc, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return crc;
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
#ifdef RISCV_FEATURES
|
||||
|
||||
#define _DEFAULT_SOURCE 1 /* For syscall() */
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "riscv_features.h"
|
||||
|
||||
#include <sys/utsname.h>
|
||||
|
||||
#if defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
# include <sys/auxv.h>
|
||||
#endif
|
||||
|
||||
#if defined(__linux__) && defined(HAVE_ASM_HWPROBE_H)
|
||||
# include <asm/hwprobe.h>
|
||||
# include <sys/syscall.h> /* For __NR_riscv_hwprobe */
|
||||
# include <unistd.h> /* For syscall() */
|
||||
#endif
|
||||
|
||||
#define ISA_V_HWCAP (1 << ('v' - 'a'))
|
||||
#define ISA_ZBC_HWCAP (1 << 29)
|
||||
|
||||
static int riscv_check_features_runtime_hwprobe(struct riscv_cpu_features *features) {
|
||||
#if defined(__NR_riscv_hwprobe) && defined(RISCV_HWPROBE_KEY_IMA_EXT_0)
|
||||
struct riscv_hwprobe probes[] = {
|
||||
{RISCV_HWPROBE_KEY_IMA_EXT_0, 0},
|
||||
};
|
||||
int ret;
|
||||
unsigned i;
|
||||
|
||||
ret = syscall(__NR_riscv_hwprobe, probes, sizeof(probes) / sizeof(probes[0]), 0, NULL, 0);
|
||||
|
||||
if (ret != 0) {
|
||||
/* Kernel does not support hwprobe */
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (i = 0; i < sizeof(probes) / sizeof(probes[0]); i++) {
|
||||
switch (probes[i].key) {
|
||||
case RISCV_HWPROBE_KEY_IMA_EXT_0:
|
||||
# ifdef RISCV_HWPROBE_IMA_V
|
||||
features->has_rvv = !!(probes[i].value & RISCV_HWPROBE_IMA_V);
|
||||
# endif
|
||||
# ifdef RISCV_HWPROBE_EXT_ZBC
|
||||
features->has_zbc = !!(probes[i].value & RISCV_HWPROBE_EXT_ZBC);
|
||||
# endif
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int riscv_check_features_runtime_hwcap(struct riscv_cpu_features *features) {
|
||||
#if defined(__linux__) && defined(HAVE_SYS_AUXV_H)
|
||||
unsigned long hw_cap = getauxval(AT_HWCAP);
|
||||
|
||||
features->has_rvv = hw_cap & ISA_V_HWCAP;
|
||||
features->has_zbc = hw_cap & ISA_ZBC_HWCAP;
|
||||
|
||||
return 1;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void riscv_check_features_runtime(struct riscv_cpu_features *features) {
|
||||
if (riscv_check_features_runtime_hwprobe(features))
|
||||
return;
|
||||
|
||||
riscv_check_features_runtime_hwcap(features);
|
||||
}
|
||||
|
||||
void Z_INTERNAL riscv_check_features(struct riscv_cpu_features *features) {
|
||||
riscv_check_features_runtime(features);
|
||||
#ifdef RISCV_RVV
|
||||
if (features->has_rvv) {
|
||||
size_t e8m1_vec_len;
|
||||
intptr_t vtype_reg_val;
|
||||
// Check that a vuint8m1_t vector is at least 16 bytes and that tail
|
||||
// agnostic and mask agnostic mode are supported
|
||||
//
|
||||
__asm__ volatile(
|
||||
"vsetvli %0, zero, e8, m1, ta, ma\n\t"
|
||||
"csrr %1, vtype"
|
||||
: "=r"(e8m1_vec_len), "=r"(vtype_reg_val));
|
||||
|
||||
// The RVV target is supported if the VILL bit of VTYPE (the MSB bit of
|
||||
// VTYPE) is not set and the length of a vuint8m1_t vector is at least 16
|
||||
// bytes
|
||||
features->has_rvv = (vtype_reg_val >= 0 && e8m1_vec_len >= 16);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
/* riscv_features.h -- check for riscv features.
|
||||
*
|
||||
* Copyright (C) 2023 SiFive, Inc. All rights reserved.
|
||||
* Contributed by Alex Chiang <alex.chiang@sifive.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef RISCV_FEATURES_H_
|
||||
#define RISCV_FEATURES_H_
|
||||
|
||||
struct riscv_cpu_features {
|
||||
int has_rvv;
|
||||
int has_zbc;
|
||||
};
|
||||
|
||||
void Z_INTERNAL riscv_check_features(struct riscv_cpu_features *features);
|
||||
|
||||
#endif /* RISCV_FEATURES_H_ */
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
/* riscv_functions.h -- RISCV implementations for arch-specific functions.
|
||||
*
|
||||
* Copyright (C) 2023 SiFive, Inc. All rights reserved.
|
||||
* Contributed by Alex Chiang <alex.chiang@sifive.com>
|
||||
*
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef RISCV_FUNCTIONS_H_
|
||||
#define RISCV_FUNCTIONS_H_
|
||||
|
||||
#include "riscv_natives.h"
|
||||
|
||||
#define CRC32_BRAID_FALLBACK /* used by crc32_zbc */
|
||||
|
||||
#ifdef RISCV_RVV
|
||||
uint32_t adler32_rvv(uint32_t adler, const uint8_t *buf, size_t len);
|
||||
uint32_t adler32_copy_rvv(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
uint8_t* chunkmemset_safe_rvv(uint8_t *out, uint8_t *from, size_t len, size_t left);
|
||||
uint32_t compare256_rvv(const uint8_t *src0, const uint8_t *src1);
|
||||
|
||||
uint32_t longest_match_rvv(deflate_state *const s, uint32_t cur_match);
|
||||
uint32_t longest_match_roll_rvv(deflate_state *const s, uint32_t cur_match);
|
||||
void slide_hash_rvv(deflate_state *s);
|
||||
void inflate_fast_rvv(PREFIX3(stream) *strm, uint32_t start);
|
||||
#endif
|
||||
|
||||
#ifndef RISCV_RVV_NATIVE
|
||||
# define ADLER32_FALLBACK
|
||||
# define CHUNKSET_FALLBACK
|
||||
# define COMPARE256_FALLBACK
|
||||
# define SLIDE_HASH_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef RISCV_CRC32_ZBC
|
||||
uint32_t crc32_riscv64_zbc(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_riscv64_zbc(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_RUNTIME_CPU_DETECTION
|
||||
// RISCV - RVV
|
||||
# ifdef RISCV_RVV_NATIVE
|
||||
# undef native_adler32
|
||||
# define native_adler32 adler32_rvv
|
||||
# undef native_adler32_copy
|
||||
# define native_adler32_copy adler32_copy_rvv
|
||||
# undef native_chunkmemset_safe
|
||||
# define native_chunkmemset_safe chunkmemset_safe_rvv
|
||||
# undef native_compare256
|
||||
# define native_compare256 compare256_rvv
|
||||
# undef native_inflate_fast
|
||||
# define native_inflate_fast inflate_fast_rvv
|
||||
# undef native_longest_match
|
||||
# define native_longest_match longest_match_rvv
|
||||
# undef native_longest_match_roll
|
||||
# define native_longest_match_roll longest_match_roll_rvv
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_rvv
|
||||
# endif
|
||||
// RISCV - CRC32
|
||||
# ifdef RISCV_ZBC_NATIVE
|
||||
# undef native_crc32
|
||||
# define native_crc32 crc32_riscv64_zbc
|
||||
# undef native_crc32_copy
|
||||
# define native_crc32_copy crc32_copy_riscv64_zbc
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* RISCV_FUNCTIONS_H_ */
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
/* riscv_natives.h -- RISCV compile-time feature detection macros.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef RISCV_NATIVES_H_
|
||||
#define RISCV_NATIVES_H_
|
||||
|
||||
#if defined(__riscv_v) && defined(__linux__)
|
||||
# ifdef RISCV_RVV
|
||||
# define RISCV_RVV_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
#if defined(__riscv_zbc)
|
||||
# ifdef RISCV_CRC32_ZBC
|
||||
# define RISCV_ZBC_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* RISCV_NATIVES_H_ */
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
/* slide_hash_rvv.c - RVV version of slide_hash
|
||||
* Copyright (C) 2023 SiFive, Inc. All rights reserved.
|
||||
* Contributed by Alex Chiang <alex.chiang@sifive.com>
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef RISCV_RVV
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "deflate.h"
|
||||
|
||||
#include <riscv_vector.h>
|
||||
|
||||
static inline void slide_hash_chain(Pos *table, uint32_t entries, uint16_t wsize) {
|
||||
size_t vl;
|
||||
while (entries > 0) {
|
||||
vl = __riscv_vsetvl_e16m4(entries);
|
||||
vuint16m4_t v_tab = __riscv_vle16_v_u16m4(table, vl);
|
||||
vuint16m4_t v_diff = __riscv_vssubu_vx_u16m4(v_tab, wsize, vl);
|
||||
__riscv_vse16_v_u16m4(table, v_diff, vl);
|
||||
table += vl, entries -= vl;
|
||||
}
|
||||
}
|
||||
|
||||
Z_INTERNAL void slide_hash_rvv(deflate_state *s) {
|
||||
Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
|
||||
slide_hash_chain(s->head, HASH_SIZE, wsize);
|
||||
slide_hash_chain(s->prev, wsize, wsize);
|
||||
}
|
||||
|
||||
#endif // RISCV_RVV
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
# Makefile for zlib-ng
|
||||
# Copyright (C) 1995-2013 Jean-loup Gailly, Mark Adler
|
||||
# For conditions of distribution and use, see copyright notice in zlib.h
|
||||
|
||||
CC=
|
||||
CFLAGS=
|
||||
SFLAGS=
|
||||
INCLUDES=
|
||||
SUFFIX=
|
||||
S390VXFLAG=
|
||||
NOLTOFLAG=
|
||||
|
||||
SRCDIR=.
|
||||
SRCTOP=../..
|
||||
TOPDIR=$(SRCTOP)
|
||||
|
||||
s390_features.o:
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/s390_features.c
|
||||
|
||||
s390_features.lo:
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/s390_features.c
|
||||
|
||||
dfltcc_deflate.o:
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/dfltcc_deflate.c
|
||||
|
||||
dfltcc_deflate.lo:
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/dfltcc_deflate.c
|
||||
|
||||
dfltcc_inflate.o:
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/dfltcc_inflate.c
|
||||
|
||||
dfltcc_inflate.lo:
|
||||
$(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/dfltcc_inflate.c
|
||||
|
||||
crc32_vx.o:
|
||||
$(CC) $(CFLAGS) $(S390VXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_vx.c
|
||||
|
||||
crc32_vx.lo:
|
||||
$(CC) $(SFLAGS) $(S390VXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/crc32_vx.c
|
||||
|
||||
slide_hash_vx.o:
|
||||
$(CC) $(CFLAGS) $(S390VXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_vx.c
|
||||
|
||||
slide_hash_vx.lo:
|
||||
$(CC) $(SFLAGS) $(S390VXFLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_vx.c
|
||||
|
||||
mostlyclean: clean
|
||||
clean:
|
||||
rm -f *.o *.lo *~
|
||||
rm -rf objs
|
||||
rm -f *.gcda *.gcno *.gcov
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
|
@ -1,265 +0,0 @@
|
|||
# Introduction
|
||||
|
||||
This directory contains SystemZ deflate hardware acceleration support.
|
||||
It can be enabled using the following build commands:
|
||||
|
||||
$ ./configure --with-dfltcc-deflate --with-dfltcc-inflate
|
||||
$ make
|
||||
|
||||
or
|
||||
|
||||
$ cmake -DWITH_DFLTCC_DEFLATE=1 -DWITH_DFLTCC_INFLATE=1 .
|
||||
$ make
|
||||
|
||||
When built like this, zlib-ng would compress using hardware on level 1,
|
||||
and using software on all other levels. Decompression will always happen
|
||||
in hardware. In order to enable hardware compression for levels 1-6
|
||||
(i.e. to make it used by default) one could add
|
||||
`-DDFLTCC_LEVEL_MASK=0x7e` to CFLAGS when building zlib-ng.
|
||||
|
||||
SystemZ deflate hardware acceleration is available on [IBM z15](
|
||||
https://www.ibm.com/products/z15) and newer machines under the name [
|
||||
"Integrated Accelerator for zEnterprise Data Compression"](
|
||||
https://www.ibm.com/support/z-content-solutions/compression/). The
|
||||
programming interface to it is a machine instruction called DEFLATE
|
||||
CONVERSION CALL (DFLTCC). It is documented in Chapter 26 of [Principles
|
||||
of Operation](https://publibfp.dhe.ibm.com/epubs/pdf/a227832c.pdf). Both
|
||||
the code and the rest of this document refer to this feature simply as
|
||||
"DFLTCC".
|
||||
|
||||
# Performance
|
||||
|
||||
Performance figures are published [here](
|
||||
https://github.com/iii-i/zlib-ng/wiki/Performance-with-dfltcc-patch-applied-and-dfltcc-support-built-on-dfltcc-enabled-machine
|
||||
). The compression speed-up can be as high as 110x and the decompression
|
||||
speed-up can be as high as 15x.
|
||||
|
||||
# Limitations
|
||||
|
||||
Two DFLTCC compression calls with identical inputs are not guaranteed to
|
||||
produce identical outputs. Therefore care should be taken when using
|
||||
hardware compression when reproducible results are desired. In
|
||||
particular, zlib-ng-specific `zng_deflateSetParams` call allows setting
|
||||
`Z_DEFLATE_REPRODUCIBLE` parameter, which disables DFLTCC support for a
|
||||
particular stream.
|
||||
|
||||
DFLTCC does not support every single zlib-ng feature, in particular:
|
||||
|
||||
* `inflate(Z_BLOCK)` and `inflate(Z_TREES)`
|
||||
* `inflateMark()`
|
||||
* `inflatePrime()`
|
||||
* `inflateSyncPoint()`
|
||||
|
||||
When used, these functions will either switch to software, or, in case
|
||||
this is not possible, gracefully fail.
|
||||
|
||||
# Code structure
|
||||
|
||||
All SystemZ-specific code lives in `arch/s390` directory and is
|
||||
integrated with the rest of zlib-ng using hook macros.
|
||||
|
||||
## Hook macros
|
||||
|
||||
DFLTCC takes as arguments a parameter block, an input buffer, an output
|
||||
buffer, and a window. Parameter blocks are stored alongside zlib states;
|
||||
buffers are forwarded from the caller; and window - which must be
|
||||
4k-aligned and is always 64k large, is managed using the `PAD_WINDOW()`,
|
||||
`WINDOW_PAD_SIZE`, `HINT_ALIGNED_WINDOW` and `DEFLATE_ADJUST_WINDOW_SIZE()`
|
||||
and `INFLATE_ADJUST_WINDOW_SIZE()` hooks.
|
||||
|
||||
Software and hardware window formats do not match, therefore,
|
||||
`deflateSetDictionary()`, `deflateGetDictionary()`, `inflateSetDictionary()`
|
||||
and `inflateGetDictionary()` need special handling, which is triggered using
|
||||
`DEFLATE_SET_DICTIONARY_HOOK()`, `DEFLATE_GET_DICTIONARY_HOOK()`,
|
||||
`INFLATE_SET_DICTIONARY_HOOK()` and `INFLATE_GET_DICTIONARY_HOOK()` macros.
|
||||
|
||||
`deflateResetKeep()` and `inflateResetKeep()` update the DFLTCC
|
||||
parameter block using `DEFLATE_RESET_KEEP_HOOK()` and
|
||||
`INFLATE_RESET_KEEP_HOOK()` macros.
|
||||
|
||||
`INFLATE_PRIME_HOOK()`, `INFLATE_MARK_HOOK()` and
|
||||
`INFLATE_SYNC_POINT_HOOK()` macros make the respective unsupported
|
||||
calls gracefully fail.
|
||||
|
||||
`DEFLATE_PARAMS_HOOK()` implements switching between hardware and
|
||||
software compression mid-stream using `deflateParams()`. Switching
|
||||
normally entails flushing the current block, which might not be possible
|
||||
in low memory situations. `deflateParams()` uses `DEFLATE_DONE()` hook
|
||||
in order to detect and gracefully handle such situations.
|
||||
|
||||
The algorithm implemented in hardware has different compression ratio
|
||||
than the one implemented in software. `DEFLATE_BOUND_ADJUST_COMPLEN()`
|
||||
and `DEFLATE_NEED_CONSERVATIVE_BOUND()` macros make `deflateBound()`
|
||||
return the correct results for the hardware implementation.
|
||||
|
||||
Actual compression and decompression are handled by `DEFLATE_HOOK()` and
|
||||
`INFLATE_TYPEDO_HOOK()` macros. Since inflation with DFLTCC manages the
|
||||
window on its own, calling `updatewindow()` is suppressed using
|
||||
`INFLATE_NEED_UPDATEWINDOW()` macro.
|
||||
|
||||
In addition to compression, DFLTCC computes CRC-32 and Adler-32
|
||||
checksums, therefore, whenever it's used, software checksumming is
|
||||
suppressed using `DEFLATE_NEED_CHECKSUM()` and `INFLATE_NEED_CHECKSUM()`
|
||||
macros.
|
||||
|
||||
While software always produces reproducible compression results, this
|
||||
is not the case for DFLTCC. Therefore, zlib-ng users are given the
|
||||
ability to specify whether or not reproducible compression results
|
||||
are required. While it is always possible to specify this setting
|
||||
before the compression begins, it is not always possible to do so in
|
||||
the middle of a deflate stream - the exact conditions for that are
|
||||
determined by `DEFLATE_CAN_SET_REPRODUCIBLE()` macro.
|
||||
|
||||
## SystemZ-specific code
|
||||
|
||||
When zlib-ng is built with DFLTCC, the hooks described above are
|
||||
converted to calls to functions, which are implemented in
|
||||
`arch/s390/dfltcc_*` files. The functions can be grouped in three broad
|
||||
categories:
|
||||
|
||||
* Base DFLTCC support, e.g. wrapping the machine instruction - `dfltcc()`.
|
||||
* Translating between software and hardware data formats, e.g.
|
||||
`dfltcc_deflate_set_dictionary()`.
|
||||
* Translating between software and hardware state machines, e.g.
|
||||
`dfltcc_deflate()` and `dfltcc_inflate()`.
|
||||
|
||||
The functions from the first two categories are fairly simple, however,
|
||||
various quirks in both software and hardware state machines make the
|
||||
functions from the third category quite complicated.
|
||||
|
||||
### `dfltcc_deflate()` function
|
||||
|
||||
This function is called by `deflate()` and has the following
|
||||
responsibilities:
|
||||
|
||||
* Checking whether DFLTCC can be used with the current stream. If this
|
||||
is not the case, then it returns `0`, making `deflate()` use some
|
||||
other function in order to compress in software. Otherwise it returns
|
||||
`1`.
|
||||
* Block management and Huffman table generation. DFLTCC ends blocks only
|
||||
when explicitly instructed to do so by the software. Furthermore,
|
||||
whether to use fixed or dynamic Huffman tables must also be determined
|
||||
by the software. Since looking at data in order to gather statistics
|
||||
would negate performance benefits, the following approach is used: the
|
||||
first `DFLTCC_FIRST_FHT_BLOCK_SIZE` bytes are placed into a fixed
|
||||
block, and every next `DFLTCC_BLOCK_SIZE` bytes are placed into
|
||||
dynamic blocks.
|
||||
* Writing EOBS. Block Closing Control bit in the parameter block
|
||||
instructs DFLTCC to write EOBS, however, certain conditions need to be
|
||||
met: input data length must be non-zero or Continuation Flag must be
|
||||
set. To put this in simpler terms, DFLTCC will silently refuse to
|
||||
write EOBS if this is the only thing that it is asked to do. Since the
|
||||
code has to be able to emit EOBS in software anyway, in order to avoid
|
||||
tricky corner cases Block Closing Control is never used. Whether to
|
||||
write EOBS is instead controlled by `soft_bcc` variable.
|
||||
* Triggering block post-processing. Depending on flush mode, `deflate()`
|
||||
must perform various additional actions when a block or a stream ends.
|
||||
`dfltcc_deflate()` informs `deflate()` about this using
|
||||
`block_state *result` parameter.
|
||||
* Converting software state fields into hardware parameter block fields,
|
||||
and vice versa. For example, `wrap` and Check Value Type or `bi_valid`
|
||||
and Sub-Byte Boundary. Certain fields cannot be translated and must
|
||||
persist untouched in the parameter block between calls, for example,
|
||||
Continuation Flag or Continuation State Buffer.
|
||||
* Handling flush modes and low-memory situations. These aspects are
|
||||
quite intertwined and pervasive. The general idea here is that the
|
||||
code must not do anything in software - whether explicitly by e.g.
|
||||
calling `send_eobs()`, or implicitly - by returning to `deflate()`
|
||||
with certain return and `*result` values, when Continuation Flag is
|
||||
set.
|
||||
* Ending streams. When a new block is started and flush mode is
|
||||
`Z_FINISH`, Block Header Final parameter block bit is used to mark
|
||||
this block as final. However, sometimes an empty final block is
|
||||
needed, and, unfortunately, just like with EOBS, DFLTCC will silently
|
||||
refuse to do this. The general idea of DFLTCC implementation is to
|
||||
rely as much as possible on the existing code. Here in order to do
|
||||
this, the code pretends that it does not support DFLTCC, which makes
|
||||
`deflate()` call a software compression function, which writes an
|
||||
empty final block. Whether this is required is controlled by
|
||||
`need_empty_block` variable.
|
||||
* Error handling. This is simply converting
|
||||
Operation-Ending-Supplemental Code to string. Errors can only happen
|
||||
due to things like memory corruption, and therefore they don't affect
|
||||
the `deflate()` return code.
|
||||
|
||||
### `dfltcc_inflate()` function
|
||||
|
||||
This function is called by `inflate()` from the `TYPEDO` state (that is,
|
||||
when all the metadata is parsed and the stream is positioned at the type
|
||||
bits of deflate block header) and it's responsible for the following:
|
||||
|
||||
* Falling back to software when flush mode is `Z_BLOCK` or `Z_TREES`.
|
||||
Unfortunately, there is no way to ask DFLTCC to stop decompressing on
|
||||
block or tree boundary.
|
||||
* `inflate()` decompression loop management. This is controlled using
|
||||
the return value, which can be either `DFLTCC_INFLATE_BREAK` or
|
||||
`DFLTCC_INFLATE_CONTINUE`.
|
||||
* Converting software state fields into hardware parameter block fields,
|
||||
and vice versa. For example, `whave` and History Length or `wnext` and
|
||||
History Offset.
|
||||
* Ending streams. This instructs `inflate()` to return `Z_STREAM_END`
|
||||
and is controlled by `last` state field.
|
||||
* Error handling. Like deflate, error handling comprises
|
||||
Operation-Ending-Supplemental Code to string conversion. Unlike
|
||||
deflate, errors may happen due to bad inputs, therefore they are
|
||||
propagated to `inflate()` by setting `mode` field to `MEM` or `BAD`.
|
||||
|
||||
# Testing
|
||||
|
||||
Given complexity of DFLTCC machine instruction, it is not clear whether
|
||||
QEMU TCG will ever support it. At the time of writing, one has to have
|
||||
access to an IBM z15+ VM or LPAR in order to test DFLTCC support. Since
|
||||
DFLTCC is a non-privileged instruction, neither special VM/LPAR
|
||||
configuration nor root are required.
|
||||
|
||||
zlib-ng CI uses an IBM-provided z15 self-hosted builder for the DFLTCC
|
||||
testing. There is no official IBM Z GitHub Actions runner, so we build
|
||||
one inspired by `anup-kodlekere/gaplib`.
|
||||
Future updates to actions-runner might need an updated patch. The .net
|
||||
version number patch has been separated into a separate file to avoid a
|
||||
need for constantly changing the patch.
|
||||
|
||||
## Configuring the builder.
|
||||
|
||||
### Install prerequisites.
|
||||
```
|
||||
sudo dnf install podman
|
||||
```
|
||||
|
||||
### Create a config file, needs github personal access token.
|
||||
Access token needs permissions; Repo Admin RW, Org Self-hosted runners RW.
|
||||
For details, consult
|
||||
https://docs.github.com/en/rest/actions/self-hosted-runners?apiVersion=2022-11-28#create-a-registration-token-for-a-repository
|
||||
|
||||
#### Create file /etc/actions-runner:
|
||||
```
|
||||
REPO=<owner>/<name>
|
||||
PAT_TOKEN=<github_pat_***>
|
||||
```
|
||||
|
||||
#### Set permissions on /etc/actions-runner:
|
||||
```
|
||||
chmod 600 /etc/actions-runner
|
||||
```
|
||||
|
||||
### Add actions-runner service.
|
||||
```
|
||||
sudo cp self-hosted-builder/actions-runner.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
### Autostart actions-runner.
|
||||
```
|
||||
$ sudo systemctl enable --now actions-runner
|
||||
```
|
||||
|
||||
### Add auto-rebuild cronjob
|
||||
```
|
||||
sudo cp self-hosted-builder/actions-runner-rebuild.sh /etc/cron.weekly/
|
||||
chmod +x /etc/cron.weekly/actions-runner-rebuild.sh
|
||||
```
|
||||
|
||||
## Building / Rebuilding the container
|
||||
```
|
||||
sudo /etc/cron.weekly/actions-runner-rebuild.sh
|
||||
```
|
||||
|
|
@ -1,228 +0,0 @@
|
|||
/*
|
||||
* Hardware-accelerated CRC-32 variants for Linux on z Systems
|
||||
*
|
||||
* Use the z/Architecture Vector Extension Facility to accelerate the
|
||||
* computing of bitreflected CRC-32 checksums.
|
||||
*
|
||||
* This CRC-32 implementation algorithm is bitreflected and processes
|
||||
* the least-significant bit first (Little-Endian).
|
||||
*
|
||||
* This code was originally written by Hendrik Brueckner
|
||||
* <brueckner@linux.vnet.ibm.com> for use in the Linux kernel and has been
|
||||
* relicensed under the zlib license.
|
||||
*/
|
||||
|
||||
#ifdef S390_VX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "arch_functions.h"
|
||||
|
||||
#include "vx_intrins.h"
|
||||
|
||||
static uint32_t crc32_le_vgfm_16(uint32_t crc, const uint8_t *buf, size_t len) {
|
||||
/*
|
||||
* The CRC-32 constant block contains reduction constants to fold and
|
||||
* process particular chunks of the input data stream in parallel.
|
||||
*
|
||||
* For the CRC-32 variants, the constants are precomputed according to
|
||||
* these definitions:
|
||||
*
|
||||
* R1 = [(x4*128+32 mod P'(x) << 32)]' << 1
|
||||
* R2 = [(x4*128-32 mod P'(x) << 32)]' << 1
|
||||
* R3 = [(x128+32 mod P'(x) << 32)]' << 1
|
||||
* R4 = [(x128-32 mod P'(x) << 32)]' << 1
|
||||
* R5 = [(x64 mod P'(x) << 32)]' << 1
|
||||
* R6 = [(x32 mod P'(x) << 32)]' << 1
|
||||
*
|
||||
* The bitreflected Barret reduction constant, u', is defined as
|
||||
* the bit reversal of floor(x**64 / P(x)).
|
||||
*
|
||||
* where P(x) is the polynomial in the normal domain and the P'(x) is the
|
||||
* polynomial in the reversed (bitreflected) domain.
|
||||
*
|
||||
* CRC-32 (IEEE 802.3 Ethernet, ...) polynomials:
|
||||
*
|
||||
* P(x) = 0x04C11DB7
|
||||
* P'(x) = 0xEDB88320
|
||||
*/
|
||||
const uv16qi perm_le2be = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; /* BE->LE mask */
|
||||
const uv2di r2r1 = {0x1C6E41596, 0x154442BD4}; /* R2, R1 */
|
||||
const uv2di r4r3 = {0x0CCAA009E, 0x1751997D0}; /* R4, R3 */
|
||||
const uv2di r5 = {0, 0x163CD6124}; /* R5 */
|
||||
const uv2di ru_poly = {0, 0x1F7011641}; /* u' */
|
||||
const uv2di crc_poly = {0, 0x1DB710641}; /* P'(x) << 1 */
|
||||
|
||||
/*
|
||||
* Load the initial CRC value.
|
||||
*
|
||||
* The CRC value is loaded into the rightmost word of the
|
||||
* vector register and is later XORed with the LSB portion
|
||||
* of the loaded input data.
|
||||
*/
|
||||
uv2di v0 = {0, 0};
|
||||
v0 = (uv2di)vec_insert(crc, (uv4si)v0, 3);
|
||||
|
||||
/* Load a 64-byte data chunk and XOR with CRC */
|
||||
uv2di v1 = vec_perm(((uv2di *)buf)[0], ((uv2di *)buf)[0], perm_le2be);
|
||||
uv2di v2 = vec_perm(((uv2di *)buf)[1], ((uv2di *)buf)[1], perm_le2be);
|
||||
uv2di v3 = vec_perm(((uv2di *)buf)[2], ((uv2di *)buf)[2], perm_le2be);
|
||||
uv2di v4 = vec_perm(((uv2di *)buf)[3], ((uv2di *)buf)[3], perm_le2be);
|
||||
|
||||
v1 ^= v0;
|
||||
buf += 64;
|
||||
len -= 64;
|
||||
|
||||
while (len >= 64) {
|
||||
/* Load the next 64-byte data chunk */
|
||||
uv16qi part1 = vec_perm(((uv16qi *)buf)[0], ((uv16qi *)buf)[0], perm_le2be);
|
||||
uv16qi part2 = vec_perm(((uv16qi *)buf)[1], ((uv16qi *)buf)[1], perm_le2be);
|
||||
uv16qi part3 = vec_perm(((uv16qi *)buf)[2], ((uv16qi *)buf)[2], perm_le2be);
|
||||
uv16qi part4 = vec_perm(((uv16qi *)buf)[3], ((uv16qi *)buf)[3], perm_le2be);
|
||||
|
||||
/*
|
||||
* Perform a GF(2) multiplication of the doublewords in V1 with
|
||||
* the R1 and R2 reduction constants in V0. The intermediate result
|
||||
* is then folded (accumulated) with the next data chunk in PART1 and
|
||||
* stored in V1. Repeat this step for the register contents
|
||||
* in V2, V3, and V4 respectively.
|
||||
*/
|
||||
v1 = (uv2di)vec_gfmsum_accum_128(r2r1, v1, part1);
|
||||
v2 = (uv2di)vec_gfmsum_accum_128(r2r1, v2, part2);
|
||||
v3 = (uv2di)vec_gfmsum_accum_128(r2r1, v3, part3);
|
||||
v4 = (uv2di)vec_gfmsum_accum_128(r2r1, v4, part4);
|
||||
|
||||
buf += 64;
|
||||
len -= 64;
|
||||
}
|
||||
|
||||
/*
|
||||
* Fold V1 to V4 into a single 128-bit value in V1. Multiply V1 with R3
|
||||
* and R4 and accumulating the next 128-bit chunk until a single 128-bit
|
||||
* value remains.
|
||||
*/
|
||||
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v2);
|
||||
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v3);
|
||||
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v4);
|
||||
|
||||
while (len >= 16) {
|
||||
/* Load next data chunk */
|
||||
v2 = vec_perm(*(uv2di *)buf, *(uv2di *)buf, perm_le2be);
|
||||
|
||||
/* Fold next data chunk */
|
||||
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v2);
|
||||
|
||||
buf += 16;
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
/*
|
||||
* Set up a vector register for byte shifts. The shift value must
|
||||
* be loaded in bits 1-4 in byte element 7 of a vector register.
|
||||
* Shift by 8 bytes: 0x40
|
||||
* Shift by 4 bytes: 0x20
|
||||
*/
|
||||
uv16qi v9 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
|
||||
v9 = vec_insert((unsigned char)0x40, v9, 7);
|
||||
|
||||
/*
|
||||
* Prepare V0 for the next GF(2) multiplication: shift V0 by 8 bytes
|
||||
* to move R4 into the rightmost doubleword and set the leftmost
|
||||
* doubleword to 0x1.
|
||||
*/
|
||||
v0 = vec_srb(r4r3, (uv2di)v9);
|
||||
v0[0] = 1;
|
||||
|
||||
/*
|
||||
* Compute GF(2) product of V1 and V0. The rightmost doubleword
|
||||
* of V1 is multiplied with R4. The leftmost doubleword of V1 is
|
||||
* multiplied by 0x1 and is then XORed with rightmost product.
|
||||
* Implicitly, the intermediate leftmost product becomes padded
|
||||
*/
|
||||
v1 = (uv2di)vec_gfmsum_128(v0, v1);
|
||||
|
||||
/*
|
||||
* Now do the final 32-bit fold by multiplying the rightmost word
|
||||
* in V1 with R5 and XOR the result with the remaining bits in V1.
|
||||
*
|
||||
* To achieve this by a single VGFMAG, right shift V1 by a word
|
||||
* and store the result in V2 which is then accumulated. Use the
|
||||
* vector unpack instruction to load the rightmost half of the
|
||||
* doubleword into the rightmost doubleword element of V1; the other
|
||||
* half is loaded in the leftmost doubleword.
|
||||
* The vector register with CONST_R5 contains the R5 constant in the
|
||||
* rightmost doubleword and the leftmost doubleword is zero to ignore
|
||||
* the leftmost product of V1.
|
||||
*/
|
||||
v9 = vec_insert((unsigned char)0x20, v9, 7);
|
||||
v2 = vec_srb(v1, (uv2di)v9);
|
||||
v1 = vec_unpackl((uv4si)v1); /* Split rightmost doubleword */
|
||||
v1 = (uv2di)vec_gfmsum_accum_128(r5, v1, (uv16qi)v2);
|
||||
|
||||
/*
|
||||
* Apply a Barret reduction to compute the final 32-bit CRC value.
|
||||
*
|
||||
* The input values to the Barret reduction are the degree-63 polynomial
|
||||
* in V1 (R(x)), degree-32 generator polynomial, and the reduction
|
||||
* constant u. The Barret reduction result is the CRC value of R(x) mod
|
||||
* P(x).
|
||||
*
|
||||
* The Barret reduction algorithm is defined as:
|
||||
*
|
||||
* 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u
|
||||
* 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x)
|
||||
* 3. C(x) = R(x) XOR T2(x) mod x^32
|
||||
*
|
||||
* Note: The leftmost doubleword of vector register containing
|
||||
* CONST_RU_POLY is zero and, thus, the intermediate GF(2) product
|
||||
* is zero and does not contribute to the final result.
|
||||
*/
|
||||
|
||||
/* T1(x) = floor( R(x) / x^32 ) GF2MUL u */
|
||||
v2 = vec_unpackl((uv4si)v1);
|
||||
v2 = (uv2di)vec_gfmsum_128(ru_poly, v2);
|
||||
|
||||
/*
|
||||
* Compute the GF(2) product of the CRC polynomial with T1(x) in
|
||||
* V2 and XOR the intermediate result, T2(x), with the value in V1.
|
||||
* The final result is stored in word element 2 of V2.
|
||||
*/
|
||||
v2 = vec_unpackl((uv4si)v2);
|
||||
v2 = (uv2di)vec_gfmsum_accum_128(crc_poly, v2, (uv16qi)v1);
|
||||
|
||||
return ((uv4si)v2)[2];
|
||||
}
|
||||
|
||||
#define VX_MIN_LEN 64
|
||||
#define VX_ALIGNMENT 16UL
|
||||
#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
|
||||
|
||||
uint32_t Z_INTERNAL crc32_s390_vx(uint32_t crc, const unsigned char *buf, size_t len) {
|
||||
size_t prealign, aligned, remaining;
|
||||
|
||||
if (len < VX_MIN_LEN + VX_ALIGN_MASK)
|
||||
return crc32_braid(crc, buf, len);
|
||||
|
||||
if ((uintptr_t)buf & VX_ALIGN_MASK) {
|
||||
prealign = (size_t)ALIGN_DIFF(buf, VX_ALIGNMENT);
|
||||
len -= prealign;
|
||||
crc = crc32_braid(crc, buf, prealign);
|
||||
buf += prealign;
|
||||
}
|
||||
aligned = ALIGN_DOWN(len, VX_ALIGNMENT);
|
||||
remaining = len & VX_ALIGN_MASK;
|
||||
|
||||
crc = ~crc32_le_vgfm_16(~crc, buf, aligned);
|
||||
|
||||
if (remaining)
|
||||
crc = crc32_braid(crc, buf + aligned, remaining);
|
||||
|
||||
return crc;
|
||||
}
|
||||
|
||||
Z_INTERNAL uint32_t crc32_copy_s390_vx(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
|
||||
crc = crc32_s390_vx(crc, src, len);
|
||||
memcpy(dst, src, len);
|
||||
return crc;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
#ifndef DFLTCC_COMMON_H
|
||||
#define DFLTCC_COMMON_H
|
||||
|
||||
#include "zutil.h"
|
||||
|
||||
/*
|
||||
Parameter Block for Query Available Functions.
|
||||
*/
|
||||
struct dfltcc_qaf_param {
|
||||
char fns[16];
|
||||
char reserved1[8];
|
||||
char fmts[2];
|
||||
char reserved2[6];
|
||||
} ALIGNED_(8);
|
||||
|
||||
/*
|
||||
Parameter Block for Generate Dynamic-Huffman Table, Compress and Expand.
|
||||
*/
|
||||
struct dfltcc_param_v0 {
|
||||
uint16_t pbvn; /* Parameter-Block-Version Number */
|
||||
uint8_t mvn; /* Model-Version Number */
|
||||
uint8_t ribm; /* Reserved for IBM use */
|
||||
uint32_t reserved32 : 31;
|
||||
uint32_t cf : 1; /* Continuation Flag */
|
||||
uint8_t reserved64[8];
|
||||
uint32_t nt : 1; /* New Task */
|
||||
uint32_t reserved129 : 1;
|
||||
uint32_t cvt : 1; /* Check Value Type */
|
||||
uint32_t reserved131 : 1;
|
||||
uint32_t htt : 1; /* Huffman-Table Type */
|
||||
uint32_t bcf : 1; /* Block-Continuation Flag */
|
||||
uint32_t bcc : 1; /* Block Closing Control */
|
||||
uint32_t bhf : 1; /* Block Header Final */
|
||||
uint32_t reserved136 : 1;
|
||||
uint32_t reserved137 : 1;
|
||||
uint32_t dhtgc : 1; /* DHT Generation Control */
|
||||
uint32_t reserved139 : 5;
|
||||
uint32_t reserved144 : 5;
|
||||
uint32_t sbb : 3; /* Sub-Byte Boundary */
|
||||
uint8_t oesc; /* Operation-Ending-Supplemental Code */
|
||||
uint32_t reserved160 : 12;
|
||||
uint32_t ifs : 4; /* Incomplete-Function Status */
|
||||
uint16_t ifl; /* Incomplete-Function Length */
|
||||
uint8_t reserved192[8];
|
||||
uint8_t reserved256[8];
|
||||
uint8_t reserved320[4];
|
||||
uint16_t hl; /* History Length */
|
||||
uint32_t reserved368 : 1;
|
||||
uint16_t ho : 15; /* History Offset */
|
||||
uint32_t cv; /* Check Value */
|
||||
uint32_t eobs : 15; /* End-of-block Symbol */
|
||||
uint32_t reserved431: 1;
|
||||
uint8_t eobl : 4; /* End-of-block Length */
|
||||
uint32_t reserved436 : 12;
|
||||
uint32_t reserved448 : 4;
|
||||
uint16_t cdhtl : 12; /* Compressed-Dynamic-Huffman Table
|
||||
Length */
|
||||
uint8_t reserved464[6];
|
||||
uint8_t cdht[288]; /* Compressed-Dynamic-Huffman Table */
|
||||
uint8_t reserved[24];
|
||||
uint8_t ribm2[8]; /* Reserved for IBM use */
|
||||
uint8_t csb[1152]; /* Continuation-State Buffer */
|
||||
} ALIGNED_(8);
|
||||
|
||||
/*
|
||||
Extension of inflate_state and deflate_state.
|
||||
*/
|
||||
struct dfltcc_state {
|
||||
struct dfltcc_param_v0 param; /* Parameter block. */
|
||||
struct dfltcc_qaf_param af; /* Available functions. */
|
||||
char msg[64]; /* Buffer for strm->msg */
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
struct dfltcc_state common;
|
||||
uint16_t level_mask; /* Levels on which to use DFLTCC */
|
||||
uint32_t block_size; /* New block each X bytes */
|
||||
size_t block_threshold; /* New block after total_in > X */
|
||||
uint32_t dht_threshold; /* New block only if avail_in >= X */
|
||||
} arch_deflate_state;
|
||||
|
||||
typedef struct {
|
||||
struct dfltcc_state common;
|
||||
} arch_inflate_state;
|
||||
|
||||
/*
|
||||
History buffer size.
|
||||
*/
|
||||
#define HB_BITS 15
|
||||
#define HB_SIZE (1 << HB_BITS)
|
||||
|
||||
/*
|
||||
Sizes of deflate block parts.
|
||||
*/
|
||||
#define DFLTCC_BLOCK_HEADER_BITS 3
|
||||
#define DFLTCC_HLITS_COUNT_BITS 5
|
||||
#define DFLTCC_HDISTS_COUNT_BITS 5
|
||||
#define DFLTCC_HCLENS_COUNT_BITS 4
|
||||
#define DFLTCC_MAX_HCLENS 19
|
||||
#define DFLTCC_HCLEN_BITS 3
|
||||
#define DFLTCC_MAX_HLITS 286
|
||||
#define DFLTCC_MAX_HDISTS 30
|
||||
#define DFLTCC_MAX_HLIT_HDIST_BITS 7
|
||||
#define DFLTCC_MAX_SYMBOL_BITS 16
|
||||
#define DFLTCC_MAX_EOBS_BITS 15
|
||||
#define DFLTCC_MAX_PADDING_BITS 7
|
||||
|
||||
#define DEFLATE_BOUND_COMPLEN(source_len) \
|
||||
((DFLTCC_BLOCK_HEADER_BITS + \
|
||||
DFLTCC_HLITS_COUNT_BITS + \
|
||||
DFLTCC_HDISTS_COUNT_BITS + \
|
||||
DFLTCC_HCLENS_COUNT_BITS + \
|
||||
DFLTCC_MAX_HCLENS * DFLTCC_HCLEN_BITS + \
|
||||
(DFLTCC_MAX_HLITS + DFLTCC_MAX_HDISTS) * DFLTCC_MAX_HLIT_HDIST_BITS + \
|
||||
(source_len) * DFLTCC_MAX_SYMBOL_BITS + \
|
||||
DFLTCC_MAX_EOBS_BITS + \
|
||||
DFLTCC_MAX_PADDING_BITS) >> 3)
|
||||
|
||||
#endif
|
||||
|
|
@ -1,390 +0,0 @@
|
|||
/* dfltcc_deflate.c - IBM Z DEFLATE CONVERSION CALL compression support. */
|
||||
|
||||
/*
|
||||
Use the following commands to build zlib-ng with DFLTCC compression support:
|
||||
|
||||
$ ./configure --with-dfltcc-deflate
|
||||
or
|
||||
|
||||
$ cmake -DWITH_DFLTCC_DEFLATE=1 .
|
||||
|
||||
and then
|
||||
|
||||
$ make
|
||||
*/
|
||||
|
||||
#ifdef S390_DFLTCC_DEFLATE
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "deflate.h"
|
||||
#include "deflate_p.h"
|
||||
#include "trees_emit.h"
|
||||
#include "dfltcc_deflate.h"
|
||||
#include "dfltcc_detail.h"
|
||||
|
||||
void Z_INTERNAL PREFIX(dfltcc_reset_deflate_state)(PREFIX3(streamp) strm) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
arch_deflate_state *dfltcc_state = &state->arch;
|
||||
|
||||
dfltcc_reset_state(&dfltcc_state->common);
|
||||
|
||||
/* Initialize tuning parameters */
|
||||
dfltcc_state->level_mask = DFLTCC_LEVEL_MASK;
|
||||
dfltcc_state->block_size = DFLTCC_BLOCK_SIZE;
|
||||
dfltcc_state->block_threshold = DFLTCC_FIRST_FHT_BLOCK_SIZE;
|
||||
dfltcc_state->dht_threshold = DFLTCC_DHT_MIN_SAMPLE_SIZE;
|
||||
}
|
||||
|
||||
static inline int dfltcc_can_deflate_with_params(PREFIX3(streamp) strm, int level, uInt window_bits, int strategy,
|
||||
int reproducible) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
arch_deflate_state *dfltcc_state = &state->arch;
|
||||
|
||||
/* Unsupported compression settings */
|
||||
if ((dfltcc_state->level_mask & (1 << level)) == 0)
|
||||
return 0;
|
||||
if (window_bits != HB_BITS)
|
||||
return 0;
|
||||
if (strategy != Z_FIXED && strategy != Z_DEFAULT_STRATEGY)
|
||||
return 0;
|
||||
if (reproducible)
|
||||
return 0;
|
||||
|
||||
/* Unsupported hardware */
|
||||
if (!is_bit_set(dfltcc_state->common.af.fns, DFLTCC_GDHT) ||
|
||||
!is_bit_set(dfltcc_state->common.af.fns, DFLTCC_CMPR) ||
|
||||
!is_bit_set(dfltcc_state->common.af.fmts, DFLTCC_FMT0))
|
||||
return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_can_deflate)(PREFIX3(streamp) strm) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
|
||||
return dfltcc_can_deflate_with_params(strm, state->level, W_BITS(state), state->strategy, state->reproducible);
|
||||
}
|
||||
|
||||
static inline void dfltcc_gdht(PREFIX3(streamp) strm) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
size_t avail_in = strm->avail_in;
|
||||
|
||||
dfltcc(DFLTCC_GDHT, param, NULL, NULL, &strm->next_in, &avail_in, NULL);
|
||||
}
|
||||
|
||||
static inline dfltcc_cc dfltcc_cmpr(PREFIX3(streamp) strm) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
size_t avail_in = strm->avail_in;
|
||||
size_t avail_out = strm->avail_out;
|
||||
dfltcc_cc cc;
|
||||
|
||||
cc = dfltcc(DFLTCC_CMPR | HBT_CIRCULAR,
|
||||
param, &strm->next_out, &avail_out,
|
||||
&strm->next_in, &avail_in, state->window);
|
||||
strm->total_in += (strm->avail_in - avail_in);
|
||||
strm->total_out += (strm->avail_out - avail_out);
|
||||
strm->avail_in = avail_in;
|
||||
strm->avail_out = avail_out;
|
||||
return cc;
|
||||
}
|
||||
|
||||
static inline void send_eobs(PREFIX3(streamp) strm, const struct dfltcc_param_v0 *param) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
|
||||
send_bits(state, bi_reverse((uint16_t)(param->eobs >> (15 - param->eobl)), param->eobl),
|
||||
param->eobl, state->bi_buf, state->bi_valid);
|
||||
|
||||
flush_pending_inline(strm);
|
||||
if (state->pending != 0) {
|
||||
/* The remaining data is located in pending_out[0:pending]. If someone
|
||||
* calls put_byte() - this might happen in deflate() - the byte will be
|
||||
* placed into pending_buf[pending], which is incorrect. Move the
|
||||
* remaining data to the beginning of pending_buf so that put_byte() is
|
||||
* usable again.
|
||||
*/
|
||||
memmove(state->pending_buf, state->pending_out, state->pending);
|
||||
state->pending_out = state->pending_buf;
|
||||
}
|
||||
#ifdef ZLIB_DEBUG
|
||||
state->compressed_len += param->eobl;
|
||||
#endif
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate)(PREFIX3(streamp) strm, int flush, block_state *result) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
arch_deflate_state *dfltcc_state = &state->arch;
|
||||
struct dfltcc_param_v0 *param = &dfltcc_state->common.param;
|
||||
uInt masked_avail_in;
|
||||
dfltcc_cc cc;
|
||||
int need_empty_block;
|
||||
int soft_bcc;
|
||||
int no_flush;
|
||||
|
||||
if (!PREFIX(dfltcc_can_deflate)(strm)) {
|
||||
/* Clear history. */
|
||||
if (flush == Z_FULL_FLUSH)
|
||||
param->hl = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
again:
|
||||
masked_avail_in = 0;
|
||||
soft_bcc = 0;
|
||||
no_flush = flush == Z_NO_FLUSH;
|
||||
|
||||
/* No input data. Return, except when Continuation Flag is set, which means
|
||||
* that DFLTCC has buffered some output in the parameter block and needs to
|
||||
* be called again in order to flush it.
|
||||
*/
|
||||
if (strm->avail_in == 0 && !param->cf) {
|
||||
/* A block is still open, and the hardware does not support closing
|
||||
* blocks without adding data. Thus, close it manually.
|
||||
*/
|
||||
if (!no_flush && param->bcf) {
|
||||
send_eobs(strm, param);
|
||||
param->bcf = 0;
|
||||
}
|
||||
/* Let one of deflate_* functions write a trailing empty block. */
|
||||
if (flush == Z_FINISH)
|
||||
return 0;
|
||||
/* Clear history. */
|
||||
if (flush == Z_FULL_FLUSH)
|
||||
param->hl = 0;
|
||||
/* Trigger block post-processing if necessary. */
|
||||
*result = no_flush ? need_more : block_done;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* There is an open non-BFINAL block, we are not going to close it just
|
||||
* yet, we have compressed more than DFLTCC_BLOCK_SIZE bytes and we see
|
||||
* more than DFLTCC_DHT_MIN_SAMPLE_SIZE bytes. Open a new block with a new
|
||||
* DHT in order to adapt to a possibly changed input data distribution.
|
||||
*/
|
||||
if (param->bcf && no_flush &&
|
||||
strm->total_in > dfltcc_state->block_threshold &&
|
||||
strm->avail_in >= dfltcc_state->dht_threshold) {
|
||||
if (param->cf) {
|
||||
/* We need to flush the DFLTCC buffer before writing the
|
||||
* End-of-block Symbol. Mask the input data and proceed as usual.
|
||||
*/
|
||||
masked_avail_in += strm->avail_in;
|
||||
strm->avail_in = 0;
|
||||
no_flush = 0;
|
||||
} else {
|
||||
/* DFLTCC buffer is empty, so we can manually write the
|
||||
* End-of-block Symbol right away.
|
||||
*/
|
||||
send_eobs(strm, param);
|
||||
param->bcf = 0;
|
||||
dfltcc_state->block_threshold = strm->total_in + dfltcc_state->block_size;
|
||||
}
|
||||
}
|
||||
|
||||
/* No space for compressed data. If we proceed, dfltcc_cmpr() will return
|
||||
* DFLTCC_CC_OP1_TOO_SHORT without buffering header bits, but we will still
|
||||
* set BCF=1, which is wrong. Avoid complications and return early.
|
||||
*/
|
||||
if (strm->avail_out == 0) {
|
||||
*result = need_more;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* The caller gave us too much data. Pass only one block worth of
|
||||
* uncompressed data to DFLTCC and mask the rest, so that on the next
|
||||
* iteration we start a new block.
|
||||
*/
|
||||
if (no_flush && strm->avail_in > dfltcc_state->block_size) {
|
||||
masked_avail_in += (strm->avail_in - dfltcc_state->block_size);
|
||||
strm->avail_in = dfltcc_state->block_size;
|
||||
}
|
||||
|
||||
/* When we have an open non-BFINAL deflate block and caller indicates that
|
||||
* the stream is ending, we need to close an open deflate block and open a
|
||||
* BFINAL one.
|
||||
*/
|
||||
need_empty_block = flush == Z_FINISH && param->bcf && !param->bhf;
|
||||
|
||||
/* Translate stream to parameter block */
|
||||
param->cvt = state->wrap == 2 ? CVT_CRC32 : CVT_ADLER32;
|
||||
if (!no_flush)
|
||||
/* We need to close a block. Always do this in software - when there is
|
||||
* no input data, the hardware will not honor BCC. */
|
||||
soft_bcc = 1;
|
||||
if (flush == Z_FINISH && !param->bcf)
|
||||
/* We are about to open a BFINAL block, set Block Header Final bit
|
||||
* until the stream ends.
|
||||
*/
|
||||
param->bhf = 1;
|
||||
/* DFLTCC-CMPR will write to next_out, so make sure that buffers with
|
||||
* higher precedence are empty.
|
||||
*/
|
||||
Assert(state->pending == 0, "There must be no pending bytes");
|
||||
Assert(state->bi_valid < 8, "There must be less than 8 pending bits");
|
||||
param->sbb = (unsigned int)state->bi_valid;
|
||||
if (param->sbb > 0)
|
||||
*strm->next_out = (unsigned char)state->bi_buf;
|
||||
/* Honor history and check value */
|
||||
param->nt = 0;
|
||||
if (state->wrap == 1)
|
||||
param->cv = strm->adler;
|
||||
else if (state->wrap == 2)
|
||||
param->cv = ZSWAP32(strm->adler);
|
||||
|
||||
/* When opening a block, choose a Huffman-Table Type */
|
||||
if (!param->bcf) {
|
||||
if (state->strategy == Z_FIXED || (strm->total_in == 0 && dfltcc_state->block_threshold > 0))
|
||||
param->htt = HTT_FIXED;
|
||||
else {
|
||||
param->htt = HTT_DYNAMIC;
|
||||
dfltcc_gdht(strm);
|
||||
}
|
||||
}
|
||||
|
||||
/* Deflate */
|
||||
do {
|
||||
cc = dfltcc_cmpr(strm);
|
||||
if (strm->avail_in < 4096 && masked_avail_in > 0)
|
||||
/* We are about to call DFLTCC with a small input buffer, which is
|
||||
* inefficient. Since there is masked data, there will be at least
|
||||
* one more DFLTCC call, so skip the current one and make the next
|
||||
* one handle more data.
|
||||
*/
|
||||
break;
|
||||
} while (cc == DFLTCC_CC_AGAIN);
|
||||
|
||||
/* Translate parameter block to stream */
|
||||
strm->msg = oesc_msg(dfltcc_state->common.msg, param->oesc);
|
||||
state->bi_valid = param->sbb;
|
||||
if (state->bi_valid == 0)
|
||||
state->bi_buf = 0; /* Avoid accessing next_out */
|
||||
else
|
||||
state->bi_buf = *strm->next_out & ((1 << state->bi_valid) - 1);
|
||||
if (state->wrap == 1)
|
||||
strm->adler = param->cv;
|
||||
else if (state->wrap == 2)
|
||||
strm->adler = ZSWAP32(param->cv);
|
||||
|
||||
/* Unmask the input data */
|
||||
strm->avail_in += masked_avail_in;
|
||||
masked_avail_in = 0;
|
||||
|
||||
/* If we encounter an error, it means there is a bug in DFLTCC call */
|
||||
Assert(cc != DFLTCC_CC_OP2_CORRUPT || param->oesc == 0, "BUG");
|
||||
|
||||
/* Update Block-Continuation Flag. It will be used to check whether to call
|
||||
* GDHT the next time.
|
||||
*/
|
||||
if (cc == DFLTCC_CC_OK) {
|
||||
if (soft_bcc) {
|
||||
send_eobs(strm, param);
|
||||
param->bcf = 0;
|
||||
dfltcc_state->block_threshold = strm->total_in + dfltcc_state->block_size;
|
||||
} else
|
||||
param->bcf = 1;
|
||||
if (flush == Z_FINISH) {
|
||||
if (need_empty_block)
|
||||
/* Make the current deflate() call also close the stream */
|
||||
return 0;
|
||||
else {
|
||||
bi_windup(state);
|
||||
*result = finish_done;
|
||||
}
|
||||
} else {
|
||||
if (flush == Z_FULL_FLUSH)
|
||||
param->hl = 0; /* Clear history */
|
||||
*result = flush == Z_NO_FLUSH ? need_more : block_done;
|
||||
}
|
||||
} else {
|
||||
param->bcf = 1;
|
||||
*result = need_more;
|
||||
}
|
||||
if (strm->avail_in != 0 && strm->avail_out != 0)
|
||||
goto again; /* deflate() must use all input or all output */
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
Switching between hardware and software compression.
|
||||
|
||||
DFLTCC does not support all zlib settings, e.g. generation of non-compressed
|
||||
blocks or alternative window sizes. When such settings are applied on the
|
||||
fly with deflateParams, we need to convert between hardware and software
|
||||
window formats.
|
||||
*/
|
||||
static int dfltcc_was_deflate_used(PREFIX3(streamp) strm) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
|
||||
return strm->total_in > 0 || param->nt == 0 || param->hl > 0;
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_params)(PREFIX3(streamp) strm, int level, int strategy, int *flush) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
int could_deflate = PREFIX(dfltcc_can_deflate)(strm);
|
||||
int can_deflate = dfltcc_can_deflate_with_params(strm, level, W_BITS(state), strategy, state->reproducible);
|
||||
|
||||
if (can_deflate == could_deflate)
|
||||
/* We continue to work in the same mode - no changes needed */
|
||||
return Z_OK;
|
||||
|
||||
if (!dfltcc_was_deflate_used(strm))
|
||||
/* DFLTCC was not used yet - no changes needed */
|
||||
return Z_OK;
|
||||
|
||||
/* For now, do not convert between window formats - simply get rid of the old data instead */
|
||||
*flush = Z_FULL_FLUSH;
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_done)(PREFIX3(streamp) strm, int flush) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
|
||||
/* When deflate(Z_FULL_FLUSH) is called with small avail_out, it might
|
||||
* close the block without resetting the compression state. Detect this
|
||||
* situation and return that deflation is not done.
|
||||
*/
|
||||
if (flush == Z_FULL_FLUSH && strm->avail_out == 0)
|
||||
return 0;
|
||||
|
||||
/* Return that deflation is not done if DFLTCC is used and either it
|
||||
* buffered some data (Continuation Flag is set), or has not written EOBS
|
||||
* yet (Block-Continuation Flag is set).
|
||||
*/
|
||||
return !PREFIX(dfltcc_can_deflate)(strm) || (!param->cf && !param->bcf);
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_can_set_reproducible)(PREFIX3(streamp) strm, int reproducible) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
|
||||
return reproducible != state->reproducible && !dfltcc_was_deflate_used(strm);
|
||||
}
|
||||
|
||||
/*
|
||||
Preloading history.
|
||||
*/
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_set_dictionary)(PREFIX3(streamp) strm,
|
||||
const unsigned char *dictionary, uInt dict_length) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
|
||||
append_history(param, state->window, dictionary, dict_length);
|
||||
state->strstart = 1; /* Add FDICT to zlib header */
|
||||
state->block_start = state->strstart; /* Make deflate_stored happy */
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_get_dictionary)(PREFIX3(streamp) strm, unsigned char *dictionary, uInt *dict_length) {
|
||||
deflate_state *state = (deflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
|
||||
if (dictionary)
|
||||
get_history(param, state->window, dictionary);
|
||||
if (dict_length)
|
||||
*dict_length = param->hl;
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
#ifndef DFLTCC_DEFLATE_H
|
||||
#define DFLTCC_DEFLATE_H
|
||||
|
||||
#include "deflate.h"
|
||||
#include "dfltcc_common.h"
|
||||
|
||||
void Z_INTERNAL PREFIX(dfltcc_reset_deflate_state)(PREFIX3(streamp));
|
||||
int Z_INTERNAL PREFIX(dfltcc_can_deflate)(PREFIX3(streamp) strm);
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate)(PREFIX3(streamp) strm, int flush, block_state *result);
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_params)(PREFIX3(streamp) strm, int level, int strategy, int *flush);
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_done)(PREFIX3(streamp) strm, int flush);
|
||||
int Z_INTERNAL PREFIX(dfltcc_can_set_reproducible)(PREFIX3(streamp) strm, int reproducible);
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_set_dictionary)(PREFIX3(streamp) strm,
|
||||
const unsigned char *dictionary, uInt dict_length);
|
||||
int Z_INTERNAL PREFIX(dfltcc_deflate_get_dictionary)(PREFIX3(streamp) strm, unsigned char *dictionary, uInt* dict_length);
|
||||
|
||||
#define DEFLATE_SET_DICTIONARY_HOOK(strm, dict, dict_len) \
|
||||
do { \
|
||||
if (PREFIX(dfltcc_can_deflate)((strm))) \
|
||||
return PREFIX(dfltcc_deflate_set_dictionary)((strm), (dict), (dict_len)); \
|
||||
} while (0)
|
||||
|
||||
#define DEFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) \
|
||||
do { \
|
||||
if (PREFIX(dfltcc_can_deflate)((strm))) \
|
||||
return PREFIX(dfltcc_deflate_get_dictionary)((strm), (dict), (dict_len)); \
|
||||
} while (0)
|
||||
|
||||
#define DEFLATE_RESET_KEEP_HOOK PREFIX(dfltcc_reset_deflate_state)
|
||||
|
||||
#define DEFLATE_PARAMS_HOOK(strm, level, strategy, hook_flush) \
|
||||
do { \
|
||||
int err; \
|
||||
\
|
||||
err = PREFIX(dfltcc_deflate_params)((strm), (level), (strategy), (hook_flush)); \
|
||||
if (err == Z_STREAM_ERROR) \
|
||||
return err; \
|
||||
} while (0)
|
||||
|
||||
#define DEFLATE_DONE PREFIX(dfltcc_deflate_done)
|
||||
|
||||
#define DEFLATE_BOUND_ADJUST_COMPLEN(strm, complen, source_len) \
|
||||
do { \
|
||||
if (deflateStateCheck((strm)) || PREFIX(dfltcc_can_deflate)((strm))) \
|
||||
(complen) = DEFLATE_BOUND_COMPLEN(source_len); \
|
||||
} while (0)
|
||||
|
||||
#define DEFLATE_NEED_CONSERVATIVE_BOUND(strm) (PREFIX(dfltcc_can_deflate)((strm)))
|
||||
|
||||
#define DEFLATE_HOOK PREFIX(dfltcc_deflate)
|
||||
|
||||
#define DEFLATE_NEED_CHECKSUM(strm) (!PREFIX(dfltcc_can_deflate)((strm)))
|
||||
|
||||
#define DEFLATE_CAN_SET_REPRODUCIBLE PREFIX(dfltcc_can_set_reproducible)
|
||||
|
||||
#define DEFLATE_ADJUST_WINDOW_SIZE(n) MAX(n, HB_SIZE)
|
||||
|
||||
#endif
|
||||
|
|
@ -1,274 +0,0 @@
|
|||
#include "zbuild.h"
|
||||
#include "zsanitizer.h"
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef HAVE_SYS_SDT_H
|
||||
#include <sys/sdt.h>
|
||||
#endif
|
||||
|
||||
/*
|
||||
Tuning parameters.
|
||||
*/
|
||||
#ifndef DFLTCC_LEVEL_MASK
|
||||
#define DFLTCC_LEVEL_MASK 0x2
|
||||
#endif
|
||||
#ifndef DFLTCC_BLOCK_SIZE
|
||||
#define DFLTCC_BLOCK_SIZE 1048576
|
||||
#endif
|
||||
#ifndef DFLTCC_FIRST_FHT_BLOCK_SIZE
|
||||
#define DFLTCC_FIRST_FHT_BLOCK_SIZE 4096
|
||||
#endif
|
||||
#ifndef DFLTCC_DHT_MIN_SAMPLE_SIZE
|
||||
#define DFLTCC_DHT_MIN_SAMPLE_SIZE 4096
|
||||
#endif
|
||||
#ifndef DFLTCC_RIBM
|
||||
#define DFLTCC_RIBM 0
|
||||
#endif
|
||||
|
||||
#define static_assert(c, msg) __attribute__((unused)) static char static_assert_failed_ ## msg[c ? 1 : -1]
|
||||
|
||||
#define DFLTCC_SIZEOF_QAF 32
|
||||
static_assert(sizeof(struct dfltcc_qaf_param) == DFLTCC_SIZEOF_QAF, qaf);
|
||||
|
||||
static inline int is_bit_set(const char *bits, int n) {
|
||||
return bits[n / 8] & (1 << (7 - (n % 8)));
|
||||
}
|
||||
|
||||
static inline void clear_bit(char *bits, int n) {
|
||||
bits[n / 8] &= ~(1 << (7 - (n % 8)));
|
||||
}
|
||||
|
||||
#define DFLTCC_FACILITY 151
|
||||
|
||||
static inline int is_dfltcc_enabled(void) {
|
||||
uint64_t facilities[(DFLTCC_FACILITY / 64) + 1];
|
||||
Z_REGISTER uint8_t r0 __asm__("r0");
|
||||
|
||||
memset(facilities, 0, sizeof(facilities));
|
||||
r0 = sizeof(facilities) / sizeof(facilities[0]) - 1;
|
||||
/* STFLE is supported since z9-109 and only in z/Architecture mode. When
|
||||
* compiling with -m31, gcc defaults to ESA mode, however, since the kernel
|
||||
* is 64-bit, it's always z/Architecture mode at runtime.
|
||||
*/
|
||||
__asm__ volatile(
|
||||
#ifndef __clang__
|
||||
".machinemode push\n"
|
||||
".machinemode zarch\n"
|
||||
#endif
|
||||
"stfle %[facilities]\n"
|
||||
#ifndef __clang__
|
||||
".machinemode pop\n"
|
||||
#endif
|
||||
: [facilities] "=Q" (facilities), [r0] "+r" (r0) :: "cc");
|
||||
return is_bit_set((const char *)facilities, DFLTCC_FACILITY);
|
||||
}
|
||||
|
||||
#define DFLTCC_FMT0 0
|
||||
|
||||
#define CVT_CRC32 0
|
||||
#define CVT_ADLER32 1
|
||||
#define HTT_FIXED 0
|
||||
#define HTT_DYNAMIC 1
|
||||
|
||||
#define DFLTCC_SIZEOF_GDHT_V0 384
|
||||
#define DFLTCC_SIZEOF_CMPR_XPND_V0 1536
|
||||
static_assert(offsetof(struct dfltcc_param_v0, csb) == DFLTCC_SIZEOF_GDHT_V0, gdht_v0);
|
||||
static_assert(sizeof(struct dfltcc_param_v0) == DFLTCC_SIZEOF_CMPR_XPND_V0, cmpr_xpnd_v0);
|
||||
|
||||
static inline z_const char *oesc_msg(char *buf, int oesc) {
|
||||
if (oesc == 0x00)
|
||||
return NULL; /* Successful completion */
|
||||
else {
|
||||
sprintf(buf, "Operation-Ending-Supplemental Code is 0x%.2X", oesc);
|
||||
return buf;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
C wrapper for the DEFLATE CONVERSION CALL instruction.
|
||||
*/
|
||||
typedef enum {
|
||||
DFLTCC_CC_OK = 0,
|
||||
DFLTCC_CC_OP1_TOO_SHORT = 1,
|
||||
DFLTCC_CC_OP2_TOO_SHORT = 2,
|
||||
DFLTCC_CC_OP2_CORRUPT = 2,
|
||||
DFLTCC_CC_AGAIN = 3,
|
||||
} dfltcc_cc;
|
||||
|
||||
#define DFLTCC_QAF 0
|
||||
#define DFLTCC_GDHT 1
|
||||
#define DFLTCC_CMPR 2
|
||||
#define DFLTCC_XPND 4
|
||||
#define HBT_CIRCULAR (1 << 7)
|
||||
#define DFLTCC_FN_MASK ((1 << 7) - 1)
|
||||
|
||||
/* Return lengths of high (starting at param->ho) and low (starting at 0) fragments of the circular history buffer. */
|
||||
static inline void get_history_lengths(struct dfltcc_param_v0 *param, size_t *hl_high, size_t *hl_low) {
|
||||
*hl_high = MIN(param->hl, HB_SIZE - param->ho);
|
||||
*hl_low = param->hl - *hl_high;
|
||||
}
|
||||
|
||||
/* Notify instrumentation about an upcoming read/write access to the circular history buffer. */
|
||||
static inline void instrument_read_write_hist(struct dfltcc_param_v0 *param, void *hist) {
|
||||
size_t hl_high, hl_low;
|
||||
|
||||
get_history_lengths(param, &hl_high, &hl_low);
|
||||
instrument_read_write(hist + param->ho, hl_high);
|
||||
instrument_read_write(hist, hl_low);
|
||||
}
|
||||
|
||||
/* Notify MSan about a completed write to the circular history buffer. */
|
||||
static inline void msan_unpoison_hist(struct dfltcc_param_v0 *param, void *hist) {
|
||||
size_t hl_high, hl_low;
|
||||
|
||||
get_history_lengths(param, &hl_high, &hl_low);
|
||||
__msan_unpoison(hist + param->ho, hl_high);
|
||||
__msan_unpoison(hist, hl_low);
|
||||
}
|
||||
|
||||
static inline dfltcc_cc dfltcc(int fn, void *param,
|
||||
unsigned char **op1, size_t *len1,
|
||||
z_const unsigned char **op2, size_t *len2, void *hist) {
|
||||
unsigned char *t2 = op1 ? *op1 : NULL;
|
||||
unsigned char *orig_t2 = t2;
|
||||
size_t t3 = len1 ? *len1 : 0;
|
||||
z_const unsigned char *t4 = op2 ? *op2 : NULL;
|
||||
size_t t5 = len2 ? *len2 : 0;
|
||||
Z_REGISTER int r0 __asm__("r0");
|
||||
Z_REGISTER void *r1 __asm__("r1");
|
||||
Z_REGISTER unsigned char *r2 __asm__("r2");
|
||||
Z_REGISTER size_t r3 __asm__("r3");
|
||||
Z_REGISTER z_const unsigned char *r4 __asm__("r4");
|
||||
Z_REGISTER size_t r5 __asm__("r5");
|
||||
int cc;
|
||||
|
||||
/* Insert pre-instrumentation for DFLTCC. */
|
||||
switch (fn & DFLTCC_FN_MASK) {
|
||||
case DFLTCC_QAF:
|
||||
instrument_write(param, DFLTCC_SIZEOF_QAF);
|
||||
break;
|
||||
case DFLTCC_GDHT:
|
||||
instrument_read_write(param, DFLTCC_SIZEOF_GDHT_V0);
|
||||
instrument_read(t4, t5);
|
||||
break;
|
||||
case DFLTCC_CMPR:
|
||||
case DFLTCC_XPND:
|
||||
instrument_read_write(param, DFLTCC_SIZEOF_CMPR_XPND_V0);
|
||||
instrument_read(t4, t5);
|
||||
instrument_write(t2, t3);
|
||||
instrument_read_write_hist(param, hist);
|
||||
break;
|
||||
}
|
||||
|
||||
r0 = fn; r1 = param; r2 = t2; r3 = t3; r4 = t4; r5 = t5;
|
||||
__asm__ volatile(
|
||||
#ifdef HAVE_SYS_SDT_H
|
||||
STAP_PROBE_ASM(zlib, dfltcc_entry, STAP_PROBE_ASM_TEMPLATE(5))
|
||||
#endif
|
||||
".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n"
|
||||
#ifdef HAVE_SYS_SDT_H
|
||||
STAP_PROBE_ASM(zlib, dfltcc_exit, STAP_PROBE_ASM_TEMPLATE(5))
|
||||
#endif
|
||||
"ipm %[cc]\n"
|
||||
: [r2] "+r" (r2)
|
||||
, [r3] "+r" (r3)
|
||||
, [r4] "+r" (r4)
|
||||
, [r5] "+r" (r5)
|
||||
, [cc] "=r" (cc)
|
||||
: [r0] "r" (r0)
|
||||
, [r1] "r" (r1)
|
||||
, [hist] "r" (hist)
|
||||
#ifdef HAVE_SYS_SDT_H
|
||||
, STAP_PROBE_ASM_OPERANDS(5, r2, r3, r4, r5, hist)
|
||||
#endif
|
||||
: "cc", "memory");
|
||||
t2 = r2; t3 = r3; t4 = r4; t5 = r5;
|
||||
|
||||
/* Insert post-instrumentation for DFLTCC. */
|
||||
switch (fn & DFLTCC_FN_MASK) {
|
||||
case DFLTCC_QAF:
|
||||
__msan_unpoison(param, DFLTCC_SIZEOF_QAF);
|
||||
break;
|
||||
case DFLTCC_GDHT:
|
||||
__msan_unpoison(param, DFLTCC_SIZEOF_GDHT_V0);
|
||||
break;
|
||||
case DFLTCC_CMPR:
|
||||
__msan_unpoison(param, DFLTCC_SIZEOF_CMPR_XPND_V0);
|
||||
__msan_unpoison(orig_t2, t2 - orig_t2 + (((struct dfltcc_param_v0 *)param)->sbb == 0 ? 0 : 1));
|
||||
msan_unpoison_hist(param, hist);
|
||||
break;
|
||||
case DFLTCC_XPND:
|
||||
__msan_unpoison(param, DFLTCC_SIZEOF_CMPR_XPND_V0);
|
||||
__msan_unpoison(orig_t2, t2 - orig_t2);
|
||||
msan_unpoison_hist(param, hist);
|
||||
break;
|
||||
}
|
||||
|
||||
if (op1)
|
||||
*op1 = t2;
|
||||
if (len1)
|
||||
*len1 = t3;
|
||||
if (op2)
|
||||
*op2 = t4;
|
||||
if (len2)
|
||||
*len2 = t5;
|
||||
return (cc >> 28) & 3;
|
||||
}
|
||||
|
||||
static inline void dfltcc_reset_state(struct dfltcc_state *dfltcc_state) {
|
||||
/* Initialize available functions */
|
||||
if (is_dfltcc_enabled()) {
|
||||
dfltcc(DFLTCC_QAF, &dfltcc_state->param, NULL, NULL, NULL, NULL, NULL);
|
||||
memmove(&dfltcc_state->af, &dfltcc_state->param, sizeof(dfltcc_state->af));
|
||||
} else
|
||||
memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af));
|
||||
|
||||
/* Initialize parameter block */
|
||||
memset(&dfltcc_state->param, 0, sizeof(dfltcc_state->param));
|
||||
dfltcc_state->param.nt = 1;
|
||||
dfltcc_state->param.ribm = DFLTCC_RIBM;
|
||||
}
|
||||
|
||||
static inline void dfltcc_copy_state(void *dst, const void *src, uInt size, uInt extension_size) {
|
||||
memcpy(dst, src, ALIGN_UP(size, 8) + extension_size);
|
||||
}
|
||||
|
||||
static inline void append_history(struct dfltcc_param_v0 *param, unsigned char *history,
|
||||
const unsigned char *buf, uInt count) {
|
||||
size_t offset;
|
||||
size_t n;
|
||||
|
||||
/* Do not use more than 32K */
|
||||
if (count > HB_SIZE) {
|
||||
buf += count - HB_SIZE;
|
||||
count = HB_SIZE;
|
||||
}
|
||||
offset = (param->ho + param->hl) % HB_SIZE;
|
||||
if (offset + count <= HB_SIZE)
|
||||
/* Circular history buffer does not wrap - copy one chunk */
|
||||
memcpy(history + offset, buf, count);
|
||||
else {
|
||||
/* Circular history buffer wraps - copy two chunks */
|
||||
n = HB_SIZE - offset;
|
||||
memcpy(history + offset, buf, n);
|
||||
memcpy(history, buf + n, count - n);
|
||||
}
|
||||
n = param->hl + count;
|
||||
if (n <= HB_SIZE)
|
||||
/* All history fits into buffer - no need to discard anything */
|
||||
param->hl = n;
|
||||
else {
|
||||
/* History does not fit into buffer - discard extra bytes */
|
||||
param->ho = (param->ho + (n - HB_SIZE)) % HB_SIZE;
|
||||
param->hl = HB_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void get_history(struct dfltcc_param_v0 *param, const unsigned char *history,
|
||||
unsigned char *buf) {
|
||||
size_t hl_high, hl_low;
|
||||
|
||||
get_history_lengths(param, &hl_high, &hl_low);
|
||||
memcpy(buf, history + param->ho, hl_high);
|
||||
memcpy(buf + hl_high, history, hl_low);
|
||||
}
|
||||
|
|
@ -1,195 +0,0 @@
|
|||
/* dfltcc_inflate.c - IBM Z DEFLATE CONVERSION CALL decompression support. */
|
||||
|
||||
/*
|
||||
Use the following commands to build zlib-ng with DFLTCC decompression support:
|
||||
|
||||
$ ./configure --with-dfltcc-inflate
|
||||
or
|
||||
|
||||
$ cmake -DWITH_DFLTCC_INFLATE=1 .
|
||||
|
||||
and then
|
||||
|
||||
$ make
|
||||
*/
|
||||
|
||||
#ifdef S390_DFLTCC_INFLATE
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "zutil.h"
|
||||
#include "inftrees.h"
|
||||
#include "inflate.h"
|
||||
#include "dfltcc_inflate.h"
|
||||
#include "dfltcc_detail.h"
|
||||
|
||||
void Z_INTERNAL PREFIX(dfltcc_reset_inflate_state)(PREFIX3(streamp) strm) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
|
||||
dfltcc_reset_state(&state->arch.common);
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_can_inflate)(PREFIX3(streamp) strm) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
struct dfltcc_state *dfltcc_state = &state->arch.common;
|
||||
|
||||
/* Unsupported hardware */
|
||||
return is_bit_set(dfltcc_state->af.fns, DFLTCC_XPND) && is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0);
|
||||
}
|
||||
|
||||
static inline dfltcc_cc dfltcc_xpnd(PREFIX3(streamp) strm) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
size_t avail_in = strm->avail_in;
|
||||
size_t avail_out = strm->avail_out;
|
||||
dfltcc_cc cc;
|
||||
|
||||
cc = dfltcc(DFLTCC_XPND | HBT_CIRCULAR,
|
||||
param, &strm->next_out, &avail_out,
|
||||
&strm->next_in, &avail_in, state->window);
|
||||
strm->avail_in = avail_in;
|
||||
strm->avail_out = avail_out;
|
||||
return cc;
|
||||
}
|
||||
|
||||
dfltcc_inflate_action Z_INTERNAL PREFIX(dfltcc_inflate)(PREFIX3(streamp) strm, int flush, int *ret) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
struct dfltcc_state *dfltcc_state = &state->arch.common;
|
||||
struct dfltcc_param_v0 *param = &dfltcc_state->param;
|
||||
dfltcc_cc cc;
|
||||
|
||||
if (flush == Z_BLOCK || flush == Z_TREES) {
|
||||
/* DFLTCC does not support stopping on block boundaries */
|
||||
if (PREFIX(dfltcc_inflate_disable)(strm)) {
|
||||
*ret = Z_STREAM_ERROR;
|
||||
return DFLTCC_INFLATE_BREAK;
|
||||
} else
|
||||
return DFLTCC_INFLATE_SOFTWARE;
|
||||
}
|
||||
|
||||
if (state->last) {
|
||||
if (state->bits != 0) {
|
||||
strm->next_in++;
|
||||
strm->avail_in--;
|
||||
state->bits = 0;
|
||||
}
|
||||
state->mode = CHECK;
|
||||
return DFLTCC_INFLATE_CONTINUE;
|
||||
}
|
||||
|
||||
if (strm->avail_in == 0 && !param->cf)
|
||||
return DFLTCC_INFLATE_BREAK;
|
||||
|
||||
/* if window not in use yet, initialize */
|
||||
if (state->wsize == 0)
|
||||
state->wsize = 1U << state->wbits;
|
||||
|
||||
/* Translate stream to parameter block */
|
||||
param->cvt = ((state->wrap & 4) && state->flags) ? CVT_CRC32 : CVT_ADLER32;
|
||||
param->sbb = state->bits;
|
||||
if (param->hl)
|
||||
param->nt = 0; /* Honor history for the first block */
|
||||
if (state->wrap & 4)
|
||||
param->cv = state->flags ? ZSWAP32(state->check) : state->check;
|
||||
|
||||
/* Inflate */
|
||||
do {
|
||||
cc = dfltcc_xpnd(strm);
|
||||
} while (cc == DFLTCC_CC_AGAIN);
|
||||
|
||||
/* Translate parameter block to stream */
|
||||
strm->msg = oesc_msg(dfltcc_state->msg, param->oesc);
|
||||
state->last = cc == DFLTCC_CC_OK;
|
||||
state->bits = param->sbb;
|
||||
if (state->wrap & 4)
|
||||
strm->adler = state->check = state->flags ? ZSWAP32(param->cv) : param->cv;
|
||||
if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0) {
|
||||
/* Report an error if stream is corrupted */
|
||||
state->mode = BAD;
|
||||
return DFLTCC_INFLATE_CONTINUE;
|
||||
}
|
||||
state->mode = TYPEDO;
|
||||
/* Break if operands are exhausted, otherwise continue looping */
|
||||
return (cc == DFLTCC_CC_OP1_TOO_SHORT || cc == DFLTCC_CC_OP2_TOO_SHORT) ?
|
||||
DFLTCC_INFLATE_BREAK : DFLTCC_INFLATE_CONTINUE;
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_was_inflate_used)(PREFIX3(streamp) strm) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
|
||||
return !state->arch.common.param.nt;
|
||||
}
|
||||
|
||||
/*
|
||||
Rotates a circular buffer.
|
||||
The implementation is based on https://cplusplus.com/reference/algorithm/rotate/
|
||||
*/
|
||||
static void rotate(unsigned char *start, unsigned char *pivot, unsigned char *end) {
|
||||
unsigned char *p = pivot;
|
||||
unsigned char tmp;
|
||||
|
||||
while (p != start) {
|
||||
tmp = *start;
|
||||
*start = *p;
|
||||
*p = tmp;
|
||||
|
||||
start++;
|
||||
p++;
|
||||
|
||||
if (p == end)
|
||||
p = pivot;
|
||||
else if (start == pivot)
|
||||
pivot = p;
|
||||
}
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_inflate_disable)(PREFIX3(streamp) strm) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
struct dfltcc_state *dfltcc_state = &state->arch.common;
|
||||
struct dfltcc_param_v0 *param = &dfltcc_state->param;
|
||||
|
||||
if (!PREFIX(dfltcc_can_inflate)(strm))
|
||||
return 0;
|
||||
if (PREFIX(dfltcc_was_inflate_used)(strm))
|
||||
/* DFLTCC has already decompressed some data. Since there is not
|
||||
* enough information to resume decompression in software, the call
|
||||
* must fail.
|
||||
*/
|
||||
return 1;
|
||||
/* DFLTCC was not used yet - decompress in software */
|
||||
memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af));
|
||||
/* Convert the window from the hardware to the software format */
|
||||
rotate(state->window, state->window + param->ho, state->window + HB_SIZE);
|
||||
state->whave = state->wnext = MIN(param->hl, state->wsize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
Preloading history.
|
||||
*/
|
||||
int Z_INTERNAL PREFIX(dfltcc_inflate_set_dictionary)(PREFIX3(streamp) strm,
|
||||
const unsigned char *dictionary, uInt dict_length) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
|
||||
/* if window not in use yet, initialize */
|
||||
if (state->wsize == 0)
|
||||
state->wsize = 1U << state->wbits;
|
||||
|
||||
append_history(param, state->window, dictionary, dict_length);
|
||||
state->havedict = 1;
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
int Z_INTERNAL PREFIX(dfltcc_inflate_get_dictionary)(PREFIX3(streamp) strm,
|
||||
unsigned char *dictionary, uInt *dict_length) {
|
||||
struct inflate_state *state = (struct inflate_state *)strm->state;
|
||||
struct dfltcc_param_v0 *param = &state->arch.common.param;
|
||||
|
||||
if (dictionary && state->window)
|
||||
get_history(param, state->window, dictionary);
|
||||
if (dict_length)
|
||||
*dict_length = param->hl;
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
#ifndef DFLTCC_INFLATE_H
|
||||
#define DFLTCC_INFLATE_H
|
||||
|
||||
#include "dfltcc_common.h"
|
||||
|
||||
void Z_INTERNAL PREFIX(dfltcc_reset_inflate_state)(PREFIX3(streamp) strm);
|
||||
int Z_INTERNAL PREFIX(dfltcc_can_inflate)(PREFIX3(streamp) strm);
|
||||
typedef enum {
|
||||
DFLTCC_INFLATE_CONTINUE,
|
||||
DFLTCC_INFLATE_BREAK,
|
||||
DFLTCC_INFLATE_SOFTWARE,
|
||||
} dfltcc_inflate_action;
|
||||
dfltcc_inflate_action Z_INTERNAL PREFIX(dfltcc_inflate)(PREFIX3(streamp) strm, int flush, int *ret);
|
||||
int Z_INTERNAL PREFIX(dfltcc_was_inflate_used)(PREFIX3(streamp) strm);
|
||||
int Z_INTERNAL PREFIX(dfltcc_inflate_disable)(PREFIX3(streamp) strm);
|
||||
int Z_INTERNAL PREFIX(dfltcc_inflate_set_dictionary)(PREFIX3(streamp) strm,
|
||||
const unsigned char *dictionary, uInt dict_length);
|
||||
int Z_INTERNAL PREFIX(dfltcc_inflate_get_dictionary)(PREFIX3(streamp) strm,
|
||||
unsigned char *dictionary, uInt* dict_length);
|
||||
|
||||
#define INFLATE_RESET_KEEP_HOOK PREFIX(dfltcc_reset_inflate_state)
|
||||
|
||||
#define INFLATE_PRIME_HOOK(strm, bits, value) \
|
||||
do { if (PREFIX(dfltcc_inflate_disable)((strm))) return Z_STREAM_ERROR; } while (0)
|
||||
|
||||
#define INFLATE_TYPEDO_HOOK(strm, flush) \
|
||||
if (PREFIX(dfltcc_can_inflate)((strm))) { \
|
||||
dfltcc_inflate_action action; \
|
||||
\
|
||||
RESTORE(); \
|
||||
action = PREFIX(dfltcc_inflate)((strm), (flush), &ret); \
|
||||
LOAD(); \
|
||||
if (action == DFLTCC_INFLATE_CONTINUE) \
|
||||
break; \
|
||||
else if (action == DFLTCC_INFLATE_BREAK) \
|
||||
goto inf_leave; \
|
||||
}
|
||||
|
||||
#define INFLATE_NEED_CHECKSUM(strm) (!PREFIX(dfltcc_can_inflate)((strm)))
|
||||
|
||||
#define INFLATE_NEED_UPDATEWINDOW(strm) (!PREFIX(dfltcc_can_inflate)((strm)))
|
||||
|
||||
#define INFLATE_MARK_HOOK(strm) \
|
||||
do { \
|
||||
if (PREFIX(dfltcc_was_inflate_used)((strm))) return -(1L << 16); \
|
||||
} while (0)
|
||||
|
||||
#define INFLATE_SYNC_POINT_HOOK(strm) \
|
||||
do { \
|
||||
if (PREFIX(dfltcc_was_inflate_used)((strm))) return Z_STREAM_ERROR; \
|
||||
} while (0)
|
||||
|
||||
#define INFLATE_SET_DICTIONARY_HOOK(strm, dict, dict_len) \
|
||||
do { \
|
||||
if (PREFIX(dfltcc_can_inflate)((strm))) \
|
||||
return PREFIX(dfltcc_inflate_set_dictionary)((strm), (dict), (dict_len)); \
|
||||
} while (0)
|
||||
|
||||
#define INFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) \
|
||||
do { \
|
||||
if (PREFIX(dfltcc_can_inflate)((strm))) \
|
||||
return PREFIX(dfltcc_inflate_get_dictionary)((strm), (dict), (dict_len)); \
|
||||
} while (0)
|
||||
|
||||
#define INFLATE_ADJUST_WINDOW_SIZE(n) MAX(n, HB_SIZE)
|
||||
|
||||
#endif
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
#ifdef S390_FEATURES
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "s390_features.h"
|
||||
|
||||
#ifdef HAVE_SYS_AUXV_H
|
||||
# include <sys/auxv.h>
|
||||
#endif
|
||||
|
||||
#ifndef HWCAP_S390_VXRS
|
||||
#define HWCAP_S390_VXRS (1 << 11)
|
||||
#endif
|
||||
|
||||
void Z_INTERNAL s390_check_features(struct s390_cpu_features *features) {
|
||||
features->has_vx = getauxval(AT_HWCAP) & HWCAP_S390_VXRS;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
/* s390_features.h -- check for s390 features.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef S390_FEATURES_H_
|
||||
#define S390_FEATURES_H_
|
||||
|
||||
struct s390_cpu_features {
|
||||
int has_vx;
|
||||
};
|
||||
|
||||
void Z_INTERNAL s390_check_features(struct s390_cpu_features *features);
|
||||
|
||||
#endif
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
/* s390_functions.h -- s390 implementations for arch-specific functions.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef S390_FUNCTIONS_H_
|
||||
#define S390_FUNCTIONS_H_
|
||||
|
||||
#include "s390_natives.h"
|
||||
|
||||
#define ADLER32_FALLBACK
|
||||
#define CHUNKSET_FALLBACK
|
||||
#define COMPARE256_FALLBACK
|
||||
#define CRC32_BRAID_FALLBACK /* used by crc32_s390_vx */
|
||||
|
||||
#ifndef S390_VX_NATIVE
|
||||
# define SLIDE_HASH_FALLBACK
|
||||
#endif
|
||||
|
||||
#ifdef S390_VX
|
||||
uint32_t crc32_s390_vx(uint32_t crc, const uint8_t *buf, size_t len);
|
||||
uint32_t crc32_copy_s390_vx(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
|
||||
void slide_hash_vx(deflate_state *s);
|
||||
|
||||
#ifdef __clang__
|
||||
# if ((__clang_major__ == 18) || (__clang_major__ == 19 && (__clang_minor__ < 1 || (__clang_minor__ == 1 && __clang_patchlevel__ < 2))))
|
||||
# error CRC32-VX optimizations are broken due to compiler bug in Clang versions: 18.0.0 <= clang_version < 19.1.2. \
|
||||
Either disable the zlib-ng S390 VX optimizations, or switch to another compiler/compiler version.
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef DISABLE_RUNTIME_CPU_DETECTION
|
||||
# ifdef S390_VX_NATIVE
|
||||
# undef native_crc32
|
||||
# define native_crc32 crc32_s390_vx
|
||||
# undef native_crc32_copy
|
||||
# define native_crc32_copy crc32_copy_s390_vx
|
||||
# undef native_slide_hash
|
||||
# define native_slide_hash slide_hash_vx
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
/* s390_natives.h -- s390 compile-time feature detection macros.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef S390_NATIVES_H_
|
||||
#define S390_NATIVES_H_
|
||||
|
||||
#if defined(__zarch__) && __ARCH__ >= 11 && defined(__VX__)
|
||||
# ifdef S390_VX
|
||||
# define S390_VX_NATIVE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif /* S390_NATIVES_H_ */
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
#
|
||||
# Ephemeral runner startup script.
|
||||
#
|
||||
# Expects the following environment variables:
|
||||
#
|
||||
# - REPO=<owner>
|
||||
# - PAT_TOKEN=<github_pat_***>
|
||||
#
|
||||
|
||||
set -e -u
|
||||
|
||||
# Validate required environment variables
|
||||
if [ -z "${REPO:-}" ] || [ -z "${PAT_TOKEN:-}" ]; then
|
||||
echo "Error: REPO and/or PAT_TOKEN environment variables not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check the cached registration token.
|
||||
TOKEN_FILE=registration-token.json
|
||||
if [ -f $TOKEN_FILE ]; then
|
||||
set +e
|
||||
EXPIRES=$(jq --raw-output .expires_at "$TOKEN_FILE" 2>/dev/null)
|
||||
STATUS=$?
|
||||
set -e
|
||||
else
|
||||
STATUS=1
|
||||
EXPIRES=""
|
||||
fi
|
||||
|
||||
if [[ $STATUS -ne 0 || -z "$EXPIRES" || "$EXPIRES" == "null" || $(date +%s) -ge $(date -d "$EXPIRES" +%s) ]]; then
|
||||
# Refresh the cached registration token.
|
||||
curl \
|
||||
-sS \
|
||||
-X POST \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "Authorization: Bearer $PAT_TOKEN" \
|
||||
"https://api.github.com/repos/$REPO/actions/runners/registration-token" \
|
||||
-o "$TOKEN_FILE"
|
||||
fi
|
||||
|
||||
REG_TOKEN=$(jq --raw-output .token "$TOKEN_FILE")
|
||||
if [ $REG_TOKEN = "null" ]; then
|
||||
echo "Failed to get registration token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# (Re-)register the runner.
|
||||
./config.sh remove --token "$REG_TOKEN" || true
|
||||
set -x
|
||||
./config.sh \
|
||||
--url "https://github.com/$REPO" \
|
||||
--token "$REG_TOKEN" \
|
||||
--unattended \
|
||||
--disableupdate \
|
||||
--replace \
|
||||
--labels z15 \
|
||||
--ephemeral
|
||||
|
||||
# Run one job.
|
||||
./run.sh
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
#!/usr/bin/bash
|
||||
set -ex
|
||||
|
||||
TMPDIR="$(mktemp -d)"
|
||||
|
||||
if [ -f actions-runner.Dockerfile ]; then
|
||||
MODE=1
|
||||
cp actions-runner.Dockerfile actions-runner entrypoint $TMPDIR
|
||||
cd $TMPDIR
|
||||
else
|
||||
MODE=2
|
||||
cd $TMPDIR
|
||||
wget https://raw.githubusercontent.com/zlib-ng/zlib-ng/refs/heads/develop/arch/s390/self-hosted-builder/actions-runner.Dockerfile
|
||||
wget https://raw.githubusercontent.com/zlib-ng/zlib-ng/refs/heads/develop/arch/s390/self-hosted-builder/actions-runner
|
||||
wget https://raw.githubusercontent.com/zlib-ng/zlib-ng/refs/heads/develop/arch/s390/self-hosted-builder/entrypoint
|
||||
fi
|
||||
|
||||
# Stop service
|
||||
systemctl stop actions-runner || true
|
||||
|
||||
# Delete old container
|
||||
podman container rm gaplib-actions-runner || true
|
||||
|
||||
# Delete old image
|
||||
podman image rm localhost/zlib-ng/actions-runner || true
|
||||
|
||||
# Prune all unused podman data
|
||||
podman system prune -f || true
|
||||
|
||||
# Build new image
|
||||
podman build --squash -f actions-runner.Dockerfile --tag zlib-ng/actions-runner . 2>&1 | tee /var/log/actions-runner-build.log
|
||||
|
||||
# Create new container
|
||||
podman create --replace --name=gaplib-actions-runner --env-file=/etc/actions-runner --init \
|
||||
zlib-ng/actions-runner 2>&1 | tee -a /var/log/actions-runner-build.log
|
||||
|
||||
# Start service
|
||||
systemctl start actions-runner || true
|
||||
|
||||
# Cleanup
|
||||
podman image prune -af || true
|
||||
|
||||
# Clean up tempfile
|
||||
if [ "$MODE" == "2" ] ; then
|
||||
cd $TMPDIR
|
||||
rm actions-runner.Dockerfile
|
||||
rm actions-runner
|
||||
rm entrypoint
|
||||
cd ..
|
||||
rmdir $TMPDIR
|
||||
echo "Deleted tempfiles."
|
||||
fi
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
# Self-Hosted IBM Z Github Actions Runner.
|
||||
|
||||
FROM almalinux:10
|
||||
|
||||
RUN dnf update -y -q && \
|
||||
dnf install -y -q --enablerepo=crb wget git which sudo jq sed \
|
||||
cmake make automake autoconf m4 libtool ninja-build \
|
||||
python3-pip python3-devel python3-lxml \
|
||||
gcc gcc-c++ clang llvm-toolset glibc-all-langpacks langpacks-en \
|
||||
glibc-static libstdc++-static libstdc++-devel libxslt-devel libxml2-devel
|
||||
|
||||
RUN dnf install -y -q dotnet-sdk-8.0 && \
|
||||
echo "Using SDK - `dotnet --version`"
|
||||
|
||||
RUN cd /tmp && \
|
||||
git clone -q https://github.com/actions/runner && \
|
||||
cd runner && \
|
||||
git checkout $(git tag --sort=-v:refname | grep '^v[0-9]' | head -n1) && \
|
||||
git log -n 1 && \
|
||||
wget https://raw.githubusercontent.com/IBM/action-runner-image-pz/refs/heads/main/patches/runner-sdk8-s390x.patch -O runner-sdk8-s390x.patch && \
|
||||
git apply --whitespace=nowarn runner-sdk8-s390x.patch && \
|
||||
|
||||
sed -i'' -e /version/s/8......\"$/$8.0.100\"/ src/global.json
|
||||
|
||||
RUN cd /tmp/runner/src && \
|
||||
./dev.sh layout && \
|
||||
./dev.sh package && \
|
||||
rm -rf /root/.dotnet /root/.nuget
|
||||
|
||||
RUN useradd -c "Action Runner" -m actions-runner && \
|
||||
usermod -L actions-runner
|
||||
|
||||
RUN tar -xf /tmp/runner/_package/*.tar.gz -C /home/actions-runner && \
|
||||
chown -R actions-runner:actions-runner /home/actions-runner
|
||||
|
||||
# Cleanup
|
||||
RUN rm -rf /tmp/runner /var/cache/dnf/* /tmp/runner.patch /tmp/global.json && \
|
||||
dnf clean all
|
||||
|
||||
USER actions-runner
|
||||
|
||||
# Scripts.
|
||||
COPY --chmod=555 entrypoint /usr/bin/
|
||||
COPY --chmod=555 actions-runner /usr/bin/
|
||||
WORKDIR /home/actions-runner
|
||||
ENTRYPOINT ["/usr/bin/entrypoint"]
|
||||
CMD ["/usr/bin/actions-runner"]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
[Unit]
|
||||
Description=Podman container: Gaplib Github Actions Runner
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
StartLimitIntervalSec=1
|
||||
RequiresMountsFor=/run/user/1001/containers
|
||||
|
||||
[Service]
|
||||
Environment=PODMAN_SYSTEMD_UNIT=%n
|
||||
Restart=always
|
||||
TimeoutStopSec=61
|
||||
ExecStart=/usr/bin/podman start gaplib-actions-runner
|
||||
ExecStop=/usr/bin/podman stop -t 30 gaplib-actions-runner
|
||||
ExecStopPost=/usr/bin/podman stop -t 10 gaplib-actions-runner
|
||||
Type=forking
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
#
|
||||
# Container entrypoint that waits for all spawned processes.
|
||||
#
|
||||
|
||||
set -e -u
|
||||
|
||||
# Create a FIFO and start reading from its read end.
|
||||
tempdir=$(mktemp -d "/tmp/done.XXXXXXXXXX")
|
||||
trap 'rm -r "$tempdir"' EXIT
|
||||
done="$tempdir/pipe"
|
||||
mkfifo "$done"
|
||||
cat "$done" & waiter=$!
|
||||
|
||||
# Start the workload. Its descendants will inherit the FIFO's write end.
|
||||
status=0
|
||||
if [ "$#" -eq 0 ]; then
|
||||
bash 9>"$done" || status=$?
|
||||
else
|
||||
"$@" 9>"$done" || status=$?
|
||||
fi
|
||||
|
||||
# When the workload and all of its descendants exit, the FIFO's write end will
|
||||
# be closed and `cat "$done"` will exit. Wait until it happens. This is needed
|
||||
# in order to handle SelfUpdater, which the workload may start in background
|
||||
# before exiting.
|
||||
wait "$waiter"
|
||||
|
||||
exit "$status"
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
/* slide_hash_vx.c - VX version of slide_hash for S390 processors
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifdef S390_VX
|
||||
|
||||
#include "zbuild.h"
|
||||
#include "deflate.h"
|
||||
|
||||
#include "vx_intrins.h"
|
||||
|
||||
static inline void slide_hash_chain(Pos *table, uint32_t entries, uint16_t wsize) {
|
||||
const uv8hi vmx_wsize = vec_splats(wsize);
|
||||
Pos *p = table;
|
||||
|
||||
do {
|
||||
uv8hi value, result;
|
||||
|
||||
value = vec_xl(0, p);
|
||||
result = vec_sub(value, vec_min(value, vmx_wsize));
|
||||
vec_xst(result, 0, p);
|
||||
|
||||
p += 8;
|
||||
entries -= 8;
|
||||
} while (entries > 0);
|
||||
}
|
||||
|
||||
Z_INTERNAL void slide_hash_vx(deflate_state *s) {
|
||||
Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
|
||||
uint16_t wsize = (uint16_t)s->w_size;
|
||||
|
||||
slide_hash_chain(s->head, HASH_SIZE, wsize);
|
||||
slide_hash_chain(s->prev, wsize, wsize);
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
#ifndef S390_VX_INTRINS_H
|
||||
#define S390_VX_INTRINS_H
|
||||
|
||||
#include <vecintrin.h>
|
||||
|
||||
typedef unsigned char uv16qi __attribute__((vector_size(16)));
|
||||
typedef unsigned short uv8hi __attribute__((vector_size(16)));
|
||||
typedef unsigned int uv4si __attribute__((vector_size(16)));
|
||||
typedef unsigned long long uv2di __attribute__((vector_size(16)));
|
||||
|
||||
#ifndef vec_sub
|
||||
#define vec_sub(a, b) ((a) - (b))
|
||||
#endif
|
||||
#ifndef vec_subs
|
||||
static inline uv8hi vec_subs_u16(uv8hi a, uv8hi b) {
|
||||
return a - vec_min(a, b);
|
||||
}
|
||||
#define vec_subs(a, b) vec_subs_u16((a), (b))
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
/* chunk_128bit_perm_idx_lut.h - shared SSSE3/NEON/LSX permutation idx lut for use with chunkmemset family of functions.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef CHUNK_128BIT_PERM_IDX_LUT_H_
|
||||
#define CHUNK_128BIT_PERM_IDX_LUT_H_
|
||||
|
||||
#include "chunk_permute_table.h"
|
||||
|
||||
static const lut_rem_pair perm_idx_lut[13] = {
|
||||
{0, 1}, /* 3 */
|
||||
{0, 0}, /* don't care */
|
||||
{1 * 32, 1}, /* 5 */
|
||||
{2 * 32, 4}, /* 6 */
|
||||
{3 * 32, 2}, /* 7 */
|
||||
{0 * 32, 0}, /* don't care */
|
||||
{4 * 32, 7}, /* 9 */
|
||||
{5 * 32, 6}, /* 10 */
|
||||
{6 * 32, 5}, /* 11 */
|
||||
{7 * 32, 4}, /* 12 */
|
||||
{8 * 32, 3}, /* 13 */
|
||||
{9 * 32, 2}, /* 14 */
|
||||
{10 * 32, 1},/* 15 */
|
||||
};
|
||||
|
||||
#endif
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
/* chunk_256bit_perm_idx_lut.h - shared AVX512/AVX2/LASX permutation idx lut for use with chunkmemset family of functions.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
#ifndef CHUNK_256BIT_PERM_IDX_LUT_H_
|
||||
#define CHUNK_256BIT_PERM_IDX_LUT_H_
|
||||
|
||||
#include "chunk_permute_table.h"
|
||||
|
||||
/* Populate don't cares so that this is a direct lookup (with some indirection into the permute table), because dist can
|
||||
* never be 0 - 2, we'll start with an offset, subtracting 3 from the input */
|
||||
static const lut_rem_pair perm_idx_lut[29] = {
|
||||
{ 0, 2}, /* 3 */
|
||||
{ 0, 0}, /* don't care */
|
||||
{ 1 * 32, 2}, /* 5 */
|
||||
{ 2 * 32, 2}, /* 6 */
|
||||
{ 3 * 32, 4}, /* 7 */
|
||||
{ 0 * 32, 0}, /* don't care */
|
||||
{ 4 * 32, 5}, /* 9 */
|
||||
{ 5 * 32, 22}, /* 10 */
|
||||
{ 6 * 32, 21}, /* 11 */
|
||||
{ 7 * 32, 20}, /* 12 */
|
||||
{ 8 * 32, 6}, /* 13 */
|
||||
{ 9 * 32, 4}, /* 14 */
|
||||
{10 * 32, 2}, /* 15 */
|
||||
{ 0 * 32, 0}, /* don't care */
|
||||
{11 * 32, 15}, /* 17 */
|
||||
{11 * 32 + 16, 14}, /* 18 */
|
||||
{11 * 32 + 16 * 2, 13}, /* 19 */
|
||||
{11 * 32 + 16 * 3, 12}, /* 20 */
|
||||
{11 * 32 + 16 * 4, 11}, /* 21 */
|
||||
{11 * 32 + 16 * 5, 10}, /* 22 */
|
||||
{11 * 32 + 16 * 6, 9}, /* 23 */
|
||||
{11 * 32 + 16 * 7, 8}, /* 24 */
|
||||
{11 * 32 + 16 * 8, 7}, /* 25 */
|
||||
{11 * 32 + 16 * 9, 6}, /* 26 */
|
||||
{11 * 32 + 16 * 10, 5}, /* 27 */
|
||||
{11 * 32 + 16 * 11, 4}, /* 28 */
|
||||
{11 * 32 + 16 * 12, 3}, /* 29 */
|
||||
{11 * 32 + 16 * 13, 2}, /* 30 */
|
||||
{11 * 32 + 16 * 14, 1} /* 31 */
|
||||
};
|
||||
|
||||
static const uint16_t half_rem_vals[13] = {
|
||||
1, 0, 1, 4, 2, 0, 7, 6, 5, 4, 3, 2, 1
|
||||
};
|
||||
|
||||
#endif
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
/* chunk_permute_table.h - shared AVX/SSSE3/NEON/LSX/LASX permutation table for use with chunkmemset family of functions.
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef CHUNK_PERMUTE_TABLE_H_
|
||||
#define CHUNK_PERMUTE_TABLE_H_
|
||||
|
||||
#include "zbuild.h"
|
||||
|
||||
/* Need entries for all numbers not an even modulus for 1, 2, 4, 8, 16 & 32 */
|
||||
static const ALIGNED_(32) uint8_t permute_table[26*32] = {
|
||||
0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, /* dist 3 */
|
||||
0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, /* dist 5 */
|
||||
0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, /* dist 6 */
|
||||
0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, /* dist 7 */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, /* dist 9 */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, /* dist 10 */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, /* dist 11 */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, /* dist 12 */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, 3, 4, 5, /* dist 13 */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, 2, 3, /* dist 14 */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, 1, /* dist 15 */
|
||||
|
||||
/* Beyond dists of 15 means we have to permute from a vector > len(m128i). Because AVX couldn't permute
|
||||
* beyond 128 bit lanes until AVX512 for sub 4-byte sequences, we have to do some math here for an eventual
|
||||
* blend with a comparison. That means we need to wrap the indices with yet another derived table. For simplicity,
|
||||
* we'll use absolute indexing here to derive a blend vector. This is actually a lot simpler with ARM's TBL, but,
|
||||
* this is what we're dealt.
|
||||
*/
|
||||
|
||||
16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* dist 17 */
|
||||
16, 17, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, /* dist 18 */
|
||||
16, 17, 18, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, /* dist 19 */
|
||||
16, 17, 18, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, /* dist 20 */
|
||||
16, 17, 18, 19, 20, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, /* dist 21 */
|
||||
16, 17, 18, 19, 20, 21, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, /* dist 22 */
|
||||
16, 17, 18, 19, 20, 21, 22, 0, 1, 2, 3, 4, 5, 6, 7, 8, /* dist 23 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7, /* dist 24 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 0, 1, 2, 3, 4, 5, 6, /* dist 25 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 1, 2, 3, 4, 5, /* dist 26 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 0, 1, 2, 3, 4, /* dist 27 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 0, 1, 2, 3, /* dist 28 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 0, 1, 2, /* dist 29 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 0, 1, /* dist 30 */
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 0, /* dist 31 */
|
||||
};
|
||||
|
||||
typedef struct lut_rem_pair_s {
|
||||
uint16_t idx;
|
||||
uint16_t remval;
|
||||
} lut_rem_pair;
|
||||
|
||||
#endif
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
/* crc32_hw_common_tpl.h -- Private shared inline CRC32 functions for CPU with native crc instructions
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#include "zbuild.h"
|
||||
|
||||
|
||||
Z_FORCEINLINE static Z_TARGET_CRC uint32_t crc32_hw_align(uint32_t crc, uint8_t **dst, const uint8_t **buf,
|
||||
size_t *len, uintptr_t align_diff, const int COPY) {
|
||||
if (*len && (align_diff & 1)) {
|
||||
uint8_t val = **buf;
|
||||
if (COPY) {
|
||||
**dst = val;
|
||||
*dst += 1;
|
||||
}
|
||||
crc = CRC32B(crc, val);
|
||||
*buf += 1;
|
||||
*len -= 1;
|
||||
}
|
||||
|
||||
if (*len >= 2 && (align_diff & 2)) {
|
||||
uint16_t val = *((uint16_t*)*buf);
|
||||
if (COPY) {
|
||||
memcpy(*dst, &val, 2);
|
||||
*dst += 2;
|
||||
}
|
||||
crc = CRC32H(crc, val);
|
||||
*buf += 2;
|
||||
*len -= 2;
|
||||
}
|
||||
|
||||
if (*len >= 4 && (align_diff & 4)) {
|
||||
uint32_t val = *((uint32_t*)*buf);
|
||||
if (COPY) {
|
||||
memcpy(*dst, &val, 4);
|
||||
*dst += 4;
|
||||
}
|
||||
crc = CRC32W(crc, val);
|
||||
*buf += 4;
|
||||
*len -= 4;
|
||||
}
|
||||
|
||||
if (*len >= 8 && (align_diff & 8)) {
|
||||
uint64_t val = *((uint64_t*)*buf);
|
||||
if (COPY) {
|
||||
memcpy(*dst, &val, 8);
|
||||
*dst += 8;
|
||||
}
|
||||
crc = CRC32D(crc, val);
|
||||
*buf += 8;
|
||||
*len -= 8;
|
||||
}
|
||||
|
||||
return crc;
|
||||
}
|
||||
|
||||
Z_FORCEINLINE static Z_TARGET_CRC uint32_t crc32_hw_tail(uint32_t crc, uint8_t *dst, const uint8_t *buf,
|
||||
size_t len, const int COPY) {
|
||||
while (len >= 8) {
|
||||
uint64_t val = *((uint64_t*)buf);
|
||||
if (COPY) {
|
||||
memcpy(dst, &val, 8);
|
||||
dst += 8;
|
||||
}
|
||||
crc = CRC32D(crc, val);
|
||||
buf += 8;
|
||||
len -= 8;
|
||||
}
|
||||
|
||||
if (len & 4) {
|
||||
uint32_t val = *((uint32_t*)buf);
|
||||
if (COPY) {
|
||||
memcpy(dst, &val, 4);
|
||||
dst += 4;
|
||||
}
|
||||
crc = CRC32W(crc, val);
|
||||
buf += 4;
|
||||
}
|
||||
|
||||
if (len & 2) {
|
||||
uint16_t val = *((uint16_t*)buf);
|
||||
if (COPY) {
|
||||
memcpy(dst, &val, 2);
|
||||
dst += 2;
|
||||
}
|
||||
crc = CRC32H(crc, val);
|
||||
buf += 2;
|
||||
}
|
||||
|
||||
if (len & 1) {
|
||||
uint8_t val = *buf;
|
||||
if (COPY)
|
||||
*dst = val;
|
||||
crc = CRC32B(crc, val);
|
||||
}
|
||||
|
||||
return ~crc;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue