Update NSS to 3.36.4-RTM

This commit is contained in:
JustOff 2018-06-09 15:11:22 +03:00 committed by Roy Tam
commit fe96962962
148 changed files with 5550 additions and 5825 deletions

View file

@ -5,11 +5,11 @@ MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com>
# the original F* formula with Daniel Fabian
# Pinned versions of HACL* (F* and KreMLin are pinned as submodules)
ENV haclrepo https://github.com/mitls/hacl-star.git
ENV haclrepo https://github.com/franziskuskiefer/hacl-star.git
# Define versions of dependencies
ENV opamv 4.04.2
ENV haclversion dcd48329d535727dbde93877b124c5ec4a7a2b20
ENV haclversion 668d6cf274c33bbe2e951e3a84b73f2b6442a51f
# Install required packages and set versions
ADD setup.sh /tmp/setup.sh

View file

@ -0,0 +1,46 @@
FROM ubuntu:latest
MAINTAINER Tim Taubert <ttaubert@mozilla.com>
RUN useradd -d /home/worker -s /bin/bash -m worker
WORKDIR /home/worker
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y \
binutils \
build-essential \
bzip2 \
clang-3.8 \
curl \
gcc-multilib \
g++-multilib \
gyp \
lib32z1-dev \
mercurial \
ninja-build \
unzip \
zlib1g-dev
# Add missing LLVM plugin for gold linker.
ADD LLVMgold.so.zip /usr/lib/llvm-3.8/lib/LLVMgold.so.zip
RUN unzip /usr/lib/llvm-3.8/lib/LLVMgold.so.zip -d /usr/lib/llvm-3.8/lib/
# Install SAW/Cryptol.
RUN curl -LO https://saw.galois.com/builds/nightly/saw-0.2-2018-01-14-Ubuntu14.04-64.tar.gz && \
tar xzvf saw-*.tar.gz -C /usr/local --strip-components=1 && \
rm saw-*.tar.gz
# Install Z3.
RUN curl -LO https://github.com/Z3Prover/z3/releases/download/z3-4.6.0/z3-4.6.0-x64-ubuntu-16.04.zip && \
unzip z3*.zip && \
cp -r z3*/* /usr/local/ && \
rm -fr z3*
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
# Change user.
USER worker
# Set a default command useful for debugging
CMD ["/bin/bash", "--login"]

View file

@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -v -e -x
# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
# Clone NSS.
for i in 0 2 5; do
sleep $i
hg clone -r $REVISION $REPOSITORY nss && exit 0
rm -rf nss
done
exit 1

View file

@ -12,6 +12,7 @@ apt-get install -y --no-install-recommends apt-utils
apt_packages=()
apt_packages+=('build-essential')
apt_packages+=('ca-certificates')
apt_packages+=('clang-5.0')
apt_packages+=('curl')
apt_packages+=('npm')
apt_packages+=('git')
@ -47,16 +48,17 @@ echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >
apt-get -y update
apt-get install -y --no-install-recommends ${apt_packages[@]}
# Download clang.
curl -LO https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
curl -LO https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
# Verify the signature.
gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
gpg --verify *.tar.xz.sig
# Install into /usr/local/.
tar xJvf *.tar.xz -C /usr/local --strip-components=1
# Cleanup.
rm *.tar.xz*
# Latest version of abigail-tools
apt-get install -y libxml2-dev autoconf libelf-dev libdw-dev libtool
git clone git://sourceware.org/git/libabigail.git
cd ./libabigail
autoreconf -fi
./configure --prefix=/usr --disable-static --disable-apidoc --disable-manual
make
make install
cd ..
apt-get remove -y libxml2-dev autoconf libtool
rm -rf libabigail
# Install latest Rust (stable).
su worker -c "curl https://sh.rustup.rs -sSf | sh -s -- -y"

View file

@ -30,6 +30,11 @@ const HACL_GEN_IMAGE = {
path: "automation/taskcluster/docker-hacl"
};
const SAW_IMAGE = {
name: "saw",
path: "automation/taskcluster/docker-saw"
};
const WINDOWS_CHECKOUT_CMD =
"bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
"(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
@ -72,7 +77,8 @@ queue.filter(task => {
}
}
if (task.tests == "fips" && task.platform == "mac") {
if (task.tests == "fips" &&
(task.platform == "mac" || task.platform == "aarch64")) {
return false;
}
@ -88,7 +94,7 @@ queue.filter(task => {
}
}
// Don't run additional hardware tests on ARM (we don't have anything there).
// Don't run all additional hardware tests on ARM.
if (task.group == "Cipher" && task.platform == "aarch64" && task.env &&
(task.env.NSS_DISABLE_PCLMUL == "1" || task.env.NSS_DISABLE_HW_AES == "1"
|| task.env.NSS_DISABLE_AVX == "1")) {
@ -187,8 +193,8 @@ export default async function main() {
UBSAN_OPTIONS: "print_stacktrace=1",
NSS_DISABLE_ARENA_FREE_LIST: "1",
NSS_DISABLE_UNLOAD: "1",
CC: "clang",
CCC: "clang++",
CC: "clang-5.0",
CCC: "clang++-5.0",
},
platform: "linux64",
collection: "asan",
@ -266,6 +272,18 @@ export default async function main() {
}, aarch64_base)
);
await scheduleLinux("Linux AArch64 (debug, make)",
merge({
env: {USE_64: "1"},
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh"
],
collection: "make",
}, aarch64_base)
);
await scheduleMac("Mac (opt)", {collection: "opt"}, "--opt");
await scheduleMac("Mac (debug)", {collection: "debug"});
}
@ -418,12 +436,12 @@ async function scheduleLinux(name, base, args = "") {
// Extra builds.
let extra_base = merge({group: "Builds"}, build_base);
queue.scheduleTask(merge(extra_base, {
name: `${name} w/ clang-4.0`,
name: `${name} w/ clang-5.0`,
env: {
CC: "clang",
CCC: "clang++",
CC: "clang-5.0",
CCC: "clang++-5.0",
},
symbol: "clang-4.0"
symbol: "clang-5.0"
}));
queue.scheduleTask(merge(extra_base, {
@ -894,6 +912,13 @@ function scheduleTests(task_build, task_cert, test_base) {
name: "Cipher tests", symbol: "NoAVX", tests: "cipher",
env: {NSS_DISABLE_AVX: "1"}, group: "Cipher"
}));
queue.scheduleTask(merge(no_cert_base, {
name: "Cipher tests", symbol: "NoSSSE3|NEON", tests: "cipher",
env: {
NSS_DISABLE_ARM_NEON: "1",
NSS_DISABLE_SSSE3: "1"
}, group: "Cipher"
}));
queue.scheduleTask(merge(no_cert_base, {
name: "EC tests", symbol: "EC", tests: "ec"
}));
@ -946,6 +971,18 @@ async function scheduleTools() {
kind: "test"
};
//ABI check task
queue.scheduleTask(merge(base, {
symbol: "abi",
name: "abi",
image: LINUX_IMAGE,
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/check_abi.sh"
],
}));
queue.scheduleTask(merge(base, {
symbol: "clang-format-3.9",
name: "clang-format-3.9",
@ -958,13 +995,13 @@ async function scheduleTools() {
}));
queue.scheduleTask(merge(base, {
symbol: "scan-build-4.0",
name: "scan-build-4.0",
symbol: "scan-build-5.0",
name: "scan-build-5.0",
image: LINUX_IMAGE,
env: {
USE_64: "1",
CC: "clang",
CCC: "clang++",
CC: "clang-5.0",
CCC: "clang++-5.0",
},
artifacts: {
public: {
@ -991,5 +1028,69 @@ async function scheduleTools() {
]
}));
let task_saw = queue.scheduleTask(merge(base, {
symbol: "B",
group: "SAW",
name: "LLVM bitcode build (32 bit)",
image: SAW_IMAGE,
kind: "build",
env: {
AR: "llvm-ar-3.8",
CC: "clang-3.8",
CCC: "clang++-3.8"
},
artifacts: {
public: {
expires: 24 * 7,
type: "directory",
path: "/home/worker/artifacts"
}
},
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --disable-tests --emit-llvm -m32"
]
}));
queue.scheduleTask(merge(base, {
parent: task_saw,
symbol: "bmul",
group: "SAW",
name: "bmul.saw",
image: SAW_IMAGE,
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/run_saw.sh bmul"
]
}));
queue.scheduleTask(merge(base, {
parent: task_saw,
symbol: "ChaCha20",
group: "SAW",
name: "chacha20.saw",
image: SAW_IMAGE,
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/run_saw.sh chacha20"
]
}));
queue.scheduleTask(merge(base, {
parent: task_saw,
symbol: "Poly1305",
group: "SAW",
name: "poly1305.saw",
image: SAW_IMAGE,
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/run_saw.sh poly1305"
]
}));
return queue.submit();
}

View file

@ -30,7 +30,7 @@ export async function buildTask({name, path}) {
let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`;
return {
name: "Image Builder",
name: `Image Builder (${name})`,
image: "nssdev/image_builder:0.1.5",
routes: ["index." + ns],
env: {

View file

@ -31,10 +31,11 @@ function parseRoutes(routes) {
];
// Notify about failures (except on try).
if (process.env.TC_PROJECT != "nss-try") {
// Turned off, too noisy.
/*if (process.env.TC_PROJECT != "nss-try") {
rv.push(`notify.email.${process.env.TC_OWNER}.on-failed`,
`notify.email.${process.env.TC_OWNER}.on-exception`);
}
}*/
return rv;
}

View file

@ -51,7 +51,7 @@ function parseOptions(opts) {
}
// Parse tools.
let allTools = ["clang-format", "scan-build", "hacl"];
let allTools = ["clang-format", "scan-build", "hacl", "saw", "abi"];
let tools = intersect(opts.tools.split(/\s*,\s*/), allTools);
// If the given value is "all" run all tools.
@ -77,7 +77,8 @@ function filter(opts) {
// are not affected by platform or build type selectors.
if (task.platform == "nss-tools") {
return opts.tools.some(tool => {
return task.symbol.toLowerCase().startsWith(tool);
return task.symbol.toLowerCase().startsWith(tool) ||
(task.group && task.group.toLowerCase().startsWith(tool));
});
}

View file

@ -0,0 +1,172 @@
#! /bin/bash
set_env()
{
cd /home/worker
HGDIR=/home/worker
OUTPUTDIR=$(pwd)$(echo "/output")
DATE=$(date "+TB [%Y-%m-%d %H:%M:%S]")
if [ ! -d "${OUTPUTDIR}" ]; then
echo "Creating output dir"
mkdir "${OUTPUTDIR}"
fi
if [ ! -d "nspr" ]; then
for i in 0 2 5; do
sleep $i
hg clone -r "default" "https://hg.mozilla.org/projects/nspr" "${HGDIR}/nspr" && break
rm -rf nspr
done
fi
cd nss
./build.sh -v -c
cd ..
}
check_abi()
{
set_env
set +e #reverses set -e from build.sh to allow possible hg clone failures
if [[ "$1" != --nobuild ]]; then # Start nobuild block
echo "######## NSS ABI CHECK ########"
echo "######## creating temporary HG clones ########"
rm -rf ${HGDIR}/baseline
mkdir ${HGDIR}/baseline
BASE_NSS=`cat ${HGDIR}/nss/automation/abi-check/previous-nss-release` #Reads the version number of the last release from the respective file
NSS_CLONE_RESULT=0
for i in 0 2 5; do
sleep $i
hg clone -u "${BASE_NSS}" "https://hg.mozilla.org/projects/nss" "${HGDIR}/baseline/nss"
if [ $? -eq 0 ]; then
NSS_CLONE_RESULT=0
break
fi
rm -rf "${HGDIR}/baseline/nss"
NSS_CLONE_RESULT=1
done
if [ ${NSS_CLONE_RESULT} -ne 0 ]; then
echo "invalid tag in automation/abi-check/previous-nss-release"
return 1
fi
BASE_NSPR=NSPR_$(head -1 ${HGDIR}/baseline/nss/automation/release/nspr-version.txt | cut -d . -f 1-2 | tr . _)_BRANCH
hg clone -u "${BASE_NSPR}" "https://hg.mozilla.org/projects/nspr" "${HGDIR}/baseline/nspr"
NSPR_CLONE_RESULT=$?
if [ ${NSPR_CLONE_RESULT} -ne 0 ]; then
rm -rf "${HGDIR}/baseline/nspr"
for i in 0 2 5; do
sleep $i
hg clone -u "default" "https://hg.mozilla.org/projects/nspr" "${HGDIR}/baseline/nspr" && break
rm -rf "${HGDIR}/baseline/nspr"
done
echo "Nonexisting tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt"
echo "Using default branch instead."
fi
echo "######## building baseline NSPR/NSS ########"
echo "${HGDIR}/baseline/nss/build.sh"
cd ${HGDIR}/baseline/nss
./build.sh -v -c
cd ${HGDIR}
else # Else nobuild block
echo "######## using existing baseline NSPR/NSS build ########"
fi # End nobuild block
set +e #reverses set -e from build.sh to allow abidiff failures
echo "######## Starting abidiff procedure ########"
abi_diff
}
#Slightly modified from builbot-slave/build.sh
abi_diff()
{
ABI_PROBLEM_FOUND=0
ABI_REPORT=${OUTPUTDIR}/abi-diff.txt
rm -f ${ABI_REPORT}
PREVDIST=${HGDIR}/baseline/dist
NEWDIST=${HGDIR}/dist
ALL_SOs="libfreebl3.so libfreeblpriv3.so libnspr4.so libnss3.so libnssckbi.so libnssdbm3.so libnsssysinit.so libnssutil3.so libplc4.so libplds4.so libsmime3.so libsoftokn3.so libssl3.so"
for SO in ${ALL_SOs}; do
if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then
touch ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt
fi
abidiff --hd1 $PREVDIST/public/ --hd2 $NEWDIST/public \
$PREVDIST/*/lib/$SO $NEWDIST/*/lib/$SO \
> ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt
RET=$?
cat ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt \
| grep -v "^Functions changes summary:" \
| grep -v "^Variables changes summary:" \
> ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt
rm -f ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt
ABIDIFF_ERROR=$((($RET & 0x01) != 0))
ABIDIFF_USAGE_ERROR=$((($RET & 0x02) != 0))
ABIDIFF_ABI_CHANGE=$((($RET & 0x04) != 0))
ABIDIFF_ABI_INCOMPATIBLE_CHANGE=$((($RET & 0x08) != 0))
ABIDIFF_UNKNOWN_BIT_SET=$((($RET & 0xf0) != 0))
# If abidiff reports an error, or a usage error, or if it sets a result
# bit value this script doesn't know yet about, we'll report failure.
# For ABI changes, we don't yet report an error. We'll compare the
# result report with our whitelist. This allows us to silence changes
# that we're already aware of and have been declared acceptable.
REPORT_RET_AS_FAILURE=0
if [ $ABIDIFF_ERROR -ne 0 ]; then
echo "abidiff reported ABIDIFF_ERROR."
REPORT_RET_AS_FAILURE=1
fi
if [ $ABIDIFF_USAGE_ERROR -ne 0 ]; then
echo "abidiff reported ABIDIFF_USAGE_ERROR."
REPORT_RET_AS_FAILURE=1
fi
if [ $ABIDIFF_UNKNOWN_BIT_SET -ne 0 ]; then
echo "abidiff reported ABIDIFF_UNKNOWN_BIT_SET."
REPORT_RET_AS_FAILURE=1
fi
if [ $ABIDIFF_ABI_CHANGE -ne 0 ]; then
echo "Ignoring abidiff result ABI_CHANGE, instead we'll check for non-whitelisted differences."
fi
if [ $ABIDIFF_ABI_INCOMPATIBLE_CHANGE -ne 0 ]; then
echo "Ignoring abidiff result ABIDIFF_ABI_INCOMPATIBLE_CHANGE, instead we'll check for non-whitelisted differences."
fi
if [ $REPORT_RET_AS_FAILURE -ne 0 ]; then
ABI_PROBLEM_FOUND=1
echo "abidiff {$PREVDIST , $NEWDIST} for $SO FAILED with result $RET, or failed writing to ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt"
fi
if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then
ABI_PROBLEM_FOUND=1
echo "FAILED to access report file: ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt"
fi
diff -wB -u ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt \
${HGDIR}/nss/automation/abi-check/new-report-$SO.txt >> ${ABI_REPORT}
if [ ! -f ${ABI_REPORT} ]; then
ABI_PROBLEM_FOUND=1
echo "FAILED to compare exepcted and new report: ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt"
fi
done
if [ -s ${ABI_REPORT} ]; then
echo "FAILED: there are new unexpected ABI changes"
cat ${ABI_REPORT}
return 1
elif [ $ABI_PROBLEM_FOUND -ne 0 ]; then
echo "FAILED: failure executing the ABI checks"
cat ${ABI_REPORT}
return 1
fi
return 0
}
check_abi $1

View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
source $(dirname "$0")/tools.sh
# Fetch artifact if needed.
fetch_dist
# Run SAW.
saw "nss/automation/saw/$1.saw"

View file

@ -34,7 +34,7 @@ for i in "${!scan[@]}"; do
done
# run scan-build (only building affected directories)
scan-build -o /home/worker/artifacts --use-cc=$CC --use-c++=$CCC make nss_build_all && cd ..
scan-build-5.0 -o /home/worker/artifacts --use-cc=$CC --use-c++=$CCC make nss_build_all && cd ..
# print errors we found
set +v +x