diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61a92d7dfaf7bb5a4c4e6b29e79b06837adfd978..5720727b3052b5fdb83fc80614697880d036c4f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,12 +24,11 @@ build: test_obfs4proxy: stage: test script: - - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./build-binary.py obfs4proxy && ./verify-binary.py obfs4proxy" + - docker run -v `pwd`/output:/opt/go-reproducer/output ${TEST_IMAGE} /bin/bash -c "./build-binary.py obfs4proxy && ./verify-binary.py obfs4proxy" allow_failure: true artifacts: paths: - - obfs4proxy-*.jar - - obfs4proxy-*.pom + - output/obfs4proxy expire_in: 1 week when: always except: @@ -38,12 +37,11 @@ test_obfs4proxy: test_snowflake: stage: test script: - - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./build-binary.py snowflake && ./verify-binary.py snowflake" + - docker run -v `pwd`/output:/opt/go-reproducer/output ${TEST_IMAGE} /bin/bash -c "./build-binary.py snowflake && ./verify-binary.py snowflake" allow_failure: true artifacts: paths: - - snowflake-*.jar - - snowflake-*.pom + - output/snowflake expire_in: 1 week when: always except: @@ -52,16 +50,12 @@ test_snowflake: test_tag: stage: test script: - - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./verify-binary.py obfs4proxy ${CI_BUILD_REF_NAME}" - - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./verify-binary.py snowflake ${CI_BUILD_REF_NAME}" + - docker run -v `pwd`/output:/opt/go-reproducer/output ${TEST_IMAGE} /bin/bash -c "./verify-binary.py obfs4proxy ${CI_BUILD_REF_NAME}" + - docker run -v `pwd`/output:/opt/go-reproducer/output ${TEST_IMAGE} /bin/bash -c "./verify-binary.py snowflake ${CI_BUILD_REF_NAME}" artifacts: paths: - - obfs4proxy-*.zip - - obfs4proxy-*.pom - - obfs4proxy-*-sources.jar - - snowflake-*.zip - - snowflake-*.pom - - snowflake-*-sources.jar + - output/obfs4proxy + - output/snowflake expire_in: 1 week when: always only: diff --git a/Dockerfile b/Dockerfile index 833334334b0b20cdab4351bacf1f2e840f705dc7..2b937f924d8c78c41199454492fdcc184aa7877b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM debian:bullseye -ARG IGNORE_EXPIRY=0 ENV LANG=C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive diff --git a/README.md b/README.md index f9a2a6f4b878d30a7476580e8189d7a1735c0fce..e84476b61bdfc35d7086253163ac331dbe1c2903 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,10 @@ Build our Docker image: docker build -t briar/go-reproducer go-reproducer -Building the image might fail due to expired Debian packages. -You can disable the expiry check by adding a build argument: - - docker build --build-arg IGNORE_EXPIRY=1 -t briar/go-reproducer go-reproducer - -However, note that this might expose the build process to MITM attacks -which inject outdated vulnerable packages. +To ensure reproducibility we build the image from a fixed snapshot of Debian +and ignore expiry warnings for Debian packages. This means the build process +inside the Docker container may use outdated packages that could contain +known vulnerabilities. ### Run the verification diff --git a/build-binary.py b/build-binary.py index 38db13eab3728af61f5e86cc2ae438ea8dfa34c6..f65560635a9c69a855d426fd13a0f346134a58a2 100755 --- a/build-binary.py +++ b/build-binary.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -import os +import os, shutil from glob import glob from subprocess import check_call -from utils import get_build_versions, ex, get_sha256, zip_files, get_final_file_name, \ - get_sources_file_name, get_pom_file_name, reset_time, get_version_number, check_go_version, \ - get_version_and_tool, GO_PATH, GO_ROOT, NDK_DIR +from utils import get_build_versions, ex, get_sha256, zip_files, get_final_file_path, \ + get_sources_file_path, get_pom_file_path, reset_time, get_version_number, check_go_version, \ + get_version_and_tool, get_output_dir, get_platform_output_dir, GO_PATH, GO_ROOT, NDK_DIR def main(): @@ -25,6 +25,11 @@ def main(): # Checkout source at specific version checkout_source_repo(tool, versions) + # Create the output directory, deleting it first if it already exists + output_dir = get_output_dir(tool) + if os.path.exists(output_dir): shutil.rmtree(output_dir) + os.makedirs(output_dir) + # Build and package for various platforms and architectures build_android(tool, versions) package_android(tool, versions) @@ -76,104 +81,113 @@ def checkout_source_repo(tool, versions): def build_android(tool, versions): + os.mkdir(get_platform_output_dir(tool, 'android')) + env = os.environ.copy() env['GOARCH'] = "arm" env['GOARM'] = "7" - build_android_arch(tool, versions, env, "arm-linux-androideabi", "arm") + build_android_arch(tool, versions, env, "armv7a-linux-androideabi", "arm", "armeabi-v7a") env = os.environ.copy() env['GOARCH'] = "arm64" - build_android_arch(tool, versions, env, "aarch64-linux-android", "arm64") + build_android_arch(tool, versions, env, "aarch64-linux-android", "arm64", "arm64-v8a") env = os.environ.copy() env['GOARCH'] = "386" - build_android_arch(tool, versions, env, "i686-linux-android", "x86") + build_android_arch(tool, versions, env, "i686-linux-android", "x86", "x86") env = os.environ.copy() env['GOARCH'] = "amd64" - build_android_arch(tool, versions, env, "x86_64-linux-android", "x86_64") + build_android_arch(tool, versions, env, "x86_64-linux-android", "x86_64", "x86_64") -def build_android_arch(tool, versions, env, clang_arch, ndk_arch): - toolchain = os.path.join("toolchain", ndk_arch) - if not os.path.isdir(toolchain): - toolchain_maker = os.path.join(NDK_DIR, "build", "tools", "make-standalone-toolchain.sh") - ex([toolchain_maker, "--arch=%s" % ndk_arch, "--install-dir=%s" % toolchain]) +def build_android_arch(tool, versions, env, clang_arch, ndk_arch, abi): + # TODO: Raise API level and upgrade NDK when we drop support for Android 4 + min_api = '21' if ndk_arch.endswith('64') else '16' + clang = "%s/toolchains/llvm/prebuilt/linux-x86_64/bin/%s%s-clang" % (NDK_DIR, clang_arch, min_api) - env['CC'] = "%s/bin/%s-clang" % (os.path.abspath(toolchain), clang_arch) + env['CC'] = os.path.abspath(clang) env['CGO_ENABLED'] = "1" env['CGO_CFLAGS'] = "-O2" # removes -g env['GOOS'] = "android" - build_mode = "pie" - extldflags = " -extldflags=-pie" - - filename = "%s_%s_pie.zip" % (tool, ndk_arch) - print("Building %s" % filename) + print("Building %s for Android %s" % (tool, abi)) + output_dir = get_platform_output_dir(tool, 'android') + arch_dir = os.path.join(output_dir, abi) + os.mkdir(arch_dir) + tool_path = os.path.join(arch_dir, "lib%s.so" % tool) output_file = os.path.abspath(os.path.join(os.path.curdir, tool)) go_flags = ['-asmflags', '-trimpath', '-o', output_file] repo_dir = get_repo_dir(versions) - ex(['go', 'build', '-buildmode=%s' % build_mode, '-ldflags', '-w -s' + extldflags] + go_flags + + ex(['go', 'build', '-buildmode=pie', '-ldflags', '-w -s -extldflags=-pie'] + go_flags + [os.path.join('.', versions['build_path'])], env=env, cwd=repo_dir) - - zip_files([tool], filename, versions) - os.remove(tool) + shutil.copy(output_file, tool_path) def build_linux(tool, versions): + os.mkdir(get_platform_output_dir(tool, 'linux')) build_desktop_arch(tool, versions, 'linux', 'armhf', 'arm', '7') build_desktop_arch(tool, versions, 'linux', 'aarch64', 'arm64') build_desktop_arch(tool, versions, 'linux', 'x86_64', 'amd64') def build_windows(tool, versions): + os.mkdir(get_platform_output_dir(tool, 'windows')) build_desktop_arch(tool, versions, 'windows', 'x86_64', 'amd64') -def build_desktop_arch(tool, versions, goos, arch, goarch, goarm=None): +def build_desktop_arch(tool, versions, platform, arch, goarch, goarm=None): env = os.environ.copy() env['CGO_ENABLED'] = "0" - env['GOOS'] = goos + env['GOOS'] = platform env['GOARCH'] = goarch if goarm: env['GOARM'] = goarm build_path = os.path.join('.', versions['build_path']) - filename = "%s_%s-%s.zip" % (tool, goos, arch) - print("Building %s" % filename) + + print("Building %s for %s %s" % (tool, platform, arch)) + output_dir = get_platform_output_dir(tool, platform) + arch_dir = os.path.join(output_dir, arch) + os.mkdir(arch_dir) + extension = '.exe' if platform == 'windows' else '' + tool_path = os.path.join(arch_dir, "%s%s" % (tool, extension)) + output_file = os.path.abspath(os.path.join(os.path.curdir, tool)) go_flags = ['-asmflags', '-trimpath', '-o', output_file] repo_dir = get_repo_dir(versions) ex(['go', 'build', '-ldflags', '-w -s'] + go_flags + [build_path], env=env, cwd=repo_dir) - zip_files([tool], filename, versions) - os.remove(tool) + shutil.copy(output_file, tool_path) def package_android(tool, versions): + output_dir = get_platform_output_dir(tool, 'android') file_list = [ - '%s_arm_pie.zip' % tool, - '%s_arm64_pie.zip' % tool, - '%s_x86_pie.zip' % tool, - '%s_x86_64_pie.zip' % tool + os.path.join(output_dir, 'armeabi-v7a', 'lib%s.so' % tool), + os.path.join(output_dir, 'arm64-v8a', 'lib%s.so' % tool), + os.path.join(output_dir, 'x86', 'lib%s.so' % tool), + os.path.join(output_dir, 'x86_64', 'lib%s.so' % tool), ] package(tool, versions, file_list, 'android') def package_linux(tool, versions): + output_dir = get_platform_output_dir(tool, 'linux') file_list = [ - '%s_linux-armhf.zip' % tool, - '%s_linux-aarch64.zip' % tool, - '%s_linux-x86_64.zip' % tool + os.path.join(output_dir, 'armhf', tool), + os.path.join(output_dir, 'aarch64', tool), + os.path.join(output_dir, 'x86_64', tool), ] package(tool, versions, file_list, 'linux') def package_windows(tool, versions): - file_list = ['%s_windows-x86_64.zip' % tool] + output_dir = get_platform_output_dir(tool, 'windows') + file_list = [os.path.join(output_dir, 'x86_64', '%s.exe' % tool)] package(tool, versions, file_list, 'windows') def package(tool, versions, file_list, platform): - zip_file = get_final_file_name(tool, versions, platform) + zip_file = get_final_file_path(tool, versions, platform) zip_files(file_list, zip_file, versions) create_sources_jar(tool, versions, platform) create_pom_file(tool, versions, platform) @@ -195,21 +209,18 @@ def create_sources_jar(tool, versions, platform): for file in glob(os.path.join(repo_dir, '*')): reset_time(file, versions) jar_files.append(os.path.relpath(file, repo_dir)) - jar_file = get_sources_file_name(tool, versions, platform) - jar_path = os.path.abspath(jar_file) + jar_path = get_sources_file_path(tool, versions, platform) check_call(['jar', 'cf', jar_path] + jar_files, cwd=repo_dir) - return jar_file def create_pom_file(tool, versions, platform): tool_version = get_version_number(versions) - pom_file = get_pom_file_name(tool, versions, platform) + pom_file = get_pom_file_path(tool, versions, platform) template = 'template-%s-%s.pom' % (tool, platform) with open(template, 'rt') as infile: with open(pom_file, 'wt') as outfile: for line in infile: outfile.write(line.replace('VERSION', tool_version)) - return pom_file if __name__ == "__main__": diff --git a/install.sh b/install.sh index dd3504b18c7e9b3ec0ee9c4a4a5e26099919e402..82dd84d2050eef6e1d803bdb33cc8a2a61ee816f 100755 --- a/install.sh +++ b/install.sh @@ -3,17 +3,14 @@ set -e set -x # use snapshot repos for deterministic package versions -DATE="20221108T000000Z" +DATE="20230209T000000Z" cat << EOF > /etc/apt/sources.list deb http://snapshot.debian.org/archive/debian/${DATE}/ bullseye main deb http://snapshot.debian.org/archive/debian/${DATE}/ bullseye-updates main EOF -# ignore expired package releases if env variable is set -if [[ "${IGNORE_EXPIRY}" = "1" ]] -then - echo 'Acquire::Check-Valid-Until "0";' >> /etc/apt/apt.conf.d/10-ignore-expiry -fi +# ignore expired package releases so we can reproduce from old snapshots +echo 'Acquire::Check-Valid-Until "0";' >> /etc/apt/apt.conf.d/10-ignore-expiry # update package sources apt-get update diff --git a/output/.gitkeep b/output/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/template-snowflake-android.pom b/template-snowflake-android.pom index 874baba8fedd43f3965434bcdea4e0f85a8e3fea..9588ce532aace3e448982781e2a4fe442187758b 100644 --- a/template-snowflake-android.pom +++ b/template-snowflake-android.pom @@ -10,7 +10,7 @@ <licenses> <license> <name>BSD-3-clause</name> - <url>https://gitweb.torproject.org/pluggable-transports/snowflake.git/tree/LICENSE</url> + <url>https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/blob/main/LICENSE</url> </license> </licenses> <developers> @@ -21,8 +21,8 @@ </developer> </developers> <scm> - <connection>scm:https://git.torproject.org/pluggable-transports/snowflake.git</connection> - <developerConnection>scm:git@gitweb.torproject.org/pluggable-transports/snowflake.git</developerConnection> - <url>scm:https://gitweb.torproject.org/pluggable-transports/snowflake.git</url> + <connection>scm:https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git</connection> + <developerConnection>scm:git@gitlab.torproject.org:tpo/anti-censorship/pluggable-transports/snowflake.git</developerConnection> + <url>scm:https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git</url> </scm> </project> diff --git a/template-snowflake-linux.pom b/template-snowflake-linux.pom index 4d2b1c7d9dfab0d18b270e305564c4aa2114e60f..e4a329257d819ed3f4b35b051aeef0ad5f9c725c 100644 --- a/template-snowflake-linux.pom +++ b/template-snowflake-linux.pom @@ -10,7 +10,7 @@ <licenses> <license> <name>BSD-3-clause</name> - <url>https://gitweb.torproject.org/pluggable-transports/snowflake.git/tree/LICENSE</url> + <url>https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/blob/main/LICENSE</url> </license> </licenses> <developers> @@ -21,8 +21,8 @@ </developer> </developers> <scm> - <connection>scm:https://git.torproject.org/pluggable-transports/snowflake.git</connection> - <developerConnection>scm:git@gitweb.torproject.org/pluggable-transports/snowflake.git</developerConnection> - <url>scm:https://gitweb.torproject.org/pluggable-transports/snowflake.git</url> + <connection>scm:https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git</connection> + <developerConnection>scm:git@gitlab.torproject.org:tpo/anti-censorship/pluggable-transports/snowflake.git</developerConnection> + <url>scm:https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git</url> </scm> </project> diff --git a/template-snowflake-windows.pom b/template-snowflake-windows.pom index 5bb59acafe9a055878113be761f1c02c9b080931..3ee94aec5db2b441e0cbe9747b385b9100e0129a 100644 --- a/template-snowflake-windows.pom +++ b/template-snowflake-windows.pom @@ -10,7 +10,7 @@ <licenses> <license> <name>BSD-3-clause</name> - <url>https://gitweb.torproject.org/pluggable-transports/snowflake.git/tree/LICENSE</url> + <url>https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/blob/main/LICENSE</url> </license> </licenses> <developers> @@ -21,8 +21,8 @@ </developer> </developers> <scm> - <connection>scm:https://git.torproject.org/pluggable-transports/snowflake.git</connection> - <developerConnection>scm:git@gitweb.torproject.org/pluggable-transports/snowflake.git</developerConnection> - <url>scm:https://gitweb.torproject.org/pluggable-transports/snowflake.git</url> + <connection>scm:https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git</connection> + <developerConnection>scm:git@gitlab.torproject.org:tpo/anti-censorship/pluggable-transports/snowflake.git</developerConnection> + <url>scm:https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git</url> </scm> </project> diff --git a/utils.py b/utils.py index d096cb6d2ebdfc6221192f9f7f06da5021018c73..c45e88dc4d28dbecad3622f9b6d8fa225fea6f3a 100644 --- a/utils.py +++ b/utils.py @@ -67,27 +67,37 @@ def zip_files(files, zip_name, versions): reset_time(file_name, versions) # use deterministic permissions to prevent differences in zip files os.chmod(file_name, 0o755) - sha256hash = get_sha256(file_name) - print("Hash before zipping %s: %s" % (file_name, sha256hash)) - ex(['zip', '-D', '-X', zip_name] + files) + (zip_dir, zip_name) = os.path.split(zip_name) + files = list(map(lambda f : os.path.relpath(f, zip_dir), files)) + ex(['zip', '-D', '-X', zip_name] + files, cwd=zip_dir) def get_version_number(versions): return versions['tag'] -def get_file_suffix(versions, platform): +def get_output_file_path(tool, versions, platform, suffix): + output_dir = get_platform_output_dir(tool, platform) version = get_version_number(versions) - return "%s-%s" % (platform, version) + filename = '%s-%s-%s%s' % (tool, platform, version, suffix) + return os.path.join(output_dir, filename) -def get_final_file_name(tool, versions, platform): - return '%s-%s.jar' % (tool, get_file_suffix(versions, platform)) +def get_final_file_path(tool, versions, platform): + return get_output_file_path(tool, versions, platform, '.jar') -def get_sources_file_name(tool, versions, platform): - return '%s-%s-sources.jar' % (tool, get_file_suffix(versions, platform)) +def get_sources_file_path(tool, versions, platform): + return get_output_file_path(tool, versions, platform, '-sources.jar') -def get_pom_file_name(tool, versions, platform): - return '%s-%s.pom' % (tool, get_file_suffix(versions, platform)) +def get_pom_file_path(tool, versions, platform): + return get_output_file_path(tool, versions, platform, '.pom') + + +def get_output_dir(tool): + return os.path.abspath(os.path.join('output', tool)) + + +def get_platform_output_dir(tool, platform): + return os.path.abspath(os.path.join('output', tool, platform)) diff --git a/verify-binary.py b/verify-binary.py index 7c69ebfef212103b4e6d9f6786c0b47eaf281821..283f04df4feedf055b429ebfefb9374f80b6408a 100755 --- a/verify-binary.py +++ b/verify-binary.py @@ -3,7 +3,7 @@ import os import sys from subprocess import check_call, CalledProcessError -from utils import get_sha256, fail, get_build_versions, get_final_file_name, \ +from utils import get_sha256, fail, get_build_versions, get_final_file_path, \ get_version_and_tool, get_version_number @@ -25,9 +25,9 @@ def verify(tool, command_line_version, platform): tool_version, versions = get_build_versions(tool, command_line_version) # download reference binary - file_name = get_final_file_name(tool, versions, platform) + file_name = get_final_file_path(tool, versions, platform) os.makedirs('reference', exist_ok=True) - reference_file_name = os.path.join('reference', file_name) + reference_file_name = os.path.join('reference', os.path.basename(file_name)) # try downloading from maven central check_call(['wget', '--no-verbose', get_url(tool, versions, platform), '-O', reference_file_name]) @@ -58,7 +58,7 @@ def verify(tool, command_line_version, platform): def get_url(tool, versions, platform): version = get_version_number(versions) directory = "%s-%s" % (tool, platform) - file = get_final_file_name(tool, versions, platform) + file = os.path.basename(get_final_file_path(tool, versions, platform)) return "https://repo.maven.apache.org/maven2/org/briarproject/%s/%s/%s" % (directory, version, file) diff --git a/versions.json b/versions.json index 1b012845be1fd9e422a525030fc003f13f07df7f..d5ae820aec95fc10e521a9261eeec977d2980495 100644 --- a/versions.json +++ b/versions.json @@ -1,5 +1,21 @@ { "obfs4proxy": { + "0.0.14-tor2": { + "repo_url": "https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/obfs4.git", + "revision": "b9e04fd9e241b0530e05511ec6895e698a25575f", + "build_path": "obfs4proxy", + "repo_dir": "obfs4", + "go": { + "version": "go1.19.3", + "sha256": "18ac263e39210bcf68d85f4370e97fb1734166995a1f63fb38b4f6e07d90d212" + }, + "ndk": { + "url": "https://dl.google.com/android/repository/android-ndk-r23c-linux.zip", + "revision": "23.2.8568313", + "sha256": "6ce94604b77d28113ecd588d425363624a5228d9662450c48d2e4053f8039242" + }, + "timestamp": "201001010000.00" + }, "0.0.14-tor1": { "repo_url": "https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/obfs4.git", "revision": "ed46c23917b55c4b274d6986daeaf6bec7963115", @@ -112,6 +128,22 @@ } }, "snowflake": { + "2.5.1": { + "repo_url": "https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git", + "revision": "7b77001eaa90e09d41172a2b170dabd3f1922b4a", + "build_path": "client", + "repo_dir": "snowflake", + "go": { + "version": "go1.19.3", + "sha256": "18ac263e39210bcf68d85f4370e97fb1734166995a1f63fb38b4f6e07d90d212" + }, + "ndk": { + "url": "https://dl.google.com/android/repository/android-ndk-r23c-linux.zip", + "revision": "23.2.8568313", + "sha256": "6ce94604b77d28113ecd588d425363624a5228d9662450c48d2e4053f8039242" + }, + "timestamp": "201001010000.00" + }, "2.3.1": { "repo_url": "https://git.torproject.org/pluggable-transports/snowflake.git", "revision": "36f03dfd4483922b3e7400dedc71df9cf2f30b6b", @@ -129,4 +161,4 @@ "timestamp": "201001010000.00" } } -} +} \ No newline at end of file