diff --git a/.gitignore b/.gitignore index 0c59a1541b9974aaafede7c8c6b5cb584f4fa4fe..e49f0be400b7268ed7f799a19cb7668833503813 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ /android-ndk -/tor-android +/tor-build +!/tor-build/Makefile +!/output/.gitkeep /android-ndk.zip -tor-*.zip +tor-*.jar tor-*.pom tor-*-sources.jar diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb09a44a5d186e2c5e36def0fb3dccbf090fcc3e..066a4a5bc885e49e4dbc68e4bf6fd4e9d20313e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,32 +21,72 @@ build: - docker build -t ${TEST_IMAGE} . - docker push $TEST_IMAGE -test_build: +.base-android: stage: test - script: - - docker run -v `pwd`:/opt/tor-reproducer ${TEST_IMAGE} /bin/bash -c "./build_tor.py && ./verify_tor.py" artifacts: paths: - - tor-*.zip - - tor-*.pom - - tor-*-sources.jar + - output/android expire_in: 1 week when: always - allow_failure: true - except: - - tags -test_tag: +.base-linux: + stage: test + artifacts: + paths: + - output/linux + expire_in: 1 week + when: always + +.base-windows: stage: test - script: - - docker run -v `pwd`:/opt/tor-reproducer ${TEST_IMAGE} ./verify_tor.py ${CI_BUILD_REF_NAME} artifacts: paths: - - tor-*.zip - - tor-*.pom - - tor-*-sources.jar + - output/windows expire_in: 1 week when: always + +test_build_android: + extends: .base-android + script: + - docker run -v `pwd`/output:/opt/tor-reproducer/output ${TEST_IMAGE} /bin/bash -c "./build_tor_android.py && ./verify_tor_android.py" + allow_failure: true + except: + - tags + +test_build_linux: + extends: .base-linux + script: + - docker run -v `pwd`/output:/opt/tor-reproducer/output ${TEST_IMAGE} /bin/bash -c "./build_tor_linux.py && ./verify_tor_linux.py" + allow_failure: true + except: + - tags + +test_build_windows: + extends: .base-windows + script: + - docker run -v `pwd`/output:/opt/tor-reproducer/output ${TEST_IMAGE} /bin/bash -c "./build_tor_windows.py && ./verify_tor_windows.py" + allow_failure: true + except: + - tags + +test_tag_android: + extends: .base-android + script: + - docker run -v `pwd`/output:/opt/tor-reproducer/output ${TEST_IMAGE} ./verify_tor.py ${CI_BUILD_REF_NAME} + only: + - tags + +test_tag_linux: + extends: .base-linux + script: + - docker run -v `pwd`/output:/opt/tor-reproducer/output ${TEST_IMAGE} ./verify_tor_linux.py ${CI_BUILD_REF_NAME} + only: + - tags + +test_tag_windows: + extends: .base-windows + script: + - docker run -v `pwd`/output:/opt/tor-reproducer/output ${TEST_IMAGE} ./verify_tor_windows.py ${CI_BUILD_REF_NAME} only: - tags diff --git a/Dockerfile b/Dockerfile index af855b54dfa21feb53caa8c3e4ee8995f4e3dd2f..1c406692f2fcfdc90399c44f1775ce6cf592f9fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,5 +22,6 @@ ADD utils.py ./ ADD template-android.pom ./ ADD template-linux.pom ./ ADD template-windows.pom ./ +ADD tor-build/Makefile ./tor-build/ CMD ./build-tor.py diff --git a/README.md b/README.md index 5b637372ede7b54295ceb2875d89f944f75df05f..402a37ab92c74535e6a919493116cc003f732348 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Build our Docker image: To verify a specific version of Tor, run - docker run briar/tor-reproducer:latest ./verify-tor.py [version] + docker run briar/tor-reproducer:latest ./verify_tor.py [version] Where `[version]` is the version of Tor you want to test, for example `0.3.3.6`. @@ -60,14 +60,19 @@ In case there is an issue with the verification of an old build, this *might* be caused by an update of the container. You can try to use the original container by running: - docker run briar/tor-reproducer:[version] ./verify-tor.py [version] + docker run briar/tor-reproducer:[version] ./verify_tor.py [version] There should be a tag with the name `[version]` in this repository that you could be used to reproduce the old container. Note that this will not work if the issue is caused by an updated Debian package. +### Historical changes + +The Tor 0.3.x series had a different build system than the 0.4.x series. +Please use a version of tor-reproducer that starts with 0.3 to reproduce those. + ### Only build Tor To build a specific version of Tor, run - docker run briar/tor-reproducer:latest ./build_tor.py [version] \ No newline at end of file + docker run briar/tor-reproducer:latest ./build_tor.py [version] diff --git a/build_tor_android.py b/build_tor_android.py index e6e07f3fb05f733e49168bd71e8907d579c7fa17..3fd75dae334c3c86bd0ebbcfe1815aff4d0aa158 100755 --- a/build_tor_android.py +++ b/build_tor_android.py @@ -4,21 +4,20 @@ from shutil import rmtree, move, copy from subprocess import check_call import utils -from utils import get_sha256, fail, REPO_DIR, EXT_DIR, reset_time +from utils import get_sha256, fail, BUILD_DIR, get_output_dir, reset_time NDK_DIR = 'android-ndk' PLATFORM = "android" def build(): - versions, _ = utils.setup(PLATFORM) + versions, jar_name = utils.setup(PLATFORM) setup_android_ndk(versions) build_android(versions) - utils.package_geoip(versions) - package_android(versions) + package_android(versions, jar_name) def setup_android_ndk(versions): @@ -58,7 +57,8 @@ def setup_android_ndk(versions): def build_android(versions): - os.environ.pop("PIEFLAGS", None) # uses default PIE flags, if not present + # use default PIE flags, if not present + os.environ.pop("PIEFLAGS", None) # build arm pie env = os.environ.copy() @@ -87,24 +87,33 @@ def build_android(versions): def build_android_arch(name, env, versions): print("Building %s" % name) - check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR, env=env) - copy(os.path.join(EXT_DIR, 'bin', 'tor'), os.path.join(REPO_DIR, 'tor')) - check_call(['strip', '-D', 'tor'], cwd=REPO_DIR) - tor_path = os.path.join(REPO_DIR, 'tor') + output_dir = get_output_dir(PLATFORM) + # TODO add extra flags to configure? + # '--enable-static-tor', + # '--enable-static-zlib', + check_call(['make', 'clean', 'tor'], cwd=BUILD_DIR, env=env) + tor_path = os.path.join(output_dir, 'tor') + # note: stripping happens in makefile for now + copy(os.path.join(BUILD_DIR, 'tor', 'src', 'app', 'tor'), tor_path) reset_time(tor_path, versions) print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path))) - check_call(['zip', '-X', '../' + name, 'tor'], cwd=REPO_DIR) - - -def package_android(versions): - # zip Android binaries together - file_list_android = ['tor_arm_pie.zip', 'tor_arm64_pie.zip', - 'tor_x86_pie.zip', 'tor_x86_64_pie.zip', - 'geoip.zip'] - zip_name_android = utils.pack(versions, file_list_android, PLATFORM) - pom_name_android = utils.create_pom_file(versions, PLATFORM) - print("Android:") - for file in file_list_android + [zip_name_android, pom_name_android]: + check_call(['zip', '--no-dir-entries', '--junk-paths', '-X', name, 'tor'], cwd=output_dir) + os.remove(tor_path) + + +def package_android(versions, jar_name): + # zip binaries together + output_dir = get_output_dir(PLATFORM) + file_list = [ + os.path.join(output_dir, 'tor_arm_pie.zip'), + os.path.join(output_dir, 'tor_arm64_pie.zip'), + os.path.join(output_dir, 'tor_x86_pie.zip'), + os.path.join(output_dir, 'tor_x86_64_pie.zip'), + ] + zip_name = utils.pack(versions, file_list, PLATFORM) + pom_name = utils.create_pom_file(versions, PLATFORM) + print("%s:" % PLATFORM) + for file in file_list + [zip_name, jar_name, pom_name]: sha256hash = get_sha256(file) print("%s: %s" % (file, sha256hash)) diff --git a/build_tor_linux.py b/build_tor_linux.py index 15164fdc25cd56b8425ce5db40470f45128e1cad..08794032b280199e0cb434a9db9660ca6e8387fe 100755 --- a/build_tor_linux.py +++ b/build_tor_linux.py @@ -4,7 +4,8 @@ from shutil import rmtree, copy from subprocess import check_call import utils -from utils import REPO_DIR, EXT_DIR, reset_time, get_sha256 +from utils import BUILD_DIR, get_output_dir, TOR_CONFIGURE_FLAGS, OPENSSL_CONFIGURE_FLAGS, REPRODUCIBLE_GCC_CFLAGS, \ + XZ_CONFIGURE_FLAGS, reset_time, get_sha256, pack, create_pom_file PLATFORM = "linux" @@ -14,7 +15,6 @@ def build(): build_linux(versions) - utils.package_geoip(versions) package_linux(versions, jar_name) @@ -27,12 +27,12 @@ def build_linux(versions): def build_linux_arch(arch, gcc_arch, cc_env, openssl_target, autogen_host, versions): name = "tor_linux-%s.zip" % arch print("Building %s" % name) - prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix')) + prefix_dir = os.path.abspath(os.path.join(BUILD_DIR, 'prefix')) lib_dir = os.path.join(prefix_dir, 'lib') include_dir = os.path.join(prefix_dir, 'include') # ensure clean build environment (again here to protect against build reordering) - utils.prepare_tor_android_repo(versions) + utils.prepare_repos(versions) if os.path.exists(prefix_dir): rmtree(prefix_dir) @@ -43,63 +43,98 @@ def build_linux_arch(arch, gcc_arch, cc_env, openssl_target, autogen_host, versi # setup environment env = os.environ.copy() + env['SOURCE_DATE_EPOCH'] = "1234567890" env['LDFLAGS'] = "-L%s" % lib_dir env['LD_LIBRARY_PATH'] = lib_dir - env['CFLAGS'] = "-fPIC -I%s" % include_dir + env['CFLAGS'] = REPRODUCIBLE_GCC_CFLAGS + ' -fPIC -I%s' % include_dir + env['PKG_CONFIG_PATH'] = os.path.join(lib_dir, 'pkgconfig') env['LIBS'] = "-ldl -L%s" % lib_dir env['CC'] = cc_env + # build lzma + xz_dir = os.path.join(BUILD_DIR, 'xz') + check_call(['./autogen.sh'], cwd=xz_dir) + check_call(['./configure', + '--prefix=%s' % prefix_dir, + '--host=%s' % autogen_host, + ] + XZ_CONFIGURE_FLAGS, cwd=xz_dir, env=env) + check_call(['make', '-j', str(os.cpu_count()), 'install'], cwd=xz_dir, env=env) + + # build zstd + zstd_dir = os.path.join(BUILD_DIR, 'zstd', "lib") + check_call(['make', '-j', str(os.cpu_count()), 'DESTDIR=%s' % prefix_dir, 'PREFIX=""', 'install'], + cwd=zstd_dir, env=env) + # build zlib - zlib_dir = os.path.join(EXT_DIR, 'zlib') + zlib_dir = os.path.join(BUILD_DIR, 'zlib') check_call(['./configure', '--prefix=%s' % prefix_dir], cwd=zlib_dir, env=env) - check_call(['make', 'install'], cwd=zlib_dir, env=env) + check_call(['make', '-j', str(os.cpu_count()), 'install'], cwd=zlib_dir, env=env) # build openssl - openssl_dir = os.path.join(EXT_DIR, 'openssl') - check_call(['perl', 'Configure', '--prefix=%s' % prefix_dir, - '--openssldir=%s' % prefix_dir, '-march=%s' % gcc_arch, - openssl_target, 'shared'], cwd=openssl_dir, env=env) - check_call(['make'], cwd=openssl_dir, env=env) + openssl_dir = os.path.join(BUILD_DIR, 'openssl') + extra_flags = [] + if autogen_host.endswith("64"): + extra_flags = ['enable-ec_nistp_64_gcc_128'] + check_call(['perl', 'Configure', + '--prefix=%s' % prefix_dir, + '--openssldir=%s' % prefix_dir, + '-march=%s' % gcc_arch, + openssl_target, + 'shared', + ] + OPENSSL_CONFIGURE_FLAGS + extra_flags, cwd=openssl_dir, env=env) + check_call(['make', '-j', str(os.cpu_count())], cwd=openssl_dir, env=env) check_call(['make', 'install_sw'], cwd=openssl_dir, env=env) # build libevent - libevent_dir = os.path.join(EXT_DIR, 'libevent') + libevent_dir = os.path.join(BUILD_DIR, 'libevent') check_call(['./autogen.sh'], cwd=libevent_dir) check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir, '--host=%s' % autogen_host], cwd=libevent_dir, env=env) - check_call(['make'], cwd=libevent_dir, env=env) + check_call(['make', '-j', str(os.cpu_count())], cwd=libevent_dir, env=env) check_call(['make', 'install'], cwd=libevent_dir, env=env) # build Tor - tor_dir = os.path.join(EXT_DIR, 'tor') + tor_dir = os.path.join(BUILD_DIR, 'tor') check_call(['./autogen.sh'], cwd=tor_dir) env['CFLAGS'] += ' -O3' # needed for FORTIFY_SOURCE - check_call(['./configure', '--disable-asciidoc', '--disable-systemd', - '--enable-static-zlib', '--with-zlib-dir=%s' % prefix_dir, - '--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir, - '--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir, - '--prefix=%s' % prefix_dir, '--host=%s' % autogen_host, - '--disable-tool-name-check'], cwd=tor_dir, env=env) - check_call(['make', 'install'], cwd=tor_dir, env=env) + # TODO check if a completely static Tor is still portable + # '--enable-static-tor', + check_call(['./configure', + '--host=%s' % autogen_host, + '--prefix=%s' % prefix_dir, + '--enable-lzma', + '--enable-zstd', + '--enable-static-zlib', + '--with-zlib-dir=%s' % prefix_dir, + '--enable-static-libevent', + '--with-libevent-dir=%s' % prefix_dir, + '--enable-static-openssl', + '--with-openssl-dir=%s' % prefix_dir, + ] + TOR_CONFIGURE_FLAGS, cwd=tor_dir, env=env) + check_call(['make', '-j', str(os.cpu_count()), 'install'], cwd=tor_dir, env=env) # copy and zip built Tor binary - tor_path = os.path.join(REPO_DIR, 'tor') - copy(os.path.join(prefix_dir, 'bin', 'tor'), tor_path) - check_call(['strip', '-D', 'tor'], cwd=REPO_DIR) + output_dir = get_output_dir(PLATFORM) + tor_path = os.path.join(output_dir, 'tor') + copy(os.path.join(BUILD_DIR, 'tor', 'src', 'app', 'tor'), tor_path) + check_call(['strip', '-D', '--strip-unneeded', '--strip-debug', '-R', '.note*', '-R', '.comment', tor_path]) reset_time(tor_path, versions) print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path))) - check_call(['zip', '-X', '../' + name, 'tor'], cwd=REPO_DIR) + check_call(['zip', '--no-dir-entries', '--junk-paths', '-X', name, 'tor'], cwd=output_dir) + os.remove(tor_path) def package_linux(versions, jar_name): # zip binaries together - file_list = ['tor_linux-aarch64.zip', 'tor_linux-armhf.zip', 'tor_linux-x86_64.zip', 'geoip.zip'] - zip_name = utils.pack(versions, file_list, PLATFORM) - - # create POM file from template - pom_name = utils.create_pom_file(versions, PLATFORM) - - # print hashes for debug purposes + output_dir = get_output_dir(PLATFORM) + file_list = [ + os.path.join(output_dir, 'tor_linux-aarch64.zip'), + os.path.join(output_dir, 'tor_linux-armhf.zip'), + os.path.join(output_dir, 'tor_linux-x86_64.zip'), + ] + zip_name = pack(versions, file_list, PLATFORM) + pom_name = create_pom_file(versions, PLATFORM) + print("%s:" % PLATFORM) for file in file_list + [zip_name, jar_name, pom_name]: sha256hash = get_sha256(file) print("%s: %s" % (file, sha256hash)) diff --git a/build_tor_windows.py b/build_tor_windows.py index afe652d0c556c31b44f9e8fb69214095da23669b..9e45ba25eca43775303420e3b4a2432557062222 100755 --- a/build_tor_windows.py +++ b/build_tor_windows.py @@ -4,7 +4,8 @@ from shutil import rmtree, copy from subprocess import check_call import utils -from utils import REPO_DIR, EXT_DIR, reset_time +from utils import BUILD_DIR, get_output_dir, TOR_CONFIGURE_FLAGS, OPENSSL_CONFIGURE_FLAGS, REPRODUCIBLE_GCC_CFLAGS, \ + XZ_CONFIGURE_FLAGS, reset_time, get_sha256 PLATFORM = "windows" @@ -14,7 +15,6 @@ def build(): build_windows(versions) - utils.package_geoip(versions) package_windows(versions, jar_name) @@ -25,12 +25,12 @@ def build_windows(versions): def build_windows_arch(arch, host, versions): name = "tor_windows-%s.zip" % arch print("Building %s" % name) - prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix')) + prefix_dir = os.path.abspath(os.path.join(BUILD_DIR, 'prefix')) lib_dir = os.path.join(prefix_dir, 'lib') include_dir = os.path.join(prefix_dir, 'include') # ensure clean build environment (again here to protect against build reordering) - utils.prepare_tor_android_repo(versions) + utils.prepare_repos(versions) if os.path.exists(prefix_dir): rmtree(prefix_dir) @@ -41,72 +41,100 @@ def build_windows_arch(arch, host, versions): # setup environment env = os.environ.copy() - - # build zlib + env['SOURCE_DATE_EPOCH'] = "1234567890" env['LDFLAGS'] = "-L%s" % prefix_dir + env['CFLAGS'] = REPRODUCIBLE_GCC_CFLAGS + ' -fPIC -I%s' % include_dir + env['PKG_CONFIG_PATH'] = os.path.join(lib_dir, 'pkgconfig') # needed to find OpenSSL env['CHOST'] = host - zlib_dir = os.path.join(EXT_DIR, 'zlib') - check_call(['make', '-f', 'win32/Makefile.gcc', 'BINARY_PATH=%s/bin' % prefix_dir, + # build lzma + xz_dir = os.path.join(BUILD_DIR, 'xz') + check_call(['./autogen.sh'], cwd=xz_dir) + check_call(['./configure', + '--prefix=%s' % prefix_dir, + '--host=%s' % host, + ] + XZ_CONFIGURE_FLAGS, cwd=xz_dir, env=env) + check_call(['make', '-j', str(os.cpu_count()), 'install'], cwd=xz_dir, env=env) + + # build zlib + zlib_dir = os.path.join(BUILD_DIR, 'zlib') + check_call(['make', '-j', str(os.cpu_count()), '-f', 'win32/Makefile.gcc', 'BINARY_PATH=%s/bin' % prefix_dir, 'INCLUDE_PATH=%s/include' % prefix_dir, 'LIBRARY_PATH=%s/lib' % prefix_dir, 'SHARED_MODE=1', 'PREFIX=%s-' % host, 'install'], cwd=zlib_dir, env=env) - # build libevent - libevent_dir = os.path.join(EXT_DIR, 'libevent') + # build openssl + env['LDFLAGS'] = REPRODUCIBLE_GCC_CFLAGS + " -static -static-libgcc -L%s" % prefix_dir + + openssl_dir = os.path.join(BUILD_DIR, 'openssl') + check_call(['perl', 'Configure', + 'mingw64', + '--cross-compile-prefix=%s-' % host, + '--prefix=%s' % prefix_dir, + '--openssldir=%s' % prefix_dir, + # '-static', # https://github.com/openssl/openssl/issues/14574 + '-static-libgcc', + 'no-shared', + 'enable-ec_nistp_64_gcc_128', + ] + OPENSSL_CONFIGURE_FLAGS, cwd=openssl_dir, env=env) + check_call(['make', '-j', str(os.cpu_count())], cwd=openssl_dir, env=env) + check_call(['make', 'install_sw'], cwd=openssl_dir, env=env) + # build libevent + libevent_dir = os.path.join(BUILD_DIR, 'libevent') check_call(['./autogen.sh'], cwd=libevent_dir) - check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir, - '--host=%s' % host, '--enable-static'], cwd=libevent_dir, env=env) - check_call(['make'], cwd=libevent_dir, env=env) + check_call(['./configure', + '--host=%s' % host, + '--disable-libevent-regress', + '--disable-samples', + '--disable-shared', + '--prefix=%s' % prefix_dir, + ], cwd=libevent_dir, env=env) + check_call(['make', '-j', str(os.cpu_count())], cwd=libevent_dir, env=env) check_call(['make', 'install'], cwd=libevent_dir, env=env) - # build openssl - env['CFLAGS'] = "-I%s" % include_dir - env['LDFLAGS'] = "-static -static-libgcc -L%s" % prefix_dir - - openssl_dir = os.path.join(EXT_DIR, 'openssl') - check_call(['perl', 'Configure', '--prefix=%s' % prefix_dir, - '--openssldir=%s' % prefix_dir, '-static', '-static-libgcc', - '--cross-compile-prefix=%s-' % host, 'mingw64', 'no-shared', 'no-asm'], - cwd=openssl_dir, env=env) - check_call(['make'], cwd=openssl_dir, env=env) - check_call(['make', 'install_sw'], cwd=openssl_dir, env=env) - # build Tor - tor_dir = os.path.join(EXT_DIR, 'tor') + tor_dir = os.path.join(BUILD_DIR, 'tor') check_call(['./autogen.sh'], cwd=tor_dir) + env['CFLAGS'] += ' -O3' env['LIBS'] = "-lcrypt32" - env['LDFLAGS'] = "-static -static-libgcc -L%s" % prefix_dir - env['CFLAGS'] = "-I%s" % include_dir - - check_call(['./configure', '--disable-asciidoc', '--disable-systemd', - '--enable-static-zlib', '--with-zlib-dir=%s' % prefix_dir, - '--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir, - '--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir, - '--prefix=%s' % prefix_dir, '--host=%s' % host, - '--disable-tool-name-check'], cwd=tor_dir, env=env) - check_call(['make'], cwd=tor_dir, env=env) + + # TODO check if a completely static Tor is still portable + # '--enable-static-tor', + check_call(['./configure', + '--host=%s' % host, + '--prefix=%s' % prefix_dir, + '--enable-lzma', + '--enable-static-zlib', + '--with-zlib-dir=%s' % prefix_dir, + '--enable-static-libevent', + '--with-libevent-dir=%s' % prefix_dir, + '--enable-static-openssl', + '--with-openssl-dir=%s' % prefix_dir, + ] + TOR_CONFIGURE_FLAGS, cwd=tor_dir, env=env) + check_call(['make', '-j', str(os.cpu_count())], cwd=tor_dir, env=env) check_call(['make', 'install'], cwd=tor_dir, env=env) # copy and zip built Tor binary - tor_path = os.path.join(REPO_DIR, 'tor') + output_dir = get_output_dir(PLATFORM) + tor_path = os.path.join(output_dir, 'tor') copy(os.path.join(prefix_dir, 'bin', 'tor.exe'), tor_path) - check_call(['strip', '-D', 'tor'], cwd=REPO_DIR) + check_call(['strip', '-D', '--strip-unneeded', '--strip-debug', '-R', '.note*', '-R', '.comment', tor_path]) reset_time(tor_path, versions) - print("Sha256 hash of tor before zipping %s: %s" % (name, utils.get_sha256(tor_path))) - check_call(['zip', '-X', '../' + name, 'tor'], cwd=REPO_DIR) + print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path))) + check_call(['zip', '--no-dir-entries', '--junk-paths', '-X', name, 'tor'], cwd=output_dir) + os.remove(tor_path) def package_windows(versions, jar_name): # zip binaries together - file_list = ['tor_windows-x86_64.zip', 'geoip.zip'] + output_dir = get_output_dir(PLATFORM) + file_list = [ + os.path.join(output_dir, 'tor_windows-x86_64.zip'), + ] zip_name = utils.pack(versions, file_list, PLATFORM) - - # create POM file from template pom_name = utils.create_pom_file(versions, PLATFORM) - - # print hashes for debug purposes + print("%s:" % PLATFORM) for file in file_list + [zip_name, jar_name, pom_name]: sha256hash = utils.get_sha256(file) print("%s: %s" % (file, sha256hash)) diff --git a/install-dependencies.sh b/install-dependencies.sh index 2677de3ee13772c5d948f252722418a20b4c5880..9e6fd22cff5edd0ec32ad0dce964da02e40e4d7d 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -10,6 +10,7 @@ apt-get install -y --no-install-recommends \ build-essential \ make \ patch \ + pkg-config \ autopoint \ libtool \ automake \ diff --git a/output/.gitkeep b/output/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/template-linux.pom b/template-linux.pom index 638b1a5f8234010abb7bedc28eab3a9ad961e6b4..bc63d81afe541969aa4f7b20f8070eb5ca40f179 100644 --- a/template-linux.pom +++ b/template-linux.pom @@ -2,8 +2,8 @@ <project> <modelVersion>4.0.0</modelVersion> <groupId>org.briarproject</groupId> - <artifactId>tor</artifactId> - <name>tor</name> + <artifactId>tor-linux</artifactId> + <name>tor-linux</name> <version>VERSION</version> <url>https://torproject.org</url> <description>Repo for building Tor for Linux.</description> diff --git a/template-windows.pom b/template-windows.pom index 725175b869d07ec683a8591559039a6567af6c13..605cbc61fc1476bed1fd5ec24d56046a30bc82ea 100644 --- a/template-windows.pom +++ b/template-windows.pom @@ -2,8 +2,8 @@ <project> <modelVersion>4.0.0</modelVersion> <groupId>org.briarproject</groupId> - <artifactId>tor</artifactId> - <name>tor</name> + <artifactId>tor-windows</artifactId> + <name>tor-windows</name> <version>VERSION</version> <url>https://torproject.org</url> <description>Repo for building Tor for Windows.</description> diff --git a/tor-build/Makefile b/tor-build/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..546f0c1c2f339bb29494fb9d812a1804802c328f --- /dev/null +++ b/tor-build/Makefile @@ -0,0 +1,311 @@ +# Based on Guardian Project's tor-android repo. Copied from: +# https://github.com/guardianproject/tor-android/blob/7e6cffd6adc5b0d62b104029148bc42694761926/external/Makefile +# Copyright (c) 2009-2010, Nathan Freitas, The Guardian Project +# Licensed under 3-clause BSD license + +EXTERNAL_ROOT := $(shell pwd) + +DEBUG ?= 0 + +MAKE ?= make -j`nproc` + +# Android now has 64-bit and 32-bit versions of the NDK for GNU/Linux. We +# assume that the build platform uses the appropriate version, otherwise the +# user building this will have to manually set NDK_PROCESSOR or NDK_TOOLCHAIN. +CPU := $(shell uname -m) +ifeq ($(CPU),x86_64) + NDK_PROCESSOR=x86_64 +else + NDK_PROCESSOR=x86 +endif + +ifeq ($(APP_ABI), ) + $(error APP_ABI must be set to an NDK-supported ABI/arch for this to run!) +endif + +# matching NDK revisions are required for reproducible builds +NDK_REVISION := $(shell sed -n 's,^Pkg.Revision *= *\([^ ]*\),\1,p' $(ANDROID_NDK_HOME)/source.properties) +NDK_REQUIRED_REVISION := 21.4.7075529 +MIN_NDK_VERSION := 19 +ifeq ($(shell test $(MIN_NDK_VERSION) -gt $(firstword $(subst ., ,$(NDK_REVISION))); echo $$?),0) + $(error NDK r$(MIN_NDK_VERSION) or newer required! r$(NDK_REVISION) is installed at $(ANDROID_NDK_HOME).) +endif +ifneq ($(NDK_REQUIRED_REVISION), $(NDK_REVISION)) + $(warning WARNING: NDK $(NDK_REQUIRED_REVISION) required for reproducible builds, $(NDK_REVISION) is installed.) +endif + +ifneq ($(filter arm64-v8a, $(APP_ABI)),) + HOST := aarch64-linux-android + ALTHOST := $(HOST) + GREP_CHECK := aarch64 + NDK_ABI := arm64 + NDK_BIT := 64 + NDK_PLATFORM_LEVEL := 21 + NDK_TOOLCHAIN := $(HOST)-$(NDK_TOOLCHAIN_VERSION) +endif +ifneq ($(filter armeabi-v7a, $(APP_ABI)),) + HOST := armv7a-linux-androideabi + ALTHOST := arm-linux-androideabi + GREP_CHECK := EABI5 + NDK_ABI := arm + NDK_BIT := 32 + NDK_PLATFORM_LEVEL := 16 + NDK_TOOLCHAIN := $(HOST)-$(NDK_TOOLCHAIN_VERSION) +endif +ifneq ($(filter x86, $(APP_ABI)),) + HOST := i686-linux-android + ALTHOST := $(HOST) + GREP_CHECK := 80386 + NDK_ABI := x86 + NDK_BIT := 32 + NDK_PLATFORM_LEVEL := 16 + NDK_TOOLCHAIN := $(NDK_ABI)-$(NDK_TOOLCHAIN_VERSION) +endif +ifneq ($(filter x86_64, $(APP_ABI)),) + HOST := x86_64-linux-android + ALTHOST := $(HOST) + GREP_CHECK := x86-64 + NDK_ABI := x86_64 + NDK_BIT := 64 + NDK_PLATFORM_LEVEL := 21 + NDK_TOOLCHAIN := $(NDK_ABI)-$(NDK_TOOLCHAIN_VERSION) +endif + +NDK_SYSROOT=$(ANDROID_NDK_HOME)/platforms/android-$(NDK_PLATFORM_LEVEL)/arch-$(NDK_ABI) +NDK_UNAME := $(shell uname -s | tr '[A-Z]' '[a-z]') +NDK_TOOLCHAIN_BASE=$(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/$(NDK_UNAME)-$(NDK_PROCESSOR) + +export CC := $(NDK_TOOLCHAIN_BASE)/bin/$(HOST)$(NDK_PLATFORM_LEVEL)-clang + + +export TZ := UTC +export LC_ALL := C.UTF-8 +export SOURCE_DATE_EPOCH := 1234567890 + +REPRODUCIBLE_CFLAGS := \ + -fbuild-session-timestamp=$(SOURCE_DATE_EPOCH) \ + -fdebug-compilation-dir . \ + -no-canonical-prefixes \ + + +ALL_CFLAGS := $(REPRODUCIBLE_CFLAGS) $(CFLAGS) -Os +export CFLAGS := $(ALL_CFLAGS) + + +OUTPUT_FILE := $(EXTERNAL_ROOT)/tor/src/app/tor +INSTALL_DIR := $(EXTERNAL_ROOT)/lib/$(APP_ABI) + +.PHONY = clean showsetup \ + openssl-clean \ + libevent-clean \ + lzma-clean \ + zstd-clean \ + tor tor-clean + +all: test-setup tor + +test-setup: + test -d $(NDK_SYSROOT) + test -x $(CC) + printf 'int main() {return 0;}\n' > .test.c + $(CC) $(CFLAGS) .test.c + rm -f .test.c a.out + + +#------------------------------------------------------------------------------# +# openssl +# +# not all of openssl is needed, only the parts that Tor needs +# https://trac.torproject.org/projects/tor/ticket/32200 +# https://gitweb.torproject.org/tor.git/tree/src/lib/tls/ciphers.inc +# https://wiki.openssl.org/index.php/Compilation_and_Installation + +PATH := $(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/linux-x86_64/bin:$(PATH) + + +openssl/Makefile: openssl/Configure $(wildcard openssl/Configurations/*.*) + cd openssl && PATH=$(PATH) \ + ./Configure \ + no-comp no-dtls no-ec2m no-psk no-srp no-ssl2 no-ssl3 \ + no-camellia no-idea no-md2 no-md4 no-mdc2 no-rc2 no-rc4 no-rc5 no-rmd160 no-whirlpool \ + no-dso no-hw no-ui-console \ + no-shared no-unit-test \ + android-$(NDK_ABI) \ + -D__ANDROID_API__=$(NDK_PLATFORM_LEVEL) \ + --prefix=/ \ + --openssldir=/ + +openssl-build-stamp: openssl/Makefile +ifeq ($(V), 0) + PATH=$(PATH) $(MAKE) --silent -C openssl install_dev DESTDIR=$(EXTERNAL_ROOT) ECHO=':' +else + PATH=$(PATH) $(MAKE) -C openssl install_dev DESTDIR=$(EXTERNAL_ROOT) +endif + touch $@ + +openssl-clean: + -rm -f openssl-build-stamp + -rm -f lib/libcrypto.a + -rm -f lib/libssl.a + -$(MAKE) -C openssl uninstall_dev > /dev/null + -$(MAKE) -C openssl clean + -cd openssl && \ + git clean -fdx > /dev/null + + +#------------------------------------------------------------------------------# +# libevent + +libevent/Makefile: libevent/Makefile.am libevent/configure.ac + cd libevent && ./autogen.sh + cd libevent && ./configure \ + LDFLAGS="-L../lib" \ + CFLAGS="-I../include" \ + --host=$(ALTHOST) \ + --disable-libevent-regress \ + --disable-samples \ + --disable-shared \ + --prefix=/ + +libevent-build-stamp: openssl-build-stamp libevent/Makefile + $(MAKE) -C libevent install DESTDIR=$(EXTERNAL_ROOT) + touch $@ + +libevent-clean: + -rm -f lib/libevent.a + -rm -f libevent-build-stamp + -$(MAKE) -C libevent uninstall DESTDIR=$(EXTERNAL_ROOT) + -$(MAKE) -C libevent clean + -cd libevent && \ + git clean -fdx > /dev/null + + +#------------------------------------------------------------------------------# +# lzma + +xz/Makefile: xz/configure.ac xz/Makefile.am + cd xz && ./autogen.sh + cd xz && ./configure \ + --host=$(HOST) \ + --enable-static \ + --disable-doc \ + --disable-lzma-links \ + --disable-lzmadec \ + --disable-lzmainfo \ + --disable-scripts \ + --disable-shared \ + --disable-xz \ + --disable-xzdec \ + --prefix=/ + +lzma-build-stamp: xz/Makefile + $(MAKE) -C xz install DESTDIR=$(EXTERNAL_ROOT) + touch $@ + +lzma-clean: + -$(MAKE) -C xz uninstall DESTDIR=$(EXTERNAL_ROOT) + -$(MAKE) -C xz clean + -rm -rf include/lzma + -rm -f include/lzma.h + -rm -f lib/liblzma.a + -rm -f lib/liblzma.la + -rm -f lzma-build-stamp + -cd xz && \ + git clean -fdx > /dev/null + + +#------------------------------------------------------------------------------# +# zstd + +zstd-build-stamp: + $(MAKE) -C zstd/lib \ + PREFIX=$(EXTERNAL_ROOT) \ + libzstd.a-mt + $(MAKE) -C zstd/lib \ + PREFIX=$(EXTERNAL_ROOT) \ + libzstd.pc + test -d lib || mkdir lib + test -d lib/pkgconfig || mkdir lib/pkgconfig + test -d include || mkdir include + cp zstd/lib/libzstd.a lib + cp zstd/lib/libzstd.pc lib/pkgconfig + cp zstd/lib/zstd.h include + cp zstd/lib/common/zstd_errors.h include + cp zstd/lib/deprecated/zbuff.h include + cp zstd/lib/dictBuilder/zdict.h include + touch $@ + +zstd-clean: + -rm -f include/zstd.h include/zstd_errors.h include/zbuff.h include/zdict.h + -rm -f lib/libzstd.a + -rm -f lib/pkgconfig/libzstd.pc + -rm -f zstd-build-stamp + -$(MAKE) -C zstd uninstall + -$(MAKE) -C zstd clean + -cd zstd && \ + git clean -fdx > /dev/null + + +#------------------------------------------------------------------------------# +# tor + +tor/Makefile: tor/configure.ac tor/Makefile.am + @which pkg-config || (echo "ERROR: pkg-config is required! apt-get install pkg-config"; exit 1) + cd tor && ./autogen.sh + cd tor && \ + ./configure \ + --host=$(ALTHOST) \ + --enable-lzma \ + --enable-pic \ + --enable-static-libevent --with-libevent-dir=$(EXTERNAL_ROOT) \ + --enable-static-openssl --with-openssl-dir=$(EXTERNAL_ROOT) \ + --enable-zstd \ + --disable-module-dirauth \ + --disable-module-relay \ + --disable-unittests \ + --disable-asciidoc \ + --disable-html-manual \ + --disable-manpage \ + --disable-system-torrc \ + --disable-tool-name-check \ + --disable-systemd \ + --prefix=$(EXTERNAL_ROOT) + +tor-build-stamp: tor/Makefile + $(MAKE) -C tor + $(NDK_TOOLCHAIN_BASE)/bin/$(ALTHOST)-strip -D --strip-unneeded -R .note* -R .comment --strip-debug $(OUTPUT_FILE) + touch $@ + +tor: lzma-build-stamp zstd-build-stamp libevent-build-stamp openssl-build-stamp tor-build-stamp + +tor-clean: + -rm -f $(OUTPUT_FILE) + -rm -f tor-build-stamp + -cd tor && \ + git clean -fdx > /dev/null + + +#------------------------------------------------------------------------------# +# cleanup, cleanup, put the toys away + +clean: openssl-clean libevent-clean lzma-clean zstd-clean tor-clean + + +#------------------------------------------------------------------------------# +# debugging stuff + +showsetup: + @echo "ALL_CFLAGS: $(ALL_CFLAGS)" + @echo "APP_ABI: $(APP_ABI)" + @echo "CC: $$CC" + @echo "CFLAGS: $$CFLAGS" + @echo "HOST: $(HOST)" + @echo "NDK_ABI: $(NDK_ABI)" + @echo "NDK_BIT: $(NDK_BIT)" + @echo "NDK_PLATFORM_LEVEL: $(NDK_PLATFORM_LEVEL)" + @echo "NDK_SYSROOT: $(NDK_SYSROOT)" + @echo "NDK_TOOLCHAIN: $(NDK_TOOLCHAIN)" + @echo "PATH: $$PATH" + @echo "REPRODUCIBLE_CFLAGS: $(REPRODUCIBLE_CFLAGS)" + @echo "SOURCE_DATE_EPOCH: $$SOURCE_DATE_EPOCH" + @echo "TZ: $$TZ" diff --git a/tor-versions.json b/tor-versions.json index a1da4f702d319da1ddf707fe817d382ddd7b3940..8bad0c766ba386d9cf2223eadd45ffaf338654f7 100644 --- a/tor-versions.json +++ b/tor-versions.json @@ -1,234 +1,66 @@ { - "0.3.5.17": { - "tor": "tor-0.3.5.17", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.5", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" + "0.4.5.12": { + "tor": { + "url": "https://git.torproject.org/git/tor.git", + "commit": "tor-0.4.5.12" }, - "timestamp": "201001010000.00" - }, - "0.3.5.15": { - "tor": "tor-0.3.5.15", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" + "libevent": { + "url": "https://github.com/libevent/libevent.git", + "commit": "release-2.1.12-stable" }, - "timestamp": "201001010000.00" - }, - "0.3.5.14": { - "tor": "tor-0.3.5.14", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" + "openssl": { + "url": "https://github.com/openssl/openssl.git", + "commit": "OpenSSL_1_1_1m" }, - "timestamp": "201001010000.00" - }, - "0.3.5.13-1": { - "tor": "tor-0.3.5.13", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" + "xz": { + "url": "https://git.tukaani.org/xz.git", + "commit": "v5.2.5" }, - "timestamp": "201001010000.00" - }, - "0.3.5.13": { - "tor": "tor-0.3.5.13", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" + "zlib": { + "url": "https://github.com/madler/zlib.git", + "commit": "v1.2.11" + }, + "zstd": { + "url": "https://github.com/facebook/zstd.git", + "commit": "v1.4.8" }, - "timestamp": "201001010000.00" - }, - "0.3.5.12": { - "tor": "tor-0.3.5.12", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" + "url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip", + "revision": "21.4.7075529", + "sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e" }, "timestamp": "201001010000.00" }, - "0.3.5.11": { - "tor": "tor-0.3.5.11", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", + "0.4.5.10": { + "tor": { + "url": "https://git.torproject.org/git/tor.git", + "commit": "tor-0.4.5.10" + }, + "libevent": { + "url": "https://github.com/libevent/libevent.git", + "commit": "release-2.1.12-stable" + }, + "openssl": { + "url": "https://github.com/openssl/openssl.git", + "commit": "OpenSSL_1_1_1k" + }, + "xz": { + "url": "https://git.tukaani.org/xz.git", + "commit": "v5.2.5" + }, + "zlib": { + "url": "https://github.com/madler/zlib.git", + "commit": "v1.2.11" + }, + "zstd": { + "url": "https://github.com/facebook/zstd.git", + "commit": "v1.4.8" + }, "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" + "url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip", + "revision": "21.4.7075529", + "sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e" }, "timestamp": "201001010000.00" - }, - "0.3.5.10": { - "tor": "tor-0.3.5.10", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2u", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } - }, - "0.3.5.9": { - "tor": "tor-0.3.5.9", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2q", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } - }, - "0.3.5.8-64": { - "tor": "tor-0.3.5.8", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2q", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "55791c108d4b732b1c12866e4db3b8b1fd826bf1", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } - }, - "0.3.5.8": { - "tor": "tor-0.3.5.8", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2q", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "8eb128aa63b97e139bc845a9426b406245770096", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } - }, - "0.3.5.7": { - "tor": "tor-0.3.5.7", - "libevent": "release-2.1.8-stable", - "openssl": "OpenSSL_1_0_2q", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.8", - "tor-android": "8eb128aa63b97e139bc845a9426b406245770096", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } - }, - "0.3.4.8": { - "tor": "tor-0.3.4.8", - "libevent": "release-2.0.22-stable", - "openssl": "OpenSSL_1_0_2p", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.5", - "tor-android": "fa1ed5505d20aae51cb4cceb51bd9c5164066259", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } - }, - "0.2.9.16": { - "tor": "tor-0.2.9.16", - "libevent": "release-2.0.22-stable", - "openssl": "OpenSSL_1_0_2o", - "xz": "v5.2.4", - "zlib": "v1.2.11", - "zstd": "v1.3.5", - "tor-android": "tor-android-binary-0.3.3.5-rc-3", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } - }, - "0.2.9.15": { - "tor": "tor-0.2.9.15", - "libevent": "release-2.0.22-stable", - "openssl": "OpenSSL_1_0_2o", - "xz": "v5.2.3", - "zlib": "v1.2.11", - "zstd": "v1.3.2", - "tor-android": "tor-android-binary-0.3.3.5-rc-3", - "tor_android_repo_url": "https://github.com/n8fr8/tor-android", - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c" - } } } diff --git a/utils.py b/utils.py index a1f8af6768fdbafc48549b2bfa20f0dcc7ab6775..63315957aa38303eb97d4744899651a64afb8969 100644 --- a/utils.py +++ b/utils.py @@ -5,13 +5,56 @@ import json import os import sys from collections import OrderedDict -from shutil import copy +from shutil import copy, rmtree from subprocess import check_call -REPO_DIR = 'tor-android' -ZLIB_REPO_URL = 'https://github.com/madler/zlib.git' -EXT_DIR = os.path.abspath(os.path.join(REPO_DIR, 'external')) - +BUILD_DIR = 'tor-build' +TOR_CONFIGURE_FLAGS = [ + '--disable-asciidoc', + '--disable-systemd', + '--disable-tool-name-check', + '--disable-module-relay', + '--disable-module-dirauth', + '--disable-unittests', + '--disable-asciidoc', + '--disable-manpage', + '--disable-html-manual', +] +XZ_CONFIGURE_FLAGS = [ + '--enable-static', + '--disable-doc', + '--disable-lzma-links', + '--disable-lzmadec', + '--disable-lzmainfo', + '--disable-scripts', + '--disable-shared', + '--disable-xz', + '--disable-xzdec', +] +OPENSSL_CONFIGURE_FLAGS = [ + 'no-unit-test', + 'no-asm', + 'no-comp', + 'no-dtls', + 'no-err', + 'no-psk', + 'no-srp', + 'no-weak-ssl-ciphers', + 'no-camellia', + 'no-idea', + 'no-md2', + 'no-md4', + 'no-rc2', + 'no-rc4', + 'no-rc5', + 'no-rmd160', + 'no-whirlpool', + 'no-ui-console', +] +REPRODUCIBLE_GCC_CFLAGS = '-fno-guess-branch-probability -frandom-seed="0"' + +def get_output_dir(platform): + return os.path.abspath(os.path.join('output', platform)) def setup(platform): # get Tor version from command or show usage information @@ -19,10 +62,16 @@ def setup(platform): # get Tor version and versions of its dependencies versions = get_build_versions(version) - print("Building Tor %s" % versions['tor']) + print("Building Tor %s" % versions['tor']['commit']) + + # remove output from previous build + output_dir = get_output_dir(platform) + if os.path.isdir(output_dir): + rmtree(output_dir) + os.makedirs(output_dir) - # clone and checkout tor-android repo based on tor-versions.json - prepare_tor_android_repo(versions) + # clone and checkout repos based on tor-versions.json + prepare_repos(versions) # create sources jar before building jar_name = create_sources_jar(versions, platform) @@ -30,48 +79,37 @@ def setup(platform): return versions, jar_name -def package_geoip(versions): - # zip geoip database - geoip_path = os.path.join(REPO_DIR, 'geoip') - copy(os.path.join(EXT_DIR, 'tor', 'src', 'config', 'geoip'), geoip_path) - reset_time(geoip_path, versions) - check_call(['zip', '-X', '../geoip.zip', 'geoip'], cwd=REPO_DIR) +def prepare_repos(versions): + prepare_repo(os.path.join(BUILD_DIR, "tor"), versions['tor']['url'], versions['tor']['commit']) + prepare_repo(os.path.join(BUILD_DIR, "libevent"), versions['libevent']['url'], versions['libevent']['commit']) + prepare_repo(os.path.join(BUILD_DIR, "openssl"), versions['openssl']['url'], versions['openssl']['commit']) + prepare_repo(os.path.join(BUILD_DIR, "xz"), versions['xz']['url'], versions['xz']['commit']) + prepare_repo(os.path.join(BUILD_DIR, "zlib"), versions['zlib']['url'], versions['zlib']['commit']) + prepare_repo(os.path.join(BUILD_DIR, "zstd"), versions['zstd']['url'], versions['zstd']['commit']) -def prepare_tor_android_repo(versions): - if os.path.isdir(REPO_DIR): +def prepare_repo(path, url, version): + if os.path.isdir(path): # get latest commits and tags from remote - check_call(['git', 'fetch', '--recurse-submodules=yes', 'origin'], cwd=REPO_DIR) + check_call(['git', 'fetch', '--recurse-submodules=yes', 'origin'], cwd=path) else: # clone repo - url = versions['tor_android_repo_url'] - check_call(['git', 'clone', url, REPO_DIR]) + check_call(['git', 'clone', url, path]) - # checkout tor-android version - check_call(['git', 'checkout', '-f', versions['tor-android']], cwd=REPO_DIR) + # checkout given version + check_call(['git', 'checkout', '-f', version], cwd=path) # initialize and/or update submodules # (after checkout, because submodules can point to non-existent commits on master) - check_call(['git', 'submodule', 'update', '--init', '--recursive', '-f'], cwd=REPO_DIR) + check_call(['git', 'submodule', 'update', '--init', '--recursive', '-f'], cwd=path) # undo all changes - check_call(['git', 'reset', '--hard'], cwd=REPO_DIR) - check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=REPO_DIR) + check_call(['git', 'reset', '--hard'], cwd=path) + check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=path) # clean all untracked files and directories (-d) from repo - check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR) - check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR) - - # add zlib - check_call(['git', 'clone', ZLIB_REPO_URL], cwd=EXT_DIR) - - # check out versions of external dependencies - checkout('tor', versions['tor'], 'tor') - checkout('libevent', versions['libevent'], 'libevent') - checkout('openssl', versions['openssl'], 'openssl') - checkout('xz', versions['xz'], 'xz') - checkout('zlib', versions['zlib'], 'zlib') - checkout('zstd', versions['zstd'], 'zstd') + check_call(['git', 'clean', '-dffx'], cwd=path) + check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=path) def get_version(): @@ -115,25 +153,18 @@ def get_file_suffix(versions, platform): def get_final_file_name(versions, platform): - version = get_version_tag(versions) - if version < '0.3.5.14': - return 'tor-%s.zip' % get_file_suffix(versions, platform) - else: - return 'tor-%s.jar' % get_file_suffix(versions, platform) + file = 'tor-%s.jar' % get_file_suffix(versions, platform) + return os.path.join(get_output_dir(platform), file) def get_sources_file_name(versions, platform): - return 'tor-%s-sources.jar' % get_file_suffix(versions, platform) + file = 'tor-%s-sources.jar' % get_file_suffix(versions, platform) + return os.path.join(get_output_dir(platform), file) def get_pom_file_name(versions, platform): - return 'tor-%s.pom' % get_file_suffix(versions, platform) - - -def checkout(name, tag, path): - print("Checking out %s: %s" % (name, tag)) - repo_path = os.path.join(EXT_DIR, path) - check_call(['git', 'checkout', '-f', tag], cwd=repo_path) + file = 'tor-%s.pom' % get_file_suffix(versions, platform) + return os.path.join(get_output_dir(platform), file) def pack(versions, file_list, platform): @@ -141,21 +172,18 @@ def pack(versions, file_list, platform): for filename in file_list: reset_time(filename, versions) zip_name = get_final_file_name(versions, platform) - check_call(['zip', '-D', '-X', zip_name] + file_list) + check_call(['zip', '--no-dir-entries', '--junk-paths', '-X', zip_name] + file_list) return zip_name def reset_time(filename, versions): - if 'timestamp' in versions: - timestamp = versions['timestamp'] - else: - timestamp = '197001010000.00' - check_call(['touch', '--no-dereference', '-t', timestamp, filename]) + check_call(['touch', '--no-dereference', '-t', versions['timestamp'], filename]) def create_sources_jar(versions, platform): + output_dir = get_output_dir(platform) jar_files = [] - for root, dir_names, filenames in os.walk(EXT_DIR): + for root, dir_names, filenames in os.walk(BUILD_DIR): for f in filenames: if '/.git' in root: continue @@ -164,8 +192,13 @@ def create_sources_jar(versions, platform): reset_time(file, versions) jar_name = get_sources_file_name(versions, platform) jar_path = os.path.abspath(jar_name) - rel_paths = [os.path.relpath(f, EXT_DIR) for f in sorted(jar_files)] - check_call(['jar', 'cf', jar_path] + rel_paths, cwd=EXT_DIR) + rel_paths = [os.path.relpath(f, BUILD_DIR) for f in sorted(jar_files)] + # create jar archive with first files + jar_step = 5000 + check_call(['jar', 'cf', jar_path] + rel_paths[0:jar_step], cwd=BUILD_DIR) + # add subsequent files in steps, because the command line can't handle all at once + for i in range(jar_step, len(rel_paths), jar_step): + check_call(['jar', 'uf', jar_path] + rel_paths[i:i + jar_step], cwd=BUILD_DIR) return jar_name diff --git a/verify_tor_utils.py b/verify_tor_utils.py index ff527f61953e7dda9bdbb2ec615ee028ccf1d17a..a0d67f93bae97a5f9b55429267eeb801a7b93b27 100755 --- a/verify_tor_utils.py +++ b/verify_tor_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os import sys -from subprocess import check_call, CalledProcessError +from subprocess import check_call from utils import get_sha256, get_build_versions, get_final_file_name, get_version_tag, get_version @@ -26,16 +26,8 @@ def verify(version, platform): os.makedirs(REF_DIR, exist_ok=True) file_name = get_final_file_name(versions, platform) ref_file = os.path.join(REF_DIR, file_name) - try: - # try downloading from maven central (or jcenter for older versions) - check_call(['wget', '--no-verbose', get_url(versions, platform), '-O', ref_file]) - except CalledProcessError as e: - # try fallback to bintray (for older versions only) - if version is None or not version < '0.3.5.14': - raise e - else: - print("Warning: Download from jcenter failed. Trying bintray directly...") - check_call(['wget', '--no-verbose', get_url(versions, platform, fallback=True), '-O', ref_file]) + # try downloading from maven central + check_call(['wget', '--no-verbose', get_url(versions, platform), '-O', ref_file]) # check if Tor was already build if not os.path.isfile(file_name): @@ -67,14 +59,8 @@ def verify(version, platform): return False -def get_url(versions, platform, fallback=False): +def get_url(versions, platform): version = get_version_tag(versions) directory = "tor-%s" % platform - file = get_final_file_name(versions, platform) - if version >= '0.3.5.14': - return "https://repo.maven.apache.org/maven2/org/briarproject/%s/%s/%s" % (directory, version, file) - elif not fallback: - return "https://jcenter.bintray.com/org/briarproject/%s/%s/%s" % (directory, version, file) - else: - return "https://dl.bintray.com/briarproject/org.briarproject/org/briarproject/%s/%s/%s" % \ - (directory, version, file) + file = os.path.basename(get_final_file_name(versions, platform)) + return "https://repo.maven.apache.org/maven2/org/briarproject/%s/%s/%s" % (directory, version, file)