diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1f5c3fd174645098a3639649328f5d77e9ab922c..a3436e498c20f41ae4be3280598a35458a799a3a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,13 +24,15 @@ build: test: stage: test script: - - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./build-obfs4proxy.py 0.0.7 && ./verify-obfs4proxy.py 0.0.7" + - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./build-obfs4proxy.py 0.0.9 && ./verify-obfs4proxy.py 0.0.9" + allow_failure: true artifacts: paths: - obfs4proxy-*.zip - obfs4proxy-*.pom - obfs4proxy-*-sources.jar expire_in: 1 week + when: always except: - tags @@ -44,6 +46,7 @@ test_tag: - obfs4proxy-*.pom - obfs4proxy-*-sources.jar expire_in: 1 week + when: always only: - tags @@ -53,5 +56,6 @@ release: - docker pull $TEST_IMAGE - docker tag $TEST_IMAGE $RELEASE_IMAGE - docker push $RELEASE_IMAGE + when: on_success only: - master diff --git a/build-obfs4proxy.py b/build-obfs4proxy.py index 37046434fb3a9787b2cfcd62de2de1f91089fe19..094e21d336e5ae1e5435d3441a30234b10f60e6d 100755 --- a/build-obfs4proxy.py +++ b/build-obfs4proxy.py @@ -7,8 +7,9 @@ from utils import get_build_versions, ex, get_sha256, fail, zip_files, get_final get_sources_file_name, get_pom_file_name, reset_time, get_obfs4_version, check_go_version, \ get_version, GO_PATH, GO_ROOT, NDK_DIR -REPO_DIR = 'obfs4' -GO_FLAGS = ['-asmflags', '-trimpath', '-o', 'obfs4proxy'] +REPO_DIR = '/tmp/obfs4' # This needs to be always the same path, otherwise breaks reproducibility +OUTPUT_FILE = os.path.abspath(os.path.join(os.path.curdir, 'obfs4proxy')) +GO_FLAGS = ['-asmflags', '-trimpath', '-o', OUTPUT_FILE] def main(): @@ -25,8 +26,8 @@ def main(): # Install Android NDK install_android_ndk(tool_version) - # Fetch and checkout dependencies at specific version - checkout_source_repos(versions) + # Checkout source at specific version + checkout_source_repo(versions) # Build for various Android versions and create package build_android(versions) @@ -45,6 +46,7 @@ def install_go(tool_version, versions): go_bin_path = os.path.join(GO_ROOT, 'bin') os.environ['GOPATH'] = GO_PATH os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH') + os.environ['GO111MODULE'] = 'on' check_go_version(versions) @@ -53,36 +55,23 @@ def install_android_ndk(tool_version): os.environ['ANDROID_NDK_HOME'] = os.path.abspath(NDK_DIR) -def checkout_source_repos(versions): - # download dependencies - ex(['go', 'get', '-d', versions['obfs4']['go-get']]) +def checkout_source_repo(versions): + if os.path.isdir(REPO_DIR): + # get latest commits and tags from remote + check_call(['git', 'fetch', 'origin'], cwd=REPO_DIR) + else: + # clone repo + check_call(['git', 'clone', versions['repo_url'], REPO_DIR]) - # find dependencies - go_src = os.path.join(GO_PATH, 'src') - dependencies = [] - for root, dirs, files in os.walk(go_src): - if root.endswith("/.git"): - dependencies.append(os.path.split(root)[0]) + # checkout version + print("Checking out %s" % versions['revision']) + check_call(['git', 'checkout', '-f', versions['revision']], cwd=REPO_DIR) - # checkout specific versions - for dependency in dependencies: - # get version (commit or tag) - key = os.path.split(dependency)[1] - try: - version = versions['obfs4']['versions'][key] - except KeyError as e: - fail("Version for dependency '%s' missing." % key) - raise e + # undo all changes + check_call(['git', 'reset', '--hard'], cwd=REPO_DIR) - # checkout commit or tag - print("Checking out %s in %s" % (version, dependency)) - check_call(['git', 'checkout', '-f', version], cwd=dependency) - - # undo all changes - check_call(['git', 'reset', '--hard'], cwd=dependency) - - # clean all untracked files and directories (-d) from repo - check_call(['git', 'clean', '-dffx'], cwd=dependency) + # clean all untracked files and directories (-d) from repo + check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR) def build_android(versions): @@ -119,9 +108,8 @@ def build_android_arch(versions, env, tool, ndk_arch, pie): build_mode = "pie" if pie else "exe" extldflags = " -extldflags=-pie" if pie else "" - ex(['go', 'clean', versions['obfs4']['go-get']]) - ex(['go', 'build', '-buildmode=%s' % build_mode, '-ldflags', - '-w -s' + extldflags] + GO_FLAGS + [versions['obfs4']['go-get']], env=env) + ex(['go', 'build', '-buildmode=%s' % build_mode, '-ldflags', '-w -s' + extldflags] + GO_FLAGS + + [os.path.join('.', versions['build_path'])], env=env, cwd=REPO_DIR) pie_suffix = '_pie' if pie else '' zip_files(['obfs4proxy'], 'obfs4proxy_%s%s.zip' % (ndk_arch, pie_suffix)) @@ -129,8 +117,10 @@ def build_android_arch(versions, env, tool, ndk_arch, pie): def build_linux(versions): - ex(['go', 'clean', versions['obfs4']['go-get']]) - ex(['go', 'build', '-ldflags', '-w -s'] + GO_FLAGS + [versions['obfs4']['go-get']]) + env = os.environ.copy() + env['CGO_ENABLED'] = "0" + build_path = os.path.join('.', versions['build_path']) + ex(['go', 'build', '-ldflags', '-w -s'] + GO_FLAGS + [build_path], env=env, cwd=REPO_DIR) zip_files(['obfs4proxy'], 'obfs4proxy_linux-x86_64.zip') os.remove('obfs4proxy') @@ -160,14 +150,17 @@ def package(versions, file_list, android): def create_sources_jar(versions): + # clean all untracked files and directories (-d) from repo + check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR) + # vendorize dependencies + ex(['go', 'mod', 'vendor'], cwd=REPO_DIR) jar_files = [] - go_src = os.path.join(GO_PATH, 'src') - for file in glob(os.path.join(go_src, '*')): + for file in glob(os.path.join(REPO_DIR, '*')): reset_time(file) - jar_files.append(os.path.relpath(file, go_src)) + jar_files.append(os.path.relpath(file, REPO_DIR)) jar_file = get_sources_file_name(versions) jar_path = os.path.abspath(jar_file) - check_call(['jar', 'cf', jar_path] + jar_files, cwd=go_src) + check_call(['jar', 'cf', jar_path] + jar_files, cwd=REPO_DIR) return jar_file diff --git a/install.sh b/install.sh index f2e5ac87050ce7fbc28de424fbefd8704f2768b0..a81614d677898d632d8c72c4de02c98f567a871c 100755 --- a/install.sh +++ b/install.sh @@ -3,7 +3,7 @@ set -e set -x # use snapshot repos for deterministic package versions -DATE="20181017T120000Z" +DATE="20190206T120000Z" cat << EOF > /etc/apt/sources.list deb http://snapshot.debian.org/archive/debian/${DATE}/ stretch main deb http://snapshot.debian.org/archive/debian-security/${DATE}/ stretch/updates main diff --git a/utils.py b/utils.py index 33550a8c9bba32ffc1e7daf5196675c721f730c7..46bbff402dfef8b38f1a0d3a33e043ff54d4258e 100644 --- a/utils.py +++ b/utils.py @@ -43,9 +43,9 @@ def check_go_version(versions): fail("You need Go version %s to reproduce this binary" % versions['go']['version']) -def ex(args, env=None): +def ex(args, env=None, cwd=None): print("+ %s" % " ".join(args)) - check_call(args, env=env) + check_call(args, env=env, cwd=cwd) def fail(msg=""): @@ -68,11 +68,15 @@ def reset_time(file_name): def zip_files(files, zip_name): for file_name in files: reset_time(file_name) + # use deterministic permissions to prevent differences in zip files + os.chmod(file_name, 0o755) + sha256hash = get_sha256(file_name) + print("Hash before zipping %s: %s" % (file_name, sha256hash)) ex(['zip', '-D', '-X', zip_name] + files) def get_obfs4_version(versions): - version = versions['obfs4']['versions']['obfs4.git'] + version = versions['revision'] if '-' in version: return version.split('-')[1] return version diff --git a/versions.json b/versions.json index 38d081d7bce6eeb3686c531ac58c7074808e0006..6f89db4e832af625deb67fb712650829d40c5992 100644 --- a/versions.json +++ b/versions.json @@ -1,20 +1,12 @@ { "obfs4proxy": { - "0.0.7": { - "obfs4": { - "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy", - "versions": { - "obfs4.git": "obfs4proxy-0.0.7", - "goptlib.git": "7d56ec4f381e", - "crypto": "0c41d7ab0a0e", - "net": "49bb7cea24b1", - "siphash": "v1.2.0", - "ed25519": "5312a6153412" - } - }, + "0.0.9": { + "repo_url": "https://git.torproject.org/pluggable-transports/obfs4.git", + "revision": "obfs4proxy-0.0.9", + "build_path": "obfs4proxy", "go": { - "version": "go1.11.1", - "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117" + "version": "go1.11.5", + "sha256": "bc1ef02bb1668835db1390a2e478dcbccb5dd16911691af9d75184bbe5aa943e" }, "ndk": { "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip",