From 74958ca24d5bcfc312b32cce2452968a37a6a3c8 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Wed, 17 Oct 2018 13:07:12 -0300 Subject: [PATCH] Factor out Android NDK install into own file Also change formation of versions.json and use fixed Go paths --- .gitignore | 1 - build-obfs4proxy.py | 59 +++++++++++------------------------------- install-android-ndk.py | 53 +++++++++++++++++++++++++++++++++++++ install-go.py | 9 +++++-- utils.py | 21 +++++++++++---- versions.json | 40 ++++++++++++++-------------- 6 files changed, 112 insertions(+), 71 deletions(-) create mode 100755 install-android-ndk.py diff --git a/.gitignore b/.gitignore index 454451c..08392d5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ /jni /android-ndk /android-ndk.zip -/go /go.tar.gz /obfs4proxy*.zip /obfs4proxy*.pom diff --git a/build-obfs4proxy.py b/build-obfs4proxy.py index 748add2..3704643 100755 --- a/build-obfs4proxy.py +++ b/build-obfs4proxy.py @@ -1,27 +1,29 @@ #!/usr/bin/env python3 import os from glob import glob -from shutil import move, rmtree from subprocess import check_call from utils import get_build_versions, ex, get_sha256, fail, zip_files, get_final_file_name, \ get_sources_file_name, get_pom_file_name, reset_time, get_obfs4_version, check_go_version, \ - GO_PATH, GO_ROOT + get_version, GO_PATH, GO_ROOT, NDK_DIR -NDK_DIR = 'android-ndk' REPO_DIR = 'obfs4' GO_FLAGS = ['-asmflags', '-trimpath', '-o', 'obfs4proxy'] def main(): + # get version from command line or show usage information + command_line_version = get_version() + # Get the latest versions for building - versions = get_build_versions(None) + tool_version, versions = get_build_versions('obfs4proxy', command_line_version) + print("Building obfs4proxy %s" % tool_version) # Install Go - install_go(versions) + install_go(tool_version, versions) - # Setup Android NDK - setup_android_ndk(versions) + # Install Android NDK + install_android_ndk(tool_version) # Fetch and checkout dependencies at specific version checkout_source_repos(versions) @@ -38,48 +40,17 @@ def main(): package_linux(versions) -def install_go(versions): - ex(['./install-go.py']) +def install_go(tool_version, versions): + ex(['./install-go.py', 'obfs4proxy', tool_version]) go_bin_path = os.path.join(GO_ROOT, 'bin') + os.environ['GOPATH'] = GO_PATH os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH') check_go_version(versions) -def setup_android_ndk(versions): - if os.path.isdir(NDK_DIR): - # check that we are using the correct NDK - from configparser import ConfigParser - config = ConfigParser() - with open(os.path.join(NDK_DIR, 'source.properties'), 'rt') as f: - config.read_string('[default]\n' + str(f.read())) - revision = config.get('default', 'Pkg.Revision') - - if revision != versions['ndk']['revision']: - print("Existing Android NDK has unexpected revision (%s). Deleting..." % revision) - rmtree(NDK_DIR) - - if not os.path.isdir(NDK_DIR): - # download Android NDK - print("Downloading Android NDK...") - check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip']) - - # check sha256 hash on downloaded file - is_hash = get_sha256('android-ndk.zip') - if is_hash != versions['ndk']['sha256']: - fail("Android NDK checksum does not match, is %s" % is_hash) - - # install the NDK - print("Unpacking Android NDK...") - ndk_dir_tmp = NDK_DIR + '-tmp' - check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp]) - content = os.listdir(ndk_dir_tmp) - if len(content) == 1 and content[0].startswith('android-ndk-r'): - move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR) - os.rmdir(ndk_dir_tmp) - else: - fail("Could not extract NDK: %s" % str(content)) - - os.putenv('ANDROID_NDK_HOME', os.path.abspath(NDK_DIR)) +def install_android_ndk(tool_version): + ex(['./install-android-ndk.py', 'obfs4proxy', tool_version]) + os.environ['ANDROID_NDK_HOME'] = os.path.abspath(NDK_DIR) def checkout_source_repos(versions): diff --git a/install-android-ndk.py b/install-android-ndk.py new file mode 100755 index 0000000..8f15398 --- /dev/null +++ b/install-android-ndk.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import os +from shutil import move, rmtree +from subprocess import check_call + +from utils import get_build_versions, get_sha256, fail, get_version_and_tool, NDK_DIR + + +def main(): + # get tool name and version from command line parameters + tool, command_line_version = get_version_and_tool() + + # Get the defined Android NDK version + tool_version, versions = get_build_versions(tool, command_line_version) + ndk = versions['ndk'] + print("Installing Android NDK %s for %s %s" % (ndk['revision'], tool, tool_version)) + + if os.path.isdir(NDK_DIR): + # check that we are using the correct NDK + from configparser import ConfigParser + config = ConfigParser() + with open(os.path.join(NDK_DIR, 'source.properties'), 'rt') as f: + config.read_string('[default]\n' + str(f.read())) + revision = config.get('default', 'Pkg.Revision') + + if revision != ndk['revision']: + print("Existing Android NDK has unexpected revision (%s). Deleting..." % revision) + rmtree(NDK_DIR) + + if not os.path.isdir(NDK_DIR): + # download Android NDK + print("Downloading Android NDK...") + check_call(['wget', '-c', '--no-verbose', ndk['url'], '-O', 'android-ndk.zip']) + + # check sha256 hash on downloaded file + is_hash = get_sha256('android-ndk.zip') + if is_hash != ndk['sha256']: + fail("Android NDK checksum does not match, is %s" % is_hash) + + # install the NDK + print("Unpacking Android NDK...") + ndk_dir_tmp = NDK_DIR + '-tmp' + check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp]) + content = os.listdir(ndk_dir_tmp) + if len(content) == 1 and content[0].startswith('android-ndk-r'): + move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR) + os.rmdir(ndk_dir_tmp) + else: + fail("Could not extract NDK: %s" % str(content)) + + +if __name__ == "__main__": + main() diff --git a/install-go.py b/install-go.py index ddae680..2d34b9a 100755 --- a/install-go.py +++ b/install-go.py @@ -2,14 +2,19 @@ import os from subprocess import check_call -from utils import GO_ROOT, get_build_versions, ex, get_sha256, fail, check_go_version +from utils import GO_ROOT, get_build_versions, ex, get_sha256, fail, check_go_version, \ + get_version_and_tool URL = 'https://golang.org/dl/%s.src.tar.gz' def main(): + # get tool name and version from command line parameters + tool, command_line_version = get_version_and_tool() + # Get the defined Go version - versions = get_build_versions(None) + tool_version, versions = get_build_versions(tool, command_line_version) + print("Installing %s for %s %s" % (versions['go']['version'], tool, tool_version)) if not os.path.isfile(os.path.join(GO_ROOT, 'bin', 'go')): # download Go source archive diff --git a/utils.py b/utils.py index df2788a..33550a8 100644 --- a/utils.py +++ b/utils.py @@ -5,8 +5,11 @@ import sys from collections import OrderedDict from subprocess import check_call, check_output -GO_ROOT = os.path.abspath('go') -GO_PATH = os.environ.get("GOPATH", os.path.expanduser('~/go')) +# WARNING: changing those affects reproducibility +GO_ROOT = '/tmp/go' +GO_PATH = '/tmp/go-path' + +NDK_DIR = 'android-ndk' def get_version(): @@ -15,15 +18,23 @@ def get_version(): return sys.argv[1] if len(sys.argv) > 1 else None -def get_build_versions(tag): +def get_version_and_tool(): + if len(sys.argv) < 2 or len(sys.argv) > 3: + fail("Usage: %s tool [version tag]" % sys.argv[0]) + tool = sys.argv[1] + command_line_version = sys.argv[2] if len(sys.argv) > 2 else None + return tool, command_line_version + + +def get_build_versions(tool, tag): # load versions and their dependencies with open('versions.json', 'r') as f: versions = json.load(f, object_pairs_hook=OrderedDict) if tag is None: # take top-most version - tag = next(iter(versions)) - return versions[tag] + tag = next(iter(versions[tool])) + return tag, versions[tool][tag] def check_go_version(versions): diff --git a/versions.json b/versions.json index 3b794ac..38d081d 100644 --- a/versions.json +++ b/versions.json @@ -1,24 +1,26 @@ { - "0.0.7": { - "obfs4": { - "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy", - "versions": { - "obfs4.git": "obfs4proxy-0.0.7", - "goptlib.git": "7d56ec4f381e", - "crypto": "0c41d7ab0a0e", - "net": "49bb7cea24b1", - "siphash": "v1.2.0", - "ed25519": "5312a6153412" + "obfs4proxy": { + "0.0.7": { + "obfs4": { + "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy", + "versions": { + "obfs4.git": "obfs4proxy-0.0.7", + "goptlib.git": "7d56ec4f381e", + "crypto": "0c41d7ab0a0e", + "net": "49bb7cea24b1", + "siphash": "v1.2.0", + "ed25519": "5312a6153412" + } + }, + "go": { + "version": "go1.11.1", + "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117" + }, + "ndk": { + "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip", + "revision": "18.0.5002713", + "sha256": "c413dd014edc37f822d0dc88fabc05b64232d07d5c6e9345224e47073fdf140b" } - }, - "go": { - "version": "go1.11.1", - "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117" - }, - "ndk": { - "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip", - "revision": "18.0.5002713", - "sha256": "c413dd014edc37f822d0dc88fabc05b64232d07d5c6e9345224e47073fdf140b" } } } -- 2.22.2