From 74958ca24d5bcfc312b32cce2452968a37a6a3c8 Mon Sep 17 00:00:00 2001
From: Torsten Grote <t@grobox.de>
Date: Wed, 17 Oct 2018 13:07:12 -0300
Subject: [PATCH] Factor out Android NDK install into own file
Also change formation of versions.json and use fixed Go paths
---
.gitignore | 1 -
build-obfs4proxy.py | 59 +++++++++++-------------------------------
install-android-ndk.py | 53 +++++++++++++++++++++++++++++++++++++
install-go.py | 9 +++++--
utils.py | 21 +++++++++++----
versions.json | 40 ++++++++++++++--------------
6 files changed, 112 insertions(+), 71 deletions(-)
create mode 100755 install-android-ndk.py
diff --git a/.gitignore b/.gitignore
index 454451c..08392d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,6 @@
/jni
/android-ndk
/android-ndk.zip
-/go
/go.tar.gz
/obfs4proxy*.zip
/obfs4proxy*.pom
diff --git a/build-obfs4proxy.py b/build-obfs4proxy.py
index 748add2..3704643 100755
--- a/build-obfs4proxy.py
+++ b/build-obfs4proxy.py
@@ -1,27 +1,29 @@
#!/usr/bin/env python3
import os
from glob import glob
-from shutil import move, rmtree
from subprocess import check_call
from utils import get_build_versions, ex, get_sha256, fail, zip_files, get_final_file_name, \
get_sources_file_name, get_pom_file_name, reset_time, get_obfs4_version, check_go_version, \
- GO_PATH, GO_ROOT
+ get_version, GO_PATH, GO_ROOT, NDK_DIR
-NDK_DIR = 'android-ndk'
REPO_DIR = 'obfs4'
GO_FLAGS = ['-asmflags', '-trimpath', '-o', 'obfs4proxy']
def main():
+ # get version from command line or show usage information
+ command_line_version = get_version()
+
# Get the latest versions for building
- versions = get_build_versions(None)
+ tool_version, versions = get_build_versions('obfs4proxy', command_line_version)
+ print("Building obfs4proxy %s" % tool_version)
# Install Go
- install_go(versions)
+ install_go(tool_version, versions)
- # Setup Android NDK
- setup_android_ndk(versions)
+ # Install Android NDK
+ install_android_ndk(tool_version)
# Fetch and checkout dependencies at specific version
checkout_source_repos(versions)
@@ -38,48 +40,17 @@ def main():
package_linux(versions)
-def install_go(versions):
- ex(['./install-go.py'])
+def install_go(tool_version, versions):
+ ex(['./install-go.py', 'obfs4proxy', tool_version])
go_bin_path = os.path.join(GO_ROOT, 'bin')
+ os.environ['GOPATH'] = GO_PATH
os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH')
check_go_version(versions)
-def setup_android_ndk(versions):
- if os.path.isdir(NDK_DIR):
- # check that we are using the correct NDK
- from configparser import ConfigParser
- config = ConfigParser()
- with open(os.path.join(NDK_DIR, 'source.properties'), 'rt') as f:
- config.read_string('[default]\n' + str(f.read()))
- revision = config.get('default', 'Pkg.Revision')
-
- if revision != versions['ndk']['revision']:
- print("Existing Android NDK has unexpected revision (%s). Deleting..." % revision)
- rmtree(NDK_DIR)
-
- if not os.path.isdir(NDK_DIR):
- # download Android NDK
- print("Downloading Android NDK...")
- check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip'])
-
- # check sha256 hash on downloaded file
- is_hash = get_sha256('android-ndk.zip')
- if is_hash != versions['ndk']['sha256']:
- fail("Android NDK checksum does not match, is %s" % is_hash)
-
- # install the NDK
- print("Unpacking Android NDK...")
- ndk_dir_tmp = NDK_DIR + '-tmp'
- check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
- content = os.listdir(ndk_dir_tmp)
- if len(content) == 1 and content[0].startswith('android-ndk-r'):
- move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
- os.rmdir(ndk_dir_tmp)
- else:
- fail("Could not extract NDK: %s" % str(content))
-
- os.putenv('ANDROID_NDK_HOME', os.path.abspath(NDK_DIR))
+def install_android_ndk(tool_version):
+ ex(['./install-android-ndk.py', 'obfs4proxy', tool_version])
+ os.environ['ANDROID_NDK_HOME'] = os.path.abspath(NDK_DIR)
def checkout_source_repos(versions):
diff --git a/install-android-ndk.py b/install-android-ndk.py
new file mode 100755
index 0000000..8f15398
--- /dev/null
+++ b/install-android-ndk.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import os
+from shutil import move, rmtree
+from subprocess import check_call
+
+from utils import get_build_versions, get_sha256, fail, get_version_and_tool, NDK_DIR
+
+
+def main():
+ # get tool name and version from command line parameters
+ tool, command_line_version = get_version_and_tool()
+
+ # Get the defined Android NDK version
+ tool_version, versions = get_build_versions(tool, command_line_version)
+ ndk = versions['ndk']
+ print("Installing Android NDK %s for %s %s" % (ndk['revision'], tool, tool_version))
+
+ if os.path.isdir(NDK_DIR):
+ # check that we are using the correct NDK
+ from configparser import ConfigParser
+ config = ConfigParser()
+ with open(os.path.join(NDK_DIR, 'source.properties'), 'rt') as f:
+ config.read_string('[default]\n' + str(f.read()))
+ revision = config.get('default', 'Pkg.Revision')
+
+ if revision != ndk['revision']:
+ print("Existing Android NDK has unexpected revision (%s). Deleting..." % revision)
+ rmtree(NDK_DIR)
+
+ if not os.path.isdir(NDK_DIR):
+ # download Android NDK
+ print("Downloading Android NDK...")
+ check_call(['wget', '-c', '--no-verbose', ndk['url'], '-O', 'android-ndk.zip'])
+
+ # check sha256 hash on downloaded file
+ is_hash = get_sha256('android-ndk.zip')
+ if is_hash != ndk['sha256']:
+ fail("Android NDK checksum does not match, is %s" % is_hash)
+
+ # install the NDK
+ print("Unpacking Android NDK...")
+ ndk_dir_tmp = NDK_DIR + '-tmp'
+ check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
+ content = os.listdir(ndk_dir_tmp)
+ if len(content) == 1 and content[0].startswith('android-ndk-r'):
+ move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
+ os.rmdir(ndk_dir_tmp)
+ else:
+ fail("Could not extract NDK: %s" % str(content))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/install-go.py b/install-go.py
index ddae680..2d34b9a 100755
--- a/install-go.py
+++ b/install-go.py
@@ -2,14 +2,19 @@
import os
from subprocess import check_call
-from utils import GO_ROOT, get_build_versions, ex, get_sha256, fail, check_go_version
+from utils import GO_ROOT, get_build_versions, ex, get_sha256, fail, check_go_version, \
+ get_version_and_tool
URL = 'https://golang.org/dl/%s.src.tar.gz'
def main():
+ # get tool name and version from command line parameters
+ tool, command_line_version = get_version_and_tool()
+
# Get the defined Go version
- versions = get_build_versions(None)
+ tool_version, versions = get_build_versions(tool, command_line_version)
+ print("Installing %s for %s %s" % (versions['go']['version'], tool, tool_version))
if not os.path.isfile(os.path.join(GO_ROOT, 'bin', 'go')):
# download Go source archive
diff --git a/utils.py b/utils.py
index df2788a..33550a8 100644
--- a/utils.py
+++ b/utils.py
@@ -5,8 +5,11 @@ import sys
from collections import OrderedDict
from subprocess import check_call, check_output
-GO_ROOT = os.path.abspath('go')
-GO_PATH = os.environ.get("GOPATH", os.path.expanduser('~/go'))
+# WARNING: changing those affects reproducibility
+GO_ROOT = '/tmp/go'
+GO_PATH = '/tmp/go-path'
+
+NDK_DIR = 'android-ndk'
def get_version():
@@ -15,15 +18,23 @@ def get_version():
return sys.argv[1] if len(sys.argv) > 1 else None
-def get_build_versions(tag):
+def get_version_and_tool():
+ if len(sys.argv) < 2 or len(sys.argv) > 3:
+ fail("Usage: %s tool [version tag]" % sys.argv[0])
+ tool = sys.argv[1]
+ command_line_version = sys.argv[2] if len(sys.argv) > 2 else None
+ return tool, command_line_version
+
+
+def get_build_versions(tool, tag):
# load versions and their dependencies
with open('versions.json', 'r') as f:
versions = json.load(f, object_pairs_hook=OrderedDict)
if tag is None:
# take top-most version
- tag = next(iter(versions))
- return versions[tag]
+ tag = next(iter(versions[tool]))
+ return tag, versions[tool][tag]
def check_go_version(versions):
diff --git a/versions.json b/versions.json
index 3b794ac..38d081d 100644
--- a/versions.json
+++ b/versions.json
@@ -1,24 +1,26 @@
{
- "0.0.7": {
- "obfs4": {
- "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy",
- "versions": {
- "obfs4.git": "obfs4proxy-0.0.7",
- "goptlib.git": "7d56ec4f381e",
- "crypto": "0c41d7ab0a0e",
- "net": "49bb7cea24b1",
- "siphash": "v1.2.0",
- "ed25519": "5312a6153412"
+ "obfs4proxy": {
+ "0.0.7": {
+ "obfs4": {
+ "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy",
+ "versions": {
+ "obfs4.git": "obfs4proxy-0.0.7",
+ "goptlib.git": "7d56ec4f381e",
+ "crypto": "0c41d7ab0a0e",
+ "net": "49bb7cea24b1",
+ "siphash": "v1.2.0",
+ "ed25519": "5312a6153412"
+ }
+ },
+ "go": {
+ "version": "go1.11.1",
+ "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117"
+ },
+ "ndk": {
+ "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip",
+ "revision": "18.0.5002713",
+ "sha256": "c413dd014edc37f822d0dc88fabc05b64232d07d5c6e9345224e47073fdf140b"
}
- },
- "go": {
- "version": "go1.11.1",
- "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117"
- },
- "ndk": {
- "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip",
- "revision": "18.0.5002713",
- "sha256": "c413dd014edc37f822d0dc88fabc05b64232d07d5c6e9345224e47073fdf140b"
}
}
}
--
GitLab