Factor out Android NDK install into own file

Also change formation of versions.json and use fixed Go paths

.gitignore

@@ -4,7 +4,6 @@
 /jni
 /android-ndk
 /android-ndk.zip
-/go
 /go.tar.gz
 /obfs4proxy*.zip
 /obfs4proxy*.pom

build-obfs4proxy.py

 #!/usr/bin/env python3
 import os
 from glob import glob
-from shutil import move, rmtree
 from subprocess import check_call
 
 from utils import get_build_versions, ex, get_sha256, fail, zip_files, get_final_file_name, \
     get_sources_file_name, get_pom_file_name, reset_time, get_obfs4_version, check_go_version, \
-    GO_PATH, GO_ROOT
+    get_version, GO_PATH, GO_ROOT, NDK_DIR
 
-NDK_DIR = 'android-ndk'
 REPO_DIR = 'obfs4'
 GO_FLAGS = ['-asmflags', '-trimpath', '-o', 'obfs4proxy']
 
 
 def main():
+    # get version from command line or show usage information
+    command_line_version = get_version()
+
     # Get the latest versions for building
-    versions = get_build_versions(None)
+    tool_version, versions = get_build_versions('obfs4proxy', command_line_version)
+    print("Building obfs4proxy %s" % tool_version)
 
     # Install Go
-    install_go(versions)
+    install_go(tool_version, versions)
 
-    # Setup Android NDK
-    setup_android_ndk(versions)
+    # Install Android NDK
+    install_android_ndk(tool_version)
 
     # Fetch and checkout dependencies at specific version
     checkout_source_repos(versions)
@@ -38,48 +40,17 @@ def main():
     package_linux(versions)
 
 
-def install_go(versions):
-    ex(['./install-go.py'])
+def install_go(tool_version, versions):
+    ex(['./install-go.py', 'obfs4proxy', tool_version])
     go_bin_path = os.path.join(GO_ROOT, 'bin')
+    os.environ['GOPATH'] = GO_PATH
     os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH')
     check_go_version(versions)
 
 
-def setup_android_ndk(versions):
-    if os.path.isdir(NDK_DIR):
-        # check that we are using the correct NDK
-        from configparser import ConfigParser
-        config = ConfigParser()
-        with open(os.path.join(NDK_DIR, 'source.properties'), 'rt') as f:
-            config.read_string('[default]\n' + str(f.read()))
-            revision = config.get('default', 'Pkg.Revision')
-
-        if revision != versions['ndk']['revision']:
-            print("Existing Android NDK has unexpected revision (%s). Deleting..." % revision)
-            rmtree(NDK_DIR)
-
-    if not os.path.isdir(NDK_DIR):
-        # download Android NDK
-        print("Downloading Android NDK...")
-        check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip'])
-
-        # check sha256 hash on downloaded file
-        is_hash = get_sha256('android-ndk.zip')
-        if is_hash != versions['ndk']['sha256']:
-            fail("Android NDK checksum does not match, is %s" % is_hash)
-
-        # install the NDK
-        print("Unpacking Android NDK...")
-        ndk_dir_tmp = NDK_DIR + '-tmp'
-        check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
-        content = os.listdir(ndk_dir_tmp)
-        if len(content) == 1 and content[0].startswith('android-ndk-r'):
-            move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
-            os.rmdir(ndk_dir_tmp)
-        else:
-            fail("Could not extract NDK: %s" % str(content))
-
-    os.putenv('ANDROID_NDK_HOME', os.path.abspath(NDK_DIR))
+def install_android_ndk(tool_version):
+    ex(['./install-android-ndk.py', 'obfs4proxy', tool_version])
+    os.environ['ANDROID_NDK_HOME'] = os.path.abspath(NDK_DIR)
 
 
 def checkout_source_repos(versions):

install-android-ndk.py

+#!/usr/bin/env python3
+import os
+from shutil import move, rmtree
+from subprocess import check_call
+
+from utils import get_build_versions, get_sha256, fail, get_version_and_tool, NDK_DIR
+
+
+def main():
+    # get tool name and version from command line parameters
+    tool, command_line_version = get_version_and_tool()
+
+    # Get the defined Android NDK version
+    tool_version, versions = get_build_versions(tool, command_line_version)
+    ndk = versions['ndk']
+    print("Installing Android NDK %s for %s %s" % (ndk['revision'], tool, tool_version))
+
+    if os.path.isdir(NDK_DIR):
+        # check that we are using the correct NDK
+        from configparser import ConfigParser
+        config = ConfigParser()
+        with open(os.path.join(NDK_DIR, 'source.properties'), 'rt') as f:
+            config.read_string('[default]\n' + str(f.read()))
+            revision = config.get('default', 'Pkg.Revision')
+
+        if revision != ndk['revision']:
+            print("Existing Android NDK has unexpected revision (%s). Deleting..." % revision)
+            rmtree(NDK_DIR)
+
+    if not os.path.isdir(NDK_DIR):
+        # download Android NDK
+        print("Downloading Android NDK...")
+        check_call(['wget', '-c', '--no-verbose', ndk['url'], '-O', 'android-ndk.zip'])
+
+        # check sha256 hash on downloaded file
+        is_hash = get_sha256('android-ndk.zip')
+        if is_hash != ndk['sha256']:
+            fail("Android NDK checksum does not match, is %s" % is_hash)
+
+        # install the NDK
+        print("Unpacking Android NDK...")
+        ndk_dir_tmp = NDK_DIR + '-tmp'
+        check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
+        content = os.listdir(ndk_dir_tmp)
+        if len(content) == 1 and content[0].startswith('android-ndk-r'):
+            move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
+            os.rmdir(ndk_dir_tmp)
+        else:
+            fail("Could not extract NDK: %s" % str(content))
+
+
+if __name__ == "__main__":
+    main()

install-go.py

@@ -2,14 +2,19 @@
 import os
 from subprocess import check_call
 
-from utils import GO_ROOT, get_build_versions, ex, get_sha256, fail, check_go_version
+from utils import GO_ROOT, get_build_versions, ex, get_sha256, fail, check_go_version, \
+    get_version_and_tool
 
 URL = 'https://golang.org/dl/%s.src.tar.gz'
 
 
 def main():
+    # get tool name and version from command line parameters
+    tool, command_line_version = get_version_and_tool()
+
     # Get the defined Go version
-    versions = get_build_versions(None)
+    tool_version, versions = get_build_versions(tool, command_line_version)
+    print("Installing %s for %s %s" % (versions['go']['version'], tool, tool_version))
 
     if not os.path.isfile(os.path.join(GO_ROOT, 'bin', 'go')):
         # download Go source archive

utils.py

@@ -5,8 +5,11 @@ import sys
 from collections import OrderedDict
 from subprocess import check_call, check_output
 
-GO_ROOT = os.path.abspath('go')
-GO_PATH = os.environ.get("GOPATH", os.path.expanduser('~/go'))
+# WARNING: changing those affects reproducibility
+GO_ROOT = '/tmp/go'
+GO_PATH = '/tmp/go-path'
+
+NDK_DIR = 'android-ndk'
 
 
 def get_version():
@@ -15,15 +18,23 @@ def get_version():
     return sys.argv[1] if len(sys.argv) > 1 else None
 
 
-def get_build_versions(tag):
+def get_version_and_tool():
+    if len(sys.argv) < 2 or len(sys.argv) > 3:
+        fail("Usage: %s tool [version tag]" % sys.argv[0])
+    tool = sys.argv[1]
+    command_line_version = sys.argv[2] if len(sys.argv) > 2 else None
+    return tool, command_line_version
+
+
+def get_build_versions(tool, tag):
     # load versions and their dependencies
     with open('versions.json', 'r') as f:
         versions = json.load(f, object_pairs_hook=OrderedDict)
 
     if tag is None:
         # take top-most version
-        tag = next(iter(versions))
-    return versions[tag]
+        tag = next(iter(versions[tool]))
+    return tag, versions[tool][tag]
 
 
 def check_go_version(versions):

versions.json

 {
-  "0.0.7": {
-    "obfs4": {
-      "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy",
-      "versions": {
-        "obfs4.git": "obfs4proxy-0.0.7",
-        "goptlib.git": "7d56ec4f381e",
-        "crypto": "0c41d7ab0a0e",
-        "net": "49bb7cea24b1",
-        "siphash": "v1.2.0",
-        "ed25519": "5312a6153412"
+  "obfs4proxy": {
+    "0.0.7": {
+      "obfs4": {
+        "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy",
+        "versions": {
+          "obfs4.git": "obfs4proxy-0.0.7",
+          "goptlib.git": "7d56ec4f381e",
+          "crypto": "0c41d7ab0a0e",
+          "net": "49bb7cea24b1",
+          "siphash": "v1.2.0",
+          "ed25519": "5312a6153412"
+        }
+      },
+      "go": {
+        "version": "go1.11.1",
+        "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117"
+      },
+      "ndk": {
+        "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip",
+        "revision": "18.0.5002713",
+        "sha256": "c413dd014edc37f822d0dc88fabc05b64232d07d5c6e9345224e47073fdf140b"
       }
-    },
-    "go": {
-      "version": "go1.11.1",
-      "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117"
-    },
-    "ndk": {
-      "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip",
-      "revision": "18.0.5002713",
-      "sha256": "c413dd014edc37f822d0dc88fabc05b64232d07d5c6e9345224e47073fdf140b"
     }
   }
 }