Add obfsproxy-0.0.9

and remove old version in favor of new go modules for dependency pinning

.gitlab-ci.yml

@@ -24,7 +24,7 @@ build:
 test:
   stage: test
   script:
-    - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./build-obfs4proxy.py 0.0.7 && ./verify-obfs4proxy.py 0.0.7"
+    - docker run -v `pwd`:/opt/go-reproducer ${TEST_IMAGE} /bin/bash -c "./build-obfs4proxy.py 0.0.9 && ./verify-obfs4proxy.py 0.0.9"
   artifacts:
     paths:
     - obfs4proxy-*.zip

build-obfs4proxy.py

@@ -7,8 +7,8 @@ from utils import get_build_versions, ex, get_sha256, fail, zip_files, get_final
     get_sources_file_name, get_pom_file_name, reset_time, get_obfs4_version, check_go_version, \
     get_version, GO_PATH, GO_ROOT, NDK_DIR
 
-REPO_DIR = 'obfs4'
-GO_FLAGS = ['-asmflags', '-trimpath', '-o', 'obfs4proxy']
+REPO_DIR = './obfs4'
+GO_FLAGS = ['-asmflags', '-trimpath', '-o', '../obfs4proxy']
 
 
 def main():
@@ -25,8 +25,8 @@ def main():
     # Install Android NDK
     install_android_ndk(tool_version)
 
-    # Fetch and checkout dependencies at specific version
-    checkout_source_repos(versions)
+    # Checkout source at specific version
+    checkout_source_repo(versions)
 
     # Build for various Android versions and create package
     build_android(versions)
@@ -45,6 +45,7 @@ def install_go(tool_version, versions):
     go_bin_path = os.path.join(GO_ROOT, 'bin')
     os.environ['GOPATH'] = GO_PATH
     os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH')
+    os.environ['GO111MODULE'] = 'on'
     check_go_version(versions)
 
 
@@ -53,36 +54,28 @@ def install_android_ndk(tool_version):
     os.environ['ANDROID_NDK_HOME'] = os.path.abspath(NDK_DIR)
 
 
-def checkout_source_repos(versions):
-    # download dependencies
-    ex(['go', 'get', '-d', versions['obfs4']['go-get']])
+def checkout_source_repo(versions):
+    if os.path.isdir(REPO_DIR):
+        # get latest commits and tags from remote
+        check_call(['git', 'fetch', 'origin'], cwd=REPO_DIR)
+    else:
+        # clone repo
+        check_call(['git', 'clone', versions['repo_url'], REPO_DIR])
 
-    # find dependencies
-    go_src = os.path.join(GO_PATH, 'src')
-    dependencies = []
-    for root, dirs, files in os.walk(go_src):
-        if root.endswith("/.git"):
-            dependencies.append(os.path.split(root)[0])
+    # checkout version
+    print("Checking out %s" % versions['revision'])
+    check_call(['git', 'checkout', '-f', versions['revision']], cwd=REPO_DIR)
 
-    # checkout specific versions
-    for dependency in dependencies:
-        # get version (commit or tag)
-        key = os.path.split(dependency)[1]
-        try:
-            version = versions['obfs4']['versions'][key]
-        except KeyError as e:
-            fail("Version for dependency '%s' missing." % key)
-            raise e
+    # initialize and/or update submodules
+    # (after checkout, because submodules can point to non-existent commits on master)
+    check_call(['git', 'submodule', 'update', '--init', '--recursive'], cwd=REPO_DIR)
 
-        # checkout commit or tag
-        print("Checking out %s in %s" % (version, dependency))
-        check_call(['git', 'checkout', '-f', version], cwd=dependency)
+    # undo all changes
+    check_call(['git', 'reset', '--hard'], cwd=REPO_DIR)
+    check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=REPO_DIR)
 
-        # undo all changes
-        check_call(['git', 'reset', '--hard'], cwd=dependency)
-
-        # clean all untracked files and directories (-d) from repo
-        check_call(['git', 'clean', '-dffx'], cwd=dependency)
+    # clean all untracked files and directories (-d) from repo
+    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
 
 
 def build_android(versions):
@@ -119,9 +112,8 @@ def build_android_arch(versions, env, tool, ndk_arch, pie):
     build_mode = "pie" if pie else "exe"
     extldflags = " -extldflags=-pie" if pie else ""
 
-    ex(['go', 'clean', versions['obfs4']['go-get']])
-    ex(['go', 'build', '-buildmode=%s' % build_mode, '-ldflags',
-        '-w -s' + extldflags] + GO_FLAGS + [versions['obfs4']['go-get']], env=env)
+    ex(['go', 'build', '-buildmode=%s' % build_mode, '-ldflags', '-w -s' + extldflags] + GO_FLAGS +
+       [os.path.join('.', versions['build_path'])], env=env, cwd=REPO_DIR)
 
     pie_suffix = '_pie' if pie else ''
     zip_files(['obfs4proxy'], 'obfs4proxy_%s%s.zip' % (ndk_arch, pie_suffix))
@@ -129,8 +121,8 @@ def build_android_arch(versions, env, tool, ndk_arch, pie):
 
 
 def build_linux(versions):
-    ex(['go', 'clean', versions['obfs4']['go-get']])
-    ex(['go', 'build', '-ldflags', '-w -s'] + GO_FLAGS + [versions['obfs4']['go-get']])
+    build_path = os.path.join('.', versions['build_path'])
+    ex(['go', 'build', '-ldflags', '-w -s'] + GO_FLAGS + [build_path], cwd=REPO_DIR)
     zip_files(['obfs4proxy'], 'obfs4proxy_linux-x86_64.zip')
     os.remove('obfs4proxy')
 
@@ -160,14 +152,14 @@ def package(versions, file_list, android):
 
 
 def create_sources_jar(versions):
+    ex(['go', 'mod', 'vendor'], cwd=REPO_DIR)
     jar_files = []
-    go_src = os.path.join(GO_PATH, 'src')
-    for file in glob(os.path.join(go_src, '*')):
+    for file in glob(os.path.join(REPO_DIR, '*')):
         reset_time(file)
-        jar_files.append(os.path.relpath(file, go_src))
+        jar_files.append(os.path.relpath(file, REPO_DIR))
     jar_file = get_sources_file_name(versions)
     jar_path = os.path.abspath(jar_file)
-    check_call(['jar', 'cf', jar_path] + jar_files, cwd=go_src)
+    check_call(['jar', 'cf', jar_path] + jar_files, cwd=REPO_DIR)
     return jar_file
 
 

utils.py

@@ -43,9 +43,9 @@ def check_go_version(versions):
         fail("You need Go version %s to reproduce this binary" % versions['go']['version'])
 
 
-def ex(args, env=None):
+def ex(args, env=None, cwd=None):
     print("+ %s" % " ".join(args))
-    check_call(args, env=env)
+    check_call(args, env=env, cwd=cwd)
 
 
 def fail(msg=""):
@@ -72,7 +72,7 @@ def zip_files(files, zip_name):
 
 
 def get_obfs4_version(versions):
-    version = versions['obfs4']['versions']['obfs4.git']
+    version = versions['revision']
     if '-' in version:
         return version.split('-')[1]
     return version

versions.json

 {
   "obfs4proxy": {
-    "0.0.7": {
-      "obfs4": {
-        "go-get": "git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy",
-        "versions": {
-          "obfs4.git": "obfs4proxy-0.0.7",
-          "goptlib.git": "7d56ec4f381e",
-          "crypto": "0c41d7ab0a0e",
-          "net": "49bb7cea24b1",
-          "siphash": "v1.2.0",
-          "ed25519": "5312a6153412"
-        }
-      },
+    "0.0.9": {
+      "repo_url": "https://git.torproject.org/pluggable-transports/obfs4.git",
+      "revision": "obfs4proxy-0.0.9",
+      "build_path": "obfs4proxy",
       "go": {
-        "version": "go1.11.1",
-        "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117"
+        "version": "go1.11.5",
+        "sha256": "bc1ef02bb1668835db1390a2e478dcbccb5dd16911691af9d75184bbe5aa943e"
       },
       "ndk": {
         "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip",