Install/bootstrap defined golang version from downloaded source

.gitignore

@@ -4,6 +4,8 @@
 /jni
 /android-ndk
 /android-ndk.zip
+/go
+/go.tar.gz
 /obfs4proxy*.zip
 /obfs4proxy*.pom
 /obfs4proxy*.jar

.gitlab-ci.yml

@@ -28,6 +28,8 @@ test_build:
   artifacts:
     paths:
     - obfs4proxy-*.zip
+    - obfs4proxy-*.pom
+    - obfs4proxy-*-sources.jar
     expire_in: 1 week
   except:
     - tags

Dockerfile

@@ -6,6 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 WORKDIR /opt/go-reproducer
 
 ADD build-*.py ./
+ADD install*.py ./
 ADD install*.sh ./
 ADD versions.json ./
 ADD utils.py ./

build-obfs4proxy.py

@@ -2,24 +2,23 @@
 import os
 from glob import glob
 from shutil import move, rmtree
-from subprocess import check_call, check_output
+from subprocess import check_call
 
 from utils import get_build_versions, ex, get_sha256, fail, zip_files, get_final_file_name, \
-    get_sources_file_name, get_pom_file_name, reset_time, get_obfs4_version
+    get_sources_file_name, get_pom_file_name, reset_time, get_obfs4_version, check_go_version, \
+    GO_PATH, GO_ROOT
 
 NDK_DIR = 'android-ndk'
 REPO_DIR = 'obfs4'
 GO_FLAGS = ['-asmflags', '-trimpath', '-o', 'obfs4proxy']
-GO_PATH = os.environ.get("GOPATH", os.path.expanduser('~/go'))
 
 
 def main():
     # Get the latest versions for building
     versions = get_build_versions(None)
 
-    # Check if proper Go version is installed (trailing space, because 'go1.10' in 'go1.10.1')
-    if versions['go']['version'] + ' ' not in check_output(['go', 'version']).decode("UTF-8"):
-        fail("You need Go version %s to reproduce this binary" % versions['go']['version'])
+    # Install Go
+    install_go(versions)
 
     # Setup Android NDK
     setup_android_ndk(versions)
@@ -39,6 +38,13 @@ def main():
     package_linux(versions)
 
 
+def install_go(versions):
+    ex(['./install-go.py'])
+    go_bin_path = os.path.join(GO_ROOT, 'bin')
+    os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH')
+    check_go_version(versions)
+
+
 def setup_android_ndk(versions):
     if os.path.isdir(NDK_DIR):
         # check that we are using the correct NDK
@@ -60,7 +66,7 @@ def setup_android_ndk(versions):
         # check sha256 hash on downloaded file
         is_hash = get_sha256('android-ndk.zip')
         if is_hash != versions['ndk']['sha256']:
-            fail("Android NDK checksum does not match, should be %s" % is_hash)
+            fail("Android NDK checksum does not match, is %s" % is_hash)
 
         # install the NDK
         print("Unpacking Android NDK...")
@@ -97,7 +103,7 @@ def checkout_source_repos(versions):
             fail("Version for dependency '%s' missing." % key)
             raise e
 
-        # checkout tag
+        # checkout commit or tag
         print("Checking out %s in %s" % (version, dependency))
         check_call(['git', 'checkout', '-f', version], cwd=dependency)
 
@@ -142,6 +148,7 @@ def build_android_arch(versions, env, tool, ndk_arch, pie):
     build_mode = "pie" if pie else "exe"
     extldflags = " -extldflags=-pie" if pie else ""
 
+    ex(['go', 'clean', versions['obfs4']['go-get']])
     ex(['go', 'build', '-buildmode=%s' % build_mode, '-ldflags',
         '-w -s' + extldflags] + GO_FLAGS + [versions['obfs4']['go-get']], env=env)
 
@@ -151,6 +158,7 @@ def build_android_arch(versions, env, tool, ndk_arch, pie):
 
 
 def build_linux(versions):
+    ex(['go', 'clean', versions['obfs4']['go-get']])
     ex(['go', 'build', '-ldflags', '-w -s'] + GO_FLAGS + [versions['obfs4']['go-get']])
     zip_files(['obfs4proxy'], 'obfs4proxy_linux-x86_64.zip')
     os.remove('obfs4proxy')

install-dependencies.sh

@@ -4,14 +4,12 @@ set -x
 
 apt-get install -y --no-install-recommends \
 	ca-certificates \
+	build-essential \
 	git \
-	binutils \
 	zip \
 	unzip \
 	wget \
 	fastjar \
 	python \
-	python3
-
-apt-get -y --no-install-recommends -t stretch-backports install \
-    golang
+	python3 \
+	golang

install-go.py

+#!/usr/bin/env python3
+import os
+from subprocess import check_call
+
+from utils import GO_ROOT, get_build_versions, ex, get_sha256, fail, check_go_version
+
+URL = 'https://golang.org/dl/%s.src.tar.gz'
+
+
+def main():
+    # Get the defined Go version
+    versions = get_build_versions(None)
+
+    if not os.path.isfile(os.path.join(GO_ROOT, 'bin', 'go')):
+        # download Go source archive
+        archive = 'go.tar.gz'
+        url = URL % versions['go']['version']
+        ex(['wget', '-c', '--no-verbose', url, '-O', archive])
+
+        # check sha256 hash on downloaded file
+        is_hash = get_sha256(archive)
+        if is_hash != versions['go']['sha256']:
+            fail("Go checksum does not match, is %s" % is_hash)
+
+        # unpack Go source archive
+        ex(['tar', '-C', os.path.dirname(GO_ROOT), '-xzf', archive])
+
+        # build
+        check_call(['./make.bash'], cwd=os.path.join(GO_ROOT, 'src'))
+
+    # final version check
+    go_bin_path = os.path.join(GO_ROOT, 'bin')
+    os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH')
+    check_go_version(versions)
+
+
+if __name__ == "__main__":
+    main()

install.sh

@@ -2,9 +2,6 @@
 set -e
 set -x
 
-# add backports repo
-echo "deb http://ftp.debian.org/debian stretch-backports main" >> /etc/apt/sources.list
-
 # update package sources
 apt-get update
 apt-get -y upgrade

utils.py

 import hashlib
 import json
+import os
 import sys
 from collections import OrderedDict
-from subprocess import check_call
+from subprocess import check_call, check_output
+
+GO_ROOT = os.path.abspath('go')
+GO_PATH = os.environ.get("GOPATH", os.path.expanduser('~/go'))
 
 
 def get_version():
@@ -22,6 +26,12 @@ def get_build_versions(tag):
     return versions[tag]
 
 
+def check_go_version(versions):
+    # Check if proper Go version is installed (trailing space, because 'go1.10' in 'go1.10.1')
+    if versions['go']['version'] + ' ' not in check_output(['go', 'version']).decode("UTF-8"):
+        fail("You need Go version %s to reproduce this binary" % versions['go']['version'])
+
+
 def ex(args, env=None):
     print("+ %s" % " ".join(args))
     check_call(args, env=env)

versions.json

@@ -12,7 +12,8 @@
       }
     },
     "go": {
-      "version": "go1.10.3"
+      "version": "go1.11.1",
+      "sha256": "558f8c169ae215e25b81421596e8de7572bd3ba824b79add22fba6e284db1117"
     },
     "ndk": {
       "url": "https://dl.google.com/android/repository/android-ndk-r18-linux-x86_64.zip",