Skip to content
Snippets Groups Projects
build-binary.py 8.72 KiB
Newer Older
#!/usr/bin/env python3
import os, shutil
from glob import glob
from subprocess import check_call

from utils import get_build_versions, ex, get_sha256, zip_files, get_final_file_path, \
    get_sources_file_path, get_pom_file_path, reset_time, get_version_number, check_go_version, \
    get_version_and_tool, get_output_dir, get_platform_output_dir, GO_PATH, GO_ROOT, NDK_DIR


def main():
    # get version from command line or show usage information
    tool, command_line_version = get_version_and_tool()

    # Get the latest versions for building
    tool_version, versions = get_build_versions(tool, command_line_version)
    print("Building %s %s" % (tool, tool_version))

    # Install Go
    install_go(tool, tool_version, versions)

    # Install Android NDK
Sebastian's avatar
Sebastian committed
    install_android_ndk(tool, tool_version)

    # Checkout source at specific version
    checkout_source_repo(tool, versions)

    # Create the output directory, deleting it first if it already exists
    output_dir = get_output_dir(tool)
    if os.path.exists(output_dir): shutil.rmtree(output_dir)
    os.makedirs(output_dir)
    # Build and package for various platforms and architectures
Sebastian's avatar
Sebastian committed
    build_android(tool, versions)
    package_android(tool, versions)
Sebastian's avatar
Sebastian committed
    build_linux(tool, versions)
    package_linux(tool, versions)
Sebastian's avatar
Sebastian committed
    build_windows(tool, versions)
    package_windows(tool, versions)

    build_macos(tool, versions)
    package_macos(tool, versions)
    # This needs to be always the same path, otherwise it breaks reproducibility
    return '/tmp/%s' % versions['repo_dir']


def install_go(tool, tool_version, versions):
    ex(['./install-go.py', tool, tool_version])
    go_bin_path = os.path.join(GO_ROOT, 'bin')
    os.environ['GOPATH'] = GO_PATH
    os.environ['PATH'] = go_bin_path + os.pathsep + os.getenv('PATH')
    os.environ['GO111MODULE'] = 'on'
    check_go_version(versions)


def install_android_ndk(tool, tool_version):
    ex(['./install-android-ndk.py', tool, tool_version])
    os.environ['ANDROID_NDK_HOME'] = os.path.abspath(NDK_DIR)


def checkout_source_repo(tool, versions):
    repo_dir = get_repo_dir(versions)
    if os.path.isdir(repo_dir):
        # get latest commits and tags from remote
        check_call(['git', 'fetch', 'origin'], cwd=repo_dir)
    else:
        # clone repo
        check_call(['git', 'clone', versions['repo_url'], repo_dir])

    # checkout version
    print("Checking out %s" % versions['revision'])
    check_call(['git', 'checkout', '-f', versions['revision']], cwd=repo_dir)

    # undo all changes
    check_call(['git', 'reset', '--hard'], cwd=repo_dir)

    # clean all untracked files and directories (-d) from repo
    check_call(['git', 'clean', '-dffx'], cwd=repo_dir)


def build_android(tool, versions):
    os.mkdir(get_platform_output_dir(tool, 'android'))

    env = os.environ.copy()
    env['GOARCH'] = "arm"
    env['GOARM'] = "7"
akwizgran's avatar
akwizgran committed
    build_android_arch(tool, versions, env, "armv7a-linux-androideabi", "arm", "armeabi-v7a")

    env = os.environ.copy()
    env['GOARCH'] = "arm64"
    build_android_arch(tool, versions, env, "aarch64-linux-android", "arm64", "arm64-v8a")

    env = os.environ.copy()
    env['GOARCH'] = "386"
    build_android_arch(tool, versions, env, "i686-linux-android", "x86", "x86")

    env = os.environ.copy()
    env['GOARCH'] = "amd64"
    build_android_arch(tool, versions, env, "x86_64-linux-android", "x86_64", "x86_64")
def build_android_arch(tool, versions, env, clang_arch, ndk_arch, abi):
akwizgran's avatar
akwizgran committed
    # TODO: Raise API level and upgrade NDK when we drop support for Android 4
    min_api = '21' if ndk_arch.endswith('64') else '16'
    clang = "%s/toolchains/llvm/prebuilt/linux-x86_64/bin/%s%s-clang" % (NDK_DIR, clang_arch, min_api)

    env['CC'] = os.path.abspath(clang)
    env['CGO_ENABLED'] = "1"
    env['CGO_CFLAGS'] = "-O2"  # removes -g
    env['GOOS'] = "android"

    print("Building %s for Android %s" % (tool, abi))
    output_dir = get_platform_output_dir(tool, 'android')
    arch_dir = os.path.join(output_dir, abi)
    os.mkdir(arch_dir)
    tool_path = os.path.join(arch_dir, "lib%s.so" % tool)

    output_file = os.path.abspath(os.path.join(os.path.curdir, tool))
    go_flags = ['-asmflags', '-trimpath', '-o', output_file]
    repo_dir = get_repo_dir(versions)
akwizgran's avatar
akwizgran committed
    ex(['go', 'build', '-buildmode=pie', '-ldflags', '-w -s -extldflags=-pie'] + go_flags +
       [os.path.join('.', versions['build_path'])], env=env, cwd=repo_dir)
    shutil.copy(output_file, tool_path)


def build_linux(tool, versions):
    os.mkdir(get_platform_output_dir(tool, 'linux'))
    build_desktop_arch(tool, versions, 'linux', 'armhf', 'arm', '7')
    build_desktop_arch(tool, versions, 'linux', 'aarch64', 'arm64')
    build_desktop_arch(tool, versions, 'linux', 'x86_64', 'amd64')


def build_windows(tool, versions):
    os.mkdir(get_platform_output_dir(tool, 'windows'))
    build_desktop_arch(tool, versions, 'windows', 'x86_64', 'amd64')


def build_macos(tool, versions):
    os.mkdir(get_platform_output_dir(tool, 'macos'))
    build_desktop_arch(tool, versions, 'darwin', 'x86_64', 'amd64', output_name = 'macos')
    build_desktop_arch(tool, versions, 'darwin', 'aarch64', 'arm64', output_name = 'macos')


def build_desktop_arch(tool, versions, platform, arch, goarch, goarm=None, output_name=None):
    env = os.environ.copy()
    env['CGO_ENABLED'] = "0"
    env['GOOS'] = platform
    env['GOARCH'] = goarch
    if goarm: env['GOARM'] = goarm
    build_path = os.path.join('.', versions['build_path'])
    if not output_name: output_name = platform

    print("Building %s for %s %s" % (tool, output_name, arch))
    output_dir = get_platform_output_dir(tool, output_name)
    arch_dir = os.path.join(output_dir, arch)
    os.mkdir(arch_dir)
    extension = '.exe' if platform == 'windows' else ''
    tool_path = os.path.join(arch_dir, "%s%s" % (tool, extension))

    output_file = os.path.abspath(os.path.join(os.path.curdir, tool))
    go_flags = ['-asmflags', '-trimpath', '-o', output_file]
    repo_dir = get_repo_dir(versions)
    ex(['go', 'build', '-ldflags', '-w -s'] + go_flags + [build_path], env=env, cwd=repo_dir)
    shutil.copy(output_file, tool_path)


def package_android(tool, versions):
    output_dir = get_platform_output_dir(tool, 'android')
    file_list = [
        os.path.join(output_dir, 'armeabi-v7a', 'lib%s.so' % tool),
        os.path.join(output_dir, 'arm64-v8a', 'lib%s.so' % tool),
        os.path.join(output_dir, 'x86', 'lib%s.so' % tool),
        os.path.join(output_dir, 'x86_64', 'lib%s.so' % tool),
    ]
    package(tool, versions, file_list, 'android')


def package_linux(tool, versions):
    output_dir = get_platform_output_dir(tool, 'linux')
    file_list = [
        os.path.join(output_dir, 'armhf', tool),
        os.path.join(output_dir, 'aarch64', tool),
        os.path.join(output_dir, 'x86_64', tool),
    ]
    package(tool, versions, file_list, 'linux')


def package_windows(tool, versions):
    output_dir = get_platform_output_dir(tool, 'windows')
    file_list = [os.path.join(output_dir, 'x86_64', '%s.exe' % tool)]
    package(tool, versions, file_list, 'windows')


def package_macos(tool, versions):
    output_dir = get_platform_output_dir(tool, 'macos')
    file_list = [
        os.path.join(output_dir, 'aarch64', tool),
        os.path.join(output_dir, 'x86_64', tool),
    ]
    package(tool, versions, file_list, 'macos')


def package(tool, versions, file_list, platform):
    zip_file = get_final_file_path(tool, versions, platform)
    zip_files(file_list, zip_file, versions)
    create_sources_jar(tool, versions, platform)
    create_pom_file(tool, versions, platform)

    # print hashes for debug purposes
    for file in file_list + [zip_file]:
        sha256hash = get_sha256(file)
        prefix = '->' if file == zip_file else ''
        print("%s %s: %s" % (prefix, file, sha256hash))


def create_sources_jar(tool, versions, platform):
    repo_dir = get_repo_dir(versions)
    # clean all untracked files and directories (-d) from repo
    check_call(['git', 'clean', '-dffx'], cwd=repo_dir)
    # vendorize dependencies
    ex(['go', 'mod', 'vendor'], cwd=repo_dir)
    jar_files = []
    for file in glob(os.path.join(repo_dir, '*')):
        reset_time(file, versions)
        jar_files.append(os.path.relpath(file, repo_dir))
    jar_path = get_sources_file_path(tool, versions, platform)
    check_call(['jar', 'cf', jar_path] + jar_files, cwd=repo_dir)


def create_pom_file(tool, versions, platform):
    tool_version = get_version_number(versions)
    pom_file = get_pom_file_path(tool, versions, platform)
    template = 'template-%s-%s.pom' % (tool, platform)
    with open(template, 'rt') as infile:
        with open(pom_file, 'wt') as outfile:
            for line in infile:
                outfile.write(line.replace('VERSION', tool_version))


if __name__ == "__main__":
    main()