build-tor.py 9.85 KB
Newer Older
Torsten Grote's avatar
Torsten Grote committed
1 2
#!/usr/bin/env python3
import os
3
from shutil import move, copy, rmtree
Torsten Grote's avatar
Torsten Grote committed
4 5
from subprocess import check_call

Torsten Grote's avatar
Torsten Grote committed
6 7
from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_tor_version, \
    get_final_file_name, get_sources_file_name, get_pom_file_name, get_version
Torsten Grote's avatar
Torsten Grote committed
8 9 10 11 12

NDK_DIR = 'android-ndk'


def main():
13 14
    # get Tor version from command or show usage information
    version = get_version()
Torsten Grote's avatar
Torsten Grote committed
15 16

    # get Tor version and versions of its dependencies
17 18
    versions = get_build_versions(version)
    print("Building Tor %s" % versions['tor'])
Torsten Grote's avatar
Torsten Grote committed
19 20 21 22 23 24 25

    # setup Android NDK
    setup_android_ndk(versions)

    # clone and checkout tor-android repo based on tor-versions.json
    prepare_tor_android_repo(versions)

26 27 28
    # build Tor for various platforms and architectures
    build()
    build_android()
Torsten Grote's avatar
Torsten Grote committed
29 30 31

    # zip geoip database
    geoip_path = os.path.join(REPO_DIR, 'external', 'tor', 'src', 'config', 'geoip')
32 33
    reset_time(geoip_path)
    check_call(['zip', '-D', '-X', os.path.join(REPO_DIR, 'geoip.zip'), geoip_path])
Torsten Grote's avatar
Torsten Grote committed
34

35 36 37
    # zip binaries together
    file_list = ['tor_linux-x86_64.zip', 'geoip.zip']
    zip_name = pack(versions, file_list)
38
    # zip Android binaries together
39 40 41
    file_list_android = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip',
                         'geoip.zip']
    zip_name_android = pack(versions, file_list_android, android=True)
akwizgran's avatar
akwizgran committed
42 43

    # create POM file from template
44 45
    pom_name = create_pom_file(versions)
    pom_name_android = create_pom_file(versions, android=True)
46 47 48

    # create sources jar
    jar_name = create_sources_jar(versions)
49 50
    jar_name_android = get_sources_file_name(versions, android=True)
    copy(os.path.join(REPO_DIR, jar_name), os.path.join(REPO_DIR, jar_name_android))
akwizgran's avatar
akwizgran committed
51

52
    # print hashes for debug purposes
akwizgran's avatar
akwizgran committed
53
    for file in file_list + [zip_name, jar_name, pom_name]:
Torsten Grote's avatar
Torsten Grote committed
54 55
        sha256hash = get_sha256(os.path.join(REPO_DIR, file))
        print("%s: %s" % (file, sha256hash))
56 57
    print("Android:")
    for file in file_list_android + [zip_name_android, jar_name_android, pom_name_android]:
58 59 60
        sha256hash = get_sha256(os.path.join(REPO_DIR, file))
        print("%s: %s" % (file, sha256hash))

Torsten Grote's avatar
Torsten Grote committed
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100

def setup_android_ndk(versions):
    if os.path.isdir(NDK_DIR):
        # check that we are using the correct NDK
        from configparser import ConfigParser
        config = ConfigParser()
        with open(os.path.join(NDK_DIR, 'source.properties'), 'r') as f:
            config.read_string('[default]\n' + f.read())
            revision = config.get('default', 'Pkg.Revision')

        if revision != versions['ndk']['revision']:
            print("Existing Android NDK has unexpected revision. Deleting...")
            rmtree(NDK_DIR)

    if not os.path.isdir(NDK_DIR):
        # download Android NDK
        print("Downloading Android NDK...")
        check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip'])

        # check sha256 hash on downloaded file
        if get_sha256('android-ndk.zip') != versions['ndk']['sha256']:
            fail("Android NDK checksum does not match")

        # install the NDK
        print("Unpacking Android NDK...")
        ndk_dir_tmp = NDK_DIR + '-tmp'
        check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
        content = os.listdir(ndk_dir_tmp)
        if len(content) == 1 and content[0].startswith('android-ndk-r'):
            move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
            os.rmdir(ndk_dir_tmp)
        else:
            fail("Could not extract NDK: %s" % str(content))

    os.putenv('ANDROID_NDK_HOME', os.path.abspath(NDK_DIR))


def prepare_tor_android_repo(versions):
    if os.path.isdir(REPO_DIR):
        # get latest commits and tags from remote
101
        check_call(['git', 'fetch', '--recurse-submodules=yes', 'origin'], cwd=REPO_DIR)
Torsten Grote's avatar
Torsten Grote committed
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
    else:
        # clone repo
        url = versions['tor_android_repo_url']
        check_call(['git', 'clone', '--recurse-submodules', url, REPO_DIR])

    # checkout tor-android version
    check_call(['git', 'checkout', '-f', versions['tor-android']], cwd=REPO_DIR)

    # undo all changes
    check_call(['git', 'reset', '--hard'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=REPO_DIR)

    # clean all untracked files and directories (-d) from repo
    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

    # check out versions of external dependencies
    checkout('tor', versions['tor'], 'external/tor')
    checkout('libevent', versions['libevent'], 'external/libevent')
    checkout('openssl', versions['openssl'], 'external/openssl')
    checkout('xz', versions['xz'], 'external/xz')
    checkout('zstd', versions['zstd'], 'external/zstd')


def checkout(name, tag, path):
    print("Checking out %s: %s" % (name, tag))
    repo_path = os.path.join(REPO_DIR, path)
    check_call(['git', 'checkout', '-f', tag], cwd=repo_path)


132
def build_android():
Torsten Grote's avatar
Torsten Grote committed
133 134 135 136
    # build arm pie
    os.unsetenv('APP_ABI')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
137
    build_android_arch('tor_arm_pie.zip')
Torsten Grote's avatar
Torsten Grote committed
138 139 140 141

    # build arm
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
142
    build_android_arch('tor_arm.zip')
Torsten Grote's avatar
Torsten Grote committed
143 144 145 146 147

    # build x86 pie
    os.putenv('APP_ABI', 'x86')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
148
    build_android_arch('tor_x86_pie.zip')
Torsten Grote's avatar
Torsten Grote committed
149 150 151 152

    # build x86
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
153
    build_android_arch('tor_x86.zip')
Torsten Grote's avatar
Torsten Grote committed
154 155


156
def build_android_arch(name):
Torsten Grote's avatar
Torsten Grote committed
157 158 159
    check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR)
    copy(os.path.join(REPO_DIR, 'external', 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
160 161 162
    tor_path = os.path.join(REPO_DIR, 'tor')
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
163 164 165
    check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)


166
def build(name='tor_linux-x86_64.zip'):
167 168 169 170
    # ensure clean build environment (again here to protect against build reordering)
    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

171 172
    # create folders for static libraries
    ext_dir = os.path.abspath(os.path.join(REPO_DIR, 'external'))
173 174 175 176
    prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix'))
    if not os.path.exists(prefix_dir):
        os.mkdir(prefix_dir)
    lib_dir = os.path.join(prefix_dir, 'lib')
177 178
    if not os.path.exists(lib_dir):
        os.mkdir(lib_dir)
179
    include_dir = os.path.join(prefix_dir, 'include')
180 181 182 183 184
    if not os.path.exists(include_dir):
        os.mkdir(include_dir)

    # setup environment
    env = os.environ.copy()
185
    env['LDFLAGS'] = "-L%s" % prefix_dir
186 187 188 189 190
    env['CFLAGS'] = "-fPIC -I%s" % include_dir
    env['LIBS'] = "-L%s" % lib_dir

    # build openssl
    openssl_dir = os.path.join(ext_dir, 'openssl')
191 192 193
    check_call(['./config', '--prefix=%s' % prefix_dir], cwd=openssl_dir, env=env)
    check_call(['make'], cwd=openssl_dir, env=env)
    check_call(['make', 'install_sw'], cwd=openssl_dir, env=env)
194 195

    # build libevent
196
    libevent_dir = os.path.join(ext_dir, 'libevent')
197
    check_call(['./autogen.sh'], cwd=libevent_dir)
198 199 200 201
    check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir], cwd=libevent_dir,
               env=env)
    check_call(['make'], cwd=libevent_dir, env=env)
    check_call(['make', 'install'], cwd=libevent_dir, env=env)
202 203

    # build Tor
204
    tor_dir = os.path.join(ext_dir, 'tor')
205 206 207
    check_call(['./autogen.sh'], cwd=tor_dir)
    env['CFLAGS'] += ' -O3'  # needed for FORTIFY_SOURCE
    check_call(['./configure', '--disable-asciidoc', '--disable-systemd',
208 209 210 211
                '--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir,
                '--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir,
                '--prefix=%s' % prefix_dir], cwd=tor_dir, env=env)
    check_call(['make', 'install'], cwd=tor_dir, env=env)
212 213 214

    # copy and zip built Tor binary
    tor_path = os.path.join(REPO_DIR, 'tor')
215
    copy(os.path.join(prefix_dir, 'bin', 'tor'), tor_path)
216 217 218 219 220 221
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
    check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)


222
def pack(versions, file_list, android=False):
223 224
    for filename in file_list:
        reset_time(os.path.join(REPO_DIR, filename))  # make file times deterministic before zipping
225
    zip_name = get_final_file_name(versions, android)
226 227 228 229
    check_call(['zip', '-D', '-X', zip_name] + file_list, cwd=REPO_DIR)
    return zip_name


230 231
def reset_time(filename):
    check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename])
Torsten Grote's avatar
Torsten Grote committed
232 233


234 235 236 237
def create_sources_jar(versions):
    external_dir = os.path.join(REPO_DIR, 'external')
    check_call(['git', 'clean', '-dfx'], cwd=external_dir)
    jar_files = []
Torsten Grote's avatar
Torsten Grote committed
238 239 240 241 242
    for root, dir_names, filenames in os.walk(external_dir):
        for f in filenames:
            jar_files.append(os.path.join(root, f))
    for file in jar_files:
        reset_time(file)
243 244 245 246
    jar_name = get_sources_file_name(versions)
    jar_path = os.path.abspath(os.path.join(REPO_DIR, jar_name))
    rel_paths = [os.path.relpath(f, external_dir) for f in sorted(jar_files)]
    check_call(['jar', 'cf', jar_path] + rel_paths, cwd=external_dir)
akwizgran's avatar
akwizgran committed
247
    return jar_name
248 249


250
def create_pom_file(versions, android=False):
251
    tor_version = get_tor_version(versions)
252 253
    pom_name = get_pom_file_name(versions, android)
    template = 'template-android.pom' if android else 'template.pom'
254
    with open(template, 'rt') as infile:
255 256 257
        with open(os.path.join(REPO_DIR, pom_name), 'wt') as outfile:
            for line in infile:
                outfile.write(line.replace('VERSION', tor_version))
akwizgran's avatar
akwizgran committed
258
    return pom_name
259 260


Torsten Grote's avatar
Torsten Grote committed
261
if __name__ == "__main__":
Torsten Grote's avatar
Torsten Grote committed
262
    main()