build-tor.py 9.99 KB
Newer Older
Torsten Grote's avatar
Torsten Grote committed
1 2
#!/usr/bin/env python3
import os
3
from shutil import move, copy, rmtree
Torsten Grote's avatar
Torsten Grote committed
4 5
from subprocess import check_call

Torsten Grote's avatar
Torsten Grote committed
6 7
from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_tor_version, \
    get_final_file_name, get_sources_file_name, get_pom_file_name, get_version
Torsten Grote's avatar
Torsten Grote committed
8

9
ZLIB_REPO_URL = 'https://github.com/madler/zlib.git'
Torsten Grote's avatar
Torsten Grote committed
10
NDK_DIR = 'android-ndk'
11
EXT_DIR = os.path.abspath(os.path.join(REPO_DIR, 'external'))
Torsten Grote's avatar
Torsten Grote committed
12 13 14


def main():
15 16
    # get Tor version from command or show usage information
    version = get_version()
Torsten Grote's avatar
Torsten Grote committed
17 18

    # get Tor version and versions of its dependencies
19 20
    versions = get_build_versions(version)
    print("Building Tor %s" % versions['tor'])
Torsten Grote's avatar
Torsten Grote committed
21 22 23 24 25 26 27

    # setup Android NDK
    setup_android_ndk(versions)

    # clone and checkout tor-android repo based on tor-versions.json
    prepare_tor_android_repo(versions)

28 29 30
    # create sources jar before building
    jar_name = create_sources_jar(versions)

31 32 33
    # build Tor for various platforms and architectures
    build()
    build_android()
Torsten Grote's avatar
Torsten Grote committed
34 35

    # zip geoip database
36 37
    geoip_path = os.path.join(REPO_DIR, 'geoip')
    copy(os.path.join(EXT_DIR, 'tor', 'src', 'config', 'geoip'), geoip_path)
38
    reset_time(geoip_path)
39
    check_call(['zip', '-X', '../geoip.zip', 'geoip'], cwd=REPO_DIR)
Torsten Grote's avatar
Torsten Grote committed
40

41 42 43
    # zip binaries together
    file_list = ['tor_linux-x86_64.zip', 'geoip.zip']
    zip_name = pack(versions, file_list)
44
    # zip Android binaries together
45 46 47
    file_list_android = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip',
                         'geoip.zip']
    zip_name_android = pack(versions, file_list_android, android=True)
akwizgran's avatar
akwizgran committed
48 49

    # create POM file from template
50 51
    pom_name = create_pom_file(versions)
    pom_name_android = create_pom_file(versions, android=True)
52

53
    # print hashes for debug purposes
akwizgran's avatar
akwizgran committed
54
    for file in file_list + [zip_name, jar_name, pom_name]:
55
        sha256hash = get_sha256(file)
Torsten Grote's avatar
Torsten Grote committed
56
        print("%s: %s" % (file, sha256hash))
57
    print("Android:")
58 59
    for file in file_list_android + [zip_name_android, pom_name_android]:
        sha256hash = get_sha256(file)
60 61
        print("%s: %s" % (file, sha256hash))

Torsten Grote's avatar
Torsten Grote committed
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101

def setup_android_ndk(versions):
    if os.path.isdir(NDK_DIR):
        # check that we are using the correct NDK
        from configparser import ConfigParser
        config = ConfigParser()
        with open(os.path.join(NDK_DIR, 'source.properties'), 'r') as f:
            config.read_string('[default]\n' + f.read())
            revision = config.get('default', 'Pkg.Revision')

        if revision != versions['ndk']['revision']:
            print("Existing Android NDK has unexpected revision. Deleting...")
            rmtree(NDK_DIR)

    if not os.path.isdir(NDK_DIR):
        # download Android NDK
        print("Downloading Android NDK...")
        check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip'])

        # check sha256 hash on downloaded file
        if get_sha256('android-ndk.zip') != versions['ndk']['sha256']:
            fail("Android NDK checksum does not match")

        # install the NDK
        print("Unpacking Android NDK...")
        ndk_dir_tmp = NDK_DIR + '-tmp'
        check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
        content = os.listdir(ndk_dir_tmp)
        if len(content) == 1 and content[0].startswith('android-ndk-r'):
            move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
            os.rmdir(ndk_dir_tmp)
        else:
            fail("Could not extract NDK: %s" % str(content))

    os.putenv('ANDROID_NDK_HOME', os.path.abspath(NDK_DIR))


def prepare_tor_android_repo(versions):
    if os.path.isdir(REPO_DIR):
        # get latest commits and tags from remote
102
        check_call(['git', 'fetch', '--recurse-submodules=yes', 'origin'], cwd=REPO_DIR)
Torsten Grote's avatar
Torsten Grote committed
103 104 105
    else:
        # clone repo
        url = versions['tor_android_repo_url']
106
        check_call(['git', 'clone', url, REPO_DIR])
Torsten Grote's avatar
Torsten Grote committed
107 108 109 110

    # checkout tor-android version
    check_call(['git', 'checkout', '-f', versions['tor-android']], cwd=REPO_DIR)

111 112 113 114
    # initialize and/or update submodules
    # (after checkout, because submodules can point to non-existent commits on master)
    check_call(['git', 'submodule', 'update', '--init', '--recursive'], cwd=REPO_DIR)

Torsten Grote's avatar
Torsten Grote committed
115 116 117 118 119 120 121 122
    # undo all changes
    check_call(['git', 'reset', '--hard'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=REPO_DIR)

    # clean all untracked files and directories (-d) from repo
    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

123 124 125
    # add zlib
    check_call(['git', 'clone', ZLIB_REPO_URL], cwd=EXT_DIR)

Torsten Grote's avatar
Torsten Grote committed
126
    # check out versions of external dependencies
127 128 129 130 131 132
    checkout('tor', versions['tor'], 'tor')
    checkout('libevent', versions['libevent'], 'libevent')
    checkout('openssl', versions['openssl'], 'openssl')
    checkout('xz', versions['xz'], 'xz')
    checkout('zlib', versions['zlib'], 'zlib')
    checkout('zstd', versions['zstd'], 'zstd')
Torsten Grote's avatar
Torsten Grote committed
133 134 135 136


def checkout(name, tag, path):
    print("Checking out %s: %s" % (name, tag))
137
    repo_path = os.path.join(EXT_DIR, path)
Torsten Grote's avatar
Torsten Grote committed
138 139 140
    check_call(['git', 'checkout', '-f', tag], cwd=repo_path)


141
def build_android():
Torsten Grote's avatar
Torsten Grote committed
142 143 144 145
    # build arm pie
    os.unsetenv('APP_ABI')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
146
    build_android_arch('tor_arm_pie.zip')
Torsten Grote's avatar
Torsten Grote committed
147 148 149 150

    # build arm
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
151
    build_android_arch('tor_arm.zip')
Torsten Grote's avatar
Torsten Grote committed
152 153 154 155 156

    # build x86 pie
    os.putenv('APP_ABI', 'x86')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
157
    build_android_arch('tor_x86_pie.zip')
Torsten Grote's avatar
Torsten Grote committed
158 159 160 161

    # build x86
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
162
    build_android_arch('tor_x86.zip')
Torsten Grote's avatar
Torsten Grote committed
163 164


165
def build_android_arch(name):
Torsten Grote's avatar
Torsten Grote committed
166
    check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR)
167
    copy(os.path.join(EXT_DIR, 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
Torsten Grote's avatar
Torsten Grote committed
168
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
169 170 171
    tor_path = os.path.join(REPO_DIR, 'tor')
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
172
    check_call(['zip', '-X', '../' + name, 'tor'], cwd=REPO_DIR)
173 174


175
def build(name='tor_linux-x86_64.zip'):
176 177 178 179
    prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix'))
    lib_dir = os.path.join(prefix_dir, 'lib')
    include_dir = os.path.join(prefix_dir, 'include')

180 181
    # ensure clean build environment (again here to protect against build reordering)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)
182 183
    if os.path.exists(prefix_dir):
        rmtree(prefix_dir)
184

185
    # create folders for static libraries
186 187 188
    os.mkdir(prefix_dir)
    os.mkdir(lib_dir)
    os.mkdir(include_dir)
189 190 191

    # setup environment
    env = os.environ.copy()
192
    env['LDFLAGS'] = "-L%s" % prefix_dir
193 194 195
    env['CFLAGS'] = "-fPIC -I%s" % include_dir
    env['LIBS'] = "-L%s" % lib_dir

196 197 198 199 200
    # build zlib
    zlib_dir = os.path.join(EXT_DIR, 'zlib')
    check_call(['./configure', '--prefix=%s' % prefix_dir], cwd=zlib_dir, env=env)
    check_call(['make', 'install'], cwd=zlib_dir, env=env)

201
    # build openssl
202
    openssl_dir = os.path.join(EXT_DIR, 'openssl')
203 204 205
    check_call(['./config', '--prefix=%s' % prefix_dir], cwd=openssl_dir, env=env)
    check_call(['make'], cwd=openssl_dir, env=env)
    check_call(['make', 'install_sw'], cwd=openssl_dir, env=env)
206 207

    # build libevent
208
    libevent_dir = os.path.join(EXT_DIR, 'libevent')
209
    check_call(['./autogen.sh'], cwd=libevent_dir)
210 211 212 213
    check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir], cwd=libevent_dir,
               env=env)
    check_call(['make'], cwd=libevent_dir, env=env)
    check_call(['make', 'install'], cwd=libevent_dir, env=env)
214 215

    # build Tor
216
    tor_dir = os.path.join(EXT_DIR, 'tor')
217 218 219
    check_call(['./autogen.sh'], cwd=tor_dir)
    env['CFLAGS'] += ' -O3'  # needed for FORTIFY_SOURCE
    check_call(['./configure', '--disable-asciidoc', '--disable-systemd',
220
                '--enable-static-zlib', '--with-zlib-dir=%s' % prefix_dir,
221 222 223 224
                '--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir,
                '--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir,
                '--prefix=%s' % prefix_dir], cwd=tor_dir, env=env)
    check_call(['make', 'install'], cwd=tor_dir, env=env)
225 226 227

    # copy and zip built Tor binary
    tor_path = os.path.join(REPO_DIR, 'tor')
228
    copy(os.path.join(prefix_dir, 'bin', 'tor'), tor_path)
229 230 231
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
232
    check_call(['zip', '-X', '../' + name, 'tor'], cwd=REPO_DIR)
233 234


235
def pack(versions, file_list, android=False):
236
    for filename in file_list:
237
        reset_time(filename)  # make file times deterministic before zipping
238
    zip_name = get_final_file_name(versions, android)
239
    check_call(['zip', '-D', '-X', zip_name] + file_list)
240 241 242
    return zip_name


243 244
def reset_time(filename):
    check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename])
Torsten Grote's avatar
Torsten Grote committed
245 246


247 248
def create_sources_jar(versions):
    jar_files = []
249
    for root, dir_names, filenames in os.walk(EXT_DIR):
Torsten Grote's avatar
Torsten Grote committed
250
        for f in filenames:
251 252
            if '/.git' in root:
                continue
Torsten Grote's avatar
Torsten Grote committed
253 254 255
            jar_files.append(os.path.join(root, f))
    for file in jar_files:
        reset_time(file)
256
    jar_name = get_sources_file_name(versions)
257
    jar_path = os.path.abspath(jar_name)
258 259
    rel_paths = [os.path.relpath(f, EXT_DIR) for f in sorted(jar_files)]
    check_call(['jar', 'cf', jar_path] + rel_paths, cwd=EXT_DIR)
akwizgran's avatar
akwizgran committed
260
    return jar_name
261 262


263
def create_pom_file(versions, android=False):
264
    tor_version = get_tor_version(versions)
265 266
    pom_name = get_pom_file_name(versions, android)
    template = 'template-android.pom' if android else 'template.pom'
267
    with open(template, 'rt') as infile:
268
        with open(pom_name, 'wt') as outfile:
269 270
            for line in infile:
                outfile.write(line.replace('VERSION', tor_version))
akwizgran's avatar
akwizgran committed
271
    return pom_name
272 273


Torsten Grote's avatar
Torsten Grote committed
274
if __name__ == "__main__":
Torsten Grote's avatar
Torsten Grote committed
275
    main()