build-tor.py 10.8 KB
Newer Older
Torsten Grote's avatar
Torsten Grote committed
1 2
#!/usr/bin/env python3
import os
3
from shutil import move, copy, rmtree
Torsten Grote's avatar
Torsten Grote committed
4 5
from subprocess import check_call

6
from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_version_tag, \
Torsten Grote's avatar
Torsten Grote committed
7
    get_final_file_name, get_sources_file_name, get_pom_file_name, get_version
Torsten Grote's avatar
Torsten Grote committed
8

9
ZLIB_REPO_URL = 'https://github.com/madler/zlib.git'
Torsten Grote's avatar
Torsten Grote committed
10
NDK_DIR = 'android-ndk'
11
EXT_DIR = os.path.abspath(os.path.join(REPO_DIR, 'external'))
Torsten Grote's avatar
Torsten Grote committed
12 13 14


def main():
15 16
    # get Tor version from command or show usage information
    version = get_version()
Torsten Grote's avatar
Torsten Grote committed
17 18

    # get Tor version and versions of its dependencies
19 20
    versions = get_build_versions(version)
    print("Building Tor %s" % versions['tor'])
Torsten Grote's avatar
Torsten Grote committed
21 22 23 24 25 26 27

    # setup Android NDK
    setup_android_ndk(versions)

    # clone and checkout tor-android repo based on tor-versions.json
    prepare_tor_android_repo(versions)

28 29 30
    # create sources jar before building
    jar_name = create_sources_jar(versions)

31 32 33
    # build Tor for various platforms and architectures
    build()
    build_android()
Torsten Grote's avatar
Torsten Grote committed
34 35

    # zip geoip database
36 37
    geoip_path = os.path.join(REPO_DIR, 'geoip')
    copy(os.path.join(EXT_DIR, 'tor', 'src', 'config', 'geoip'), geoip_path)
38
    reset_time(geoip_path)
39
    check_call(['zip', '-X', '../geoip.zip', 'geoip'], cwd=REPO_DIR)
Torsten Grote's avatar
Torsten Grote committed
40

41 42 43
    # zip binaries together
    file_list = ['tor_linux-x86_64.zip', 'geoip.zip']
    zip_name = pack(versions, file_list)
44
    # zip Android binaries together
45 46
    file_list_android = ['tor_arm.zip', 'tor_arm_pie.zip', 'tor_arm64_pie.zip',
                         'tor_x86.zip', 'tor_x86_pie.zip', 'tor_x86_64_pie.zip',
47 48
                         'geoip.zip']
    zip_name_android = pack(versions, file_list_android, android=True)
akwizgran's avatar
akwizgran committed
49 50

    # create POM file from template
51 52
    pom_name = create_pom_file(versions)
    pom_name_android = create_pom_file(versions, android=True)
53

54
    # print hashes for debug purposes
akwizgran's avatar
akwizgran committed
55
    for file in file_list + [zip_name, jar_name, pom_name]:
56
        sha256hash = get_sha256(file)
Torsten Grote's avatar
Torsten Grote committed
57
        print("%s: %s" % (file, sha256hash))
58
    print("Android:")
59 60
    for file in file_list_android + [zip_name_android, pom_name_android]:
        sha256hash = get_sha256(file)
61 62
        print("%s: %s" % (file, sha256hash))

Torsten Grote's avatar
Torsten Grote committed
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96

def setup_android_ndk(versions):
    if os.path.isdir(NDK_DIR):
        # check that we are using the correct NDK
        from configparser import ConfigParser
        config = ConfigParser()
        with open(os.path.join(NDK_DIR, 'source.properties'), 'r') as f:
            config.read_string('[default]\n' + f.read())
            revision = config.get('default', 'Pkg.Revision')

        if revision != versions['ndk']['revision']:
            print("Existing Android NDK has unexpected revision. Deleting...")
            rmtree(NDK_DIR)

    if not os.path.isdir(NDK_DIR):
        # download Android NDK
        print("Downloading Android NDK...")
        check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip'])

        # check sha256 hash on downloaded file
        if get_sha256('android-ndk.zip') != versions['ndk']['sha256']:
            fail("Android NDK checksum does not match")

        # install the NDK
        print("Unpacking Android NDK...")
        ndk_dir_tmp = NDK_DIR + '-tmp'
        check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
        content = os.listdir(ndk_dir_tmp)
        if len(content) == 1 and content[0].startswith('android-ndk-r'):
            move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
            os.rmdir(ndk_dir_tmp)
        else:
            fail("Could not extract NDK: %s" % str(content))

97
    os.environ['ANDROID_NDK_HOME'] = os.path.abspath(NDK_DIR)
Torsten Grote's avatar
Torsten Grote committed
98 99 100 101 102


def prepare_tor_android_repo(versions):
    if os.path.isdir(REPO_DIR):
        # get latest commits and tags from remote
103
        check_call(['git', 'fetch', '--recurse-submodules=yes', 'origin'], cwd=REPO_DIR)
Torsten Grote's avatar
Torsten Grote committed
104 105 106
    else:
        # clone repo
        url = versions['tor_android_repo_url']
107
        check_call(['git', 'clone', url, REPO_DIR])
Torsten Grote's avatar
Torsten Grote committed
108 109 110 111

    # checkout tor-android version
    check_call(['git', 'checkout', '-f', versions['tor-android']], cwd=REPO_DIR)

112 113 114 115
    # initialize and/or update submodules
    # (after checkout, because submodules can point to non-existent commits on master)
    check_call(['git', 'submodule', 'update', '--init', '--recursive'], cwd=REPO_DIR)

Torsten Grote's avatar
Torsten Grote committed
116 117 118 119 120 121 122 123
    # undo all changes
    check_call(['git', 'reset', '--hard'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=REPO_DIR)

    # clean all untracked files and directories (-d) from repo
    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

124 125 126
    # add zlib
    check_call(['git', 'clone', ZLIB_REPO_URL], cwd=EXT_DIR)

Torsten Grote's avatar
Torsten Grote committed
127
    # check out versions of external dependencies
128 129 130 131 132 133
    checkout('tor', versions['tor'], 'tor')
    checkout('libevent', versions['libevent'], 'libevent')
    checkout('openssl', versions['openssl'], 'openssl')
    checkout('xz', versions['xz'], 'xz')
    checkout('zlib', versions['zlib'], 'zlib')
    checkout('zstd', versions['zstd'], 'zstd')
Torsten Grote's avatar
Torsten Grote committed
134 135 136 137


def checkout(name, tag, path):
    print("Checking out %s: %s" % (name, tag))
138
    repo_path = os.path.join(EXT_DIR, path)
Torsten Grote's avatar
Torsten Grote committed
139 140 141
    check_call(['git', 'checkout', '-f', tag], cwd=repo_path)


142
def build_android():
143
    os.environ.pop("PIEFLAGS", None)  # uses default PIE flags, if not present
Torsten Grote's avatar
Torsten Grote committed
144 145

    # build arm
146 147 148 149 150
    env = os.environ.copy()
    env['APP_ABI'] = "armeabi-v7a"
    env['NDK_PLATFORM_LEVEL'] = "14"
    env['PIEFLAGS'] = ""  # do not use default PIE flags
    build_android_arch('tor_arm.zip', env)
Torsten Grote's avatar
Torsten Grote committed
151

152 153 154 155 156 157 158 159 160 161 162
    # build arm pie
    env = os.environ.copy()
    env['APP_ABI'] = "armeabi-v7a"
    env['NDK_PLATFORM_LEVEL'] = "16"  # first level supporting PIE
    build_android_arch('tor_arm_pie.zip', env)

    # build arm64 pie
    env = os.environ.copy()
    env['APP_ABI'] = "arm64-v8a"
    env['NDK_PLATFORM_LEVEL'] = "21"  # first level supporting 64-bit
    build_android_arch('tor_arm64_pie.zip', env)
Torsten Grote's avatar
Torsten Grote committed
163 164

    # build x86
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
    env = os.environ.copy()
    env['APP_ABI'] = "x86"
    env['NDK_PLATFORM_LEVEL'] = "14"
    env['PIEFLAGS'] = ""  # do not use default PIE flags
    build_android_arch('tor_x86.zip', env)

    # build x86 pie
    env = os.environ.copy()
    env['APP_ABI'] = "x86"
    env['NDK_PLATFORM_LEVEL'] = "16"  # first level supporting PIE
    build_android_arch('tor_x86_pie.zip', env)

    # build x86_64 pie
    env = os.environ.copy()
    env['APP_ABI'] = "x86_64"
    env['NDK_PLATFORM_LEVEL'] = "21"  # first level supporting 64-bit
    build_android_arch('tor_x86_64_pie.zip', env)
Torsten Grote's avatar
Torsten Grote committed
182 183


184 185 186
def build_android_arch(name, env):
    print("Building %s" % name)
    check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR, env=env)
187
    copy(os.path.join(EXT_DIR, 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
Torsten Grote's avatar
Torsten Grote committed
188
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
189 190 191
    tor_path = os.path.join(REPO_DIR, 'tor')
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
192
    check_call(['zip', '-X', '../' + name, 'tor'], cwd=REPO_DIR)
193 194


195
def build(name='tor_linux-x86_64.zip'):
196 197 198 199
    prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix'))
    lib_dir = os.path.join(prefix_dir, 'lib')
    include_dir = os.path.join(prefix_dir, 'include')

200 201
    # ensure clean build environment (again here to protect against build reordering)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)
202 203
    if os.path.exists(prefix_dir):
        rmtree(prefix_dir)
204

205
    # create folders for static libraries
206 207 208
    os.mkdir(prefix_dir)
    os.mkdir(lib_dir)
    os.mkdir(include_dir)
209 210 211

    # setup environment
    env = os.environ.copy()
212
    env['LDFLAGS'] = "-L%s" % prefix_dir
213 214 215
    env['CFLAGS'] = "-fPIC -I%s" % include_dir
    env['LIBS'] = "-L%s" % lib_dir

216 217 218 219 220
    # build zlib
    zlib_dir = os.path.join(EXT_DIR, 'zlib')
    check_call(['./configure', '--prefix=%s' % prefix_dir], cwd=zlib_dir, env=env)
    check_call(['make', 'install'], cwd=zlib_dir, env=env)

221
    # build openssl
222
    openssl_dir = os.path.join(EXT_DIR, 'openssl')
223 224 225
    check_call(['./config', '--prefix=%s' % prefix_dir], cwd=openssl_dir, env=env)
    check_call(['make'], cwd=openssl_dir, env=env)
    check_call(['make', 'install_sw'], cwd=openssl_dir, env=env)
226 227

    # build libevent
228
    libevent_dir = os.path.join(EXT_DIR, 'libevent')
229
    check_call(['./autogen.sh'], cwd=libevent_dir)
230 231 232 233
    check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir], cwd=libevent_dir,
               env=env)
    check_call(['make'], cwd=libevent_dir, env=env)
    check_call(['make', 'install'], cwd=libevent_dir, env=env)
234 235

    # build Tor
236
    tor_dir = os.path.join(EXT_DIR, 'tor')
237 238 239
    check_call(['./autogen.sh'], cwd=tor_dir)
    env['CFLAGS'] += ' -O3'  # needed for FORTIFY_SOURCE
    check_call(['./configure', '--disable-asciidoc', '--disable-systemd',
240
                '--enable-static-zlib', '--with-zlib-dir=%s' % prefix_dir,
241 242 243 244
                '--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir,
                '--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir,
                '--prefix=%s' % prefix_dir], cwd=tor_dir, env=env)
    check_call(['make', 'install'], cwd=tor_dir, env=env)
245 246 247

    # copy and zip built Tor binary
    tor_path = os.path.join(REPO_DIR, 'tor')
248
    copy(os.path.join(prefix_dir, 'bin', 'tor'), tor_path)
249 250 251
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
252
    check_call(['zip', '-X', '../' + name, 'tor'], cwd=REPO_DIR)
253 254


255
def pack(versions, file_list, android=False):
256
    for filename in file_list:
257
        reset_time(filename)  # make file times deterministic before zipping
258
    zip_name = get_final_file_name(versions, android)
259
    check_call(['zip', '-D', '-X', zip_name] + file_list)
260 261 262
    return zip_name


263 264
def reset_time(filename):
    check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename])
Torsten Grote's avatar
Torsten Grote committed
265 266


267 268
def create_sources_jar(versions):
    jar_files = []
269
    for root, dir_names, filenames in os.walk(EXT_DIR):
Torsten Grote's avatar
Torsten Grote committed
270
        for f in filenames:
271 272
            if '/.git' in root:
                continue
Torsten Grote's avatar
Torsten Grote committed
273 274 275
            jar_files.append(os.path.join(root, f))
    for file in jar_files:
        reset_time(file)
276
    jar_name = get_sources_file_name(versions)
277
    jar_path = os.path.abspath(jar_name)
278 279
    rel_paths = [os.path.relpath(f, EXT_DIR) for f in sorted(jar_files)]
    check_call(['jar', 'cf', jar_path] + rel_paths, cwd=EXT_DIR)
akwizgran's avatar
akwizgran committed
280
    return jar_name
281 282


283
def create_pom_file(versions, android=False):
284
    version = get_version_tag(versions)
285 286
    pom_name = get_pom_file_name(versions, android)
    template = 'template-android.pom' if android else 'template.pom'
287
    with open(template, 'rt') as infile:
288
        with open(pom_name, 'wt') as outfile:
289
            for line in infile:
290
                outfile.write(line.replace('VERSION', version))
akwizgran's avatar
akwizgran committed
291
    return pom_name
292 293


Torsten Grote's avatar
Torsten Grote committed
294
if __name__ == "__main__":
Torsten Grote's avatar
Torsten Grote committed
295
    main()