build-tor.py 9.9 KB
Newer Older
Torsten Grote's avatar
Torsten Grote committed
1 2
#!/usr/bin/env python3
import os
3
from shutil import move, copy, rmtree
Torsten Grote's avatar
Torsten Grote committed
4 5
from subprocess import check_call

Torsten Grote's avatar
Torsten Grote committed
6 7
from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_tor_version, \
    get_final_file_name, get_sources_file_name, get_pom_file_name, get_version
Torsten Grote's avatar
Torsten Grote committed
8 9 10 11 12

NDK_DIR = 'android-ndk'


def main():
13 14
    # get Tor version from command or show usage information
    version = get_version()
Torsten Grote's avatar
Torsten Grote committed
15 16

    # get Tor version and versions of its dependencies
17 18
    versions = get_build_versions(version)
    print("Building Tor %s" % versions['tor'])
Torsten Grote's avatar
Torsten Grote committed
19 20 21 22 23 24 25 26 27 28 29 30

    # setup Android NDK
    setup_android_ndk(versions)

    # clone and checkout tor-android repo based on tor-versions.json
    prepare_tor_android_repo(versions)

    # build Tor for various architectures
    build_architectures()

    # zip geoip database
    geoip_path = os.path.join(REPO_DIR, 'external', 'tor', 'src', 'config', 'geoip')
31 32
    reset_time(geoip_path)
    check_call(['zip', '-D', '-X', os.path.join(REPO_DIR, 'geoip.zip'), geoip_path])
Torsten Grote's avatar
Torsten Grote committed
33

34
    # zip Android binaries together
Torsten Grote's avatar
Torsten Grote committed
35
    file_list = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip', 'geoip.zip']
36
    zip_name = pack(versions, file_list, 'android')
Torsten Grote's avatar
Torsten Grote committed
37

38 39 40
    # zip Linux binaries together
    file_list_linux = ['tor_linux-x86_64.zip', 'geoip.zip']
    zip_name_linux = pack(versions, file_list_linux, 'linux')
akwizgran's avatar
akwizgran committed
41 42

    # create POM file from template
43 44 45 46 47 48 49
    pom_name = create_pom_file(versions, 'android')
    pom_name_linux = create_pom_file(versions, 'linux')

    # create sources jar
    jar_name = create_sources_jar(versions)
    jar_name_linux = get_sources_file_name(versions, 'linux')
    copy(os.path.join(REPO_DIR, jar_name), os.path.join(REPO_DIR, jar_name_linux))
akwizgran's avatar
akwizgran committed
50

51
    # print Android hashes for debug purposes
akwizgran's avatar
akwizgran committed
52
    for file in file_list + [zip_name, jar_name, pom_name]:
Torsten Grote's avatar
Torsten Grote committed
53 54 55
        sha256hash = get_sha256(os.path.join(REPO_DIR, file))
        print("%s: %s" % (file, sha256hash))

56 57 58 59 60
    # print Linux hashes for debug purposes
    for file in file_list_linux + [zip_name_linux, jar_name_linux, pom_name_linux]:
        sha256hash = get_sha256(os.path.join(REPO_DIR, file))
        print("%s: %s" % (file, sha256hash))

Torsten Grote's avatar
Torsten Grote committed
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100

def setup_android_ndk(versions):
    if os.path.isdir(NDK_DIR):
        # check that we are using the correct NDK
        from configparser import ConfigParser
        config = ConfigParser()
        with open(os.path.join(NDK_DIR, 'source.properties'), 'r') as f:
            config.read_string('[default]\n' + f.read())
            revision = config.get('default', 'Pkg.Revision')

        if revision != versions['ndk']['revision']:
            print("Existing Android NDK has unexpected revision. Deleting...")
            rmtree(NDK_DIR)

    if not os.path.isdir(NDK_DIR):
        # download Android NDK
        print("Downloading Android NDK...")
        check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip'])

        # check sha256 hash on downloaded file
        if get_sha256('android-ndk.zip') != versions['ndk']['sha256']:
            fail("Android NDK checksum does not match")

        # install the NDK
        print("Unpacking Android NDK...")
        ndk_dir_tmp = NDK_DIR + '-tmp'
        check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
        content = os.listdir(ndk_dir_tmp)
        if len(content) == 1 and content[0].startswith('android-ndk-r'):
            move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
            os.rmdir(ndk_dir_tmp)
        else:
            fail("Could not extract NDK: %s" % str(content))

    os.putenv('ANDROID_NDK_HOME', os.path.abspath(NDK_DIR))


def prepare_tor_android_repo(versions):
    if os.path.isdir(REPO_DIR):
        # get latest commits and tags from remote
101
        check_call(['git', 'fetch', '--recurse-submodules=yes', 'origin'], cwd=REPO_DIR)
Torsten Grote's avatar
Torsten Grote committed
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
    else:
        # clone repo
        url = versions['tor_android_repo_url']
        check_call(['git', 'clone', '--recurse-submodules', url, REPO_DIR])

    # checkout tor-android version
    check_call(['git', 'checkout', '-f', versions['tor-android']], cwd=REPO_DIR)

    # undo all changes
    check_call(['git', 'reset', '--hard'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=REPO_DIR)

    # clean all untracked files and directories (-d) from repo
    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

    # check out versions of external dependencies
    checkout('tor', versions['tor'], 'external/tor')
    checkout('libevent', versions['libevent'], 'external/libevent')
    checkout('openssl', versions['openssl'], 'external/openssl')
    checkout('xz', versions['xz'], 'external/xz')
    checkout('zstd', versions['zstd'], 'external/zstd')


def checkout(name, tag, path):
    print("Checking out %s: %s" % (name, tag))
    repo_path = os.path.join(REPO_DIR, path)
    check_call(['git', 'checkout', '-f', tag], cwd=repo_path)


def build_architectures():
133 134
    build_linux()

Torsten Grote's avatar
Torsten Grote committed
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
    # build arm pie
    os.unsetenv('APP_ABI')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
    build_arch('tor_arm_pie.zip')

    # build arm
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
    build_arch('tor_arm.zip')

    # build x86 pie
    os.putenv('APP_ABI', 'x86')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
    build_arch('tor_x86_pie.zip')

    # build x86
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
    build_arch('tor_x86.zip')


def build_arch(name):
    check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR)
    copy(os.path.join(REPO_DIR, 'external', 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
162 163 164
    tor_path = os.path.join(REPO_DIR, 'tor')
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
165 166 167
    check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)


168
def build_linux(name='tor_linux-x86_64.zip'):
169 170 171 172
    # ensure clean build environment (again here to protect against build reordering)
    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

173 174
    # create folders for static libraries
    ext_dir = os.path.abspath(os.path.join(REPO_DIR, 'external'))
175 176 177 178
    prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix'))
    if not os.path.exists(prefix_dir):
        os.mkdir(prefix_dir)
    lib_dir = os.path.join(prefix_dir, 'lib')
179 180
    if not os.path.exists(lib_dir):
        os.mkdir(lib_dir)
181
    include_dir = os.path.join(prefix_dir, 'include')
182 183 184 185 186
    if not os.path.exists(include_dir):
        os.mkdir(include_dir)

    # setup environment
    env = os.environ.copy()
187 188
    env['PREFIX'] = prefix_dir
    env['LDFLAGS'] = "-L%s" % prefix_dir
189 190 191 192 193
    env['CFLAGS'] = "-fPIC -I%s" % include_dir
    env['LIBS'] = "-L%s" % lib_dir

    # build openssl
    openssl_dir = os.path.join(ext_dir, 'openssl')
194 195 196
    check_call(['./config', '--prefix=%s' % prefix_dir], cwd=openssl_dir, env=env)
    check_call(['make'], cwd=openssl_dir, env=env)
    check_call(['make', 'install_sw'], cwd=openssl_dir, env=env)
197 198

    # build libevent
199
    libevent_dir = os.path.join(ext_dir, 'libevent')
200
    check_call(['./autogen.sh'], cwd=libevent_dir)
201 202 203 204
    check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir], cwd=libevent_dir,
               env=env)
    check_call(['make'], cwd=libevent_dir, env=env)
    check_call(['make', 'install'], cwd=libevent_dir, env=env)
205 206

    # build Tor
207
    tor_dir = os.path.join(ext_dir, 'tor')
208 209 210
    check_call(['./autogen.sh'], cwd=tor_dir)
    env['CFLAGS'] += ' -O3'  # needed for FORTIFY_SOURCE
    check_call(['./configure', '--disable-asciidoc', '--disable-systemd',
211 212 213 214
                '--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir,
                '--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir,
                '--prefix=%s' % prefix_dir], cwd=tor_dir, env=env)
    check_call(['make', 'install'], cwd=tor_dir, env=env)
215 216 217

    # copy and zip built Tor binary
    tor_path = os.path.join(REPO_DIR, 'tor')
218
    copy(os.path.join(prefix_dir, 'bin', 'tor'), tor_path)
219 220 221 222 223 224 225 226 227 228 229 230 231 232
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
    check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)


def pack(versions, file_list, platform):
    for filename in file_list:
        reset_time(os.path.join(REPO_DIR, filename))  # make file times deterministic before zipping
    zip_name = get_final_file_name(versions, platform)
    check_call(['zip', '-D', '-X', zip_name] + file_list, cwd=REPO_DIR)
    return zip_name


233 234
def reset_time(filename):
    check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename])
Torsten Grote's avatar
Torsten Grote committed
235 236


237 238 239 240
def create_sources_jar(versions):
    external_dir = os.path.join(REPO_DIR, 'external')
    check_call(['git', 'clean', '-dfx'], cwd=external_dir)
    jar_files = []
Torsten Grote's avatar
Torsten Grote committed
241 242 243 244 245
    for root, dir_names, filenames in os.walk(external_dir):
        for f in filenames:
            jar_files.append(os.path.join(root, f))
    for file in jar_files:
        reset_time(file)
246 247 248 249
    jar_name = get_sources_file_name(versions)
    jar_path = os.path.abspath(os.path.join(REPO_DIR, jar_name))
    rel_paths = [os.path.relpath(f, external_dir) for f in sorted(jar_files)]
    check_call(['jar', 'cf', jar_path] + rel_paths, cwd=external_dir)
akwizgran's avatar
akwizgran committed
250
    return jar_name
251 252


253
def create_pom_file(versions, platform='android'):
254
    tor_version = get_tor_version(versions)
255 256
    pom_name = get_pom_file_name(versions, platform)
    if platform == 'android':
257
        template = 'template-android.pom'
258
    else:
259
        template = 'template.pom'
260
    with open(template, 'rt') as infile:
261 262 263
        with open(os.path.join(REPO_DIR, pom_name), 'wt') as outfile:
            for line in infile:
                outfile.write(line.replace('VERSION', tor_version))
akwizgran's avatar
akwizgran committed
264
    return pom_name
265 266


Torsten Grote's avatar
Torsten Grote committed
267
if __name__ == "__main__":
Torsten Grote's avatar
Torsten Grote committed
268
    main()