build-tor.py 11 KB
Newer Older
Torsten Grote's avatar
Torsten Grote committed
1 2
#!/usr/bin/env python3
import os
3
from shutil import move, copy, rmtree, copytree
Torsten Grote's avatar
Torsten Grote committed
4 5
from subprocess import check_call

Torsten Grote's avatar
Torsten Grote committed
6 7
from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_tor_version, \
    get_final_file_name, get_sources_file_name, get_pom_file_name, get_version
Torsten Grote's avatar
Torsten Grote committed
8 9 10 11 12

NDK_DIR = 'android-ndk'


def main():
13 14
    # get Tor version from command or show usage information
    version = get_version()
Torsten Grote's avatar
Torsten Grote committed
15 16

    # get Tor version and versions of its dependencies
17 18
    versions = get_build_versions(version)
    print("Building Tor %s" % versions['tor'])
Torsten Grote's avatar
Torsten Grote committed
19 20 21 22 23 24 25 26 27 28 29 30

    # setup Android NDK
    setup_android_ndk(versions)

    # clone and checkout tor-android repo based on tor-versions.json
    prepare_tor_android_repo(versions)

    # build Tor for various architectures
    build_architectures()

    # zip geoip database
    geoip_path = os.path.join(REPO_DIR, 'external', 'tor', 'src', 'config', 'geoip')
31 32
    reset_time(geoip_path)
    check_call(['zip', '-D', '-X', os.path.join(REPO_DIR, 'geoip.zip'), geoip_path])
Torsten Grote's avatar
Torsten Grote committed
33

34
    # zip Android binaries together
Torsten Grote's avatar
Torsten Grote committed
35
    file_list = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip', 'geoip.zip']
36
    zip_name = pack(versions, file_list, 'android')
Torsten Grote's avatar
Torsten Grote committed
37

38 39 40
    # zip Linux binaries together
    file_list_linux = ['tor_linux-x86_64.zip', 'geoip.zip']
    zip_name_linux = pack(versions, file_list_linux, 'linux')
akwizgran's avatar
akwizgran committed
41 42

    # create POM file from template
43 44 45 46 47 48 49
    pom_name = create_pom_file(versions, 'android')
    pom_name_linux = create_pom_file(versions, 'linux')

    # create sources jar
    jar_name = create_sources_jar(versions)
    jar_name_linux = get_sources_file_name(versions, 'linux')
    copy(os.path.join(REPO_DIR, jar_name), os.path.join(REPO_DIR, jar_name_linux))
akwizgran's avatar
akwizgran committed
50

51
    # print Android hashes for debug purposes
akwizgran's avatar
akwizgran committed
52
    for file in file_list + [zip_name, jar_name, pom_name]:
Torsten Grote's avatar
Torsten Grote committed
53 54 55
        sha256hash = get_sha256(os.path.join(REPO_DIR, file))
        print("%s: %s" % (file, sha256hash))

56 57 58 59 60
    # print Linux hashes for debug purposes
    for file in file_list_linux + [zip_name_linux, jar_name_linux, pom_name_linux]:
        sha256hash = get_sha256(os.path.join(REPO_DIR, file))
        print("%s: %s" % (file, sha256hash))

Torsten Grote's avatar
Torsten Grote committed
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100

def setup_android_ndk(versions):
    if os.path.isdir(NDK_DIR):
        # check that we are using the correct NDK
        from configparser import ConfigParser
        config = ConfigParser()
        with open(os.path.join(NDK_DIR, 'source.properties'), 'r') as f:
            config.read_string('[default]\n' + f.read())
            revision = config.get('default', 'Pkg.Revision')

        if revision != versions['ndk']['revision']:
            print("Existing Android NDK has unexpected revision. Deleting...")
            rmtree(NDK_DIR)

    if not os.path.isdir(NDK_DIR):
        # download Android NDK
        print("Downloading Android NDK...")
        check_call(['wget', '-c', '--no-verbose', versions['ndk']['url'], '-O', 'android-ndk.zip'])

        # check sha256 hash on downloaded file
        if get_sha256('android-ndk.zip') != versions['ndk']['sha256']:
            fail("Android NDK checksum does not match")

        # install the NDK
        print("Unpacking Android NDK...")
        ndk_dir_tmp = NDK_DIR + '-tmp'
        check_call(['unzip', '-q', 'android-ndk.zip', '-d', ndk_dir_tmp])
        content = os.listdir(ndk_dir_tmp)
        if len(content) == 1 and content[0].startswith('android-ndk-r'):
            move(os.path.join(ndk_dir_tmp, content[0]), NDK_DIR)
            os.rmdir(ndk_dir_tmp)
        else:
            fail("Could not extract NDK: %s" % str(content))

    os.putenv('ANDROID_NDK_HOME', os.path.abspath(NDK_DIR))


def prepare_tor_android_repo(versions):
    if os.path.isdir(REPO_DIR):
        # get latest commits and tags from remote
101
        check_call(['git', 'fetch', '--recurse-submodules=yes', 'origin'], cwd=REPO_DIR)
Torsten Grote's avatar
Torsten Grote committed
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
    else:
        # clone repo
        url = versions['tor_android_repo_url']
        check_call(['git', 'clone', '--recurse-submodules', url, REPO_DIR])

    # checkout tor-android version
    check_call(['git', 'checkout', '-f', versions['tor-android']], cwd=REPO_DIR)

    # undo all changes
    check_call(['git', 'reset', '--hard'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'reset', '--hard'], cwd=REPO_DIR)

    # clean all untracked files and directories (-d) from repo
    check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

    # check out versions of external dependencies
    checkout('tor', versions['tor'], 'external/tor')
    checkout('libevent', versions['libevent'], 'external/libevent')
    checkout('openssl', versions['openssl'], 'external/openssl')
    checkout('xz', versions['xz'], 'external/xz')
    checkout('zstd', versions['zstd'], 'external/zstd')


def checkout(name, tag, path):
    print("Checking out %s: %s" % (name, tag))
    repo_path = os.path.join(REPO_DIR, path)
    check_call(['git', 'checkout', '-f', tag], cwd=repo_path)


def build_architectures():
133 134
    build_linux()

Torsten Grote's avatar
Torsten Grote committed
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
    # build arm pie
    os.unsetenv('APP_ABI')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
    build_arch('tor_arm_pie.zip')

    # build arm
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
    build_arch('tor_arm.zip')

    # build x86 pie
    os.putenv('APP_ABI', 'x86')
    os.unsetenv('NDK_PLATFORM_LEVEL')
    os.unsetenv('PIEFLAGS')
    build_arch('tor_x86_pie.zip')

    # build x86
    os.putenv('NDK_PLATFORM_LEVEL', '14')
    os.putenv('PIEFLAGS', '')
    build_arch('tor_x86.zip')


def build_arch(name):
    check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR)
    copy(os.path.join(REPO_DIR, 'external', 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
162 163 164
    tor_path = os.path.join(REPO_DIR, 'tor')
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
165 166 167
    check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)


168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
def build_linux(name='tor_linux-x86_64.zip'):
    # create folders for static libraries
    ext_dir = os.path.abspath(os.path.join(REPO_DIR, 'external'))
    lib_dir = os.path.join(ext_dir, 'lib')
    if not os.path.exists(lib_dir):
        os.mkdir(lib_dir)
    include_dir = os.path.join(ext_dir, 'include')
    if not os.path.exists(include_dir):
        os.mkdir(include_dir)

    # setup environment
    env = os.environ.copy()
    env['LDFLAGS'] = "-L%s" % ext_dir
    env['CFLAGS'] = "-fPIC -I%s" % include_dir
    env['LIBS'] = "-L%s" % lib_dir
    env['PKG_PATH'] = include_dir

    # ensure clean build environment
    check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)

    # build lzma
    xz_dir = os.path.join(ext_dir, 'xz')
    check_call(['./autogen.sh'], cwd=xz_dir)
    check_call(['./configure', '--disable-shared', '--enable-static', '--disable-doc',
                '--disable-xz', '--disable-xzdec', '--disable-lzmadec', '--disable-lzmainfo',
                '--disable-lzma-links', '--disable-scripts', '--prefix=%s' % ext_dir],
               cwd=xz_dir, env=env)
    check_call(['make', 'install'], cwd=xz_dir)

    # build zstd
    zstd_dir = os.path.join(ext_dir, 'zstd', 'lib')
    check_call(['make', 'libzstd.a-mt'], cwd=zstd_dir)
    check_call(['make', 'libzstd.pc'], cwd=zstd_dir)
    copy(os.path.join(zstd_dir, 'libzstd.a'), lib_dir)
    copy(os.path.join(zstd_dir, 'libzstd.pc'), os.path.join(lib_dir, 'pkgconfig'))
    copy(os.path.join(zstd_dir, 'zstd.h'), include_dir)
    copy(os.path.join(zstd_dir, 'common', 'zstd_errors.h'), include_dir)
    copy(os.path.join(zstd_dir, 'deprecated', 'zbuff.h'), include_dir)
    copy(os.path.join(zstd_dir, 'dictBuilder', 'zdict.h'), include_dir)

    # build openssl
    openssl_dir = os.path.join(ext_dir, 'openssl')
    check_call(['perl', 'Configure', 'linux-x86_64', '-fPIC'], cwd=openssl_dir, env=env)
    check_call(['make', 'depend'], cwd=openssl_dir)
    check_call(['make', 'build_libs'], cwd=openssl_dir)
    copy(os.path.join(openssl_dir, 'libcrypto.a'), os.path.join(lib_dir, 'libcrypto.a'))
    copy(os.path.join(openssl_dir, 'libssl.a'), os.path.join(lib_dir, 'libssl.a'))
    copytree(os.path.join(openssl_dir, 'include', 'openssl'), os.path.join(include_dir, 'openssl'))

    # build libevent
    libevent_dir = os.path.join(REPO_DIR, 'external', 'libevent')
    check_call(['./autogen.sh'], cwd=libevent_dir)
    check_call(['./configure', '--disable-shared'], cwd=libevent_dir, env=env)
    check_call(['make', './include/event2/event-config.h', 'all-am'], cwd=libevent_dir)
    copy(os.path.join(libevent_dir, '.libs', 'libevent.a'), os.path.join(lib_dir, 'libevent.a'))
    copytree(os.path.join(libevent_dir, 'include', 'event2'), os.path.join(include_dir, 'event2'))

    # build Tor
    tor_dir = os.path.join(REPO_DIR, 'external', 'tor')
    check_call(['./autogen.sh'], cwd=tor_dir)
    env['CFLAGS'] += ' -O3'  # needed for FORTIFY_SOURCE
    check_call(['./configure', '--disable-asciidoc', '--disable-systemd',
                '--enable-static-libevent', '--with-libevent-dir=%s' % ext_dir,
                '--enable-static-openssl', '--with-openssl-dir=%s' % ext_dir], cwd=tor_dir, env=env)
    check_call(['make', 'all-am'], cwd=tor_dir)

    # copy and zip built Tor binary
    tor_path = os.path.join(REPO_DIR, 'tor')
    copy(os.path.join(tor_dir, 'src', 'or', 'tor'), tor_path)
    check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
    reset_time(tor_path)
    print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
    check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)


def pack(versions, file_list, platform):
    for filename in file_list:
        reset_time(os.path.join(REPO_DIR, filename))  # make file times deterministic before zipping
    zip_name = get_final_file_name(versions, platform)
    check_call(['zip', '-D', '-X', zip_name] + file_list, cwd=REPO_DIR)
    return zip_name


251 252
def reset_time(filename):
    check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename])
Torsten Grote's avatar
Torsten Grote committed
253 254


255 256 257 258
def create_sources_jar(versions):
    external_dir = os.path.join(REPO_DIR, 'external')
    check_call(['git', 'clean', '-dfx'], cwd=external_dir)
    jar_files = []
Torsten Grote's avatar
Torsten Grote committed
259 260 261 262 263
    for root, dir_names, filenames in os.walk(external_dir):
        for f in filenames:
            jar_files.append(os.path.join(root, f))
    for file in jar_files:
        reset_time(file)
264 265 266 267
    jar_name = get_sources_file_name(versions)
    jar_path = os.path.abspath(os.path.join(REPO_DIR, jar_name))
    rel_paths = [os.path.relpath(f, external_dir) for f in sorted(jar_files)]
    check_call(['jar', 'cf', jar_path] + rel_paths, cwd=external_dir)
akwizgran's avatar
akwizgran committed
268
    return jar_name
269 270


271
def create_pom_file(versions, platform='android'):
272
    tor_version = get_tor_version(versions)
273 274
    pom_name = get_pom_file_name(versions, platform)
    if platform == 'android':
275
        template = 'template-android.pom'
276
    else:
277
        template = 'template.pom'
278
    with open(template, 'rt') as infile:
279 280 281
        with open(os.path.join(REPO_DIR, pom_name), 'wt') as outfile:
            for line in infile:
                outfile.write(line.replace('VERSION', tor_version))
akwizgran's avatar
akwizgran committed
282
    return pom_name
283 284


Torsten Grote's avatar
Torsten Grote committed
285
if __name__ == "__main__":
Torsten Grote's avatar
Torsten Grote committed
286
    main()