Build macOS binaries using Tor upstream build system

.gitlab-ci.yml

@@ -11,6 +11,7 @@ stages:
 variables:
   TEST_IMAGE: briar/tor-reproducer:${CI_BUILD_REF_NAME}
   RELEASE_IMAGE: briar/tor-reproducer:latest
+  UPSTREAM_IMAGE: briar/tor-upstream-builder
 
 before_script:
   - echo ${DOCKER_HUB_PASS} | docker login -u ${DOCKER_HUB_USER} --password-stdin
@@ -45,6 +46,14 @@ build:
     expire_in: 1 week
     when: always
 
+.base-mac:
+  stage: test
+  artifacts:
+    paths:
+      - output/macos
+    expire_in: 1 week
+    when: always
+
 test_build_android:
   extends: .base-android
   script:
@@ -69,6 +78,15 @@ test_build_windows:
   except:
     - tags
 
+test_build_mac:
+  extends: .base-mac
+  script:
+    - docker build -t ${UPSTREAM_IMAGE} -f upstream/Dockerfile .
+    - docker run --privileged -v `pwd`/output:/opt/tor-reproducer/output ${UPSTREAM_IMAGE} /bin/bash -c "touch /dev/console && su builduser -c \"./build_tor_macos.py && ./verify_tor_macos.py\""
+  allow_failure: true
+  except:
+    - tags
+
 test_tag_android:
   extends: .base-android
   script:
@@ -90,6 +108,13 @@ test_tag_windows:
   only:
     - tags
 
+test_tag_macos:
+  extends: .base-mac
+  script:
+    - docker run --privileged -v `pwd`/output:/opt/tor-reproducer/output ${UPSTREAM_IMAGE} ./verify_tor_macos.py ${CI_BUILD_REF_NAME}
+  only:
+    - tags
+
 release:
   stage: release
   script:

template-macos.pom

+<?xml version="1.0" encoding="UTF-8"?>
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.briarproject</groupId>
+  <artifactId>tor-macos</artifactId>
+  <name>tor-macos</name>
+  <version>VERSION</version>
+  <url>https://torproject.org</url>
+  <description>Repo for building Tor for macOS.</description>
+  <licenses>
+   <license>
+     <name>BSD-3-clause</name>
+     <url>https://gitweb.torproject.org/tor.git/tree/LICENSE</url>
+   </license>
+  </licenses>
+  <developers>
+    <developer>
+      <id>dingledine</id>
+      <name>Roger Dingledine</name>
+      <email>arma@mit.edu</email>
+    </developer>
+    <developer>
+      <id>mathewson</id>
+      <name>Nick Mathewson</name>
+      <email>nickm@torproject.org</email>
+    </developer>
+    <developer>
+      <id>torproject</id>
+      <name>Tor Project</name>
+      <email>frontdesk@rt.torproject.org</email>
+    </developer>
+  </developers>
+  <scm>
+    <connection>scm:https://gitweb.torproject.org/tor.git/</connection>
+    <developerConnection>scm:git@gitweb.torproject.org/tor.git</developerConnection>
+    <url>scm:https://gitweb.torproject.org/tor.git</url>
+  </scm>
+</project>

tor-versions.json

@@ -29,6 +29,12 @@
       "revision": "21.4.7075529",
       "sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e"
     },
+    "upstream": {
+      "url": "https://gitlab.torproject.org/tpo/applications/tor-browser-build.git",
+      "commit": "b1b4bf77",
+      "tor-browser": "12.0.6",
+      "libevent": "2.1.7"
+    },
     "timestamp": "201001010000.00"
   },
   "0.4.7.13-1": {

upstream/Dockerfile

+FROM briar/tor-upstream-reproducer:latest
+
+ENV LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /opt/tor-reproducer
+
+ADD upstream/build_tor_macos.py ./
+ADD tor-versions.json ./
+ADD utils.py ./
+ADD template-macos.pom ./
+ADD verify_tor_macos.py ./
+ADD verify_tor_utils.py ./
+
+CMD ./build_tor_macos.py

upstream/build_tor_macos.py

+#!/usr/bin/env python3
+import os
+from shutil import copytree, rmtree
+from subprocess import check_call
+import hashlib
+
+import utils
+from utils import get_version, get_build_versions, reset_time, \
+    get_sources_file_name, get_output_dir, get_sha256, pack, create_pom_file
+from pathlib import Path
+import tarfile
+
+PLATFORM = "macos"
+BUILD_DIR = "tor-browser-build"
+
+
+def setup():
+    # get Tor version from command or show usage information
+    version = get_version()
+
+    # get versions of upstream repository and Tor browser for comparison
+    versions = get_build_versions(version)
+    print("Building Tor from upstream repository at commit %s" % versions['upstream']['commit'])
+
+    # remove output from previous build
+    output_dir = get_output_dir(PLATFORM)
+    print(output_dir)
+    os.makedirs(output_dir, exist_ok=True)
+
+    # clone tor-browser-build repo
+    check_call(['git', 'clone', versions['upstream']['url'], BUILD_DIR])
+    check_call(['git', 'checkout', versions['upstream']['commit']], cwd=Path(BUILD_DIR))
+    check_call(['git', 'submodule', 'init'], cwd=Path(BUILD_DIR))
+    check_call(['git', 'submodule', 'update'], cwd=Path(BUILD_DIR))
+
+    # create sources jar before building
+    jar_name = create_sources_jar(versions)
+
+    return versions, jar_name
+
+
+def build():
+    versions, jar_name = setup()
+
+    copytree('pre-out', os.path.join(BUILD_DIR, 'out'))
+    copytree('pre-clones', os.path.join(BUILD_DIR, 'git_clones'))
+
+    build_arch(versions, 'aarch64')
+    build_arch(versions, 'x86_64')
+
+    package_macos(versions, jar_name)
+    return versions
+
+
+def build_arch(versions, arch):
+    target = PLATFORM + '-' + arch
+    libevent_version = versions['upstream']['libevent']
+    # build using rbm
+    check_call(['./rbm/rbm', 'build', 'tor', '--target', 'release', '--target', 'torbrowser-' + target], cwd=Path(BUILD_DIR))
+    # extract tar.gz file
+    arch_dir = Path('output') / PLATFORM / arch
+    arch_dir.mkdir(parents=True, exist_ok=True)
+    tar_file = next(Path(BUILD_DIR).glob('out/tor/tor-*-' + target + '-*.tar.gz'))
+    tar = tarfile.open(tar_file, "r:gz")
+    tar.extractall(path=arch_dir)
+    tar.close()
+    # move contents out of tor/ directory
+    (arch_dir / 'tor').rename(arch_dir / 'tordir')
+    for file in (arch_dir / 'tordir').glob('*'):
+        file.rename(arch_dir / file.name)
+    (arch_dir / 'tordir').rmdir
+    rmtree(arch_dir / 'data')
+    # print hashsums
+    tor_file = arch_dir / 'tor'
+    libevent_file = arch_dir / ('libevent-' + libevent_version + '.dylib')
+    for file in [tor_file, libevent_file]:
+        sha256hash = get_sha256(file)
+        print("%s: %s" % (file, sha256hash))
+
+
+def package_macos(versions, jar_name):
+    libevent_version = versions['upstream']['libevent']
+    # zip binaries together
+    output_dir = get_output_dir(PLATFORM)
+    file_list = [
+        os.path.join(output_dir, 'aarch64', 'tor'),
+        os.path.join(output_dir, 'aarch64', 'libevent-' + libevent_version + '.dylib'),
+        os.path.join(output_dir, 'x86_64', 'tor'),
+        os.path.join(output_dir, 'x86_64', 'libevent-' + libevent_version + '.dylib'),
+    ]
+    zip_name = pack(versions, file_list, PLATFORM)
+    pom_name = create_pom_file(versions, PLATFORM)
+    print("%s:" % PLATFORM)
+    for file in file_list + [zip_name, jar_name, pom_name]:
+        sha256hash = get_sha256(file)
+        print("%s: %s" % (file, sha256hash))
+
+
+def create_sources_jar(versions):
+    output_dir = get_output_dir(PLATFORM)
+    jar_files = []
+    for root, dir_names, filenames in os.walk(BUILD_DIR):
+        for f in filenames:
+            if '/.git' in root:
+                continue
+            jar_files.append(os.path.join(root, f))
+    for file in jar_files:
+        reset_time(file, versions)
+    jar_name = get_sources_file_name(versions, PLATFORM)
+    jar_path = os.path.abspath(jar_name)
+    rel_paths = [os.path.relpath(f, BUILD_DIR) for f in sorted(jar_files)]
+    # create jar archive with first files
+    jar_step = 5000
+    check_call(['jar', 'cf', jar_path] + rel_paths[0:jar_step], cwd=BUILD_DIR)
+    # add subsequent files in steps, because the command line can't handle all at once
+    for i in range(jar_step, len(rel_paths), jar_step):
+        check_call(['jar', 'uf', jar_path] + rel_paths[i:i + jar_step], cwd=BUILD_DIR)
+    return jar_name
+
+
+def compare_output_with_upstream(versions):
+    compare_with_upstream(versions, "aarch64")
+    compare_with_upstream(versions, "x86_64")
+
+
+def compare_with_upstream(versions, arch):
+    print('comparing hashsums for {0}'.format(arch))
+    tor_browser_version = versions['upstream']['tor-browser']
+    libevent_version = versions['upstream']['libevent']
+    check_call(['wget', '-c', ('https://archive.torproject.org/tor-package-archive/torbrowser/{0}/'
+                + 'tor-expert-bundle-{0}-macos-{1}.tar.gz').format(tor_browser_version, arch)])
+    check_call(['tar', 'xvfz', 'tor-expert-bundle-{0}-macos-{1}.tar.gz'.format(tor_browser_version, arch),
+                '--one-top-level=upstream-' + arch, '--strip-components=1',
+                'tor/tor', 'tor/libevent-' + libevent_version + '.dylib'])
+    hash_tor_upstream = get_sha256(os.path.join('upstream-' + arch, 'tor'))
+    hash_libevent_upstream = get_sha256(os.path.join('upstream-' + arch, 'libevent-' + libevent_version + '.dylib'))
+    print('upstream tor: {0}'.format(hash_tor_upstream))
+    print('upstream libevent: {0}'.format(hash_libevent_upstream))
+
+    hash_tor_built = get_sha256(os.path.join('output', 'macos', arch, 'tor'))
+    hash_libevent_built = get_sha256(os.path.join('output', 'macos', arch, 'libevent-' + libevent_version + '.dylib'))
+    print('built tor: {0}'.format(hash_tor_built))
+    print('built libevent: {0}'.format(hash_libevent_built))
+
+    if hash_tor_upstream != hash_tor_built:
+        print("tor hash does not match")
+        exit(1)
+
+    if hash_libevent_upstream != hash_libevent_built:
+        print("libevent hash does not match")
+        exit(1)
+
+if __name__ == "__main__":
+    versions = build()
+    compare_output_with_upstream(versions)

verify_tor_macos.py

+#!/usr/bin/env python3
+from verify_tor_utils import main
+
+if __name__ == "__main__":
+    main("macos")