Skip to content
Snippets Groups Projects
Verified Commit 0d19da2f authored by Sebastian's avatar Sebastian
Browse files

Build macOS binaries using Tor upstream build system

parent 2fba3166
No related branches found
No related tags found
1 merge request!27macOS artifacts
Pipeline #14834 passed with warnings
...@@ -11,6 +11,7 @@ stages: ...@@ -11,6 +11,7 @@ stages:
variables: variables:
TEST_IMAGE: briar/tor-reproducer:${CI_BUILD_REF_NAME} TEST_IMAGE: briar/tor-reproducer:${CI_BUILD_REF_NAME}
RELEASE_IMAGE: briar/tor-reproducer:latest RELEASE_IMAGE: briar/tor-reproducer:latest
UPSTREAM_IMAGE: briar/tor-upstream-builder
before_script: before_script:
- echo ${DOCKER_HUB_PASS} | docker login -u ${DOCKER_HUB_USER} --password-stdin - echo ${DOCKER_HUB_PASS} | docker login -u ${DOCKER_HUB_USER} --password-stdin
...@@ -45,6 +46,14 @@ build: ...@@ -45,6 +46,14 @@ build:
expire_in: 1 week expire_in: 1 week
when: always when: always
.base-mac:
stage: test
artifacts:
paths:
- output/macos
expire_in: 1 week
when: always
test_build_android: test_build_android:
extends: .base-android extends: .base-android
script: script:
...@@ -69,6 +78,15 @@ test_build_windows: ...@@ -69,6 +78,15 @@ test_build_windows:
except: except:
- tags - tags
test_build_mac:
extends: .base-mac
script:
- docker build -t ${UPSTREAM_IMAGE} -f upstream/Dockerfile .
- docker run --privileged -v `pwd`/output:/opt/tor-reproducer/output ${UPSTREAM_IMAGE} /bin/bash -c "touch /dev/console && su builduser -c \"./build_tor_macos.py && ./verify_tor_macos.py\""
allow_failure: true
except:
- tags
test_tag_android: test_tag_android:
extends: .base-android extends: .base-android
script: script:
...@@ -90,6 +108,13 @@ test_tag_windows: ...@@ -90,6 +108,13 @@ test_tag_windows:
only: only:
- tags - tags
test_tag_macos:
extends: .base-mac
script:
- docker run --privileged -v `pwd`/output:/opt/tor-reproducer/output ${UPSTREAM_IMAGE} ./verify_tor_macos.py ${CI_BUILD_REF_NAME}
only:
- tags
release: release:
stage: release stage: release
script: script:
......
<?xml version="1.0" encoding="UTF-8"?>
<project>
<modelVersion>4.0.0</modelVersion>
<groupId>org.briarproject</groupId>
<artifactId>tor-macos</artifactId>
<name>tor-macos</name>
<version>VERSION</version>
<url>https://torproject.org</url>
<description>Repo for building Tor for macOS.</description>
<licenses>
<license>
<name>BSD-3-clause</name>
<url>https://gitweb.torproject.org/tor.git/tree/LICENSE</url>
</license>
</licenses>
<developers>
<developer>
<id>dingledine</id>
<name>Roger Dingledine</name>
<email>arma@mit.edu</email>
</developer>
<developer>
<id>mathewson</id>
<name>Nick Mathewson</name>
<email>nickm@torproject.org</email>
</developer>
<developer>
<id>torproject</id>
<name>Tor Project</name>
<email>frontdesk@rt.torproject.org</email>
</developer>
</developers>
<scm>
<connection>scm:https://gitweb.torproject.org/tor.git/</connection>
<developerConnection>scm:git@gitweb.torproject.org/tor.git</developerConnection>
<url>scm:https://gitweb.torproject.org/tor.git</url>
</scm>
</project>
...@@ -29,6 +29,12 @@ ...@@ -29,6 +29,12 @@
"revision": "21.4.7075529", "revision": "21.4.7075529",
"sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e" "sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e"
}, },
"upstream": {
"url": "https://gitlab.torproject.org/tpo/applications/tor-browser-build.git",
"commit": "b1b4bf77",
"tor-browser": "12.0.6",
"libevent": "2.1.7"
},
"timestamp": "201001010000.00" "timestamp": "201001010000.00"
}, },
"0.4.7.13-1": { "0.4.7.13-1": {
......
FROM briar/tor-upstream-reproducer:latest
ENV LANG=C.UTF-8
ENV DEBIAN_FRONTEND=noninteractive
WORKDIR /opt/tor-reproducer
ADD upstream/build_tor_macos.py ./
ADD tor-versions.json ./
ADD utils.py ./
ADD template-macos.pom ./
ADD verify_tor_macos.py ./
ADD verify_tor_utils.py ./
CMD ./build_tor_macos.py
#!/usr/bin/env python3
import os
from shutil import copytree, rmtree
from subprocess import check_call
import hashlib
import utils
from utils import get_version, get_build_versions, reset_time, \
get_sources_file_name, get_output_dir, get_sha256, pack, create_pom_file
from pathlib import Path
import tarfile
PLATFORM = "macos"
BUILD_DIR = "tor-browser-build"
def setup():
# get Tor version from command or show usage information
version = get_version()
# get versions of upstream repository and Tor browser for comparison
versions = get_build_versions(version)
print("Building Tor from upstream repository at commit %s" % versions['upstream']['commit'])
# remove output from previous build
output_dir = get_output_dir(PLATFORM)
print(output_dir)
os.makedirs(output_dir, exist_ok=True)
# clone tor-browser-build repo
check_call(['git', 'clone', versions['upstream']['url'], BUILD_DIR])
check_call(['git', 'checkout', versions['upstream']['commit']], cwd=Path(BUILD_DIR))
check_call(['git', 'submodule', 'init'], cwd=Path(BUILD_DIR))
check_call(['git', 'submodule', 'update'], cwd=Path(BUILD_DIR))
# create sources jar before building
jar_name = create_sources_jar(versions)
return versions, jar_name
def build():
versions, jar_name = setup()
copytree('pre-out', os.path.join(BUILD_DIR, 'out'))
copytree('pre-clones', os.path.join(BUILD_DIR, 'git_clones'))
build_arch(versions, 'aarch64')
build_arch(versions, 'x86_64')
package_macos(versions, jar_name)
return versions
def build_arch(versions, arch):
target = PLATFORM + '-' + arch
libevent_version = versions['upstream']['libevent']
# build using rbm
check_call(['./rbm/rbm', 'build', 'tor', '--target', 'release', '--target', 'torbrowser-' + target], cwd=Path(BUILD_DIR))
# extract tar.gz file
arch_dir = Path('output') / PLATFORM / arch
arch_dir.mkdir(parents=True, exist_ok=True)
tar_file = next(Path(BUILD_DIR).glob('out/tor/tor-*-' + target + '-*.tar.gz'))
tar = tarfile.open(tar_file, "r:gz")
tar.extractall(path=arch_dir)
tar.close()
# move contents out of tor/ directory
(arch_dir / 'tor').rename(arch_dir / 'tordir')
for file in (arch_dir / 'tordir').glob('*'):
file.rename(arch_dir / file.name)
(arch_dir / 'tordir').rmdir
rmtree(arch_dir / 'data')
# print hashsums
tor_file = arch_dir / 'tor'
libevent_file = arch_dir / ('libevent-' + libevent_version + '.dylib')
for file in [tor_file, libevent_file]:
sha256hash = get_sha256(file)
print("%s: %s" % (file, sha256hash))
def package_macos(versions, jar_name):
libevent_version = versions['upstream']['libevent']
# zip binaries together
output_dir = get_output_dir(PLATFORM)
file_list = [
os.path.join(output_dir, 'aarch64', 'tor'),
os.path.join(output_dir, 'aarch64', 'libevent-' + libevent_version + '.dylib'),
os.path.join(output_dir, 'x86_64', 'tor'),
os.path.join(output_dir, 'x86_64', 'libevent-' + libevent_version + '.dylib'),
]
zip_name = pack(versions, file_list, PLATFORM)
pom_name = create_pom_file(versions, PLATFORM)
print("%s:" % PLATFORM)
for file in file_list + [zip_name, jar_name, pom_name]:
sha256hash = get_sha256(file)
print("%s: %s" % (file, sha256hash))
def create_sources_jar(versions):
output_dir = get_output_dir(PLATFORM)
jar_files = []
for root, dir_names, filenames in os.walk(BUILD_DIR):
for f in filenames:
if '/.git' in root:
continue
jar_files.append(os.path.join(root, f))
for file in jar_files:
reset_time(file, versions)
jar_name = get_sources_file_name(versions, PLATFORM)
jar_path = os.path.abspath(jar_name)
rel_paths = [os.path.relpath(f, BUILD_DIR) for f in sorted(jar_files)]
# create jar archive with first files
jar_step = 5000
check_call(['jar', 'cf', jar_path] + rel_paths[0:jar_step], cwd=BUILD_DIR)
# add subsequent files in steps, because the command line can't handle all at once
for i in range(jar_step, len(rel_paths), jar_step):
check_call(['jar', 'uf', jar_path] + rel_paths[i:i + jar_step], cwd=BUILD_DIR)
return jar_name
def compare_output_with_upstream(versions):
compare_with_upstream(versions, "aarch64")
compare_with_upstream(versions, "x86_64")
def compare_with_upstream(versions, arch):
print('comparing hashsums for {0}'.format(arch))
tor_browser_version = versions['upstream']['tor-browser']
libevent_version = versions['upstream']['libevent']
check_call(['wget', '-c', ('https://archive.torproject.org/tor-package-archive/torbrowser/{0}/'
+ 'tor-expert-bundle-{0}-macos-{1}.tar.gz').format(tor_browser_version, arch)])
check_call(['tar', 'xvfz', 'tor-expert-bundle-{0}-macos-{1}.tar.gz'.format(tor_browser_version, arch),
'--one-top-level=upstream-' + arch, '--strip-components=1',
'tor/tor', 'tor/libevent-' + libevent_version + '.dylib'])
hash_tor_upstream = get_sha256(os.path.join('upstream-' + arch, 'tor'))
hash_libevent_upstream = get_sha256(os.path.join('upstream-' + arch, 'libevent-' + libevent_version + '.dylib'))
print('upstream tor: {0}'.format(hash_tor_upstream))
print('upstream libevent: {0}'.format(hash_libevent_upstream))
hash_tor_built = get_sha256(os.path.join('output', 'macos', arch, 'tor'))
hash_libevent_built = get_sha256(os.path.join('output', 'macos', arch, 'libevent-' + libevent_version + '.dylib'))
print('built tor: {0}'.format(hash_tor_built))
print('built libevent: {0}'.format(hash_libevent_built))
if hash_tor_upstream != hash_tor_built:
print("tor hash does not match")
exit(1)
if hash_libevent_upstream != hash_libevent_built:
print("libevent hash does not match")
exit(1)
if __name__ == "__main__":
versions = build()
compare_output_with_upstream(versions)
#!/usr/bin/env python3
from verify_tor_utils import main
if __name__ == "__main__":
main("macos")
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment