Commit fced5640 authored by akwizgran's avatar akwizgran

Merge branch 'linux' into 'master'

Add support for building 64-bit static Linux binary

See merge request !3
parents f1623f6f 96f91a8c
Pipeline #2152 canceled with stages
in 2 minutes and 55 seconds
...@@ -11,6 +11,7 @@ ADD install*.sh ./ ...@@ -11,6 +11,7 @@ ADD install*.sh ./
ADD tor-versions.json ./ ADD tor-versions.json ./
ADD utils.py ./ ADD utils.py ./
ADD template.pom ./ ADD template.pom ./
ADD template-android.pom ./
RUN ./install.sh RUN ./install.sh
......
...@@ -70,4 +70,4 @@ Note that this will not work if the issue is caused by an updated Debian package ...@@ -70,4 +70,4 @@ Note that this will not work if the issue is caused by an updated Debian package
To build a specific version of Tor for Android, run To build a specific version of Tor for Android, run
docker run briar/tor-reproducer:latest ./build-tor.py [version] docker run briar/tor-reproducer:latest ./verify-tor.py [version]
\ No newline at end of file \ No newline at end of file
...@@ -6,7 +6,9 @@ from subprocess import check_call ...@@ -6,7 +6,9 @@ from subprocess import check_call
from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_tor_version, \ from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_tor_version, \
get_final_file_name, get_sources_file_name, get_pom_file_name, get_version get_final_file_name, get_sources_file_name, get_pom_file_name, get_version
ZLIB_REPO_URL = 'https://github.com/madler/zlib.git'
NDK_DIR = 'android-ndk' NDK_DIR = 'android-ndk'
EXT_DIR = os.path.abspath(os.path.join(REPO_DIR, 'external'))
def main(): def main():
...@@ -23,31 +25,41 @@ def main(): ...@@ -23,31 +25,41 @@ def main():
# clone and checkout tor-android repo based on tor-versions.json # clone and checkout tor-android repo based on tor-versions.json
prepare_tor_android_repo(versions) prepare_tor_android_repo(versions)
# build Tor for various architectures # build Tor for various platforms and architectures
build_architectures() build()
build_android()
# zip geoip database # zip geoip database
geoip_path = os.path.join(REPO_DIR, 'external', 'tor', 'src', 'config', 'geoip') geoip_path = os.path.join(EXT_DIR, 'tor', 'src', 'config', 'geoip')
reset_time(geoip_path) reset_time(geoip_path)
check_call(['zip', '-D', '-X', os.path.join(REPO_DIR, 'geoip.zip'), geoip_path]) copy(geoip_path, os.path.join(REPO_DIR, 'geoip')) # copy first to not zip directory
check_call(['zip', '-X', os.path.join(REPO_DIR, 'geoip.zip'), os.path.join(REPO_DIR, 'geoip')])
# zip everything together # zip binaries together
file_list = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip', 'geoip.zip'] file_list = ['tor_linux-x86_64.zip', 'geoip.zip']
for filename in file_list: zip_name = pack(versions, file_list)
reset_time(os.path.join(REPO_DIR, filename)) # make file times deterministic before zipping # zip Android binaries together
zip_name = get_final_file_name(versions) file_list_android = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip',
check_call(['zip', '-D', '-X', zip_name] + file_list, cwd=REPO_DIR) 'geoip.zip']
zip_name_android = pack(versions, file_list_android, android=True)
# create sources jar
jar_name = create_sources_jar(versions)
# create POM file from template # create POM file from template
pom_name = create_pom_file(versions) pom_name = create_pom_file(versions)
pom_name_android = create_pom_file(versions, android=True)
# create sources jar
jar_name = create_sources_jar(versions)
jar_name_android = get_sources_file_name(versions, android=True)
copy(os.path.join(REPO_DIR, jar_name), os.path.join(REPO_DIR, jar_name_android))
# print hashes for debug purposes # print hashes for debug purposes
for file in file_list + [zip_name, jar_name, pom_name]: for file in file_list + [zip_name, jar_name, pom_name]:
sha256hash = get_sha256(os.path.join(REPO_DIR, file)) sha256hash = get_sha256(os.path.join(REPO_DIR, file))
print("%s: %s" % (file, sha256hash)) print("%s: %s" % (file, sha256hash))
print("Android:")
for file in file_list_android + [zip_name_android, jar_name_android, pom_name_android]:
sha256hash = get_sha256(os.path.join(REPO_DIR, file))
print("%s: %s" % (file, sha256hash))
def setup_android_ndk(versions): def setup_android_ndk(versions):
...@@ -106,47 +118,51 @@ def prepare_tor_android_repo(versions): ...@@ -106,47 +118,51 @@ def prepare_tor_android_repo(versions):
check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR) check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR) check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)
# add zlib
check_call(['git', 'clone', ZLIB_REPO_URL], cwd=EXT_DIR)
# check out versions of external dependencies # check out versions of external dependencies
checkout('tor', versions['tor'], 'external/tor') checkout('tor', versions['tor'], 'tor')
checkout('libevent', versions['libevent'], 'external/libevent') checkout('libevent', versions['libevent'], 'libevent')
checkout('openssl', versions['openssl'], 'external/openssl') checkout('openssl', versions['openssl'], 'openssl')
checkout('xz', versions['xz'], 'external/xz') checkout('xz', versions['xz'], 'xz')
checkout('zstd', versions['zstd'], 'external/zstd') checkout('zlib', versions['zlib'], 'zlib')
checkout('zstd', versions['zstd'], 'zstd')
def checkout(name, tag, path): def checkout(name, tag, path):
print("Checking out %s: %s" % (name, tag)) print("Checking out %s: %s" % (name, tag))
repo_path = os.path.join(REPO_DIR, path) repo_path = os.path.join(EXT_DIR, path)
check_call(['git', 'checkout', '-f', tag], cwd=repo_path) check_call(['git', 'checkout', '-f', tag], cwd=repo_path)
def build_architectures(): def build_android():
# build arm pie # build arm pie
os.unsetenv('APP_ABI') os.unsetenv('APP_ABI')
os.unsetenv('NDK_PLATFORM_LEVEL') os.unsetenv('NDK_PLATFORM_LEVEL')
os.unsetenv('PIEFLAGS') os.unsetenv('PIEFLAGS')
build_arch('tor_arm_pie.zip') build_android_arch('tor_arm_pie.zip')
# build arm # build arm
os.putenv('NDK_PLATFORM_LEVEL', '14') os.putenv('NDK_PLATFORM_LEVEL', '14')
os.putenv('PIEFLAGS', '') os.putenv('PIEFLAGS', '')
build_arch('tor_arm.zip') build_android_arch('tor_arm.zip')
# build x86 pie # build x86 pie
os.putenv('APP_ABI', 'x86') os.putenv('APP_ABI', 'x86')
os.unsetenv('NDK_PLATFORM_LEVEL') os.unsetenv('NDK_PLATFORM_LEVEL')
os.unsetenv('PIEFLAGS') os.unsetenv('PIEFLAGS')
build_arch('tor_x86_pie.zip') build_android_arch('tor_x86_pie.zip')
# build x86 # build x86
os.putenv('NDK_PLATFORM_LEVEL', '14') os.putenv('NDK_PLATFORM_LEVEL', '14')
os.putenv('PIEFLAGS', '') os.putenv('PIEFLAGS', '')
build_arch('tor_x86.zip') build_android_arch('tor_x86.zip')
def build_arch(name): def build_android_arch(name):
check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR) check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR)
copy(os.path.join(REPO_DIR, 'external', 'bin', 'tor'), os.path.join(REPO_DIR, 'tor')) copy(os.path.join(EXT_DIR, 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
check_call(['strip', '-D', 'tor'], cwd=REPO_DIR) check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
tor_path = os.path.join(REPO_DIR, 'tor') tor_path = os.path.join(REPO_DIR, 'tor')
reset_time(tor_path) reset_time(tor_path)
...@@ -154,30 +170,98 @@ def build_arch(name): ...@@ -154,30 +170,98 @@ def build_arch(name):
check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR) check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)
def build(name='tor_linux-x86_64.zip'):
prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix'))
lib_dir = os.path.join(prefix_dir, 'lib')
include_dir = os.path.join(prefix_dir, 'include')
# ensure clean build environment (again here to protect against build reordering)
check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)
if os.path.exists(prefix_dir):
rmtree(prefix_dir)
# create folders for static libraries
os.mkdir(prefix_dir)
os.mkdir(lib_dir)
os.mkdir(include_dir)
# setup environment
env = os.environ.copy()
env['LDFLAGS'] = "-L%s" % prefix_dir
env['CFLAGS'] = "-fPIC -I%s" % include_dir
env['LIBS'] = "-L%s" % lib_dir
# build zlib
zlib_dir = os.path.join(EXT_DIR, 'zlib')
check_call(['./configure', '--prefix=%s' % prefix_dir], cwd=zlib_dir, env=env)
check_call(['make', 'install'], cwd=zlib_dir, env=env)
# build openssl
openssl_dir = os.path.join(EXT_DIR, 'openssl')
check_call(['./config', '--prefix=%s' % prefix_dir], cwd=openssl_dir, env=env)
check_call(['make'], cwd=openssl_dir, env=env)
check_call(['make', 'install_sw'], cwd=openssl_dir, env=env)
# build libevent
libevent_dir = os.path.join(EXT_DIR, 'libevent')
check_call(['./autogen.sh'], cwd=libevent_dir)
check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir], cwd=libevent_dir,
env=env)
check_call(['make'], cwd=libevent_dir, env=env)
check_call(['make', 'install'], cwd=libevent_dir, env=env)
# build Tor
tor_dir = os.path.join(EXT_DIR, 'tor')
check_call(['./autogen.sh'], cwd=tor_dir)
env['CFLAGS'] += ' -O3' # needed for FORTIFY_SOURCE
check_call(['./configure', '--disable-asciidoc', '--disable-systemd',
'--enable-static-zlib', '--with-zlib-dir=%s' % prefix_dir,
'--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir,
'--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir,
'--prefix=%s' % prefix_dir], cwd=tor_dir, env=env)
check_call(['make', 'install'], cwd=tor_dir, env=env)
# copy and zip built Tor binary
tor_path = os.path.join(REPO_DIR, 'tor')
copy(os.path.join(prefix_dir, 'bin', 'tor'), tor_path)
check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
reset_time(tor_path)
print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)
def pack(versions, file_list, android=False):
for filename in file_list:
reset_time(os.path.join(REPO_DIR, filename)) # make file times deterministic before zipping
zip_name = get_final_file_name(versions, android)
check_call(['zip', '-D', '-X', zip_name] + file_list, cwd=REPO_DIR)
return zip_name
def reset_time(filename): def reset_time(filename):
check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename]) check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename])
def create_sources_jar(versions): def create_sources_jar(versions):
external_dir = os.path.join(REPO_DIR, 'external') check_call(['git', 'clean', '-dfx'], cwd=EXT_DIR)
check_call(['git', 'clean', '-dfx'], cwd=external_dir)
jar_files = [] jar_files = []
for root, dir_names, filenames in os.walk(external_dir): for root, dir_names, filenames in os.walk(EXT_DIR):
for f in filenames: for f in filenames:
jar_files.append(os.path.join(root, f)) jar_files.append(os.path.join(root, f))
for file in jar_files: for file in jar_files:
reset_time(file) reset_time(file)
jar_name = get_sources_file_name(versions) jar_name = get_sources_file_name(versions)
jar_path = os.path.abspath(os.path.join(REPO_DIR, jar_name)) jar_path = os.path.abspath(os.path.join(REPO_DIR, jar_name))
rel_paths = [os.path.relpath(f, external_dir) for f in sorted(jar_files)] rel_paths = [os.path.relpath(f, EXT_DIR) for f in sorted(jar_files)]
check_call(['jar', 'cf', jar_path] + rel_paths, cwd=external_dir) check_call(['jar', 'cf', jar_path] + rel_paths, cwd=EXT_DIR)
return jar_name return jar_name
def create_pom_file(versions): def create_pom_file(versions, android=False):
tor_version = get_tor_version(versions) tor_version = get_tor_version(versions)
pom_name = get_pom_file_name(versions) pom_name = get_pom_file_name(versions, android)
with open('template.pom', 'rt') as infile: template = 'template-android.pom' if android else 'template.pom'
with open(template, 'rt') as infile:
with open(os.path.join(REPO_DIR, pom_name), 'wt') as outfile: with open(os.path.join(REPO_DIR, pom_name), 'wt') as outfile:
for line in infile: for line in infile:
outfile.write(line.replace('VERSION', tor_version)) outfile.write(line.replace('VERSION', tor_version))
......
...@@ -7,6 +7,7 @@ apt-get install -y --no-install-recommends \ ...@@ -7,6 +7,7 @@ apt-get install -y --no-install-recommends \
zip \ zip \
unzip \ unzip \
wget \ wget \
build-essential \
make \ make \
patch \ patch \
autopoint \ autopoint \
......
<?xml version="1.0" encoding="UTF-8"?>
<project>
<modelVersion>4.0.0</modelVersion>
<groupId>org.briarproject</groupId>
<artifactId>tor-android</artifactId>
<version>VERSION</version>
<url>https://github.com/n8fr8/tor-android</url>
<description>Repo for building Tor for Android.</description>
<licenses>
<license>
<name>BSD-3-clause</name>
<url>https://github.com/n8fr8/tor-android/blob/master/LICENSE</url>
</license>
</licenses>
<developers>
<developer>
<id>dingledine</id>
<name>Roger Dingledine</name>
<email>arma@mit.edu</email>
</developer>
<developer>
<id>mathewson</id>
<name>Nick Mathewson</name>
<email>nickm@torproject.org</email>
</developer>
<developer>
<id>torproject</id>
<name>Tor Project</name>
<email>frontdesk@rt.torproject.org</email>
</developer>
<developer>
<id>n8fr8</id>
<name>Nathan Freitas</name>
<email>nathan@guardianproject.info</email>
</developer>
</developers>
<scm>
<connection>scm:https://github.com/n8fr8/tor-android.git</connection>
<developerConnection>scm:git@github.com:n8fr8/tor-android.git</developerConnection>
<url>scm:https://github.com/n8fr8/tor-android</url>
</scm>
</project>
...@@ -2,14 +2,14 @@ ...@@ -2,14 +2,14 @@
<project> <project>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.briarproject</groupId> <groupId>org.briarproject</groupId>
<artifactId>tor-android</artifactId> <artifactId>tor</artifactId>
<version>VERSION</version> <version>VERSION</version>
<url>https://github.com/n8fr8/tor-android</url> <url>https://torproject.org</url>
<description>Repo for building Tor for Android.</description> <description>Repo for building Tor for Linux.</description>
<licenses> <licenses>
<license> <license>
<name>BSD-3-clause</name> <name>BSD-3-clause</name>
<url>https://github.com/n8fr8/tor-android/blob/master/LICENSE</url> <url>https://gitweb.torproject.org/tor.git/tree/LICENSE</url>
</license> </license>
</licenses> </licenses>
<developers> <developers>
...@@ -28,15 +28,10 @@ ...@@ -28,15 +28,10 @@
<name>Tor Project</name> <name>Tor Project</name>
<email>frontdesk@rt.torproject.org</email> <email>frontdesk@rt.torproject.org</email>
</developer> </developer>
<developer>
<id>n8fr8</id>
<name>Nathan Freitas</name>
<email>nathan@guardianproject.info</email>
</developer>
</developers> </developers>
<scm> <scm>
<connection>scm:https://github.com/n8fr8/tor-android.git</connection> <connection>scm:https://gitweb.torproject.org/tor.git/</connection>
<developerConnection>scm:git@github.com:n8fr8/tor-android.git</developerConnection> <developerConnection>scm:git@gitweb.torproject.org/tor.git</developerConnection>
<url>scm:https://github.com/n8fr8/tor-android</url> <url>scm:https://gitweb.torproject.org/tor.git</url>
</scm> </scm>
</project> </project>
...@@ -4,7 +4,8 @@ ...@@ -4,7 +4,8 @@
"libevent": "release-2.0.22-stable", "libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o", "openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.4", "xz": "v5.2.4",
"zstd": "v1.3.4", "zlib": "v1.2.11",
"zstd": "v1.3.5",
"tor-android": "tor-android-binary-0.3.3.5-rc-3", "tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android", "tor_android_repo_url": "https://github.com/n8fr8/tor-android",
"ndk": { "ndk": {
...@@ -18,6 +19,7 @@ ...@@ -18,6 +19,7 @@
"libevent": "release-2.0.22-stable", "libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o", "openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.4", "xz": "v5.2.4",
"zlib": "v1.2.11",
"zstd": "v1.3.4", "zstd": "v1.3.4",
"tor-android": "tor-android-binary-0.3.3.5-rc-3", "tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android", "tor_android_repo_url": "https://github.com/n8fr8/tor-android",
...@@ -32,6 +34,7 @@ ...@@ -32,6 +34,7 @@
"libevent": "release-2.0.22-stable", "libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o", "openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.4", "xz": "v5.2.4",
"zlib": "v1.2.11",
"zstd": "v1.3.5", "zstd": "v1.3.5",
"tor-android": "tor-android-binary-0.3.3.5-rc-3", "tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android", "tor_android_repo_url": "https://github.com/n8fr8/tor-android",
...@@ -46,6 +49,7 @@ ...@@ -46,6 +49,7 @@
"libevent": "release-2.0.22-stable", "libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o", "openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.3", "xz": "v5.2.3",
"zlib": "v1.2.11",
"zstd": "v1.3.2", "zstd": "v1.3.2",
"tor-android": "tor-android-binary-0.3.3.5-rc-3", "tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android", "tor_android_repo_url": "https://github.com/n8fr8/tor-android",
...@@ -56,4 +60,3 @@ ...@@ -56,4 +60,3 @@
} }
} }
} }
...@@ -42,13 +42,18 @@ def get_tor_version(versions): ...@@ -42,13 +42,18 @@ def get_tor_version(versions):
return versions['tor'].split('-')[1] return versions['tor'].split('-')[1]
def get_final_file_name(versions): def get_file_suffix(versions, android=False):
return 'tor-android-%s.zip' % get_tor_version(versions) version = get_tor_version(versions)
return "%s-%s" % ("android", version) if android else version
def get_sources_file_name(versions): def get_final_file_name(versions, android=False):
return 'tor-android-%s-sources.jar' % get_tor_version(versions) return 'tor-%s.zip' % get_file_suffix(versions, android)
def get_pom_file_name(versions): def get_sources_file_name(versions, android=False):
return 'tor-android-%s.pom' % get_tor_version(versions) return 'tor-%s-sources.jar' % get_file_suffix(versions, android)
def get_pom_file_name(versions, android=False):
return 'tor-%s.pom' % get_file_suffix(versions, android)
...@@ -15,7 +15,7 @@ def main(): ...@@ -15,7 +15,7 @@ def main():
versions = get_build_versions(version) versions = get_build_versions(version)
# download reference binary # download reference binary
file_name = get_final_file_name(versions) file_name = get_final_file_name(versions, android=True)
try: try:
# try downloading from jcenter # try downloading from jcenter
check_call(['wget', '--no-verbose', get_url(versions), '-O', file_name]) check_call(['wget', '--no-verbose', get_url(versions), '-O', file_name])
...@@ -49,7 +49,7 @@ def main(): ...@@ -49,7 +49,7 @@ def main():
def get_url(versions, fallback=False): def get_url(versions, fallback=False):
version = get_tor_version(versions) version = get_tor_version(versions)
file = get_final_file_name(versions) file = get_final_file_name(versions, android=True)
if not fallback: if not fallback:
return "https://jcenter.bintray.com/org/briarproject/tor-android/%s/%s" % (version, file) return "https://jcenter.bintray.com/org/briarproject/tor-android/%s/%s" % (version, file)
else: else:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment