Commit fced5640 authored by akwizgran's avatar akwizgran

Merge branch 'linux' into 'master'

Add support for building 64-bit static Linux binary

See merge request !3
parents f1623f6f 96f91a8c
Pipeline #2152 canceled with stages
in 2 minutes and 55 seconds
......@@ -11,6 +11,7 @@ ADD install*.sh ./
ADD tor-versions.json ./
ADD utils.py ./
ADD template.pom ./
ADD template-android.pom ./
RUN ./install.sh
......
......@@ -70,4 +70,4 @@ Note that this will not work if the issue is caused by an updated Debian package
To build a specific version of Tor for Android, run
docker run briar/tor-reproducer:latest ./build-tor.py [version]
\ No newline at end of file
docker run briar/tor-reproducer:latest ./verify-tor.py [version]
\ No newline at end of file
......@@ -6,7 +6,9 @@ from subprocess import check_call
from utils import REPO_DIR, get_sha256, fail, get_build_versions, get_tor_version, \
get_final_file_name, get_sources_file_name, get_pom_file_name, get_version
ZLIB_REPO_URL = 'https://github.com/madler/zlib.git'
NDK_DIR = 'android-ndk'
EXT_DIR = os.path.abspath(os.path.join(REPO_DIR, 'external'))
def main():
......@@ -23,31 +25,41 @@ def main():
# clone and checkout tor-android repo based on tor-versions.json
prepare_tor_android_repo(versions)
# build Tor for various architectures
build_architectures()
# build Tor for various platforms and architectures
build()
build_android()
# zip geoip database
geoip_path = os.path.join(REPO_DIR, 'external', 'tor', 'src', 'config', 'geoip')
geoip_path = os.path.join(EXT_DIR, 'tor', 'src', 'config', 'geoip')
reset_time(geoip_path)
check_call(['zip', '-D', '-X', os.path.join(REPO_DIR, 'geoip.zip'), geoip_path])
copy(geoip_path, os.path.join(REPO_DIR, 'geoip')) # copy first to not zip directory
check_call(['zip', '-X', os.path.join(REPO_DIR, 'geoip.zip'), os.path.join(REPO_DIR, 'geoip')])
# zip everything together
file_list = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip', 'geoip.zip']
for filename in file_list:
reset_time(os.path.join(REPO_DIR, filename)) # make file times deterministic before zipping
zip_name = get_final_file_name(versions)
check_call(['zip', '-D', '-X', zip_name] + file_list, cwd=REPO_DIR)
# create sources jar
jar_name = create_sources_jar(versions)
# zip binaries together
file_list = ['tor_linux-x86_64.zip', 'geoip.zip']
zip_name = pack(versions, file_list)
# zip Android binaries together
file_list_android = ['tor_arm_pie.zip', 'tor_arm.zip', 'tor_x86_pie.zip', 'tor_x86.zip',
'geoip.zip']
zip_name_android = pack(versions, file_list_android, android=True)
# create POM file from template
pom_name = create_pom_file(versions)
pom_name_android = create_pom_file(versions, android=True)
# create sources jar
jar_name = create_sources_jar(versions)
jar_name_android = get_sources_file_name(versions, android=True)
copy(os.path.join(REPO_DIR, jar_name), os.path.join(REPO_DIR, jar_name_android))
# print hashes for debug purposes
for file in file_list + [zip_name, jar_name, pom_name]:
sha256hash = get_sha256(os.path.join(REPO_DIR, file))
print("%s: %s" % (file, sha256hash))
print("Android:")
for file in file_list_android + [zip_name_android, jar_name_android, pom_name_android]:
sha256hash = get_sha256(os.path.join(REPO_DIR, file))
print("%s: %s" % (file, sha256hash))
def setup_android_ndk(versions):
......@@ -106,47 +118,51 @@ def prepare_tor_android_repo(versions):
check_call(['git', 'clean', '-dffx'], cwd=REPO_DIR)
check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)
# add zlib
check_call(['git', 'clone', ZLIB_REPO_URL], cwd=EXT_DIR)
# check out versions of external dependencies
checkout('tor', versions['tor'], 'external/tor')
checkout('libevent', versions['libevent'], 'external/libevent')
checkout('openssl', versions['openssl'], 'external/openssl')
checkout('xz', versions['xz'], 'external/xz')
checkout('zstd', versions['zstd'], 'external/zstd')
checkout('tor', versions['tor'], 'tor')
checkout('libevent', versions['libevent'], 'libevent')
checkout('openssl', versions['openssl'], 'openssl')
checkout('xz', versions['xz'], 'xz')
checkout('zlib', versions['zlib'], 'zlib')
checkout('zstd', versions['zstd'], 'zstd')
def checkout(name, tag, path):
print("Checking out %s: %s" % (name, tag))
repo_path = os.path.join(REPO_DIR, path)
repo_path = os.path.join(EXT_DIR, path)
check_call(['git', 'checkout', '-f', tag], cwd=repo_path)
def build_architectures():
def build_android():
# build arm pie
os.unsetenv('APP_ABI')
os.unsetenv('NDK_PLATFORM_LEVEL')
os.unsetenv('PIEFLAGS')
build_arch('tor_arm_pie.zip')
build_android_arch('tor_arm_pie.zip')
# build arm
os.putenv('NDK_PLATFORM_LEVEL', '14')
os.putenv('PIEFLAGS', '')
build_arch('tor_arm.zip')
build_android_arch('tor_arm.zip')
# build x86 pie
os.putenv('APP_ABI', 'x86')
os.unsetenv('NDK_PLATFORM_LEVEL')
os.unsetenv('PIEFLAGS')
build_arch('tor_x86_pie.zip')
build_android_arch('tor_x86_pie.zip')
# build x86
os.putenv('NDK_PLATFORM_LEVEL', '14')
os.putenv('PIEFLAGS', '')
build_arch('tor_x86.zip')
build_android_arch('tor_x86.zip')
def build_arch(name):
def build_android_arch(name):
check_call(['make', '-C', 'external', 'clean', 'tor'], cwd=REPO_DIR)
copy(os.path.join(REPO_DIR, 'external', 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
copy(os.path.join(EXT_DIR, 'bin', 'tor'), os.path.join(REPO_DIR, 'tor'))
check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
tor_path = os.path.join(REPO_DIR, 'tor')
reset_time(tor_path)
......@@ -154,30 +170,98 @@ def build_arch(name):
check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)
def build(name='tor_linux-x86_64.zip'):
prefix_dir = os.path.abspath(os.path.join(REPO_DIR, 'prefix'))
lib_dir = os.path.join(prefix_dir, 'lib')
include_dir = os.path.join(prefix_dir, 'include')
# ensure clean build environment (again here to protect against build reordering)
check_call(['git', 'submodule', 'foreach', 'git', 'clean', '-dffx'], cwd=REPO_DIR)
if os.path.exists(prefix_dir):
rmtree(prefix_dir)
# create folders for static libraries
os.mkdir(prefix_dir)
os.mkdir(lib_dir)
os.mkdir(include_dir)
# setup environment
env = os.environ.copy()
env['LDFLAGS'] = "-L%s" % prefix_dir
env['CFLAGS'] = "-fPIC -I%s" % include_dir
env['LIBS'] = "-L%s" % lib_dir
# build zlib
zlib_dir = os.path.join(EXT_DIR, 'zlib')
check_call(['./configure', '--prefix=%s' % prefix_dir], cwd=zlib_dir, env=env)
check_call(['make', 'install'], cwd=zlib_dir, env=env)
# build openssl
openssl_dir = os.path.join(EXT_DIR, 'openssl')
check_call(['./config', '--prefix=%s' % prefix_dir], cwd=openssl_dir, env=env)
check_call(['make'], cwd=openssl_dir, env=env)
check_call(['make', 'install_sw'], cwd=openssl_dir, env=env)
# build libevent
libevent_dir = os.path.join(EXT_DIR, 'libevent')
check_call(['./autogen.sh'], cwd=libevent_dir)
check_call(['./configure', '--disable-shared', '--prefix=%s' % prefix_dir], cwd=libevent_dir,
env=env)
check_call(['make'], cwd=libevent_dir, env=env)
check_call(['make', 'install'], cwd=libevent_dir, env=env)
# build Tor
tor_dir = os.path.join(EXT_DIR, 'tor')
check_call(['./autogen.sh'], cwd=tor_dir)
env['CFLAGS'] += ' -O3' # needed for FORTIFY_SOURCE
check_call(['./configure', '--disable-asciidoc', '--disable-systemd',
'--enable-static-zlib', '--with-zlib-dir=%s' % prefix_dir,
'--enable-static-libevent', '--with-libevent-dir=%s' % prefix_dir,
'--enable-static-openssl', '--with-openssl-dir=%s' % prefix_dir,
'--prefix=%s' % prefix_dir], cwd=tor_dir, env=env)
check_call(['make', 'install'], cwd=tor_dir, env=env)
# copy and zip built Tor binary
tor_path = os.path.join(REPO_DIR, 'tor')
copy(os.path.join(prefix_dir, 'bin', 'tor'), tor_path)
check_call(['strip', '-D', 'tor'], cwd=REPO_DIR)
reset_time(tor_path)
print("Sha256 hash of tor before zipping %s: %s" % (name, get_sha256(tor_path)))
check_call(['zip', '-X', name, 'tor'], cwd=REPO_DIR)
def pack(versions, file_list, android=False):
for filename in file_list:
reset_time(os.path.join(REPO_DIR, filename)) # make file times deterministic before zipping
zip_name = get_final_file_name(versions, android)
check_call(['zip', '-D', '-X', zip_name] + file_list, cwd=REPO_DIR)
return zip_name
def reset_time(filename):
check_call(['touch', '--no-dereference', '-t', '197001010000.00', filename])
def create_sources_jar(versions):
external_dir = os.path.join(REPO_DIR, 'external')
check_call(['git', 'clean', '-dfx'], cwd=external_dir)
check_call(['git', 'clean', '-dfx'], cwd=EXT_DIR)
jar_files = []
for root, dir_names, filenames in os.walk(external_dir):
for root, dir_names, filenames in os.walk(EXT_DIR):
for f in filenames:
jar_files.append(os.path.join(root, f))
for file in jar_files:
reset_time(file)
jar_name = get_sources_file_name(versions)
jar_path = os.path.abspath(os.path.join(REPO_DIR, jar_name))
rel_paths = [os.path.relpath(f, external_dir) for f in sorted(jar_files)]
check_call(['jar', 'cf', jar_path] + rel_paths, cwd=external_dir)
rel_paths = [os.path.relpath(f, EXT_DIR) for f in sorted(jar_files)]
check_call(['jar', 'cf', jar_path] + rel_paths, cwd=EXT_DIR)
return jar_name
def create_pom_file(versions):
def create_pom_file(versions, android=False):
tor_version = get_tor_version(versions)
pom_name = get_pom_file_name(versions)
with open('template.pom', 'rt') as infile:
pom_name = get_pom_file_name(versions, android)
template = 'template-android.pom' if android else 'template.pom'
with open(template, 'rt') as infile:
with open(os.path.join(REPO_DIR, pom_name), 'wt') as outfile:
for line in infile:
outfile.write(line.replace('VERSION', tor_version))
......
......@@ -7,6 +7,7 @@ apt-get install -y --no-install-recommends \
zip \
unzip \
wget \
build-essential \
make \
patch \
autopoint \
......
<?xml version="1.0" encoding="UTF-8"?>
<project>
<modelVersion>4.0.0</modelVersion>
<groupId>org.briarproject</groupId>
<artifactId>tor-android</artifactId>
<version>VERSION</version>
<url>https://github.com/n8fr8/tor-android</url>
<description>Repo for building Tor for Android.</description>
<licenses>
<license>
<name>BSD-3-clause</name>
<url>https://github.com/n8fr8/tor-android/blob/master/LICENSE</url>
</license>
</licenses>
<developers>
<developer>
<id>dingledine</id>
<name>Roger Dingledine</name>
<email>arma@mit.edu</email>
</developer>
<developer>
<id>mathewson</id>
<name>Nick Mathewson</name>
<email>nickm@torproject.org</email>
</developer>
<developer>
<id>torproject</id>
<name>Tor Project</name>
<email>frontdesk@rt.torproject.org</email>
</developer>
<developer>
<id>n8fr8</id>
<name>Nathan Freitas</name>
<email>nathan@guardianproject.info</email>
</developer>
</developers>
<scm>
<connection>scm:https://github.com/n8fr8/tor-android.git</connection>
<developerConnection>scm:git@github.com:n8fr8/tor-android.git</developerConnection>
<url>scm:https://github.com/n8fr8/tor-android</url>
</scm>
</project>
......@@ -2,14 +2,14 @@
<project>
<modelVersion>4.0.0</modelVersion>
<groupId>org.briarproject</groupId>
<artifactId>tor-android</artifactId>
<artifactId>tor</artifactId>
<version>VERSION</version>
<url>https://github.com/n8fr8/tor-android</url>
<description>Repo for building Tor for Android.</description>
<url>https://torproject.org</url>
<description>Repo for building Tor for Linux.</description>
<licenses>
<license>
<name>BSD-3-clause</name>
<url>https://github.com/n8fr8/tor-android/blob/master/LICENSE</url>
<url>https://gitweb.torproject.org/tor.git/tree/LICENSE</url>
</license>
</licenses>
<developers>
......@@ -28,15 +28,10 @@
<name>Tor Project</name>
<email>frontdesk@rt.torproject.org</email>
</developer>
<developer>
<id>n8fr8</id>
<name>Nathan Freitas</name>
<email>nathan@guardianproject.info</email>
</developer>
</developers>
<scm>
<connection>scm:https://github.com/n8fr8/tor-android.git</connection>
<developerConnection>scm:git@github.com:n8fr8/tor-android.git</developerConnection>
<url>scm:https://github.com/n8fr8/tor-android</url>
<connection>scm:https://gitweb.torproject.org/tor.git/</connection>
<developerConnection>scm:git@gitweb.torproject.org/tor.git</developerConnection>
<url>scm:https://gitweb.torproject.org/tor.git</url>
</scm>
</project>
......@@ -4,7 +4,8 @@
"libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.4",
"zstd": "v1.3.4",
"zlib": "v1.2.11",
"zstd": "v1.3.5",
"tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android",
"ndk": {
......@@ -18,6 +19,7 @@
"libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.4",
"zlib": "v1.2.11",
"zstd": "v1.3.4",
"tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android",
......@@ -32,6 +34,7 @@
"libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.4",
"zlib": "v1.2.11",
"zstd": "v1.3.5",
"tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android",
......@@ -46,6 +49,7 @@
"libevent": "release-2.0.22-stable",
"openssl": "OpenSSL_1_0_2o",
"xz": "v5.2.3",
"zlib": "v1.2.11",
"zstd": "v1.3.2",
"tor-android": "tor-android-binary-0.3.3.5-rc-3",
"tor_android_repo_url": "https://github.com/n8fr8/tor-android",
......@@ -56,4 +60,3 @@
}
}
}
......@@ -42,13 +42,18 @@ def get_tor_version(versions):
return versions['tor'].split('-')[1]
def get_final_file_name(versions):
return 'tor-android-%s.zip' % get_tor_version(versions)
def get_file_suffix(versions, android=False):
version = get_tor_version(versions)
return "%s-%s" % ("android", version) if android else version
def get_sources_file_name(versions):
return 'tor-android-%s-sources.jar' % get_tor_version(versions)
def get_final_file_name(versions, android=False):
return 'tor-%s.zip' % get_file_suffix(versions, android)
def get_pom_file_name(versions):
return 'tor-android-%s.pom' % get_tor_version(versions)
def get_sources_file_name(versions, android=False):
return 'tor-%s-sources.jar' % get_file_suffix(versions, android)
def get_pom_file_name(versions, android=False):
return 'tor-%s.pom' % get_file_suffix(versions, android)
......@@ -15,7 +15,7 @@ def main():
versions = get_build_versions(version)
# download reference binary
file_name = get_final_file_name(versions)
file_name = get_final_file_name(versions, android=True)
try:
# try downloading from jcenter
check_call(['wget', '--no-verbose', get_url(versions), '-O', file_name])
......@@ -49,7 +49,7 @@ def main():
def get_url(versions, fallback=False):
version = get_tor_version(versions)
file = get_final_file_name(versions)
file = get_final_file_name(versions, android=True)
if not fallback:
return "https://jcenter.bintray.com/org/briarproject/tor-android/%s/%s" % (version, file)
else:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment