Build wheels for windows and macos with conan

This commit is contained in:
laggykiller 2024-02-24 03:28:30 +08:00
parent bb6becd3ef
commit 1bc6af1dc4
14 changed files with 390 additions and 280 deletions

202
.github/workflows/pythonpackage.yml vendored Normal file
View file

@ -0,0 +1,202 @@
name: Python package
on: [push]
jobs:
prepare-sqlite:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
with:
submodules: true
- if: steps.sqlite-amalgamation.outputs.cache-hit != 'true'
run: |
cd sqlcipher/
LIBS="-lm" ./configure --disable-tcl --enable-tempstore=always
make sqlite3.c
- uses: actions/upload-artifact@v3
with:
name: sqlite-amalgamation
path: |
sqlcipher/sqlite3.c
sqlcipher/sqlite3.h
tests:
needs: [prepare-sqlite]
strategy:
fail-fast: true
matrix:
python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11']
os: [ubuntu-20.04, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
with:
submodules: false
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/download-artifact@v3
with:
name: sqlite-amalgamation
path: ./src/sqlcipher
- name: Build module
run: |
python setup.py build_ext -i
- name: Run tests
run: |
python -m test
build-wheels:
needs: [prepare-sqlite, tests]
strategy:
matrix:
include:
- os: ubuntu-20.04
cibw_archs: x86_64
cibw_build: "cp3*-manylinux_x86_64"
compile_target: x86_64
- os: ubuntu-20.04
cibw_archs: x86_64
cibw_build: "cp3*-musllinux_x86_64"
cibw_skip: "cp38-musllinux_*"
compile_target: x86_64
- os: ubuntu-20.04
cibw_archs: aarch64
cibw_build: "cp3*-manylinux_aarch64"
compile_target: armv8
- os: ubuntu-20.04
cibw_archs: aarch64
cibw_build: "cp3*-musllinux_aarch64"
compile_target: armv8
- os: ubuntu-20.04
cibw_archs: i686
cibw_build: "cp3*-manylinux_i686"
compile_target: x86
- os: ubuntu-20.04
cibw_archs: i686
cibw_build: "cp3*-musllinux_i686"
compile_target: x86
- os: ubuntu-20.04
cibw_archs: ppc64le
cibw_build: "cp3*-manylinux_ppc64le"
compile_target: ppc64le
- os: ubuntu-20.04
cibw_archs: ppc64le
cibw_build: "cp3*-musllinux_ppc64le"
compile_target: ppc64le
- os: ubuntu-20.04
cibw_archs: s390x
cibw_build: "cp3*-manylinux_s390x"
compile_target: s390x
- os: ubuntu-20.04
cibw_archs: s390x
cibw_build: "cp3*-musllinux_s390x"
compile_target: s390x
- os: windows-2019
cibw_archs: AMD64
cibw_build: "cp3*-win_amd64"
compile_target: x86_64
- os: windows-2019
cibw_archs: x86
cibw_build: "cp3*-win32"
compile_target: x86
- os: windows-2019
cibw_archs: ARM64
cibw_build: "cp3*-win_arm64"
compile_target: armv8
- os: macos-11
cibw_archs: x86_64
cibw_build: "cp3*-macosx_x86_64"
compile_target: x86_64
- os: macos-11
cibw_archs: arm64
cibw_build: "cp3*-macosx_arm64"
compile_target: armv8
- os: macos-11
cibw_archs: universal2
cibw_build: "cp3*-macosx_universal2"
compile_target: universal2
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
with:
submodules: false
- uses: actions/setup-python@v4
with:
python-version: 3.8
- uses: actions/download-artifact@v3
with:
name: sqlite-amalgamation
path: ./
- name: Install dependencies
run: |
python -m pip install -U pip setuptools wheel cibuildwheel
- name: Build wheels for ${{ matrix.os }} ${{ matrix.cibw_archs }} ${{ matrix.cibw_build }}
uses: pypa/cibuildwheel@v2.15.0
env:
CIBW_BUILD_FRONTEND: build
CIBW_BUILD: ${{ matrix.cibw_build }}
CIBW_SKIP: ${{ matrix.cibw_skip }}
CIBW_ARCHS: ${{ matrix.cibw_archs }}
CIBW_ENVIRONMENT: SQLCIPHER3_COMPILE_TARGET=${{ matrix.compile_target }}
- uses: actions/upload-artifact@v3
with:
name: wheels-${{ github.sha }}
path: ./wheelhouse/*.whl
retention-days: 7
build-sdist:
needs: [prepare-sqlite, tests]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
with:
submodules: false
- uses: actions/setup-python@v4
with:
python-version: 3.8
- uses: actions/download-artifact@v3
with:
name: sqlite-amalgamation
path: ./
- name: Install dependencies
run: |
python -m pip install -U pip setuptools wheel
- name: Build sdist
run: |
python setup.py sdist
- uses: actions/upload-artifact@v3
with:
path: dist/*.tar.gz
# upload-pypi:
# needs: [build-wheels, build-sdist]
# runs-on: ubuntu-20.04
# steps:
# - uses: actions/download-artifact@v3
# with:
# name: wheels
# path: dist/
# - uses: pypa/gh-action-pypi-publish@release/v1
# with:
# user: __token__
# password: ${{ secrets.pypi_password }}

View file

@ -1,23 +0,0 @@
name: Tests
on: [push]
jobs:
tests:
name: ${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: package deps
run: |
sudo apt-get install libsqlcipher-dev
python setup.py build_ext -i
- name: runtests
env:
PYTHONPATH: '.:$PYTHONPATH'
run: python test/

1
.gitignore vendored
View file

@ -3,3 +3,4 @@ build/*
dist/*
sqlcipher3.egg-info/*
_sqlite*.so
conan_output/*

6
.gitmodules vendored Normal file
View file

@ -0,0 +1,6 @@
[submodule "lipo-dir-merge"]
path = lipo-dir-merge
url = https://github.com/faaxm/lipo-dir-merge.git
[submodule "sqlcipher"]
path = sqlcipher
url = https://github.com/sqlcipher/sqlcipher.git

View file

@ -2,7 +2,9 @@ include MANIFEST.in
include README.md
include LICENSE
include setup.py
include src/*.h
include src/*.c
recursive-include src *.h
recursive-include src *.c
include lipo-dir-merge/*.py
include conanfile.py
global-exclude *~ *.pyc

View file

@ -1,16 +0,0 @@
### Building wheels
This directory contains a utility script (`build.sh`) which can be used to
create completely self-contained manylinux wheels for Python 3.6 and 3.7. The
build script fetches the latest release of Sqlcipher and generates the source
amalgamation and header file, which are then compiled into `sqlcipher3`. The
resulting Python package can be deployed to any linux environment and will
utilize the latest Sqlcipher source code with extensions compiled in.
The package name for the wheels is `sqlcipher3-binary` to differentiate it from
the standard `sqlcipher3` source distribution. The source distribution will
link against the system sqlcipher by default, though you can provide your own
`sqlite3.c` and `sqlite3.h` and use `setup.py build_static` to create a
self-contained package as well.
Build artifacts are placed in the `wheelhouse/` directory.

View file

@ -1,77 +0,0 @@
#!/bin/bash
# manylinux uses ancient Centos 5, which ships openssl 0.9.8. We need to build
# our own in order to then build sqlcipher. The SSL headers will be installed
# to /usr/local/.
#
# The code for installing Perl and OpenSSL is derived from the psycopg2-binary
# build scripts.
yum install -y zlib-devel
OPENSSL_VERSION="1.1.1b"
OPENSSL_TAG="OpenSSL_${OPENSSL_VERSION//./_}"
cd /io
if [ ! -d "openssl-${OPENSSL_TAG}/" ]; then
# Building openssl 1.1 requires perl 5.10 or newer.
curl -L https://install.perlbrew.pl | bash
source ~/perl5/perlbrew/etc/bashrc
perlbrew install --notest perl-5.16.0
perlbrew switch perl-5.16.0
curl -sL https://github.com/openssl/openssl/archive/${OPENSSL_TAG}.tar.gz \
| tar xzf -
cd "openssl-${OPENSSL_TAG}"
# Expose the lib version number in the .so file name.
sed -i "s/SHLIB_VERSION_NUMBER\s\+\".*\"/SHLIB_VERSION_NUMBER \"${OPENSSL_VERSION}\"/" \
./include/openssl/opensslv.h
sed -i "s|if (\$shlib_version_number =~ /(^\[0-9\]\*)\\\.(\[0-9\\\.\]\*)/)|if (\$shlib_version_number =~ /(^[0-9]*)\.([0-9\.]*[a-z]?)/)|" \
./Configure
./config --prefix=/usr/local/ --openssldir=/usr/local/ zlib -fPIC shared
make depend && make && make install
fi
# Volume (cwd of build script) is mounted at /io.
# A checkout of sqlcipher3 is cloned beforehand by the build.sh script.
cd /io/sqlcipher3
sed -i "s|name='sqlcipher3-binary'|name=PACKAGE_NAME|g" setup.py
export CFLAGS="-I/usr/local/include -L/usr/local/lib"
PY36="/opt/python/cp36-cp36m/bin"
"${PY36}/python" setup.py build_static
PY37="/opt/python/cp37-cp37m/bin"
"${PY37}/python" setup.py build_static
PY38="/opt/python/cp38-cp38/bin"
"${PY38}/python" setup.py build_static
PY39="/opt/python/cp39-cp39/bin"
"${PY39}/python" setup.py build_static
PY310="/opt/python/cp310-cp310/bin"
"${PY310}/python" setup.py build_static
PY311="/opt/python/cp311-cp311/bin"
"${PY311}/python" setup.py build_static
# Replace the package name defined in setup.py so we can push this to PyPI
# without stomping on the source dist.
sed -i "s|name=PACKAGE_NAME,|name='sqlcipher3-binary',|g" setup.py
"${PY36}/pip" wheel /io/sqlcipher3 -w /io/wheelhouse
"${PY37}/pip" wheel /io/sqlcipher3 -w /io/wheelhouse
"${PY38}/pip" wheel /io/sqlcipher3 -w /io/wheelhouse
"${PY39}/pip" wheel /io/sqlcipher3 -w /io/wheelhouse
"${PY310}/pip" wheel /io/sqlcipher3 -w /io/wheelhouse
"${PY311}/pip" wheel /io/sqlcipher3 -w /io/wheelhouse
for whl in /io/wheelhouse/*.whl; do
auditwheel repair "$whl" -w /io/wheelhouse/
done

View file

@ -1,27 +0,0 @@
#!/bin/bash
set -e -x
# Fetch the source code for SQLCipher.
if [[ ! -d "sqlcipher" ]]; then
git clone --depth=1 git@github.com:sqlcipher/sqlcipher
cd sqlcipher/
./configure --disable-tcl --enable-tempstore=yes LDFLAGS="-lcrypto -lm"
make sqlite3.c
cd ../
fi
# Grab the sqlcipher3 source code.
if [[ ! -d "./sqlcipher3" ]]; then
git clone git@github.com:coleifer/sqlcipher3
fi
# Copy the sqlcipher source amalgamation into the pysqlite3 directory so we can
# create a self-contained extension module.
cp "sqlcipher/sqlite3.c" sqlcipher3/
cp "sqlcipher/sqlite3.h" sqlcipher3/
# Create the wheels and strip symbols to produce manylinux wheels.
docker run -it -v $(pwd):/io quay.io/pypa/manylinux_2_24_x86_64 /io/_build_wheels.sh
sudo rm ./wheelhouse/*-linux_*

View file

@ -1,8 +0,0 @@
#!/bin/bash
cleanup="openssl-OpenSSL_1_1_1b sqlcipher sqlcipher3 wheelhouse"
for p in $cleanup; do
if [[ -d "$p" ]]; then
sudo rm -rf "$p"
fi
done

6
conanfile.py Normal file
View file

@ -0,0 +1,6 @@
from conan import ConanFile
import shutil
class OpensslRecipe(ConanFile):
def requirements(self):
self.requires('openssl/1.1.1w')

1
lipo-dir-merge Submodule

@ -0,0 +1 @@
Subproject commit 45fb925fd44345986696f5f901ee1eafdaddd260

View file

@ -1,3 +1,7 @@
[build_ext]
include_dirs=/usr/include
library_dirs=/usr/lib
[options]
setup_requires =
conan >= 2.0

282
setup.py
View file

@ -2,24 +2,32 @@
# setup.py: the distutils script
#
import os
import setuptools
import shutil
import subprocess
import json
import sys
from distutils import log
from distutils.command.build_ext import build_ext
from setuptools import Extension
import platform
from glob import glob
from setuptools import setup, Extension
# If you need to change anything, it should be enough to change setup.cfg.
PACKAGE_NAME = 'sqlcipher3'
VERSION = '0.5.2'
# Mapping from Conan architectures to Python machine types
CONAN_ARCHS = {
'x86_64': ['amd64', 'x86_64', 'x64'],
'x86': ['i386', 'i686', 'x86'],
'armv8': ['arm64', 'aarch64', 'aarch64_be', 'armv8b', 'armv8l'],
'ppc64le': ['ppc64le', 'powerpc'],
's390x': ['s390', 's390x'],
}
# define sqlite sources
sources = [os.path.join('src', source)
for source in ["module.c", "connection.c", "cursor.c", "cache.c",
"microprotocols.c", "prepare_protocol.c",
"statement.c", "util.c", "row.c", "blob.c"]]
sources = glob("src/*.c") + ["src/sqlcipher/sqlite3.c"]
include_dirs = ["./src"]
# define packages
packages = [PACKAGE_NAME]
@ -28,128 +36,170 @@ EXTENSION_MODULE_NAME = "._sqlite3"
# Work around clang raising hard error for unused arguments
if sys.platform == "darwin":
os.environ['CFLAGS'] = "-Qunused-arguments"
log.info("CFLAGS: " + os.environ['CFLAGS'])
def get_arch() -> str:
"""Get the Conan compilation target architecture.
If not explicitly set using the `SQLCIPHER3_COMPILE_TARGET` environment variable, this will be
determined using the host machine's platform information.
"""
env_arch = os.getenv('SQLCIPHER3_COMPILE_TARGET', '')
if env_arch:
return env_arch
if (
platform.architecture()[0] == '32bit'
and platform.machine().lower() in (CONAN_ARCHS['x86'] + CONAN_ARCHS['x86_64'])
):
return 'x86'
for k, v in CONAN_ARCHS.items():
if platform.machine().lower() in v:
return k
raise RuntimeError('Unable to determine the compilation target architecture')
def install_openssl(arch: str) -> dict:
"""Install openssl using Conan.
"""
settings = []
if platform.system() == 'Windows':
settings.append('os=Windows')
elif platform.system() == 'Darwin':
settings.append('os=Macos')
if arch == 'x86_64':
settings.append('os.version=10.9')
else:
settings.append('os.version=11.0')
settings.append('compiler=apple-clang')
settings.append('compiler.libcxx=libc++')
elif platform.system() == 'Linux':
settings.append('os=Linux')
settings.append(f'arch={arch}')
build = ['missing']
if os.path.isdir('/lib') and any(e.startswith('libc.musl') for e in os.listdir('/lib')):
# Need to compile openssl if musllinux
build.append('openssl*')
subprocess.run(['conan', 'profile', 'detect'])
conan_output = os.path.join('conan_output', arch)
result = subprocess.run([
'conan', 'install',
*[x for s in settings for x in ('-s', s)],
*[x for b in build for x in ('-b', b)],
'-of', conan_output, '--deployer=direct_deploy', '--format=json', '.'
], stdout=subprocess.PIPE).stdout.decode()
conan_info = json.loads(result)
return conan_info
def fetch_openssl_dir(conan_info: dict) -> str:
"""Find directory of openssl.
"""
for dep in conan_info['graph']['nodes'].values():
if dep.get('name') == 'openssl':
return dep.get('package_folder')
def quote_argument(arg):
q = '\\"' if sys.platform == 'win32' and sys.version_info < (3, 8) else '"'
return q + arg + q
define_macros = [('MODULE_NAME', quote_argument(PACKAGE_NAME + '.dbapi2'))]
class SystemLibSqliteBuilder(build_ext):
description = "Builds a C extension linking against libsqlcipher library"
def build_extension(self, ext):
log.info(self.description)
ext.libraries.append('sqlcipher')
ext.define_macros.append(('SQLITE_HAS_CODEC', '1'))
build_ext.build_extension(self, ext)
class AmalgationLibSqliteBuilder(build_ext):
description = "Builds a C extension using a sqlcipher amalgamation"
amalgamation_root = "."
amalgamation_header = os.path.join(amalgamation_root, 'sqlite3.h')
amalgamation_source = os.path.join(amalgamation_root, 'sqlite3.c')
header_dir = os.path.join(amalgamation_root, 'sqlcipher')
header_file = os.path.join(header_dir, 'sqlite3.h')
amalgamation_message = ('Sqlcipher amalgamation not found. Please download'
' or build the amalgamation and make sure the '
'following files are present in the sqlcipher3 '
'folder: sqlite3.h, sqlite3.c')
def check_amalgamation(self):
header_exists = os.path.exists(self.amalgamation_header)
source_exists = os.path.exists(self.amalgamation_source)
if not header_exists or not source_exists:
raise RuntimeError(self.amalgamation_message)
if not os.path.exists(self.header_dir):
os.mkdir(self.header_dir)
if not os.path.exists(self.header_file):
shutil.copy(self.amalgamation_header, self.header_file)
def build_extension(self, ext):
log.info(self.description)
# it is responsibility of user to provide amalgamation
self.check_amalgamation()
# Feature-ful library.
features = (
'ENABLE_FTS3',
'ENABLE_FTS3_PARENTHESIS',
'ENABLE_FTS4',
'ENABLE_FTS5',
'ENABLE_JSON1',
'ENABLE_LOAD_EXTENSION',
'ENABLE_RTREE',
'ENABLE_STAT4',
'ENABLE_UPDATE_DELETE_LIMIT',
'HAS_CODEC', # Required for SQLCipher.
'SOUNDEX',
'USE_URI',
)
for feature in features:
ext.define_macros.append(('SQLITE_%s' % feature, '1'))
define_macros = [
('MODULE_NAME', quote_argument(PACKAGE_NAME + '.dbapi2')),
('ENABLE_FTS3', '1'),
('ENABLE_FTS3_PARENTHESIS', '1'),
('ENABLE_FTS4', '1'),
('ENABLE_FTS5', '1'),
('ENABLE_JSON1', '1'),
('ENABLE_LOAD_EXTENSION', '1'),
('ENABLE_RTREE', '1'),
('ENABLE_STAT4', '1'),
('ENABLE_UPDATE_DELETE_LIMIT', '1'),
('SOUNDEX', '1'),
('USE_URI', '1'),
# Required for SQLCipher.
ext.define_macros.append(("SQLITE_TEMP_STORE", "2"))
('SQLITE_HAS_CODEC', '1'),
('HAS_CODEC', '1'),
("SQLITE_TEMP_STORE", "2"),
# Increase the maximum number of "host parameters".
ext.define_macros.append(("SQLITE_MAX_VARIABLE_NUMBER", "250000"))
("SQLITE_MAX_VARIABLE_NUMBER", "250000"),
# Additional nice-to-have.
ext.define_macros.extend((
('SQLITE_DEFAULT_PAGE_SIZE', '4096'),
('SQLITE_DEFAULT_CACHE_SIZE', '-8000'))) # 8MB.
('SQLITE_DEFAULT_CACHE_SIZE', '-8000'),
]
ext.include_dirs.append(self.amalgamation_root)
ext.sources.append(os.path.join(self.amalgamation_root, "sqlite3.c"))
arch = get_arch()
if arch == 'universal2':
conan_info_x64 = install_openssl('x86_64')
openssl_dir_x64 = fetch_openssl_dir(conan_info_x64)
conan_info_arm = install_openssl('armv8')
openssl_dir_arm = fetch_openssl_dir(conan_info_arm)
openssl_dir_universal2 = openssl_dir_arm.replace('armv8', 'universal2')
subprocess.run(
[
'python3',
'./lipo-dir-merge/lipo-dir-merge.py',
openssl_dir_x64,
openssl_dir_arm,
openssl_dir_universal2
]
)
shutil.rmtree(openssl_dir_x64)
shutil.move(openssl_dir_universal2, openssl_dir_x64)
openssl_dir = openssl_dir_x64
else:
conan_info = install_openssl(arch)
openssl_dir = fetch_openssl_dir(conan_info)
if sys.platform != "win32":
extra_link_args = []
if sys.platform != "win32":
# Include math library, required for fts5, and crypto.
ext.extra_link_args.extend(["-lm", "-lcrypto"])
else:
# Try to locate openssl.
openssl_conf = os.environ.get('OPENSSL_CONF')
if not openssl_conf:
error_message = 'Fatal error: OpenSSL could not be detected!'
raise RuntimeError(error_message)
extra_link_args.extend(["-lm", "-lcrypto"])
openssl = os.path.dirname(os.path.dirname(openssl_conf))
openssl_lib_path = os.path.join(openssl, "lib")
openssl_lib_path = os.path.join(openssl_dir, "lib")
# Configure the compiler
ext.include_dirs.append(os.path.join(openssl, "include"))
ext.define_macros.append(("inline", "__inline"))
# Configure the compiler
include_dirs.append(os.path.join(openssl_dir, "include"))
define_macros.append(("inline", "__inline"))
# Configure the linker
openssl_libname = os.environ.get('OPENSSL_LIBNAME') or 'libeay32.lib'
ext.extra_link_args.append(openssl_libname)
ext.extra_link_args.append('/LIBPATH:' + openssl_lib_path)
# Configure the linker
if sys.platform == "win32":
# https://github.com/openssl/openssl/blob/master/NOTES-WINDOWS.md#linking-native-applications
extra_link_args.append("WS2_32.LIB")
extra_link_args.append("GDI32.LIB")
extra_link_args.append("ADVAPI32.LIB")
extra_link_args.append("CRYPT32.LIB")
extra_link_args.append("USER32.LIB")
extra_link_args.append('libcrypto.lib')
else:
extra_link_args.append('libcrypto.a')
build_ext.build_extension(self, ext)
module = Extension(
name=PACKAGE_NAME + EXTENSION_MODULE_NAME,
sources=sources,
define_macros=define_macros,
library_dirs=[openssl_lib_path],
include_dirs=include_dirs,
extra_link_args=extra_link_args,
language="c",
)
def __setattr__(self, k, v):
# Make sure we don't link against the SQLite
# library, no matter what setup.cfg says
if k == "libraries":
v = None
self.__dict__[k] = v
with open("README.md", "r", encoding="utf-8") as fr:
long_description = fr.read()
def get_setup_args():
return dict(
if __name__ == "__main__":
setup(
name=PACKAGE_NAME,
version=VERSION,
description="DB-API 2.0 interface for SQLCipher 3.x",
long_description='',
long_description=long_description,
author="Charles Leifer",
author_email="coleifer@gmail.com",
license="zlib/libpng",
@ -157,11 +207,7 @@ def get_setup_args():
url="https://github.com/coleifer/sqlcipher3",
package_dir={PACKAGE_NAME: "sqlcipher3"},
packages=packages,
ext_modules=[Extension(
name=PACKAGE_NAME + EXTENSION_MODULE_NAME,
sources=sources,
define_macros=define_macros)
],
ext_modules=[module],
classifiers=[
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
@ -173,12 +219,4 @@ def get_setup_args():
"Programming Language :: Python",
"Topic :: Database :: Database Engines/Servers",
"Topic :: Software Development :: Libraries :: Python Modules"],
cmdclass={
"build_static": AmalgationLibSqliteBuilder,
"build_ext": SystemLibSqliteBuilder
}
)
if __name__ == "__main__":
setuptools.setup(**get_setup_args())

1
sqlcipher Submodule

@ -0,0 +1 @@
Subproject commit df092f0a7af1c8e3558a743036c089e6ef8e6307