Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-31 13:41:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-131531: android.py enhancements to support cibuildwheel (#132870)

Browse source 
Modifies the environment handling and execution arguments of the Android management
script to support the compilation of third-party binaries, and the use of the testbed to 
invoke third-party test code.

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Russell Keith-Magee <russell@keith-magee.com>
This commit is contained in:
Malcolm Smith 2025-06-05 06:46:16 +01:00 • committed by GitHub
parent 6b77af257c
commit 2e1544fd2b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 239 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Android/README.md
View file

@ -156,6 +156,10 @@ ## Testing
and architecture-specific files such as sysconfigdata, will not take effect
until you re-run `android.py make-host` or `build`.
The testbed app can also be used to test third-party packages. For more details,
run `android.py test --help`, paying attention to the options `--site-packages`,
`--cwd`, `-c` and `-m`.
## Using in your own app

6
Android/android-env.sh
View file

@ -3,7 +3,7 @@
: "${HOST:?}" # GNU target triplet
# You may also override the following:
: "${api_level:=24}" # Minimum Android API level the build will run on
: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on
: "${PREFIX:-}" # Path in which to find required libraries
@ -24,7 +24,7 @@ fail() {
# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md
# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.:
# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md
ndk_version=27.1.12297006
ndk_version=27.2.12479018
ndk=$ANDROID_HOME/ndk/$ndk_version
if ! [ -e "$ndk" ]; then
@ -43,7 +43,7 @@ fi
toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*)
export AR="$toolchain/bin/llvm-ar"
export AS="$toolchain/bin/llvm-as"
export CC="$toolchain/bin/${clang_triplet}${api_level}-clang"
export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang"
export CXX="${CC}++"
export LD="$toolchain/bin/ld"
export NM="$toolchain/bin/llvm-nm"

231
Android/android.py
View file

@ -14,7 +14,7 @@
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from glob import glob
from os.path import basename, relpath
from os.path import abspath, basename, relpath
from pathlib import Path
from subprocess import CalledProcessError
from tempfile import TemporaryDirectory
@ -22,9 +22,13 @@
SCRIPT_NAME = Path(__file__).name
ANDROID_DIR = Path(__file__).resolve().parent
CHECKOUT = ANDROID_DIR.parent
PYTHON_DIR = ANDROID_DIR.parent
in_source_tree = (
ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists()
)
TESTBED_DIR = ANDROID_DIR / "testbed"
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
HOSTS = ["aarch64-linux-android", "x86_64-linux-android"]
APP_ID = "org.python.testbed"
@ -76,39 +80,68 @@ def run(command, *, host=None, env=None, log=True, **kwargs):
kwargs.setdefault("check", True)
if env is None:
env = os.environ.copy()
original_env = env.copy()
if host:
env_script = ANDROID_DIR / "android-env.sh"
env_output = subprocess.run(
f"set -eu; "
f"HOST={host}; "
f"PREFIX={subdir(host)}/prefix; "
f". {env_script}; "
f"export",
check=True, shell=True, text=True, stdout=subprocess.PIPE
).stdout
for line in env_output.splitlines():
# We don't require every line to match, as there may be some other
# output from installing the NDK.
if match := re.search(
"^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line
):
key, value = match[2], match[3]
if env.get(key) != value:
print(line)
env[key] = value
if env == original_env:
raise ValueError(f"Found no variables in {env_script.name} output:\n"
+ env_output)
host_env = android_env(host)
print_env(host_env)
env.update(host_env)
if log:
print(">", " ".join(map(str, command)))
print(">", join_command(command))
return subprocess.run(command, env=env, **kwargs)
# Format a command so it can be copied into a shell. Like shlex.join, but also
# accepts arguments which are Paths, or a single string/Path outside of a list.
def join_command(args):
if isinstance(args, (str, Path)):
return str(args)
else:
return shlex.join(map(str, args))
# Format the environment so it can be pasted into a shell.
def print_env(env):
for key, value in sorted(env.items()):
print(f"export {key}={shlex.quote(value)}")
def android_env(host):
if host:
prefix = subdir(host) / "prefix"
else:
prefix = ANDROID_DIR / "prefix"
sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py")
sysconfig_filename = next(sysconfig_files).name
host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1]
env_script = ANDROID_DIR / "android-env.sh"
env_output = subprocess.run(
f"set -eu; "
f"export HOST={host}; "
f"PREFIX={prefix}; "
f". {env_script}; "
f"export",
check=True, shell=True, capture_output=True, encoding='utf-8',
).stdout
env = {}
for line in env_output.splitlines():
# We don't require every line to match, as there may be some other
# output from installing the NDK.
if match := re.search(
"^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line
):
key, value = match[2], match[3]
if os.environ.get(key) != value:
env[key] = value
if not env:
raise ValueError(f"Found no variables in {env_script.name} output:\n"
+ env_output)
return env
def build_python_path():
"""The path to the build Python binary."""
build_dir = subdir("build")
@ -127,7 +160,7 @@ def configure_build_python(context):
clean("build")
os.chdir(subdir("build", create=True))
command = [relpath(CHECKOUT / "configure")]
command = [relpath(PYTHON_DIR / "configure")]
if context.args:
command.extend(context.args)
run(command)
@ -139,12 +172,13 @@ def make_build_python(context):
def unpack_deps(host, prefix_dir):
os.chdir(prefix_dir)
deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4",
for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4",
"sqlite-3.49.1-0", "xz-5.4.6-1"]:
filename = f"{name_ver}-{host}.tar.gz"
download(f"{deps_url}/{name_ver}/{filename}")
shutil.unpack_archive(filename, prefix_dir)
shutil.unpack_archive(filename)
os.remove(filename)
@ -167,7 +201,7 @@ def configure_host_python(context):
os.chdir(host_dir)
command = [
# Basic cross-compiling configuration
relpath(CHECKOUT / "configure"),
relpath(PYTHON_DIR / "configure"),
f"--host={context.host}",
f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}",
f"--with-build-python={build_python_path()}",
@ -196,9 +230,12 @@ def make_host_python(context):
for pattern in ("include/python*", "lib/libpython*", "lib/python*"):
delete_glob(f"{prefix_dir}/{pattern}")
# The Android environment variables were already captured in the Makefile by
# `configure`, and passing them again when running `make` may cause some
# flags to be duplicated. So we don't use the `host` argument here.
os.chdir(host_dir)
run(["make", "-j", str(os.cpu_count())], host=context.host)
run(["make", "install", f"prefix={prefix_dir}"], host=context.host)
run(["make", "-j", str(os.cpu_count())])
run(["make", "install", f"prefix={prefix_dir}"])
def build_all(context):
@ -228,7 +265,12 @@ def setup_sdk():
if not all((android_home / "licenses" / path).exists() for path in [
"android-sdk-arm-dbt-license", "android-sdk-license"
]):
run([sdkmanager, "--licenses"], text=True, input="y\n" * 100)
run(
[sdkmanager, "--licenses"],
text=True,
capture_output=True,
input="y\n" * 100,
)
# Gradle may install this automatically, but we can't rely on that because
# we need to run adb within the logcat task.
@ -474,24 +516,49 @@ async def gradle_task(context):
task_prefix = "connected"
env["ANDROID_SERIAL"] = context.connected
hidden_output = []
def log(line):
# Gradle may take several minutes to install SDK packages, so it's worth
# showing those messages even in non-verbose mode.
if context.verbose or line.startswith('Preparing "Install'):
sys.stdout.write(line)
else:
hidden_output.append(line)
if context.command:
mode = "-c"
module = context.command
else:
mode = "-m"
module = context.module or "test"
args = [
gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest",
"-Pandroid.testInstrumentationRunnerArguments.pythonArgs="
+ shlex.join(context.args),
] + [
# Build-time properties
f"-Ppython.{name}={value}"
for name, value in [
("sitePackages", context.site_packages), ("cwd", context.cwd)
] if value
] + [
# Runtime properties
f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}"
for name, value in [
("Mode", mode), ("Module", module), ("Args", join_command(context.args))
] if value
]
hidden_output = []
if context.verbose >= 2:
args.append("--info")
log("> " + join_command(args))
try:
async with async_process(
*args, cwd=TESTBED_DIR, env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
) as process:
while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
# Gradle may take several minutes to install SDK packages, so
# it's worth showing those messages even in non-verbose mode.
if context.verbose or line.startswith('Preparing "Install'):
sys.stdout.write(line)
else:
hidden_output.append(line)
log(line)
status = await wait_for(process.wait(), timeout=1)
if status == 0:
@ -604,6 +671,10 @@ def package(context):
print(f"Wrote {package_path}")
def env(context):
print_env(android_env(getattr(context, "host", None)))
# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated
# by the buildbot worker, we'll make an attempt to clean up our subprocesses.
def install_signal_handler():
@ -615,36 +686,41 @@ def signal_handler(*args):
def parse_args():
parser = argparse.ArgumentParser()
subcommands = parser.add_subparsers(dest="subcommand")
subcommands = parser.add_subparsers(dest="subcommand", required=True)
# Subcommands
build = subcommands.add_parser("build", help="Build everything")
configure_build = subcommands.add_parser("configure-build",
help="Run `configure` for the "
"build Python")
make_build = subcommands.add_parser("make-build",
help="Run `make` for the build Python")
configure_host = subcommands.add_parser("configure-host",
help="Run `configure` for Android")
make_host = subcommands.add_parser("make-host",
help="Run `make` for Android")
build = subcommands.add_parser(
"build", help="Run configure-build, make-build, configure-host and "
"make-host")
configure_build = subcommands.add_parser(
"configure-build", help="Run `configure` for the build Python")
subcommands.add_parser(
"clean", help="Delete all build and prefix directories")
subcommands.add_parser(
"build-testbed", help="Build the testbed app")
test = subcommands.add_parser(
"test", help="Run the test suite")
"make-build", help="Run `make` for the build Python")
configure_host = subcommands.add_parser(
"configure-host", help="Run `configure` for Android")
make_host = subcommands.add_parser(
"make-host", help="Run `make` for Android")
subcommands.add_parser("clean", help="Delete all build directories")
subcommands.add_parser("build-testbed", help="Build the testbed app")
test = subcommands.add_parser("test", help="Run the testbed app")
package = subcommands.add_parser("package", help="Make a release package")
env = subcommands.add_parser("env", help="Print environment variables")
# Common arguments
for subcommand in build, configure_build, configure_host:
subcommand.add_argument(
"--clean", action="store_true", default=False, dest="clean",
help="Delete the relevant build and prefix directories first")
for subcommand in [build, configure_host, make_host, package]:
help="Delete the relevant build directories first")
host_commands = [build, configure_host, make_host, package]
if in_source_tree:
host_commands.append(env)
for subcommand in host_commands:
subcommand.add_argument(
"host", metavar="HOST", choices=HOSTS,
help="Host triplet: choices=[%(choices)s]")
for subcommand in build, configure_build, configure_host:
subcommand.add_argument("args", nargs="*",
help="Extra arguments to pass to `configure`")
@ -654,6 +730,7 @@ def parse_args():
"-v", "--verbose", action="count", default=0,
help="Show Gradle output, and non-Python logcat messages. "
"Use twice to include high-volume messages which are rarely useful.")
device_group = test.add_mutually_exclusive_group(required=True)
device_group.add_argument(
"--connected", metavar="SERIAL", help="Run on a connected device. "
@ -661,8 +738,24 @@ def parse_args():
device_group.add_argument(
"--managed", metavar="NAME", help="Run on a Gradle-managed device. "
"These are defined in `managedDevices` in testbed/app/build.gradle.kts.")
test.add_argument(
"args", nargs="*", help=f"Arguments for `python -m test`. "
"--site-packages", metavar="DIR", type=abspath,
help="Directory to copy as the app's site-packages.")
test.add_argument(
"--cwd", metavar="DIR", type=abspath,
help="Directory to copy as the app's working directory.")
mode_group = test.add_mutually_exclusive_group()
mode_group.add_argument(
"-c", dest="command", help="Execute the given Python code.")
mode_group.add_argument(
"-m", dest="module", help="Execute the module with the given name.")
test.epilog = (
"If neither -c nor -m are passed, the default is '-m test', which will "
"run Python's own test suite.")
test.add_argument(
"args", nargs="*", help=f"Arguments to add to sys.argv. "
f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.")
return parser.parse_args()
@ -688,6 +781,7 @@ def main():
"build-testbed": build_testbed,
"test": run_testbed,
"package": package,
"env": env,
}
try:
@ -708,14 +802,9 @@ def print_called_process_error(e):
if not content.endswith("\n"):
stream.write("\n")
# Format the command so it can be copied into a shell. shlex uses single
# quotes, so we surround the whole command with double quotes.
args_joined = (
e.cmd if isinstance(e.cmd, str)
else " ".join(shlex.quote(str(arg)) for arg in e.cmd)
)
# shlex uses single quotes, so we surround the command with double quotes.
print(
f'Command "{args_joined}" returned exit status {e.returncode}'
f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}'
)

20
Android/testbed/app/build.gradle.kts
View file

@ -85,7 +85,7 @@ android {
minSdk = androidEnvFile.useLines {
for (line in it) {
"""api_level:=(\d+)""".toRegex().find(line)?.let {
"""ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let {
return@useLines it.groupValues[1].toInt()
}
}
@ -205,11 +205,29 @@ androidComponents.onVariants { variant ->
into("site-packages") {
from("$projectDir/src/main/python")
val sitePackages = findProperty("python.sitePackages") as String?
if (!sitePackages.isNullOrEmpty()) {
if (!file(sitePackages).exists()) {
throw GradleException("$sitePackages does not exist")
}
from(sitePackages)
}
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
exclude("**/__pycache__")
}
into("cwd") {
val cwd = findProperty("python.cwd") as String?
if (!cwd.isNullOrEmpty()) {
if (!file(cwd).exists()) {
throw GradleException("$cwd does not exist")
}
from(cwd)
}
}
}
}

8
Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt
View file

@ -17,11 +17,11 @@ class PythonSuite {
fun testPython() {
val start = System.currentTimeMillis()
try {
val context =
val status = PythonTestRunner(
InstrumentationRegistry.getInstrumentation().targetContext
val args =
InstrumentationRegistry.getArguments().getString("pythonArgs", "")
val status = PythonTestRunner(context).run(args)
).run(
InstrumentationRegistry.getArguments()
)
assertEquals(0, status)
} finally {
// Make sure the process lives long enough for the test script to

27
Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
View file

@ -15,17 +15,29 @@ class MainActivity : AppCompatActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_main)
val status = PythonTestRunner(this).run("-W -uall")
val status = PythonTestRunner(this).run("-m", "test", "-W -uall")
findViewById<TextView>(R.id.tvHello).text = "Exit status $status"
}
}
class PythonTestRunner(val context: Context) {
/** @param args Extra arguments for `python -m test`.
* @return The Python exit status: zero if the tests passed, nonzero if
* they failed. */
fun run(args: String = "") : Int {
fun run(instrumentationArgs: Bundle) = run(
instrumentationArgs.getString("pythonMode")!!,
instrumentationArgs.getString("pythonModule")!!,
instrumentationArgs.getString("pythonArgs") ?: "",
)
/** Run Python.
*
* @param mode Either "-c" or "-m".
* @param module Python statements for "-c" mode, or a module name for
* "-m" mode.
* @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`.
* @return The Python exit status: zero on success, nonzero on failure. */
fun run(mode: String, module: String, args: String) : Int {
Os.setenv("PYTHON_MODE", mode, true)
Os.setenv("PYTHON_MODULE", module, true)
Os.setenv("PYTHON_ARGS", args, true)
// Python needs this variable to help it find the temporary directory,
@ -36,8 +48,9 @@ class PythonTestRunner(val context: Context) {
System.loadLibrary("main_activity")
redirectStdioToLogcat()
// The main module is in src/main/python/main.py.
return runPython(pythonHome.toString(), "main")
// The main module is in src/main/python. We don't simply call it
// "main", as that could clash with third-party test code.
return runPython(pythonHome.toString(), "android_testbed_main")
}
private fun extractAssets() : File {

20
Android/testbed/app/src/main/python/main.py → Android/testbed/app/src/main/python/android_testbed_main.py
View file

@ -26,7 +26,23 @@
# test_signals in test_threadsignals.py.
signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1])
mode = os.environ["PYTHON_MODE"]
module = os.environ["PYTHON_MODULE"]
sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"])
# The test module will call sys.exit to indicate whether the tests passed.
runpy.run_module("test")
cwd = f"{sys.prefix}/cwd"
if not os.path.exists(cwd):
# Empty directories are lost in the asset packing/unpacking process.
os.mkdir(cwd)
os.chdir(cwd)
if mode == "-c":
# In -c mode, sys.path starts with an empty string, which means whatever the current
# working directory is at the moment of each import.
sys.path.insert(0, "")
exec(module, {})
elif mode == "-m":
sys.path.insert(0, os.getcwd())
runpy.run_module(module, run_name="__main__", alter_sys=True)
else:
raise ValueError(f"unknown mode: {mode}")

2
Android/testbed/build.gradle.kts
View file

@ -1,5 +1,5 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
plugins {
id("com.android.application") version "8.6.1" apply false
id("com.android.application") version "8.10.0" apply false
id("org.jetbrains.kotlin.android") version "1.9.22" apply false
}

2
Android/testbed/gradle/wrapper/gradle-wrapper.properties vendored
View file

@ -1,6 +1,6 @@
#Mon Feb 19 20:29:06 GMT 2024
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

9
Doc/using/android.rst
View file

@ -63,3 +63,12 @@ link to the relevant file.
* Add code to your app to :source:`start Python in embedded mode
<Android/testbed/app/src/main/c/main_activity.c>`. This will need to be C code
called via JNI.
Building a Python package for Android
-------------------------------------
Python packages can be built for Android as wheels and released on PyPI. The
recommended tool for doing this is `cibuildwheel
<https://cibuildwheel.pypa.io/en/stable/platforms/#android>`__, which automates
all the details of setting up a cross-compilation environment, building the
wheel, and testing it on an emulator.