Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-02-05 17:35:34 +00:00
Code Issues Projects Releases Packages Wiki Activity

[3.13] gh-144415: Android testbed fixes (GH-142912) (#144417)

Browse source 
Modifies handling of `.gz` files in Android app payloads, and ensures that
when the Android testbed streams logs, stream flushes aren't treated as
newlines. This improves the output of test suites that use "one dot per test"
progress indicators.
(cherry picked from commit cb1dc91dcb)

Co-authored-by: Malcolm Smith <smith@chaquo.com>
This commit is contained in:
Miss Islington (bot) 2026-02-03 10:28:43 +01:00 • committed by GitHub
parent b4a84bc555
commit ebc047f717
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 129 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

106
Android/android.py
View file

@ -15,6 +15,7 @@
from asyncio import wait_for
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from enum import IntEnum, auto
from glob import glob
from os.path import abspath, basename, relpath
from pathlib import Path
@ -61,6 +62,19 @@
hidden_output = []
# Based on android/log.h in the NDK.
class LogPriority(IntEnum):
UNKNOWN = 0
DEFAULT = auto()
VERBOSE = auto()
DEBUG = auto()
INFO = auto()
WARN = auto()
ERROR = auto()
FATAL = auto()
SILENT = auto()
def log_verbose(context, line, stream=sys.stdout):
if context.verbose:
stream.write(line)
@ -505,21 +519,23 @@ async def logcat_task(context, initial_devices):
pid = await wait_for(find_pid(serial), startup_timeout)
# `--pid` requires API level 24 or higher.
args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"]
#
# `--binary` mode is used in order to detect which messages end with a
# newline, which most of the other modes don't indicate (except `--format
# long`). For example, every time pytest runs a test, it prints a "." and
# flushes the stream. Each "." becomes a separate log message, but we should
# show them all on the same line.
args = [adb, "-s", serial, "logcat", "--pid", pid, "--binary"]
logcat_started = False
async with async_process(
*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
*args, stdout=subprocess.PIPE, stderr=None
) as process:
while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL):
while True:
try:
priority, tag, message = await read_logcat(process.stdout)
logcat_started = True
level, message = match.groups()
else:
# If the regex doesn't match, this is either a logcat startup
# error, or the second or subsequent line of a multi-line
# message. Python won't produce multi-line messages, but other
# components might.
level, message = None, line
except asyncio.IncompleteReadError:
break
# Exclude high-volume messages which are rarely useful.
if context.verbose < 2 and "from python test_syslog" in message:
@ -527,25 +543,23 @@ async def logcat_task(context, initial_devices):
# Put high-level messages on stderr so they're highlighted in the
# buildbot logs. This will include Python's own stderr.
stream = (
sys.stderr
if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT)
else sys.stdout
)
stream = sys.stderr if priority >= LogPriority.WARN else sys.stdout
# To simplify automated processing of the output, e.g. a buildbot
# posting a failure notice on a GitHub PR, we strip the level and
# tag indicators from Python's stdout and stderr.
for prefix in ["python.stdout: ", "python.stderr: "]:
if message.startswith(prefix):
global python_started
python_started = True
stream.write(message.removeprefix(prefix))
break
# The app's stdout and stderr should be passed through transparently
# to our own corresponding streams.
if tag in ["python.stdout", "python.stderr"]:
global python_started
python_started = True
stream.write(message)
stream.flush()
else:
# Non-Python messages add a lot of noise, but they may
# sometimes help explain a failure.
log_verbose(context, line, stream)
# sometimes help explain a failure. Format them in the same way
# as `logcat --format tag`.
formatted = f"{priority.name[0]}/{tag}: {message}"
if not formatted.endswith("\n"):
formatted += "\n"
log_verbose(context, formatted, stream)
# If the device disconnects while logcat is running, which always
# happens in --managed mode, some versions of adb return non-zero.
@ -556,6 +570,44 @@ async def logcat_task(context, initial_devices):
raise CalledProcessError(status, args)
# Read one binary log message from the given StreamReader. The message format is
# described at https://android.stackexchange.com/a/74660. All supported versions
# of Android use format version 2 or later.
async def read_logcat(stream):
async def read_bytes(size):
return await stream.readexactly(size)
async def read_int(size):
return int.from_bytes(await read_bytes(size), "little")
payload_len = await read_int(2)
if payload_len < 2:
# 1 byte for priority, 1 byte for null terminator of tag.
raise ValueError(f"payload length {payload_len} is too short")
header_len = await read_int(2)
if header_len < 4:
raise ValueError(f"header length {header_len} is too short")
await read_bytes(header_len - 4) # Ignore other header fields.
priority_int = await read_int(1)
try:
priority = LogPriority(priority_int)
except ValueError:
priority = LogPriority.UNKNOWN
payload_fields = (await read_bytes(payload_len - 1)).split(b"\0")
if len(payload_fields) < 2:
raise ValueError(
f"payload {payload!r} does not contain at least 2 "
f"null-separated fields"
)
tag, message, *_ = [
field.decode(*DECODE_ARGS) for field in payload_fields
]
return priority, tag, message
def stop_app(serial):
run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False)

24
Android/testbed/app/build.gradle.kts
View file

@ -94,10 +94,13 @@ android {
}
// This controls the API level of the maxVersion managed emulator, which is used
// by CI and cibuildwheel. 34 takes up too much disk space (#142289), 35 has
// issues connecting to the internet (#142387), and 36 and later are not
// available as aosp_atd images yet.
targetSdk = 33
// by CI and cibuildwheel.
// * 33 has excessive buffering in the logcat client
// (https://cs.android.com/android/_/android/platform/system/logging/+/d340721894f223327339010df59b0ac514308826).
// * 34 consumes too much disk space on GitHub Actions (#142289).
// * 35 has issues connecting to the internet (#142387).
// * 36 and later are not available as aosp_atd images yet.
targetSdk = 32
versionCode = 1
versionName = "1.0"
@ -130,9 +133,10 @@ android {
path("src/main/c/CMakeLists.txt")
}
// Set this property to something non-empty, otherwise it'll use the default
// list, which ignores asset directories beginning with an underscore.
aaptOptions.ignoreAssetsPattern = ".git"
// Set this property to something nonexistent but non-empty. Otherwise it'll use the
// default list, which ignores asset directories beginning with an underscore, and
// maybe also other files required by tests.
aaptOptions.ignoreAssetsPattern = "android-testbed-dont-ignore-anything"
compileOptions {
sourceCompatibility = JavaVersion.VERSION_1_8
@ -234,6 +238,12 @@ androidComponents.onVariants { variant ->
from(cwd)
}
}
// A filename ending with .gz will be automatically decompressed
// while building the APK. Avoid this by adding a dash to the end,
// and add an extra dash to any filenames that already end with one.
// This will be undone in MainActivity.kt.
rename(""".*(\.gz|-)""", "$0-")
}
}

4
Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
View file

@ -80,7 +80,9 @@ class PythonTestRunner(val context: Context) {
continue
}
input.use {
File(targetSubdir, name).outputStream().use { output ->
// Undo the .gz workaround from build.gradle.kts.
val outputName = name.replace(Regex("""(.*)-"""), "$1")
File(targetSubdir, outputName).outputStream().use { output ->
input.copyTo(output)
}
}

7
Lib/_android_support.py
View file

@ -168,6 +168,13 @@ def write(self, prio, tag, message):
# message.
message = message.replace(b"\x00", b"\xc0\x80")
# On API level 30 and higher, Logcat will strip any number of leading
# newlines. This is visible in all `logcat` modes, even --binary. Work
# around this by adding a leading space, which shouldn't make any
# difference to the log's usability.
if message.startswith(b"\n"):
message = b" " + message
with self._lock:
now = time()
self._bucket_level += (

38
Lib/test/test_android.py
View file

@ -183,14 +183,18 @@ def write(s, lines=None, *, write_len=None):
# Multi-line messages. Avoid identical consecutive lines, as
# they may activate "chatty" filtering and break the tests.
write("\nx", [""])
#
# Additional spaces will appear in the output where necessary to
# protect leading newlines.
write("\nx", [" "])
write("\na\n", ["x", "a"])
write("\n", [""])
write("\n", [" "])
write("\n\n", [" ", " "])
write("b\n", ["b"])
write("c\n\n", ["c", ""])
write("c\n\n", ["c", " "])
write("d\ne", ["d"])
write("xx", [])
write("f\n\ng", ["exxf", ""])
write("f\n\ng", ["exxf", " "])
write("\n", ["g"])
# Since this is a line-based logging system, line buffering
@ -201,15 +205,16 @@ def write(s, lines=None, *, write_len=None):
# However, buffering can be turned off completely if you want a
# flush after every write.
with self.reconfigure(stream, write_through=True):
write("\nx", ["", "x"])
write("\na\n", ["", "a"])
write("\n", [""])
write("\nx", [" ", "x"])
write("\na\n", [" ", "a"])
write("\n", [" "])
write("\n\n", [" ", " "])
write("b\n", ["b"])
write("c\n\n", ["c", ""])
write("c\n\n", ["c", " "])
write("d\ne", ["d", "e"])
write("xx", ["xx"])
write("f\n\ng", ["f", "", "g"])
write("\n", [""])
write("f\n\ng", ["f", " ", "g"])
write("\n", [" "])
# "\r\n" should be translated into "\n".
write("hello\r\n", ["hello"])
@ -329,19 +334,16 @@ def write(b, lines=None, *, write_len=None):
# currently use `logcat -v tag`, which shows each line as if it
# was a separate log entry, but strips a single trailing
# newline.
#
# On newer versions of Android, all three of the above tools (or
# maybe Logcat itself) will also strip any number of leading
# newlines.
write(b"\nx", ["", "x"] if api_level < 30 else ["x"])
write(b"\na\n", ["", "a"] if api_level < 30 else ["a"])
write(b"\n", [""])
write(b"\nx", [" ", "x"])
write(b"\na\n", [" ", "a"])
write(b"\n", [" "])
write(b"\n\n", [" ", ""])
write(b"b\n", ["b"])
write(b"c\n\n", ["c", ""])
write(b"d\ne", ["d", "e"])
write(b"xx", ["xx"])
write(b"f\n\ng", ["f", "", "g"])
write(b"\n", [""])
write(b"\n", [" "])
# "\r\n" should be translated into "\n".
write(b"hello\r\n", ["hello"])

3
Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst Normal file
View file

@ -0,0 +1,3 @@
The Android testbed now distinguishes between stdout/stderr messages which
were triggered by a newline, and those triggered by a manual call to
``flush``. This fixes logging of progress indicators and similar content.