Skip to content

Commit 258038c

Browse files
hallvictoriahallvictoria
committed
Address feedback: bugs, osxarm build, inc test cov
1 parent 5747062 commit 258038c

16 files changed

Lines changed: 714 additions & 106 deletions

File tree

‎eng/pack/scripts/r2p2_deps.sh‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Copyright (c) Microsoft Corporation. All rights reserved.
33
# Licensed under the MIT License.
44
#
5-
# Build the R2P2 release artifact tree for packaging (Linux/X64 and Linux/Arm64).
5+
# Build the R2P2 release artifact tree for packaging (Linux and macOS).
66
#
77
# Unlike the Python workers (which are pure pip installs), the R2P2 is a
88
# compiled binary that embeds CPython via PyO3. This script:
@@ -24,6 +24,7 @@ PYTHON_VERSION="${1:?usage: r2p2_deps.sh <python-version> [x64|arm64]}"
2424
ARCHITECTURE="${2:-x64}"
2525
DEPS="$BUILD_SOURCESDIRECTORY/deps"
2626
WORKER="$BUILD_SOURCESDIRECTORY/workers/r2p2"
27+
HOST_OS="$(uname -s)"
2728

2829
if [[ "$ARCHITECTURE" != "x64" && "$ARCHITECTURE" != "arm64" ]]; then
2930
echo "Unsupported architecture: $ARCHITECTURE" >&2
@@ -47,7 +48,7 @@ cargo --version
4748
# --- Compile the release binary against the target interpreter ------------
4849
CARGO_ARGS=(build --release --locked)
4950
BINARY_PATH="$WORKER/target/release/r2p2"
50-
if [[ "$ARCHITECTURE" == "arm64" ]]; then
51+
if [[ "$HOST_OS" == "Linux" && "$ARCHITECTURE" == "arm64" ]]; then
5152
RUST_TARGET="aarch64-unknown-linux-gnu"
5253
rustup target add "$RUST_TARGET"
5354
sudo apt-get update
@@ -117,12 +118,30 @@ EOF
117118
export PYO3_CROSS_PYTHON_VERSION="$PYTHON_MAJOR_MINOR"
118119
export PYO3_CROSS_PYTHON_IMPLEMENTATION=CPython
119120
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
121+
CARGO_ARGS+=(--target "$RUST_TARGET")
122+
BINARY_PATH="$WORKER/target/$RUST_TARGET/release/r2p2"
123+
elif [[ "$HOST_OS" == "Darwin" && "$ARCHITECTURE" == "arm64" ]]; then
124+
RUST_TARGET="aarch64-apple-darwin"
125+
rustup target add "$RUST_TARGET"
126+
127+
export PYO3_PYTHON="$(command -v python)"
128+
PYTHON_FRAMEWORK_BINARY="$(python -c 'import sysconfig; framework = sysconfig.get_config_var("PYTHONFRAMEWORK"); prefix = sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX"); version = sysconfig.get_config_var("VERSION"); print(f"{prefix}/{framework}.framework/Versions/{version}/{framework}")')"
129+
PYTHON_ARCHITECTURES="$(lipo -archs "$PYTHON_FRAMEWORK_BINARY")"
130+
if [[ " $PYTHON_ARCHITECTURES " != *" arm64 "* ]]; then
131+
echo "The selected Python framework cannot target macOS ARM64; found: $PYTHON_ARCHITECTURES" >&2
132+
exit 1
133+
fi
134+
120135
CARGO_ARGS+=(--target "$RUST_TARGET")
121136
BINARY_PATH="$WORKER/target/$RUST_TARGET/release/r2p2"
122137
else
123138
export PYO3_PYTHON="$(command -v python)"
124139
PY_LIBDIR="$(python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))')"
125-
export LD_LIBRARY_PATH="${PY_LIBDIR}:${LD_LIBRARY_PATH:-}"
140+
if [[ "$HOST_OS" == "Darwin" ]]; then
141+
export DYLD_LIBRARY_PATH="${PY_LIBDIR}:${DYLD_LIBRARY_PATH:-}"
142+
else
143+
export LD_LIBRARY_PATH="${PY_LIBDIR}:${LD_LIBRARY_PATH:-}"
144+
fi
126145
fi
127146
( cd "$WORKER" && cargo "${CARGO_ARGS[@]}" )
128147

‎eng/pack/templates/r2p2_env_gen.yml‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,5 +121,61 @@ steps:
121121
Write-Error "Validation failed. Pruned packages unexpectedly present:`n$($present -join "`n")"
122122
exit 1
123123
}
124-
Write-Host "✅ Validation passed. R2P2 artifact contents are correct."
124+
125+
$binaryPath = Join-Path $root $binary
126+
$binaryBytes = [System.IO.File]::ReadAllBytes($binaryPath)
127+
$osName = '${{ parameters.osName }}'
128+
$architecture = '${{ parameters.architecture }}'
129+
if ($osName -eq 'WINDOWS') {
130+
if ($binaryBytes.Length -lt 64 -or $binaryBytes[0] -ne 0x4d -or $binaryBytes[1] -ne 0x5a) {
131+
throw "Validation failed. $binary is not a PE executable."
132+
}
133+
$peOffset = [BitConverter]::ToInt32($binaryBytes, 0x3c)
134+
if ($peOffset -lt 0 -or $peOffset + 6 -gt $binaryBytes.Length -or
135+
$binaryBytes[$peOffset] -ne 0x50 -or $binaryBytes[$peOffset + 1] -ne 0x45 -or
136+
$binaryBytes[$peOffset + 2] -ne 0 -or $binaryBytes[$peOffset + 3] -ne 0) {
137+
throw "Validation failed. $binary has an invalid PE header."
138+
}
139+
$actualMachine = [BitConverter]::ToUInt16($binaryBytes, $peOffset + 4)
140+
$expectedMachine = switch ($architecture) {
141+
'x86' { 0x014c }
142+
'x64' { 0x8664 }
143+
'arm64' { 0xaa64 }
144+
default { throw "Unsupported Windows architecture: $architecture" }
145+
}
146+
if ($actualMachine -ne $expectedMachine) {
147+
throw "Validation failed. Expected $binary PE machine 0x$($expectedMachine.ToString('x4')) for $architecture; found 0x$($actualMachine.ToString('x4'))."
148+
}
149+
} elseif ($osName -eq 'LINUX') {
150+
if ($binaryBytes.Length -lt 20 -or $binaryBytes[0] -ne 0x7f -or
151+
$binaryBytes[1] -ne 0x45 -or $binaryBytes[2] -ne 0x4c -or $binaryBytes[3] -ne 0x46) {
152+
throw "Validation failed. $binary is not an ELF executable."
153+
}
154+
if ($binaryBytes[5] -ne 1) {
155+
throw "Validation failed. $binary is not a little-endian ELF executable."
156+
}
157+
$actualMachine = [BitConverter]::ToUInt16($binaryBytes, 18)
158+
$expectedMachine = switch ($architecture) {
159+
'x86' { 0x0003 }
160+
'x64' { 0x003e }
161+
'arm64' { 0x00b7 }
162+
default { throw "Unsupported Linux architecture: $architecture" }
163+
}
164+
if ($actualMachine -ne $expectedMachine) {
165+
throw "Validation failed. Expected $binary ELF machine 0x$($expectedMachine.ToString('x4')) for $architecture; found 0x$($actualMachine.ToString('x4'))."
166+
}
167+
} elseif ($osName -eq 'OSX') {
168+
$expectedArchitecture = if ($architecture -eq 'arm64') { 'arm64' } else { 'x86_64' }
169+
$lipoOutput = & lipo -archs $binaryPath
170+
if ($LASTEXITCODE -ne 0) {
171+
throw "Validation failed. lipo could not inspect $binary."
172+
}
173+
$architectures = $lipoOutput.Trim() -split '\s+'
174+
if ($architectures -notcontains $expectedArchitecture) {
175+
throw "Validation failed. Expected $binary to contain $expectedArchitecture; found $($architectures -join ', ')."
176+
}
177+
} else {
178+
throw "Unsupported R2P2 artifact OS: $osName"
179+
}
180+
Write-Host "✅ Validation passed. R2P2 artifact contents and $architecture binary are correct."
125181
displayName: "Validate R2P2 artifact contents"

‎eng/templates/official/jobs/build-artifacts.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,8 @@ jobs:
337337
pythonVersion: '$(pythonVersion)'
338338
artifactName: '$(normalizedPythonVersion)_OSX_ARM64'
339339
osName: 'OSX'
340+
architecture: 'arm64'
341+
hostArchitecture: 'x64'
340342
- job: Build_R2P2_LINUX_ARM64
341343
timeoutInMinutes: 240
342344
pool:

‎runtimes/v1/azure_functions_runtime_v1/native_invocation.py‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@
3131
function_id: str,
3232
inputs: list[tuple[str, Datum]], # (param name, decoded Datum)
3333
trigger_metadata: dict[str, Datum],
34-
) -> tuple[bool, Optional[Datum], list[tuple[str, Datum]], Optional[str]]
35-
# (ok, return_datum, output_data, exception_text)
34+
) -> tuple[bool, Optional[Datum], list[tuple[str, Datum]],
35+
Optional[str], Optional[str]]
36+
# (ok, return_datum, output_data, exception_text, exception_traceback)
3637
"""
3738
import sys
39+
import traceback
3840

3941
from .bindings.context import get_context
4042
from .bindings.meta import get_binding, get_datum, is_trigger_binding
@@ -154,7 +156,7 @@ async def invocation_request_native(invocation_id, function_id, inputs,
154156
threadpool = get_threadpool_executor()
155157
fi = _functions.get_function(function_id)
156158
if fi is None:
157-
return (False, None, [], "function %s not loaded" % function_id)
159+
return (False, None, [], "function %s not loaded" % function_id, "")
158160

159161
try:
160162
args = _decode_inputs(fi, inputs, trigger_metadata)
@@ -188,47 +190,53 @@ async def invocation_request_native(invocation_id, function_id, inputs,
188190
return_datum = _encode_return(fi, call_result)
189191

190192
sys.stdout.flush()
191-
return (True, return_datum, output_data, None)
193+
return (True, return_datum, output_data, None, None)
192194

193195
except Exception as ex: # noqa - surfaced to the Host as a failed invocation
194196
logger.exception("Native invocation failed")
195-
return (False, None, [], repr(ex))
197+
return (False, None, [], repr(ex), traceback.format_exc())
196198

197199

198200
def run_invocation_sync(invocation_id, function_id, inputs, trigger_metadata,
199201
trace_parent="", trace_state="", retry_count=0,
200202
max_retry_count=0, retry_exception=None):
201203
"""Synchronous native invocation for *sync* customer functions.
202204
203-
Runs the handler directly on the calling thread (no asyncio loop, no
204-
``run_in_executor`` bounce). With the Rust transport dispatching each call
205-
on its own thread, this avoids funneling sync handlers through a single
206-
event loop, trimming per-invocation overhead on the hot path.
205+
Queues the handler on the runtime's configured threadpool without
206+
funneling the invocation through the asyncio loop. This preserves
207+
``PYTHON_THREADPOOL_THREAD_COUNT`` for the native path while keeping datum
208+
conversion on the Rust-dispatched calling thread.
207209
208-
Returns ``(handled, ok, return_datum, output_data, exception_text)``.
210+
Returns ``(handled, ok, return_datum, output_data, exception_text,
211+
exception_traceback)``.
209212
``handled=False`` means the function is ``async`` and the caller should use
210213
the coroutine path (:func:`invocation_request_native`) instead.
211214
"""
212215
fi = _functions.get_function(function_id)
213216
if fi is None:
214-
return (True, False, None, [], "function %s not loaded" % function_id)
217+
return (True, False, None, [],
218+
"function %s not loaded" % function_id, "")
215219
if fi.is_async:
216-
return (False, None, None, [], None)
220+
return (False, None, None, [], None, None)
217221

218222
try:
219223
args = _decode_inputs(fi, inputs, trigger_metadata)
220224
fi_context = _prepare_context(fi, invocation_id, args,
221225
trace_parent, trace_state, retry_count,
222226
max_retry_count, retry_exception)
223227

224-
call_result = run_sync_func(invocation_id, fi_context, fi.func, args)
228+
threadpool = get_threadpool_executor()
229+
if threadpool is None:
230+
raise RuntimeError("threadpool executor is not initialized")
231+
call_result = threadpool.submit(
232+
run_sync_func, invocation_id, fi_context, fi.func, args).result()
225233

226234
_check_return(fi, call_result)
227235
output_data = _collect_output(fi, args)
228236
return_datum = _encode_return(fi, call_result)
229237

230-
return (True, True, return_datum, output_data, None)
238+
return (True, True, return_datum, output_data, None, None)
231239

232240
except Exception as ex: # noqa - surfaced to the Host as a failed invocation
233241
logger.exception("Native sync invocation failed")
234-
return (True, False, None, [], repr(ex))
242+
return (True, False, None, [], repr(ex), traceback.format_exc())

‎runtimes/v2/azure_functions_runtime/native_invocation.py‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,17 @@
2020
function_id: str,
2121
inputs: list[tuple[str, Datum]], # (param name, decoded Datum)
2222
trigger_metadata: dict[str, Datum],
23-
) -> tuple[bool, Optional[Datum], list[tuple[str, Datum]], Optional[str]]
24-
# (ok, return_datum, output_data, exception_text)
23+
) -> tuple[bool, Optional[Datum], list[tuple[str, Datum]],
24+
Optional[str], Optional[str]]
25+
# (ok, return_datum, output_data, exception_text, exception_traceback)
2526
2627
Scope: this native path supports the classic (non http-v2) binding path,
2728
sync and async functions, input/output bindings and $return. Deferred (SDK-type)
2829
bindings and the http-v2 streaming path are intentionally out of scope here and
2930
fall back through the protobuf control path if ever needed.
3031
"""
3132
import sys
33+
import traceback
3234

3335
from .bindings.context import get_context
3436
from .bindings.meta import get_binding, get_datum, is_trigger_binding
@@ -151,7 +153,7 @@ async def invocation_request_native(invocation_id, function_id, inputs,
151153
threadpool = get_threadpool_executor()
152154
fi = _functions.get_function(function_id)
153155
if fi is None:
154-
return (False, None, [], "function %s not loaded" % function_id)
156+
return (False, None, [], "function %s not loaded" % function_id, "")
155157

156158
try:
157159
args = _decode_inputs(fi, inputs, trigger_metadata)
@@ -188,32 +190,34 @@ async def invocation_request_native(invocation_id, function_id, inputs,
188190
return_datum = _encode_return(fi, call_result)
189191

190192
sys.stdout.flush()
191-
return (True, return_datum, output_data, None)
193+
return (True, return_datum, output_data, None, None)
192194

193195
except Exception as ex: # noqa - surfaced to the Host as a failed invocation
194196
logger.exception("Native invocation failed")
195-
return (False, None, [], repr(ex))
197+
return (False, None, [], repr(ex), traceback.format_exc())
196198

197199

198200
def run_invocation_sync(invocation_id, function_id, inputs, trigger_metadata,
199201
trace_parent="", trace_state="", retry_count=0,
200202
max_retry_count=0, retry_exception=None):
201203
"""Synchronous native invocation for *sync* customer functions.
202204
203-
Runs the handler directly on the calling thread (no asyncio loop, no
204-
``run_in_executor`` bounce). With the Rust transport dispatching each call
205-
on its own thread, this avoids funneling sync handlers through a single
206-
event loop, trimming per-invocation overhead on the hot path.
205+
Queues the handler on the runtime's configured threadpool without
206+
funneling the invocation through the asyncio loop. This preserves
207+
``PYTHON_THREADPOOL_THREAD_COUNT`` for the native path while keeping datum
208+
conversion on the Rust-dispatched calling thread.
207209
208-
Returns ``(handled, ok, return_datum, output_data, exception_text)``.
210+
Returns ``(handled, ok, return_datum, output_data, exception_text,
211+
exception_traceback)``.
209212
``handled=False`` means the function is ``async`` and the caller should use
210213
the coroutine path (:func:`invocation_request_native`) instead.
211214
"""
212215
fi = _functions.get_function(function_id)
213216
if fi is None:
214-
return (True, False, None, [], "function %s not loaded" % function_id)
217+
return (True, False, None, [],
218+
"function %s not loaded" % function_id, "")
215219
if fi.is_async:
216-
return (False, None, None, [], None)
220+
return (False, None, None, [], None, None)
217221

218222
try:
219223
args = _decode_inputs(fi, inputs, trigger_metadata)
@@ -223,14 +227,18 @@ def run_invocation_sync(invocation_id, function_id, inputs, trigger_metadata,
223227

224228
# run_sync_func calls configure_opentelemetry(fi_context) when otel is
225229
# available, so the trace context prepared above parents customer spans.
226-
call_result = run_sync_func(invocation_id, fi_context, fi.func, args)
230+
threadpool = get_threadpool_executor()
231+
if threadpool is None:
232+
raise RuntimeError("threadpool executor is not initialized")
233+
call_result = threadpool.submit(
234+
run_sync_func, invocation_id, fi_context, fi.func, args).result()
227235

228236
_check_return(fi, call_result)
229237
output_data = _collect_output(fi, args)
230238
return_datum = _encode_return(fi, call_result)
231239

232-
return (True, True, return_datum, output_data, None)
240+
return (True, True, return_datum, output_data, None, None)
233241

234242
except Exception as ex: # noqa - surfaced to the Host as a failed invocation
235243
logger.exception("Native sync invocation failed")
236-
return (True, False, None, [], repr(ex))
244+
return (True, False, None, [], repr(ex), traceback.format_exc())

0 commit comments

Comments
 (0)