diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d1ef0f1a..e2296a6c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed `term_is_uint32` accepting big integers whose low 64 bits are within range on 32-bit builds, which made `erlang:crc32/2`, `erlang:crc32_combine/3` and `crypto:pbkdf2_hmac/5` silently truncate huge integer arguments instead of raising `badarg` +- Fixed a bug where AtomVM could only have 256 loaded modules ## [0.7.0-alpha.1] - 2026-04-06 diff --git a/libs/jit/src/jit.erl b/libs/jit/src/jit.erl index f8d4f6a6df..ce2e55c19c 100644 --- a/libs/jit/src/jit.erl +++ b/libs/jit/src/jit.erl @@ -272,7 +272,8 @@ first_pass(<>, MMod, MSt0, #state{tail_cache = TC} false -> Offset0 = MMod:offset(MSt0), MSt1 = MMod:move_to_cp(MSt0, {y_reg, NWords}), - MSt2 = MMod:increment_sp(MSt1, NWords + 1), + % The saved cp occupies one stack slot on 64-bit, two on 32-bit. + MSt2 = MMod:increment_sp(MSt1, NWords + (8 div MMod:word_size())), TailCacheKey1 = {op_call_only, Label}, case tail_cache_find(TailCacheKey1, TC) of false -> @@ -443,25 +444,48 @@ first_pass(<>, MMod, MSt0, State0) -> first_pass(<>, MMod, MSt0, #state{tail_cache = TC} = State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), ?TRACE("OP_RETURN\n", []), - % Optimized return: check if returning within same module - {MSt1, CpReg0} = MMod:move_to_native_register(MSt0, cp), - {MSt2, ModuleIndexReg} = MMod:get_module_index(MSt1), - % Extract module index from cp (upper 8 bits: cp >> 24) - {MSt3, CpReg1} = MMod:shift_right(MSt2, CpReg0, 24), - % Compare extracted module index with current module index - MSt4 = MMod:if_block( - MSt3, - {{free, CpReg1}, '==', {free, ModuleIndexReg}}, - % Same module: fast intra-module return - fun(BSt0) -> - % Mask to get lower 24 bits and shift right by 2 for offset - {BSt1, CpReg0} = MMod:and_(BSt0, {free, CpReg0}, 16#FFFFFF), - {BSt3, CPReg1} = MMod:shift_right(BSt1, {free, CpReg0}, 2), - % Jump to continuation (this is a tail call) - MMod:jump_to_continuation(BSt3, {free, CPReg1}) - end - ), - MSt5 = MMod:free_native_registers(MSt4, [CpReg0]), + % Optimized return: check if returning within the same module, in which case + % we jump directly to the continuation rather than going through PRIM_RETURN. + MSt5 = + case MMod:word_size() of + 8 -> + % 64-bit: cp packs (module_index << 24) | (offset << 2) in one word. + {MSt1, CpReg0} = MMod:move_to_native_register(MSt0, cp), + {MSt2, ModuleIndexReg} = MMod:get_module_index(MSt1), + % Extract module index from cp (upper 8 bits: cp >> 24) + {MSt3, CpReg1} = MMod:shift_right(MSt2, CpReg0, 24), + % Compare extracted module index with current module index + MSt4 = MMod:if_block( + MSt3, + {{free, CpReg1}, '==', {free, ModuleIndexReg}}, + % Same module: fast intra-module return + fun(BSt0) -> + % Mask to get lower 24 bits and shift right by 2 for offset + {BSt1, CpReg0} = MMod:and_(BSt0, {free, CpReg0}, 16#FFFFFF), + {BSt3, CPReg1} = MMod:shift_right(BSt1, {free, CpReg0}, 2), + % Jump to continuation (this is a tail call) + MMod:jump_to_continuation(BSt3, {free, CPReg1}) + end + ), + MMod:free_native_registers(MSt4, [CpReg0]); + 4 -> + % 32-bit: cp spans two words, the Module pointer (?CP_MODULE) and + % the offset << 2 (?CP_OFFSET). Compare the saved Module pointer + % with the current module pointer (jit_state->module). + {MSt1, CpModReg} = MMod:get_cp_module(MSt0), + {MSt2, CurModReg} = MMod:get_module(MSt1), + MMod:if_block( + MSt2, + {{free, CpModReg}, '==', {free, CurModReg}}, + % Same module: fast intra-module return + fun(BSt0) -> + {BSt1, OffReg} = MMod:get_cp_offset(BSt0), + {BSt2, OffReg2} = MMod:shift_right(BSt1, {free, OffReg}, 2), + % Jump to continuation (this is a tail call) + MMod:jump_to_continuation(BSt2, {free, OffReg2}) + end + ) + end, % Different module: use existing slow path TailCacheKey = {call_primitive_last, ?PRIM_RETURN}, case tail_cache_find(TailCacheKey, TC) of @@ -1241,7 +1265,8 @@ first_pass(<>, MMod, MSt0, State0) -> MSt4 = verify_is_atom(Module, 0, MMod, MSt3), MSt5 = verify_is_atom(Function, 0, MMod, MSt4), MSt6 = MMod:move_to_cp(MSt5, {y_reg, NWords}), - MSt7 = MMod:increment_sp(MSt6, NWords + 1), + % The saved cp occupies one stack slot on 64-bit, two on 32-bit. + MSt7 = MMod:increment_sp(MSt6, NWords + (8 div MMod:word_size())), MSt8 = MMod:call_primitive_last(MSt7, ?PRIM_APPLY, [ ctx, jit_state, offset, {free, Module}, {free, Function}, Arity ]), @@ -4185,10 +4210,11 @@ term_alloc_bin_match_state(Live, Src, Dest, MMod, MSt0) -> MSt6 = MMod:free_native_registers(MSt5, [AllocMatchStateReg]), {MSt6, NewSrc}. -term_from_catch_label(Dest, Label, MMod, MSt1) -> - {MSt2, Reg} = MMod:get_module_index(MSt1), - MSt3 = MMod:shift_left(MSt2, Reg, 24), - MSt4 = MMod:or_(MSt3, Reg, (Label bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_CATCH), +term_from_catch_label(Dest, Label, MMod, MSt0) -> + {MSt1, Reg} = MMod:get_module_catch_labels_base(MSt0), + MSt2 = MMod:add(MSt1, Reg, Label), + MSt3 = MMod:shift_left(MSt2, Reg, ?TERM_IMMED2_TAG_SIZE), + MSt4 = MMod:or_(MSt3, Reg, ?TERM_IMMED2_CATCH), MSt5 = MMod:move_to_vm_register(MSt4, Reg, Dest), MMod:free_native_registers(MSt5, [Reg, Dest]). diff --git a/libs/jit/src/jit_aarch64.erl b/libs/jit/src/jit_aarch64.erl index b8f96d746b..994e719f10 100644 --- a/libs/jit/src/jit_aarch64.erl +++ b/libs/jit/src/jit_aarch64.erl @@ -59,6 +59,7 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, and_/3, or_/3, add/3, @@ -208,6 +209,7 @@ -define(JITSTATE_REDUCTIONCOUNT, {?JITSTATE_REG, 16#10}). -define(PRIMITIVE(N), {?NATIVE_INTERFACE_REG, N * ?WORD_SIZE}). -define(MODULE_INDEX(ModuleReg), {ModuleReg, 0}). +-define(MODULE_CATCH_LABELS_BASE(ModuleReg), {ModuleReg, 4}). % aarch64 ABI specific -define(LR_REG, r30). @@ -2376,6 +2378,37 @@ get_module_index( Reg }. +%%----------------------------------------------------------------------------- +%% @doc Load the catch id of the current module's label 0 into a native +%% register. +%% @end +%% @param State current backend state +%% @return Tuple of {Updated backend state, Native register containing the base} +%%----------------------------------------------------------------------------- +-spec get_module_catch_labels_base(state()) -> {state(), aarch64_register()}. +get_module_catch_labels_base( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + Bit = reg_bit(Reg), + I1 = jit_aarch64_asm:ldr(Reg, ?JITSTATE_MODULE), + I2 = jit_aarch64_asm:ldr_w(Reg, ?MODULE_CATCH_LABELS_BASE(Reg)), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:set_contents(Regs0, Reg, catch_labels_base), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, Bit) + }, + Reg + }. + %% @private -spec op_imm(state(), atom(), aarch64_register(), aarch64_register(), integer()) -> state(). op_imm(#state{stream_module = StreamModule, stream = Stream0} = State, Op, Reg, Reg, Val) -> diff --git a/libs/jit/src/jit_arm32.erl b/libs/jit/src/jit_arm32.erl index 324df7ea56..531a0289d7 100644 --- a/libs/jit/src/jit_arm32.erl +++ b/libs/jit/src/jit_arm32.erl @@ -60,6 +60,10 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -179,10 +183,13 @@ -define(NATIVE_INTERFACE_REG, r2). -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). +% ctx->cp is a 64-bit cp_t occupying two slots (little-endian targets): +% ?CP holds the low word (offset << 2), ?CP_MODULE holds the high word (Module*). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). % JITSTATE is on stack, accessed via stack offset % These macros now expect a register that contains the jit_state pointer -define(JITSTATE_MODULE(Reg), {Reg, 0}). @@ -190,6 +197,7 @@ -define(JITSTATE_REDUCTIONCOUNT(Reg), {Reg, 16#8}). -define(PRIMITIVE(N), {?NATIVE_INTERFACE_REG, N * 4}). -define(MODULE_INDEX(ModuleReg), {ModuleReg, 0}). +-define(MODULE_CATCH_LABELS_BASE(ModuleReg), {ModuleReg, 4}). -define(JUMP_TABLE_ENTRY_SIZE, 8). @@ -2835,10 +2843,14 @@ move_to_cp( Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), AvailT = Avail band (bnot reg_bit(Reg)), + % The saved cp spans two slots: y[Y] = offset word (-> ?CP), y[Y+1] = Module* + % (-> ?CP_MODULE). Copy both into ctx->cp. State1 = ldr_y_reg(State, Reg, Y, AvailT), - I2 = jit_arm32_asm:str(al, Reg, ?CP), - Stream1 = (State1#state.stream_module):append(State1#state.stream, I2), - State1#state{stream = Stream1}. + SM = State1#state.stream_module, + Stream1 = SM:append(State1#state.stream, jit_arm32_asm:str(al, Reg, ?CP)), + State2 = ldr_y_reg(State1#state{stream = Stream1}, Reg, Y + 1, AvailT), + Stream2 = SM:append(State2#state.stream, jit_arm32_asm:str(al, Reg, ?CP_MODULE)), + State2#state{stream = Stream2}. increment_sp( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = @@ -2967,6 +2979,78 @@ get_module_index( Reg }. +%% @doc Load the catch id of the current module's label 0 into a native register. +get_module_catch_labels_base( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + Avail1 = Avail band (bnot RegBit), + TempJitState = first_avail(Avail1), + % Load jit_state pointer from stack, then load module + I1a = jit_arm32_asm:ldr(al, TempJitState, {sp, ?STACK_OFFSET_JITSTATE}), + I1b = jit_arm32_asm:ldr(al, Reg, ?JITSTATE_MODULE(TempJitState)), + I2 = jit_arm32_asm:ldr(al, Reg, ?MODULE_CATCH_LABELS_BASE(Reg)), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:invalidate_reg(Regs0, TempJitState), + Regs2 = jit_regs:set_contents(Regs1, Reg, catch_labels_base), + Regs3 = jit_regs:alloc_reg(Regs2, RegBit), + { + State#state{ + stream = Stream1, + regs = Regs3 + }, + Reg + }. + +%% @doc Load the current module pointer (jit_state->module) into a register. +get_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + Avail1 = Avail band (bnot RegBit), + TempJitState = first_avail(Avail1), + I1a = jit_arm32_asm:ldr(al, TempJitState, {sp, ?STACK_OFFSET_JITSTATE}), + I1b = jit_arm32_asm:ldr(al, Reg, ?JITSTATE_MODULE(TempJitState)), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, TempJitState), Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the Module pointer stored in ctx->cp (?CP_MODULE) into a register. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_arm32_asm:ldr(al, Reg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the offset word (offset << 2) stored in ctx->cp (?CP) into a register. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_arm32_asm:ldr(al, Reg, ?CP), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. %% @doc Perform an AND of a register with an immediate. %% JIT currently calls this with two values: ?TERM_PRIMARY_CLEAR_MASK (-4) to %% clear bits and ?TERM_BOXED_TAG_MASK (0x3F). We can avoid any literal pool @@ -3518,30 +3602,24 @@ call_primitive_with_cp(State0, Primitive, Args) -> -spec set_cp(state()) -> {state(), non_neg_integer(), arm32_register()}. set_cp(State0) -> - % get module index (dynamically) - { - #state{stream_module = StreamModule, stream = Stream0, regs = Regs1} = State1, - Reg - } = get_module_index( - State0 - ), - AvailRegs = jit_regs:available_regs(Regs1), - % Get a temporary register from available registers + % cp is two words: store the Module pointer (jit_state->module) at ?CP_MODULE, + % and the return offset << 2 at ?CP (patched by rewrite_cp_offset below). + {#state{stream_module = StreamModule, stream = Stream0} = State1, ModReg} = + get_module(State0), + IModStore = jit_arm32_asm:str(al, ModReg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + AvailRegs = jit_regs:available_regs(State2#state.regs), + % Get a temporary register to hold the offset value TempReg = first_avail(AvailRegs), - - Offset = StreamModule:offset(Stream0), - % build cp with module_index << 24 - I1 = jit_arm32_asm:lsl(al, Reg, Reg, 24), - % Placeholder for offset load instruction - I2 = <<16#FFFFFFFF:32>>, - MOVOffset = Offset + byte_size(I1), - % OR the module index with the offset (loaded in temp register) - I3 = jit_arm32_asm:orr(al, Reg, Reg, TempReg), - I4 = jit_arm32_asm:str(al, Reg, ?CP), - Code = <>, - Stream1 = StreamModule:append(Stream0, Code), - State2 = State1#state{stream = Stream1}, - State3 = free_native_register(State2, Reg), + Offset = StreamModule:offset(Stream1), + % Placeholder for the offset load instruction (patched by rewrite_cp_offset) + I1 = <<16#FFFFFFFF:32>>, + MOVOffset = Offset, + I2 = jit_arm32_asm:str(al, TempReg, ?CP), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, {State3, MOVOffset, TempReg}. -spec rewrite_cp_offset(state(), non_neg_integer(), arm32_register()) -> state(). diff --git a/libs/jit/src/jit_armv6m.erl b/libs/jit/src/jit_armv6m.erl index 5b74cc9409..9e3af4858b 100644 --- a/libs/jit/src/jit_armv6m.erl +++ b/libs/jit/src/jit_armv6m.erl @@ -60,6 +60,10 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -199,10 +203,12 @@ -define(NATIVE_INTERFACE_REG, r2). -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). +% ?CP holds the low word (offset << 2), ?CP_MODULE holds the high word (Module*). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). % JITSTATE is on stack, accessed via stack offset % These macros now expect a register that contains the jit_state pointer -define(JITSTATE_MODULE(Reg), {Reg, 0}). @@ -210,6 +216,7 @@ -define(JITSTATE_REDUCTIONCOUNT(Reg), {Reg, 16#8}). -define(PRIMITIVE(N), {?NATIVE_INTERFACE_REG, N * 4}). -define(MODULE_INDEX(ModuleReg), {ModuleReg, 0}). +-define(MODULE_CATCH_LABELS_BASE(ModuleReg), {ModuleReg, 4}). -define(JUMP_TABLE_ENTRY_SIZE, 12). -define(JUMP_TABLE_ENTRY_SIZE_THUMB2, 6). @@ -3202,10 +3209,14 @@ move_to_cp( Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), AvailT = Avail band (bnot reg_bit(Reg)), + % The saved cp spans two slots: y[Y] = offset word (-> ?CP), y[Y+1] = Module* + % (-> ?CP_MODULE). Copy both into ctx->cp. State1 = ldr_y_reg(State, Reg, Y, AvailT), - I2 = jit_armv6m_asm:str(Reg, ?CP), - Stream1 = (State1#state.stream_module):append(State1#state.stream, I2), - State1#state{stream = Stream1}. + SM = State1#state.stream_module, + Stream1 = SM:append(State1#state.stream, jit_armv6m_asm:str(Reg, ?CP)), + State2 = ldr_y_reg(State1#state{stream = Stream1}, Reg, Y + 1, AvailT), + Stream2 = SM:append(State2#state.stream, jit_armv6m_asm:str(Reg, ?CP_MODULE)), + State2#state{stream = Stream2}. increment_sp( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = @@ -3347,6 +3358,77 @@ get_module_index( Reg }. +%% @doc Load the catch id of the current module's label 0 into a native register. +get_module_catch_labels_base( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + Avail1 = Avail band (bnot RegBit), + TempJitState = first_avail(Avail1), + % Load jit_state pointer from stack, then load module + I1a = jit_armv6m_asm:ldr(TempJitState, {sp, ?STACK_OFFSET_JITSTATE}), + I1b = jit_armv6m_asm:ldr(Reg, ?JITSTATE_MODULE(TempJitState)), + I2 = jit_armv6m_asm:ldr(Reg, ?MODULE_CATCH_LABELS_BASE(Reg)), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:invalidate_reg(Regs0, TempJitState), + Regs2 = jit_regs:set_contents(Regs1, Reg, catch_labels_base), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs2, RegBit) + }, + Reg + }. + +%% @doc Load the current module pointer (jit_state->module) into a register. +get_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + Avail1 = Avail band (bnot RegBit), + TempJitState = first_avail(Avail1), + I1a = jit_armv6m_asm:ldr(TempJitState, {sp, ?STACK_OFFSET_JITSTATE}), + I1b = jit_armv6m_asm:ldr(Reg, ?JITSTATE_MODULE(TempJitState)), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, TempJitState), Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the Module pointer stored in ctx->cp (?CP_MODULE) into a register. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_armv6m_asm:ldr(Reg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the offset word (offset << 2) stored in ctx->cp (?CP) into a register. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_armv6m_asm:ldr(Reg, ?CP), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. %% @doc Perform an AND of a register with an immediate. %% JIT currentl calls this with two values: ?TERM_PRIMARY_CLEAR_MASK (-4) to %% clear bits and ?TERM_BOXED_TAG_MASK (0x3F). We can avoid any literal pool @@ -4062,29 +4144,24 @@ call_primitive_with_cp(State0, Primitive, Args) -> -spec set_cp(state()) -> {state(), non_neg_integer(), armv6m_register()}. set_cp(State0) -> - % get module index (dynamically) - { - #state{stream_module = StreamModule, stream = Stream0, regs = AvailRegs0} = State1, - Reg - } = get_module_index( - State0 - ), - % Get a temporary register from available registers - TempReg = first_avail(jit_regs:available_regs(AvailRegs0)), - - Offset = StreamModule:offset(Stream0), - % build cp with module_index << 24 - I1 = jit_armv6m_asm:lsls(Reg, Reg, 24), - % Placeholder for offset load instruction - I2 = <<16#FFFF:16>>, - MOVOffset = Offset + byte_size(I1), - % OR the module index with the offset (loaded in temp register) - I3 = jit_armv6m_asm:orrs(Reg, TempReg), - I4 = jit_armv6m_asm:str(Reg, ?CP), - Code = <>, - Stream1 = StreamModule:append(Stream0, Code), - State2 = State1#state{stream = Stream1}, - State3 = free_native_register(State2, Reg), + % cp is two words: store the Module pointer (jit_state->module) at ?CP_MODULE, + % and the return offset << 2 at ?CP (patched by rewrite_cp_offset below). + {#state{stream_module = StreamModule, stream = Stream0} = State1, ModReg} = + get_module(State0), + IModStore = jit_armv6m_asm:str(ModReg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + AvailRegs = jit_regs:available_regs(State2#state.regs), + % Get a temporary register to hold the offset value + TempReg = first_avail(AvailRegs), + Offset = StreamModule:offset(Stream1), + % Placeholder for the offset load instruction (patched by rewrite_cp_offset) + I1 = <<16#FFFF:16>>, + MOVOffset = Offset, + I2 = jit_armv6m_asm:str(TempReg, ?CP), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, {State3, MOVOffset, TempReg}. -spec rewrite_cp_offset(state(), non_neg_integer(), armv6m_register()) -> state(). diff --git a/libs/jit/src/jit_regs.erl b/libs/jit/src/jit_regs.erl index 1fff453441..84b80fdb2e 100644 --- a/libs/jit/src/jit_regs.erl +++ b/libs/jit/src/jit_regs.erl @@ -81,6 +81,8 @@ | cp %% Register holds the module index | module_index + %% Register holds the catch id of the module's label 0 + | catch_labels_base %% Unknown / clobbered | unknown. diff --git a/libs/jit/src/jit_riscv32.erl b/libs/jit/src/jit_riscv32.erl index d301d7022b..4fafecbf13 100644 --- a/libs/jit/src/jit_riscv32.erl +++ b/libs/jit/src/jit_riscv32.erl @@ -59,6 +59,10 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -210,9 +214,10 @@ -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). -define(JITSTATE_REG, a1). -define(RA_REG, ra). -define(JITSTATE_MODULE_OFFSET, 0). @@ -220,6 +225,7 @@ -define(JITSTATE_REDUCTIONCOUNT_OFFSET, 16#8). -define(PRIMITIVE(N), {?NATIVE_INTERFACE_REG, N * 4}). -define(MODULE_INDEX(ModuleReg), {ModuleReg, 0}). +-define(MODULE_CATCH_LABELS_BASE_OFFSET, 4). -define(JUMP_TABLE_ENTRY_SIZE, 8). @@ -311,6 +317,61 @@ rem_( Regs1 = jit_regs:invalidate_reg(Regs0, DividendReg), {State#state{stream = Stream1, regs = Regs1}, DividendReg}. +%% @doc Load the current module pointer (jit_state->module) into a fresh +%% register. This is the Module* itself (unlike get_module_index/1 which then +%% dereferences module->module_index). Used by set_cp to store the Module* +%% half of the two-word (32-bit) cp, and by the 32-bit OP_RETURN fast path. +-spec get_module(state()) -> {state(), riscv32_register()}. +get_module( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + %% Reg = jit_state->module (jit_state is in a1) + I = ?LOAD_WORD(Reg, ?JITSTATE_REG, ?JITSTATE_MODULE_OFFSET), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the Module* half of the saved cp (ctx->cp high word, ?CP_MODULE) +%% into a fresh register. 32-bit only: cp spans two words. +-spec get_cp_module(state()) -> {state(), riscv32_register()}. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP_MODULE, + I = ?LOAD_WORD(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the offset half of the saved cp (ctx->cp low word, ?CP) into a +%% fresh register. The value is offset << 2 (TERM_PRIMARY_CP tag); the caller +%% shifts it right by 2. 32-bit only. +-spec get_cp_offset(state()) -> {state(), riscv32_register()}. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP, + I = ?LOAD_WORD(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + % ILP32: 64-bit arguments require double-word alignment (even register number) parameter_regs0_avm_int64_t(T, [a0, a1 | Rest], Acc) -> parameter_regs0(T, Rest, [a1, a0 | Acc]); diff --git a/libs/jit/src/jit_riscv64.erl b/libs/jit/src/jit_riscv64.erl index 4ed6c8eb0d..61d1c86ba9 100644 --- a/libs/jit/src/jit_riscv64.erl +++ b/libs/jit/src/jit_riscv64.erl @@ -59,6 +59,7 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, and_/3, or_/3, add/3, @@ -229,6 +230,7 @@ -define(JITSTATE_REDUCTIONCOUNT_OFFSET, 16#10). -define(PRIMITIVE(N), {?NATIVE_INTERFACE_REG, N * 8}). -define(MODULE_INDEX(ModuleReg), {ModuleReg, 0}). +-define(MODULE_CATCH_LABELS_BASE_OFFSET, 4). -define(JUMP_TABLE_ENTRY_SIZE, 8). diff --git a/libs/jit/src/jit_riscv_impl.hrl b/libs/jit/src/jit_riscv_impl.hrl index 132127d498..aa9593935f 100644 --- a/libs/jit/src/jit_riscv_impl.hrl +++ b/libs/jit/src/jit_riscv_impl.hrl @@ -2499,6 +2499,35 @@ copy_to_native_register( copy_to_native_register(State, Reg) -> move_to_native_register(State, Reg). +-if(?WORD_SIZE_BYTES =:= 4). +move_to_cp( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = + State, + {y_reg, Y} +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + AvailT = Avail band (bnot reg_bit(Reg)), + % The saved cp spans two slots: y[Y] = offset word (-> ?CP), y[Y+1] = Module* + % (-> ?CP_MODULE). Copy both into ctx->cp. + I1 = ldr_y_reg(Reg, Y, AvailT), + {CpBase, CpOff} = ?CP, + I2 = ?STORE_WORD(CpBase, Reg, CpOff), + I3 = ldr_y_reg(Reg, Y + 1, AvailT), + {CpModBase, CpModOff} = ?CP_MODULE, + I4 = ?STORE_WORD(CpModBase, Reg, CpModOff), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + % Reg ends holding y[Y+1] (the Module*); ldr_y_reg also clobbers + % first_avail(AvailT) as a hidden temp for loading the Y_REGS pointer. + Regs1a = jit_regs:invalidate_reg(Regs0, Reg), + Regs1 = + case AvailT of + 0 -> Regs1a; + _ -> jit_regs:invalidate_reg(Regs1a, first_avail(AvailT)) + end, + State#state{stream = Stream1, regs = Regs1}. +-else. move_to_cp( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, @@ -2520,6 +2549,7 @@ move_to_cp( _ -> jit_regs:invalidate_reg(Regs1a, first_avail(AvailT)) end, State#state{stream = Stream1, regs = Regs1}. +-endif. increment_sp( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = @@ -2632,6 +2662,31 @@ get_module_index( Reg }. +%% @doc Load the catch id of the current module's label 0 into a native register. +get_module_catch_labels_base( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + % Load module pointer from jit_state (which is in a1) + I1 = ?LOAD_WORD(Reg, ?JITSTATE_REG, ?JITSTATE_MODULE_OFFSET), + I2 = ?ASM:lw(Reg, Reg, ?MODULE_CATCH_LABELS_BASE_OFFSET), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:set_contents(Regs0, Reg, catch_labels_base), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. + %% @doc Perform an AND of a register with an immediate. %% JIT currently calls this with two values: ?TERM_PRIMARY_CLEAR_MASK (-4) to %% clear bits and ?TERM_BOXED_TAG_MASK (0x3F). We can avoid any literal pool @@ -3093,6 +3148,35 @@ call_primitive_with_cp(State0, Primitive, Args) -> State2 = call_primitive_last(State1, Primitive, Args), rewrite_cp_offset(State2, RewriteOffset, TempReg). +-if(?WORD_SIZE_BYTES =:= 4). +set_cp(State0) -> + % 32-bit: cp spans two words. Store the Module pointer (jit_state->module) + % to ?CP_MODULE, and a placeholder offset << 2 to ?CP (the placeholder is + % patched by rewrite_cp_offset once the resume offset is known). + { + #state{stream_module = StreamModule, stream = Stream0} = State1, + ModReg + } = get_module(State0), + {CpModBase, CpModOff} = ?CP_MODULE, + IModStore = ?STORE_WORD(CpModBase, ModReg, CpModOff), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + Avail = jit_regs:available_regs(State2#state.regs), + TempReg = first_avail(Avail), + Offset = StreamModule:offset(Stream1), + % Reserve 8 bytes (two instructions) for the offset load; li may expand to + % one or two instructions, so always reserve the larger form. The 0xFFFFFFFF + % placeholders are flash-friendly (bits can only flip 1->0). + I1 = <<16#FFFFFFFF:32/little>>, + I2 = <<16#FFFFFFFF:32/little>>, + MOVOffset = Offset, + {CpBase, CpOff} = ?CP, + I3 = ?STORE_WORD(CpBase, TempReg, CpOff), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, + {State3, MOVOffset, TempReg}. +-else. set_cp(#state{regs = RegsSC} = State0) -> Avail = jit_regs:available_regs(RegsSC), TempReg = first_avail(Avail), @@ -3133,6 +3217,7 @@ set_cp(#state{regs = RegsSC} = State0) -> State3 = free_native_register(State2, Reg), State4 = free_native_register(State3, TempReg), {State4, MOVOffset, TempReg}. +-endif. rewrite_cp_offset( #state{stream_module = StreamModule, stream = Stream0, offset = CodeOffset} = State0, diff --git a/libs/jit/src/jit_wasm32.erl b/libs/jit/src/jit_wasm32.erl index 3e9d4f729f..0a3b61ea04 100644 --- a/libs/jit/src/jit_wasm32.erl +++ b/libs/jit/src/jit_wasm32.erl @@ -85,6 +85,10 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -131,10 +135,16 @@ %% Context struct offsets (32-bit architecture, same as armv6m/riscv32) -define(CTX_E_OFFSET, 16#14). -define(CTX_X_OFFSET, 16#18). +%% ctx->cp is a 64-bit cp_t spanning two words (little-endian): low word at +%% 0x5C = offset << 2 (?CTX_CP_OFFSET), high word at 0x60 = Module* (?CTX_CP_MODULE_OFFSET). -define(CTX_CP_OFFSET, 16#5C). --define(CTX_FR_OFFSET, 16#60). --define(CTX_BS_OFFSET, 16#64). --define(CTX_BS_OFFSET_OFFSET, 16#68). +-define(CTX_CP_MODULE_OFFSET, 16#60). +-define(CTX_FR_OFFSET, 16#64). +-define(CTX_BS_OFFSET, 16#68). +-define(CTX_BS_OFFSET_OFFSET, 16#6C). + +%% Module struct offsets +-define(MODULE_CATCH_LABELS_BASE_OFFSET, 16#4). %% JITState struct offsets -define(JITSTATE_MODULE_OFFSET, 16#0). @@ -800,24 +810,32 @@ move_to_native_register(State0, Value, Local) -> State1#state{regs = Regs1}. move_to_cp(State0, {y_reg, Y}) -> - %% Load y register and store to ctx->cp - {State1, TempLocal} = alloc_local(State0), + %% The saved cp spans two slots: e[Y] = offset word (-> ctx->cp), e[Y+1] = + %% Module* (-> ctx->cp_module). Copy both into ctx->cp. + {State1, BaseLocal} = alloc_local(State0), + {State2, TempLocal} = alloc_local(State1), Code = << - %% Load ctx->e + %% BaseLocal = ctx->e (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:i32_load(2, ?CTX_E_OFFSET))/binary, + (jit_wasm32_asm:local_set(BaseLocal))/binary, + %% ctx->cp = e[Y] (offset word) + (jit_wasm32_asm:local_get(BaseLocal))/binary, + (jit_wasm32_asm:i32_load(2, Y * 4))/binary, (jit_wasm32_asm:local_set(TempLocal))/binary, - %% Load e[Y] + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:local_get(TempLocal))/binary, - (jit_wasm32_asm:i32_load(2, Y * 4))/binary, + (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary, + %% ctx->cp_module = e[Y+1] (Module*) + (jit_wasm32_asm:local_get(BaseLocal))/binary, + (jit_wasm32_asm:i32_load(2, (Y + 1) * 4))/binary, (jit_wasm32_asm:local_set(TempLocal))/binary, - %% Store to ctx->cp (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:local_get(TempLocal))/binary, - (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary + (jit_wasm32_asm:i32_store(2, ?CTX_CP_MODULE_OFFSET))/binary >>, - State2 = emit(State1, Code), - free_native_register(State2, TempLocal). + State3 = emit(State2, Code), + free_native_register(free_native_register(State3, TempLocal), BaseLocal). move_array_element(State0, Base, Index, {ptr, Dest}) when is_integer(Index) -> %% Load Base[Index] and store to [Dest] @@ -1025,6 +1043,20 @@ get_module_index(State0) -> State2 = emit(State1, Code), {State2#state{regs = Regs1}, Local}. +%% @doc Load the catch id of the current module's label 0 into a native register. +get_module_catch_labels_base(State0) -> + {State1, Local} = alloc_local(State0), + %% Load jit_state->module, then load module->catch_labels_base + Code = << + (jit_wasm32_asm:local_get(?JITSTATE_LOCAL))/binary, + (jit_wasm32_asm:i32_load(2, ?JITSTATE_MODULE_OFFSET))/binary, + (jit_wasm32_asm:i32_load(2, ?MODULE_CATCH_LABELS_BASE_OFFSET))/binary, + (jit_wasm32_asm:local_set(Local))/binary + >>, + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Local), + State2 = emit(State1, Code), + {State2#state{regs = Regs1}, Local}. + %%============================================================================= %% Scheduling and reductions %%============================================================================= @@ -1478,30 +1510,61 @@ emit_set_continuation_for_label(State, _RefLabel) -> emit(State, Code). %% Emit code to set up CP (continuation pointer for returns). -%% CP format: (module_index << 24) | (label_offset << 2) +%% cp is two words: ctx->cp = label_offset << 2, ctx->cp_module = Module*. emit_set_cp_for_label(State0, Label) -> LabelOffset = Label * ?JUMP_TABLE_ENTRY_SIZE, emit_set_cp_for_offset(State0, LabelOffset). emit_set_cp_for_offset(State0, LabelOffset) -> - {State1, ModIdxLocal} = get_module_index(State0), - State2 = shift_left(State1, ModIdxLocal, 24), - {State3, OffsetLocal} = alloc_local(State2), + {State1, ModLocal} = get_module(State0), Code = << + %% ctx->cp = label_offset << 2 + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:i32_const(LabelOffset bsl 2))/binary, - (jit_wasm32_asm:local_set(OffsetLocal))/binary + (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary, + %% ctx->cp_module = Module* + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, + (jit_wasm32_asm:local_get(ModLocal))/binary, + (jit_wasm32_asm:i32_store(2, ?CTX_CP_MODULE_OFFSET))/binary >>, - State4 = emit(State3, Code), - State5 = or_(State4, ModIdxLocal, OffsetLocal), - State6 = free_native_register(State5, OffsetLocal), - %% Store CP to ctx->cp - Code2 = << + State2 = emit(State1, Code), + free_native_register(State2, ModLocal). + +%% Load the current module pointer (jit_state->module) into a fresh local. +get_module(State0) -> + {State1, Local} = alloc_local(State0), + Code = << + (jit_wasm32_asm:local_get(?JITSTATE_LOCAL))/binary, + (jit_wasm32_asm:i32_load(2, ?JITSTATE_MODULE_OFFSET))/binary, + (jit_wasm32_asm:local_set(Local))/binary + >>, + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Local), + State2 = emit(State1, Code), + {State2#state{regs = Regs1}, Local}. + +%% Load the Module pointer stored in ctx->cp_module into a fresh local. +get_cp_module(State0) -> + {State1, Local} = alloc_local(State0), + Code = << + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, + (jit_wasm32_asm:i32_load(2, ?CTX_CP_MODULE_OFFSET))/binary, + (jit_wasm32_asm:local_set(Local))/binary + >>, + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Local), + State2 = emit(State1, Code), + {State2#state{regs = Regs1}, Local}. + +%% Load the offset word (offset << 2) stored in ctx->cp into a fresh local. +get_cp_offset(State0) -> + {State1, Local} = alloc_local(State0), + Code = << (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, - (jit_wasm32_asm:local_get(ModIdxLocal))/binary, - (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary + (jit_wasm32_asm:i32_load(2, ?CTX_CP_OFFSET))/binary, + (jit_wasm32_asm:local_set(Local))/binary >>, - State7 = emit(State6, Code2), - free_native_register(State7, ModIdxLocal). + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Local), + State2 = emit(State1, Code), + {State2#state{regs = Regs1}, Local}. emit_call_primitive(State0, Primitive, Args, ResultLocal, IsTailCall) -> WasmArgCount = count_wasm_args(Args), diff --git a/libs/jit/src/jit_x86_64.erl b/libs/jit/src/jit_x86_64.erl index 7bc907b974..56cf63a0e6 100644 --- a/libs/jit/src/jit_x86_64.erl +++ b/libs/jit/src/jit_x86_64.erl @@ -60,6 +60,7 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, and_/3, or_/3, add/3, @@ -197,6 +198,7 @@ -define(JITSTATE_REMAINING_REDUCTIONS, {16#10, ?JITSTATE_REG}). -define(PRIMITIVE(N), {N * ?WORD_SIZE, ?NATIVE_INTERFACE_REG}). -define(MODULE_INDEX(ModuleReg), {0, ModuleReg}). +-define(MODULE_CATCH_LABELS_BASE(ModuleReg), {4, ModuleReg}). -define(IS_SINT8_T(X), is_integer(X) andalso X >= -128 andalso X =< 127). -define(IS_SINT32_T(X), is_integer(X) andalso X >= -16#80000000 andalso X < 16#80000000). @@ -2407,6 +2409,30 @@ get_module_index( Reg }. +%% @doc Load the catch id of the current module's label 0 into a native register. +get_module_catch_labels_base( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + Bit = reg_bit(Reg), + I1 = jit_x86_64_asm:movq(?JITSTATE_MODULE, Reg), + I2 = jit_x86_64_asm:movl(?MODULE_CATCH_LABELS_BASE(Reg), Reg), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:set_contents(Regs0, Reg, catch_labels_base), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, Bit) + }, + Reg + }. + and_( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, diff --git a/libs/jit/src/jit_x86_64_asm.erl b/libs/jit/src/jit_x86_64_asm.erl index 39d16d0b48..26905a19a6 100644 --- a/libs/jit/src/jit_x86_64_asm.erl +++ b/libs/jit/src/jit_x86_64_asm.erl @@ -294,6 +294,31 @@ movl({0, SrcReg}, DestReg) when is_atom(SrcReg), is_atom(DestReg) -> (case {REX_R, REX_B} of {0, 0} -> <<16#8B, 0:2, MODRM_REG:3, MODRM_RM:3>>; _ -> <> + end); +movl({Offset, SrcReg}, DestReg) when is_atom(SrcReg), is_atom(DestReg), ?IS_SINT8_T(Offset) -> + {REX_B, MODRM_RM} = x86_64_x_reg(SrcReg), + {REX_R, MODRM_REG} = x86_64_x_reg(DestReg), + % disp8 + (case {REX_R, REX_B} of + {0, 0} -> <<16#8B, 1:2, MODRM_REG:3, MODRM_RM:3, Offset>>; + _ -> <> + end); +movl({Offset, SrcReg}, DestReg) when is_atom(SrcReg), is_atom(DestReg), ?IS_SINT32_T(Offset) -> + {REX_B, MODRM_RM} = x86_64_x_reg(SrcReg), + {REX_R, MODRM_REG} = x86_64_x_reg(DestReg), + % disp32 + (case {REX_R, REX_B} of + {0, 0} -> + <<16#8B, 2:2, MODRM_REG:3, MODRM_RM:3, Offset:32/little>>; + _ -> + << + ?X86_64_REX(0, REX_R, 0, REX_B), + 16#8B, + 2:2, + MODRM_REG:3, + MODRM_RM:3, + Offset:32/little + >> end). shlq(Imm, Reg) when ?IS_UINT8_T(Imm) -> diff --git a/libs/jit/src/jit_xtensa.erl b/libs/jit/src/jit_xtensa.erl index 8b808fcfb8..3506a99e3b 100644 --- a/libs/jit/src/jit_xtensa.erl +++ b/libs/jit/src/jit_xtensa.erl @@ -60,6 +60,10 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module_catch_labels_base/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -202,9 +206,11 @@ -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). +-define(MODULE_CATCH_LABELS_BASE_OFFSET, 4). -define(JITSTATE_REG, a3). -define(JITSTATE_MODULE_OFFSET, 0). -define(JITSTATE_CONTINUATION_OFFSET, 16#4). @@ -3014,10 +3020,15 @@ move_to_cp( Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), AvailT = Avail band (bnot reg_bit(Reg)), + %% cp is two words: store y[Y] (offset << 2) at ?CP and y[Y+1] (Module*) + %% at ?CP_MODULE. I1 = ldr_y_reg(Reg, Y, AvailT), {BaseReg, Off} = ?CP, I2 = jit_xtensa_asm:s32i(Reg, BaseReg, Off), - Code = <>, + I3 = ldr_y_reg(Reg, Y + 1, AvailT), + {BaseRegM, OffM} = ?CP_MODULE, + I4 = jit_xtensa_asm:s32i(Reg, BaseRegM, OffM), + Code = <>, Stream1 = StreamModule:append(Stream0, Code), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), Regs2 = @@ -3129,6 +3140,96 @@ get_module_index( Reg }. +%% @doc Load the catch id of the current module's label 0 into a native register. +get_module_catch_labels_base( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + % Load module from jit_state (which is in a3) + I1 = jit_xtensa_asm:l32i(Reg, ?JITSTATE_REG, ?JITSTATE_MODULE_OFFSET), + I2 = jit_xtensa_asm:l32i(Reg, Reg, ?MODULE_CATCH_LABELS_BASE_OFFSET), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:set_contents(Regs0, Reg, catch_labels_base), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. + +%% @doc Load the current module pointer (jit_state->module) into a fresh +%% register. Like get_module_index without the final module->index load, and +%% without caching the result. +-spec get_module(state()) -> {state(), xtensa_register()}. +get_module( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + % Load module from jit_state (which is in a3) + I1 = jit_xtensa_asm:l32i(Reg, ?JITSTATE_REG, ?JITSTATE_MODULE_OFFSET), + Stream1 = StreamModule:append(Stream0, I1), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. + +%% @doc Load the Module pointer stored in ctx->cp (?CP_MODULE) into a register. +-spec get_cp_module(state()) -> {state(), xtensa_register()}. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP_MODULE, + I = jit_xtensa_asm:l32i(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. + +%% @doc Load the offset word (offset << 2) stored in ctx->cp (?CP) into a register. +-spec get_cp_offset(state()) -> {state(), xtensa_register()}. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP, + I = jit_xtensa_asm:l32i(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. and_( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {free, Reg}, @@ -3969,33 +4070,32 @@ call_primitive_with_cp(State0, Primitive, Args) -> rewrite_cp_offset(State2, RewriteOffset, TempReg). -spec set_cp(state()) -> {state(), non_neg_integer(), xtensa_register()}. -set_cp(#state{regs = Regs0} = State0) -> - Avail = jit_regs:available_regs(Regs0), - TempReg = first_avail(Avail), - TempBit = reg_bit(TempReg), - %% Reserve TempReg for the offset BEFORE get_module_index consumes available registers. - State1 = State0#state{ - regs = jit_regs:alloc_reg(Regs0, TempBit) - }, - {State2, Reg} = get_module_index(State1), - #state{stream_module = StreamModule, stream = Stream0} = State2, - - Offset = StreamModule:offset(Stream0), - I1 = jit_xtensa_asm:slli(Reg, Reg, 24), - %% Reserve 15 bytes for offset load (li generates 3..21 bytes, patched by rewrite_cp_offset). - I2 = +set_cp(State0) -> + %% cp is two words: store the Module pointer (jit_state->module) at + %% ?CP_MODULE, and the return offset << 2 at ?CP (patched by + %% rewrite_cp_offset below). + {#state{stream_module = StreamModule, stream = Stream0} = State1, ModReg} = + get_module(State0), + {BaseRegM, OffM} = ?CP_MODULE, + IModStore = jit_xtensa_asm:s32i(ModReg, BaseRegM, OffM), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + %% Get a temporary register to hold the offset value + TempReg = first_avail(jit_regs:available_regs(State2#state.regs)), + Offset = StreamModule:offset(Stream1), + %% Reserve 15 bytes for offset load (li generates 3..21 bytes, patched by + %% rewrite_cp_offset). The placeholder is loaded into TempReg, which is then + %% stored to ?CP. + I1 = <<16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF>>, - MOVOffset = Offset + byte_size(I1), - I4 = jit_xtensa_asm:or_(Reg, Reg, TempReg), + MOVOffset = Offset, {BaseReg, Off} = ?CP, - I5 = jit_xtensa_asm:s32i(Reg, BaseReg, Off), - Code = <>, - Stream1 = StreamModule:append(Stream0, Code), - State3 = State2#state{stream = Stream1}, - State4 = free_native_register(State3, Reg), - State5 = free_native_register(State4, TempReg), - {State5, MOVOffset, TempReg}. + I2 = jit_xtensa_asm:s32i(TempReg, BaseReg, Off), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, + {State3, MOVOffset, TempReg}. -spec rewrite_cp_offset(state(), non_neg_integer(), xtensa_register()) -> state(). rewrite_cp_offset( diff --git a/src/libAtomVM/context.c b/src/libAtomVM/context.c index 1066183cb2..7f967b216d 100644 --- a/src/libAtomVM/context.c +++ b/src/libAtomVM/context.c @@ -616,8 +616,8 @@ void context_process_code_server_resume_signal(Context *ctx) #endif #endif // Fix CP to OP_INT_CALL_END - if (ctx->cp == module_address(module->module_index, 0)) { - ctx->cp = module_address(module->module_index, module->end_instruction_ii); + if (ctx->cp == make_cp(module, 0)) { + ctx->cp = make_cp(module, module->end_instruction_ii); } #endif context_update_flags(ctx, ~Trap, NoFlags); @@ -1415,10 +1415,10 @@ int context_get_catch_label(Context *ctx, Module **mod) while (ct != ctx->heap.heap_end) { if (term_is_catch_label(*ct)) { - int target_module; - int target_label = term_to_catch_label_and_module(*ct, &target_module); - TRACE("- found catch: label: %i, module: %i\n", target_label, target_module); - *mod = globalcontext_get_module_by_index(ctx->global, target_module); + int target_label; + Module *target_module = globalcontext_get_module_by_catch_id(ctx->global, term_to_catch_id(*ct), &target_label); + TRACE("- found catch: label: %i, module: %i\n", target_label, target_module->module_index); + *mod = target_module; DEBUG_DUMP_STACK(ctx); ctx->e = last_frame; @@ -1471,15 +1471,18 @@ COLD_FUNC void context_dump(Context *ctx) while (ct != ctx->heap.heap_end) { if (term_is_catch_label(*ct)) { - int target_module; - int target_label = term_to_catch_label_and_module(*ct, &target_module); - fprintf(stderr, "catch: %i:%i\n", target_label, target_module); + int target_label; + Module *target_module = globalcontext_get_module_by_catch_id(ctx->global, term_to_catch_id(*ct), &target_label); + fprintf(stderr, "catch: %i:%i\n", target_label, target_module->module_index); } else if (term_is_cp(*ct)) { Module *cp_mod; int label; size_t offset; - module_cp_to_label_offset(*ct, &cp_mod, &label, &offset, NULL, ctx->global); + // A saved cp spans CP_SIZE_IN_TERMS slots (2 on 32-bit: offset then + // Module*). Reconstruct it from all its slots and skip the extra one. + module_cp_to_label_offset(load_cp(ct), &cp_mod, &label, &offset, NULL, ctx->global); + ct += CP_SIZE_IN_TERMS - 1; // Cast offset to unsigned as some embedded libc implementations do not support %zu fprintf(stderr, "#CP\n", cp_mod->module_index, label, (unsigned) offset); diff --git a/src/libAtomVM/context.h b/src/libAtomVM/context.h index b1f0f3b5bc..4a92f849f7 100644 --- a/src/libAtomVM/context.h +++ b/src/libAtomVM/context.h @@ -95,7 +95,17 @@ struct Context // Following fields offsets are also hard-coded in jit backends term x[MAX_REG + 1]; - term cp; + // Continuation pointer: 64-bit wide, so on 32-bit term platforms it spans + // two stack slots and can hold a full Module pointer plus a code offset. + // On 32-bit term platforms uint64_t is naturally 8-byte aligned, which would + // insert 4 bytes of padding after x[] and push cp to 0x60; the JIT backends + // expect the offset word at 0x5C and the Module* at 0x60, so keep it packed + // and 4-byte aligned (it always lands on a 4-byte boundary anyway). +#if TERM_BYTES == 4 + cp_t cp __attribute__((packed, aligned(4))); +#else + cp_t cp; +#endif avm_float_t *fr; term bs; size_t bs_offset; diff --git a/src/libAtomVM/debug.c b/src/libAtomVM/debug.c index a74da858c1..773f955f64 100644 --- a/src/libAtomVM/debug.c +++ b/src/libAtomVM/debug.c @@ -20,6 +20,8 @@ #include "debug.h" +#include "module.h" + static COLD_FUNC void debug_display_type(term t, const Context *ctx) { if (term_is_atom(t) || term_is_integer(t) || term_is_nil(t) || term_is_pid(t)) { @@ -31,9 +33,9 @@ static COLD_FUNC void debug_display_type(term t, const Context *ctx) } else if ((t & 0x3) == 0x1) { fprintf(stderr, "list(0x%lx)", (unsigned long) term_to_term_ptr(t)); } else if (term_is_catch_label(t)) { - int module_index; - int catch_label = term_to_catch_label_and_module(t, &module_index); - fprintf(stderr, "catch label(%i:%i)", module_index, catch_label); + int catch_label; + Module *catch_module = globalcontext_get_module_by_catch_id(ctx->global, term_to_catch_id(t), &catch_label); + fprintf(stderr, "catch label(%i:%i)", catch_module->module_index, catch_label); } else if (term_is_cp(t)) { fprintf(stderr, "continuation pointer"); } else { diff --git a/src/libAtomVM/globalcontext.c b/src/libAtomVM/globalcontext.c index cc85e4620b..0c605477c0 100644 --- a/src/libAtomVM/globalcontext.c +++ b/src/libAtomVM/globalcontext.c @@ -99,6 +99,7 @@ GlobalContext *globalcontext_new(void) glb->modules_by_index = NULL; glb->loaded_modules_count = 0; + glb->catch_labels_count = 0; glb->modules_table = valueshashtable_new(); if (IS_NULL_PTR(glb->modules_table)) { atom_table_destroy(glb->atom_table); @@ -687,6 +688,13 @@ int globalcontext_insert_module(GlobalContext *global, Module *module) { term module_name = module_get_name(module); SMP_RWLOCK_WRLOCK(global->modules_lock); + + if (UNLIKELY(module->labels_count > TERM_MAX_CATCH_ID - global->catch_labels_count)) { + fprintf(stderr, "Too many labels in loaded modules to install exception handlers.\n"); + SMP_RWLOCK_UNLOCK(global->modules_lock); + return -1; + } + if (!valueshashtable_insert(global->modules_table, term_to_atom_index(module_name), TO_VALUESHASHTABLE_VALUE(module))) { SMP_RWLOCK_UNLOCK(global->modules_lock); return -1; @@ -708,6 +716,8 @@ int globalcontext_insert_module(GlobalContext *global, Module *module) } module->module_index = module_index; + module->catch_labels_base = global->catch_labels_count; + global->catch_labels_count += module->labels_count; global->modules_by_index = new_modules_by_index; global->modules_by_index[module_index] = module; @@ -717,6 +727,26 @@ int globalcontext_insert_module(GlobalContext *global, Module *module) return module_index; } +Module *globalcontext_get_module_by_catch_id(GlobalContext *global, unsigned int catch_id, int *label) +{ + SMP_RWLOCK_RDLOCK(global->modules_lock); + int low = 0; + int high = global->loaded_modules_count - 1; + while (low < high) { + int middle = low + (high - low + 1) / 2; + if (global->modules_by_index[middle]->catch_labels_base <= catch_id) { + low = middle; + } else { + high = middle - 1; + } + } + Module *result = global->modules_by_index[low]; + SMP_RWLOCK_UNLOCK(global->modules_lock); + + *label = (int) (catch_id - result->catch_labels_base); + return result; +} + Module *globalcontext_load_module_from_avm(GlobalContext *global, const char *module_name) { const void *beam_module = NULL; diff --git a/src/libAtomVM/globalcontext.h b/src/libAtomVM/globalcontext.h index b36465e28c..4972c678bd 100644 --- a/src/libAtomVM/globalcontext.h +++ b/src/libAtomVM/globalcontext.h @@ -131,6 +131,7 @@ struct GlobalContext #endif Module **modules_by_index; int ATOMIC loaded_modules_count; + unsigned int catch_labels_count; struct SyncList avmpack_data; @@ -538,6 +539,19 @@ int globalcontext_insert_module(GlobalContext *global, Module *module); */ Module *globalcontext_get_module_by_index(GlobalContext *global, int index); +/** + * @brief Get the module and the label a catch id refers to. + * + * @details Catch ids are handed out at load time, each module getting as many + * consecutive ids as it has labels, so this resolves an id by looking for the + * module with the greatest base that is not greater than the id. + * @param global the global context. + * @param catch_id the catch id, as obtained from term_to_catch_id. + * @param label on return, the label of the exception handler in the module. + * @returns the module the catch id belongs to + */ +Module *globalcontext_get_module_by_catch_id(GlobalContext *global, unsigned int catch_id, int *label); + /** * @brief Returns the module with the given name * diff --git a/src/libAtomVM/jit.c b/src/libAtomVM/jit.c index 30798f5cf1..2a998ff168 100644 --- a/src/libAtomVM/jit.c +++ b/src/libAtomVM/jit.c @@ -161,10 +161,12 @@ _Static_assert(offsetof(JITState, remaining_reductions) == 0x10, "jit_state->rem #elif JIT_ARCH_TARGET == JIT_ARCH_ARMV6M || JIT_ARCH_TARGET == JIT_ARCH_ARM32 || JIT_ARCH_TARGET == JIT_ARCH_RISCV32 || JIT_ARCH_TARGET == JIT_ARCH_WASM32 || JIT_ARCH_TARGET == JIT_ARCH_XTENSA _Static_assert(offsetof(Context, e) == 0x14, "ctx->e is 0x14 in 32-bit backends"); _Static_assert(offsetof(Context, x) == 0x18, "ctx->x is 0x18 in 32-bit backends"); +// cp is now a 64-bit cp_t spanning two 32-bit words (low word = offset << 2 at +// 0x5C, high word = Module* at 0x60); the following fields shift up by one word. _Static_assert(offsetof(Context, cp) == 0x5C, "ctx->cp is 0x5C in 32-bit backends"); -_Static_assert(offsetof(Context, fr) == 0x60, "ctx->fr is 0x60 in 32-bit backends"); -_Static_assert(offsetof(Context, bs) == 0x64, "ctx->bs is 0x64 in 32-bit backends"); -_Static_assert(offsetof(Context, bs_offset) == 0x68, "ctx->bs_offset is 0x68 in 32-bit backends"); +_Static_assert(offsetof(Context, fr) == 0x64, "ctx->fr is 0x64 in 32-bit backends"); +_Static_assert(offsetof(Context, bs) == 0x68, "ctx->bs is 0x68 in 32-bit backends"); +_Static_assert(offsetof(Context, bs_offset) == 0x6C, "ctx->bs_offset is 0x6C in 32-bit backends"); _Static_assert(offsetof(JITState, module) == 0x0, "jit_state->module is 0x0 in 32-bit backends"); _Static_assert(offsetof(JITState, continuation) == 0x4, "jit_state->continuation is 0x4 in 32-bit backends"); @@ -260,26 +262,25 @@ static void jit_trim_live_regs(Context *ctx, uint32_t live) // Update jit_state->module and jit_state->continuation static Context *jit_return(Context *ctx, JITState *jit_state) { - int module_index = ctx->cp >> 24; - TRACE("jit_return: ctx->cp = %d, module_index = %d, offset = %d\n", (int) ctx->cp, module_index, (int) (ctx->cp & 0xFFFFFF) >> 2); - Module *mod = globalcontext_get_module_by_index(ctx->global, module_index); + Module *mod = cp_to_module(ctx->cp, ctx->global); + unsigned int offset = cp_to_offset(ctx->cp); + TRACE("jit_return: mod = %p, offset = %u\n", (void *) mod, offset); // Native case #ifndef AVM_NO_EMU if (mod->native_code == NULL) { // return to emulated const uint8_t *code = mod->code->code; - const uint8_t *pc = code + ((ctx->cp & 0xFFFFFF) >> 2); - jit_state->continuation_pc = pc; + jit_state->continuation_pc = code + offset; } else { #endif #ifdef JIT_JUMPTABLE_IS_DATA // WASM: continuation stores (label + 1) for the dispatch loop to convert. - int label = ((ctx->cp & 0xFFFFFF) >> 2) / JIT_JUMPTABLE_ENTRY_SIZE; - TRACE("jit_return: cp=0x%x mod=%d label=%d\n", (unsigned) ctx->cp, module_index, label); + int label = (int) offset / JIT_JUMPTABLE_ENTRY_SIZE; + TRACE("jit_return: mod=%p label=%d\n", (void *) mod, label); jit_state->continuation = (NativeContinuation) (label + 1); #else - uintptr_t native_pc = (uintptr_t) mod->native_code + ((ctx->cp & 0xFFFFFF) >> 2); + uintptr_t native_pc = (uintptr_t) mod->native_code + offset; jit_state->continuation = (NativeContinuation) native_pc; #endif #ifndef AVM_NO_EMU @@ -510,8 +511,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int // workaround for issue // https://github.com/erlang/otp/issues/7152 if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } if (ctx->heap.root->next) { @@ -519,7 +520,7 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); } } - if ((long) ctx->cp == -1) { + if (cp_is_terminate(ctx->cp)) { return 0; } @@ -531,8 +532,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int // not access ctx->e or ctx->cp) if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } // Native case @@ -567,8 +568,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int } case ModuleNativeFunction: { if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); @@ -582,8 +583,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int } case BIFFunctionType: { if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func); @@ -615,8 +616,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int // even on OTP28, so it is required to allow calling them using // CALL_EXT_ONLY even on OTP28: BIFs are used for try ... catch. if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } const struct GCBif *gcbif = EXPORTED_FUNCTION_TO_GCBIF(func); @@ -653,15 +654,15 @@ static term jit_module_get_atom_term_by_id(JITState *jit_state, int atom_index) static bool jit_allocate(Context *ctx, JITState *jit_state, uint32_t stack_need, uint32_t heap_need, uint32_t live) { TRACE("jit_allocate: ENTRY ctx=%p jit_state=%p stack_need=%" PRIu32 " heap_need=%" PRIu32 " live=%" PRIu32 "\n", (void *) ctx, (void *) jit_state, stack_need, heap_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need > ctx->e - (stack_need + 1)))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need > ctx->e - (stack_need + CP_SIZE_IN_TERMS)))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + 1, live, ctx->x, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + CP_SIZE_IN_TERMS, live, ctx->x, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { set_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); return false; } } - ctx->e -= stack_need + 1; - ctx->e[stack_need] = ctx->cp; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; + store_cp(ctx->e + stack_need, ctx->cp); return true; } @@ -676,8 +677,8 @@ static BifImpl0 jit_get_imported_bif(JITState *jit_state, uint32_t bif) static bool jit_deallocate(Context *ctx, JITState *jit_state, uint32_t n_words) { TRACE("jit_deallocate: n_words=%" PRIu32 "\n", n_words); - ctx->cp = ctx->e[n_words]; - ctx->e += n_words + 1; + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += n_words + CP_SIZE_IN_TERMS; // Hopefully, we only need x[0] if (ctx->heap.root->next) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { diff --git a/src/libAtomVM/module.c b/src/libAtomVM/module.c index eecb92f940..8bc3977ec2 100644 --- a/src/libAtomVM/module.c +++ b/src/libAtomVM/module.c @@ -1117,6 +1117,10 @@ Module *module_new_from_iff_binary(GlobalContext *global, const void *iff_binary fprintf(stderr, "Error: Failed to allocate memory: %s:%i.\n", __FILE__, __LINE__); return NULL; } + // On 32-bit, continuation pointers store this Module pointer directly in a + // stack slot, relying on its low 2 bits being clear (TERM_PRIMARY_CP tag) so + // the GC skips it. malloc guarantees suitable alignment; assert it anyway. + assert(((uintptr_t) mod & TERM_PRIMARY_MASK) == 0); memset(mod, 0, sizeof(Module)); mod->module_index = -1; @@ -1139,6 +1143,7 @@ Module *module_new_from_iff_binary(GlobalContext *global, const void *iff_binary if (offsets[CODE]) { mod->code = (CodeChunk *) (beam_file + offsets[CODE]); + mod->labels_count = ENDIAN_SWAP_32(mod->code->labels); } mod->import_table = beam_file + offsets[IMPT]; mod->export_table = beam_file + offsets[EXPT]; @@ -2104,10 +2109,10 @@ bool module_find_line(Module *mod, size_t offset, uint32_t *line, size_t *filena #endif } -COLD_FUNC void module_cp_to_label_offset(term cp, Module **cp_mod, int *label, size_t *l_off, size_t *out_mod_offset, GlobalContext *global) +COLD_FUNC void module_cp_to_label_offset(cp_t cp, Module **cp_mod, int *label, size_t *l_off, size_t *out_mod_offset, GlobalContext *global) { - Module *mod = globalcontext_get_module_by_index(global, ((uintptr_t) cp) >> 24); - size_t mod_offset = (cp & 0xFFFFFF) >> 2; + Module *mod = cp_to_module(cp, global); + size_t mod_offset = cp_to_offset(cp); if (out_mod_offset) { *out_mod_offset = mod_offset; } @@ -2252,6 +2257,7 @@ uint32_t module_label_code_offset(Module *mod, int label) void module_set_native_code(Module *mod, uint32_t labels_count, ModuleNativeEntryPoint entry_point) { mod->native_code = entry_point; + mod->labels_count = labels_count; // Extra function is OP_INT_CALL_END mod->end_instruction_ii = JIT_JUMPTABLE_OFFSET + JIT_JUMPTABLE_ENTRY_SIZE * labels_count; } diff --git a/src/libAtomVM/module.h b/src/libAtomVM/module.h index 02f4d84fc6..5e6c2ea51c 100644 --- a/src/libAtomVM/module.h +++ b/src/libAtomVM/module.h @@ -109,6 +109,8 @@ struct Module { int module_index; + unsigned int catch_labels_base; + uint32_t labels_count; CodeChunk *code; void *import_table; void *export_table; @@ -330,16 +332,135 @@ static inline const struct ExportedFunction *module_resolve_function(Module *mod } /* - * @brief Casts an instruction index and module index to a return address + * Number of `term` stack slots occupied by a saved continuation pointer. + * On 64-bit a cp fits in one slot; on 32-bit it spans two (offset + Module*). + */ +#if TERM_BITS == 64 +#define CP_SIZE_IN_TERMS 1 +#else +#define CP_SIZE_IN_TERMS 2 +#endif + +/* + * @brief Builds a continuation pointer (return address) from a module and an instruction index. + * + * @details The continuation pointer encodes which module to resume and at which + * code offset. The offset is shifted left by 2 (TERM_PRIMARY_CP tag, 0b00) so the + * garbage collector skips it when it appears on the stack. The offset is mode- + * interpreted at the destination module (BEAM bytecode offset when emulated, + * native code offset when jit-compiled). + * + * On 64-bit, the module is identified by its index packed in the high bits (as + * historically). On 32-bit, the Module pointer is stored directly in the high 32 + * bits: AtomVM never unloads modules so the pointer is stable, this removes the + * 256-module ceiling of index packing, and the return path needs no index lookup. + * A malloc-aligned Module pointer has its low 2 bits clear, so once stored in its + * own stack slot it carries the TERM_PRIMARY_CP tag and is also skipped by the GC. + * + * @param mod the module to resume into + * @param instruction_index the code offset (0 is the first module instruction) + * @return the continuation pointer + */ +static inline cp_t make_cp(const Module *mod, unsigned int instruction_index) +{ +#if TERM_BITS == 64 + return ((cp_t) (unsigned int) mod->module_index << 24) | ((cp_t) instruction_index << 2); +#else + return ((cp_t) (uintptr_t) mod << 32) | ((cp_t) (instruction_index << 2)); +#endif +} + +/* + * @brief Builds a continuation pointer from a module index and instruction index. + * + * @details Cold-path counterpart of make_cp() for callers that only have a module + * index (e.g. rebuilding a cp from a stored raw stacktrace). On 32-bit it resolves + * the index to a Module pointer; on 64-bit it packs the index as make_cp() would. + */ +static inline cp_t make_cp_from_index(unsigned int module_index, unsigned int instruction_index, GlobalContext *global) +{ +#if TERM_BITS == 64 + (void) global; + return ((cp_t) module_index << 24) | ((cp_t) instruction_index << 2); +#else + Module *mod = globalcontext_get_module_by_index(global, (int) module_index); + return make_cp(mod, instruction_index); +#endif +} + +/* + * @brief Stores a continuation pointer to a stack location (CP_SIZE_IN_TERMS slots). + */ +static inline void store_cp(term *dst, cp_t cp) +{ +#if TERM_BITS == 64 + dst[0] = (term) cp; +#else + dst[0] = (term) (uint32_t) cp; // offset << 2 (TERM_PRIMARY_CP tag) + dst[1] = (term) (uintptr_t) (cp >> 32); // Module pointer (aligned, TERM_PRIMARY_CP tag) +#endif +} + +/* + * @brief Loads a continuation pointer previously saved with store_cp. + */ +static inline cp_t load_cp(const term *src) +{ +#if TERM_BITS == 64 + return (cp_t) src[0]; +#else + return (cp_t) (uint32_t) src[0] | ((cp_t) (uintptr_t) src[1] << 32); +#endif +} + +/* + * @brief Returns the module a continuation pointer resumes into. + */ +static inline Module *cp_to_module(cp_t cp, GlobalContext *global) +{ +#if TERM_BITS == 64 + return globalcontext_get_module_by_index(global, (int) (cp >> 24)); +#else + (void) global; + return (Module *) (uintptr_t) (cp >> 32); +#endif +} + +/* + * @brief Returns the code offset a continuation pointer resumes at. + */ +static inline unsigned int cp_to_offset(cp_t cp) +{ +#if TERM_BITS == 64 + return (unsigned int) ((cp & 0xFFFFFF) >> 2); +#else + return (unsigned int) ((uint32_t) cp >> 2); +#endif +} + +/* + * @brief Tells whether a continuation pointer is the process-termination sentinel. + */ +static inline bool cp_is_terminate(cp_t cp) +{ + return ((int64_t) cp) == -1; +} + +/* + * @brief Builds the catch term installing an exception handler at a label. + * + * @details Exception handlers are identified on the stack by a catch id rather + * than by a (module index, label) pair, so that neither the number of modules + * nor the number of labels per module is capped by how the two would have to + * share the bits of a 32-bit term. * - * @details Casts an instruction index and module index to a value that return instruction can restore later. - * @param module_index the module index - * @param the instruction index (0 is the first module instruction) - * @return casted return address + * @param mod the module the handler belongs to + * @param label the label of the handler + * @return the catch term to store in the stack slot of the try/catch */ -static inline term module_address(unsigned int module_index, unsigned int instruction_index) +static inline term module_term_from_catch_label(const Module *mod, unsigned int label) { - return (term) ((module_index << 24) | (instruction_index << 2)); + return term_from_catch_id(mod->catch_labels_base + label); } static inline uint32_t module_get_fun_freeze(const Module *this_module, int fun_index) @@ -477,7 +598,7 @@ static inline bool module_has_line_chunk(Module *mod) * @param mod_offset if not null, set to offset of cp from module start * @param global the global context */ -void module_cp_to_label_offset(term cp, Module **cp_mod, int *label, size_t *l_off, size_t *mod_offset, GlobalContext *global); +void module_cp_to_label_offset(cp_t cp, Module **cp_mod, int *label, size_t *l_off, size_t *mod_offset, GlobalContext *global); /** * @brief Get the offset of a given label from the beginning of the code, emulated or native diff --git a/src/libAtomVM/nifs.c b/src/libAtomVM/nifs.c index 1e425d6b17..96efc2981c 100644 --- a/src/libAtomVM/nifs.c +++ b/src/libAtomVM/nifs.c @@ -1758,7 +1758,7 @@ static term nif_erlang_spawn_fun_opt(Context *ctx, int argc, term argv[]) #ifndef AVM_NO_JIT } #endif - new_ctx->cp = module_address(fun_module->module_index, fun_module->end_instruction_ii); + new_ctx->cp = make_cp(fun_module, fun_module->end_instruction_ii); return do_spawn(ctx, new_ctx, arity, n_freeze, opts_term); } @@ -1815,7 +1815,7 @@ term nif_erlang_spawn_opt(Context *ctx, int argc, term argv[]) #ifndef AVM_NO_JIT } #endif - new_ctx->cp = module_address(found_module->module_index, found_module->end_instruction_ii); + new_ctx->cp = make_cp(found_module, found_module->end_instruction_ii); // TODO: check available registers count int reg_index = 0; diff --git a/src/libAtomVM/opcodesswitch.h b/src/libAtomVM/opcodesswitch.h index 413311fba5..6b1783b8e5 100644 --- a/src/libAtomVM/opcodesswitch.h +++ b/src/libAtomVM/opcodesswitch.h @@ -978,9 +978,12 @@ static void destroy_extended_registers(Context *ctx, unsigned int live) } \ } -#if AVM_NO_JIT - -#define DO_RETURN() \ +// Resolve the destination module of a return from ctx->cp, keeping the +// prev_mod/mod cache so an intra-module return needs no lookup. On 64-bit the +// module is identified by its packed index (historical layout); on 32-bit the +// cp carries the Module pointer directly (high word), so no index lookup at all. +#if TERM_BITS == 64 +#define DO_RETURN_RESOLVE_MODULE() \ { \ int module_index = ((uintptr_t) ctx->cp) >> 24; \ if (module_index == prev_mod->module_index) { \ @@ -993,7 +996,30 @@ static void destroy_extended_registers(Context *ctx, unsigned int live) mod = globalcontext_get_module_by_index(glb, module_index); \ code = mod->code->code; \ } \ - pc = code + ((((uintptr_t) ctx->cp) & 0xFFFFFF) >> 2); \ + } +#else +#define DO_RETURN_RESOLVE_MODULE() \ + { \ + Module *cp_mod = (Module *) (uintptr_t) (ctx->cp >> 32); \ + if (cp_mod == prev_mod) { \ + Module *t = mod; \ + mod = prev_mod; \ + prev_mod = t; \ + code = mod->code->code; \ + } else if (cp_mod != mod) { \ + prev_mod = mod; \ + mod = cp_mod; \ + code = mod->code->code; \ + } \ + } +#endif + +#if AVM_NO_JIT + +#define DO_RETURN() \ + { \ + DO_RETURN_RESOLVE_MODULE(); \ + pc = code + cp_to_offset(ctx->cp); \ } #else @@ -1001,35 +1027,25 @@ static void destroy_extended_registers(Context *ctx, unsigned int live) #ifdef JIT_JUMPTABLE_IS_DATA static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) { - int label = (int) ((ctx->cp & 0xFFFFFF) >> 2) / JIT_JUMPTABLE_ENTRY_SIZE; + int label = (int) cp_to_offset(ctx->cp) / JIT_JUMPTABLE_ENTRY_SIZE; return module_get_native_entry_point(mod, label); } #else static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) { return (ModuleNativeEntryPoint) ((const uint8_t *) mod->native_code) - + ((ctx->cp & 0xFFFFFF) >> 2); + + cp_to_offset(ctx->cp); } #endif #define DO_RETURN() \ { \ - int module_index = ((uintptr_t) ctx->cp) >> 24; \ - if (module_index == prev_mod->module_index) { \ - Module *t = mod; \ - mod = prev_mod; \ - prev_mod = t; \ - code = mod->code->code; \ - } else if (module_index != mod->module_index) { \ - prev_mod = mod; \ - mod = globalcontext_get_module_by_index(glb, module_index); \ - code = mod->code->code; \ - } \ + DO_RETURN_RESOLVE_MODULE(); \ if (mod->native_code) { \ native_pc = do_return_native(mod, ctx); \ } else { \ native_pc = NULL; \ - pc = code + ((((uintptr_t) ctx->cp) & 0xFFFFFF) >> 2); \ + pc = code + cp_to_offset(ctx->cp); \ } \ } @@ -1147,7 +1163,7 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) dreg_t ext_reg = extended_register_ptr(ctx, i); \ *ext_reg = boxed_value[i - fun_arity + 3]; \ } \ - ctx->cp = module_address(mod->module_index, pc - code); \ + ctx->cp = make_cp(mod, pc - code); \ JUMP_TO_LABEL(fun_module, label); #define DECODE_FLAGS_LIST(flags_value, flags, opcode) \ @@ -1653,7 +1669,7 @@ int context_execute_loop(Context *ctx, Module *mod, const char *function_name, i return 0; } - ctx->cp = module_address(mod->module_index, mod->end_instruction_ii); + ctx->cp = make_cp(mod, mod->end_instruction_ii); ctx->saved_module = mod; #if AVM_NO_JIT @@ -1865,7 +1881,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("call/2, arity=%i, label=%i\n", arity, label); USED_BY_TRACE(arity); - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); remaining_reductions--; if (LIKELY(remaining_reductions)) { @@ -1889,8 +1905,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("call_last/3, arity=%i, label=%i, dellocate=%i\n", arity, label, n_words); USED_BY_TRACE(arity); - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); DEBUG_DUMP_STACK(ctx); @@ -1962,7 +1978,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ModuleFunction: { const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); JUMP_TO_LABEL(jump->target, jump->label); break; @@ -1971,7 +1987,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ModuleNativeFunction: { const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); if (jump->target != mod) { prev_mod = mod; mod = jump->target; @@ -2074,8 +2090,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // workaround for issue // https://github.com/erlang/otp/issues/7152 - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); if (ctx->heap.root->next) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { @@ -2091,8 +2107,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // (and it doesn't matter as the code below does // not access ctx->e or ctx->cp) - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); JUMP_TO_LABEL(jump->target, jump->label); @@ -2103,8 +2119,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ModuleNativeFunction: { const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); if (jump->target != mod) { prev_mod = mod; @@ -2116,8 +2132,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif case BIFFunctionType: { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func); term return_value; @@ -2149,8 +2165,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // Regular CALL_EXT_LASTs to those functions are generated as well // even on OTP28, so it is required to allow calling them using // CALL_EXT_LAST even on OTP28: BIFs are used for try ... catch. - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); const struct GCBif *gcbif = EXPORTED_FUNCTION_TO_GCBIF(func); term return_value; @@ -2264,14 +2280,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(live, pc); TRACE("allocate/2 stack_need=%i, live=%i\n", stack_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + 1)))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + CP_SIZE_IN_TERMS)))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + 1, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + CP_SIZE_IN_TERMS, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - ctx->e -= stack_need + 1; - ctx->e[stack_need] = ctx->cp; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; + store_cp(ctx->e + stack_need, ctx->cp); break; } @@ -2284,14 +2300,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(live, pc); TRACE("allocate_heap/2 stack_need=%i, heap_need=%i, live=%i\n", stack_need, heap_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need) > ctx->e - (stack_need + 1))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need) > ctx->e - (stack_need + CP_SIZE_IN_TERMS))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + 1, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + CP_SIZE_IN_TERMS, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - ctx->e -= stack_need + 1; - ctx->e[stack_need] = ctx->cp; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; + store_cp(ctx->e + stack_need, ctx->cp); break; } @@ -2302,18 +2318,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(live, pc); TRACE("allocate_zero/2 stack_need=%i, live=%i\n", stack_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + 1)))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + CP_SIZE_IN_TERMS)))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + 1, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + CP_SIZE_IN_TERMS, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - ctx->e -= stack_need + 1; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; for (uint32_t s = 0; s < stack_need; s++) { ctx->e[s] = term_nil(); } - ctx->e[stack_need] = ctx->cp; + store_cp(ctx->e + stack_need, ctx->cp); break; } @@ -2362,8 +2378,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DEBUG_DUMP_STACK(ctx); - ctx->cp = ctx->e[n_words]; - ctx->e += n_words + 1; + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += n_words + CP_SIZE_IN_TERMS; DEBUG_DUMP_STACK(ctx); // Hopefully, we only need x[0] if (ctx->heap.root->next) { @@ -2379,7 +2395,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE_RETURN(ctx); - if ((intptr_t) ctx->cp == -1) { + if (cp_is_terminate(ctx->cp)) { return 0; } @@ -3180,7 +3196,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - if ((intptr_t) ctx->cp == -1) { + if (cp_is_terminate(ctx->cp)) { return 0; } @@ -3275,7 +3291,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("try/2, label=%i, reg=%c%i\n", label, T_DEST_REG(dreg)); - term catch_term = term_from_catch_label(mod->module_index, label); + term catch_term = module_term_from_catch_label(mod, label); WRITE_REGISTER(dreg, catch_term); break; } @@ -3345,7 +3361,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("catch/2, label=%i, reg=%c%i\n", label, T_DEST_REG(dreg)); - term catch_term = term_from_catch_label(mod->module_index, label); + term catch_term = module_term_from_catch_label(mod, label); WRITE_REGISTER(dreg, catch_term); break; } @@ -4621,7 +4637,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) SET_ERROR(UNDEF_ATOM); HANDLE_ERROR(); } - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); JUMP_TO_LABEL(target_module, target_label); } break; @@ -4646,8 +4662,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(BADARG_ATOM); } - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); atom_index_t module_name = term_to_atom_index(module); atom_index_t function_name = term_to_atom_index(function); diff --git a/src/libAtomVM/stacktrace.c b/src/libAtomVM/stacktrace.c index 42fe665bc3..8bec7c9307 100644 --- a/src/libAtomVM/stacktrace.c +++ b/src/libAtomVM/stacktrace.c @@ -163,7 +163,10 @@ term stacktrace_create_raw_mfa(Context *ctx, Module *mod, size_t current_offset, Module *cp_mod; size_t mod_offset; - module_cp_to_label_offset(*ct, &cp_mod, NULL, NULL, &mod_offset, ctx->global); + // A saved cp spans CP_SIZE_IN_TERMS slots (2 on 32-bit: offset then + // Module*). Both are CP-tagged; reconstruct it and skip the extra slot. + module_cp_to_label_offset(load_cp(ct), &cp_mod, NULL, NULL, &mod_offset, ctx->global); + ct += CP_SIZE_IN_TERMS - 1; // TODO: investigate // mod_offset is currently never equal to cp_mod->end_instruction_ii with native code if (mod_offset != cp_mod->end_instruction_ii && !(prev_mod == cp_mod && mod_offset == prev_mod_offset)) { @@ -183,10 +186,8 @@ term stacktrace_create_raw_mfa(Context *ctx, Module *mod, size_t current_offset, } } } else if (term_is_catch_label(*ct)) { - int module_index; - int label = term_to_catch_label_and_module(*ct, &module_index); - - Module *cl_mod = globalcontext_get_module_by_index(ctx->global, module_index); + int label; + Module *cl_mod = globalcontext_get_module_by_catch_id(ctx->global, term_to_catch_id(*ct), &label); size_t mod_offset = module_label_code_offset(cl_mod, label); if (!(prev_mod == cl_mod && mod_offset == prev_mod_offset)) { @@ -298,7 +299,8 @@ term stacktrace_create_raw_mfa(Context *ctx, Module *mod, size_t current_offset, Module *cp_mod; size_t mod_offset; - module_cp_to_label_offset(*ct, &cp_mod, NULL, NULL, &mod_offset, ctx->global); + module_cp_to_label_offset(load_cp(ct), &cp_mod, NULL, NULL, &mod_offset, ctx->global); + ct += CP_SIZE_IN_TERMS - 1; if (mod_offset != cp_mod->end_instruction_ii && !(prev_mod == cp_mod && mod_offset == prev_mod_offset)) { prev_mod = cp_mod; @@ -312,9 +314,8 @@ term stacktrace_create_raw_mfa(Context *ctx, Module *mod, size_t current_offset, } } else if (term_is_catch_label(*ct)) { - int module_index; - int label = term_to_catch_label_and_module(*ct, &module_index); - Module *cl_mod = globalcontext_get_module_by_index(ctx->global, module_index); + int label; + Module *cl_mod = globalcontext_get_module_by_catch_id(ctx->global, term_to_catch_id(*ct), &label); size_t mod_offset = module_label_code_offset(cl_mod, label); if (!(prev_mod == cl_mod && mod_offset == prev_mod_offset)) { @@ -323,7 +324,7 @@ term stacktrace_create_raw_mfa(Context *ctx, Module *mod, size_t current_offset, prev_mod_offset = mod_offset; term frame_info = term_alloc_tuple(2, &ctx->heap); - term_put_tuple_element(frame_info, 0, term_from_int(module_index)); + term_put_tuple_element(frame_info, 0, term_from_int(cl_mod->module_index)); term_put_tuple_element(frame_info, 1, term_from_int(mod_offset)); raw_stacktrace = term_list_prepend(frame_info, raw_stacktrace, &ctx->heap); @@ -428,9 +429,10 @@ term stacktrace_build(Context *ctx, term *stack_info, uint32_t live) size_t mod_index_tuple_arity = term_get_tuple_arity(mod_index_tuple); assert((mod_index_tuple_arity == 2) || (mod_index_tuple_arity == 5)); - term cp = module_address( + cp_t cp = make_cp_from_index( term_to_int(term_get_tuple_element(mod_index_tuple, 0)), - term_to_int(term_get_tuple_element(mod_index_tuple, 1))); + term_to_int(term_get_tuple_element(mod_index_tuple, 1)), + ctx->global); Module *cp_mod; int label; diff --git a/src/libAtomVM/term.h b/src/libAtomVM/term.h index 46038f6a1e..ab7130b355 100644 --- a/src/libAtomVM/term.h +++ b/src/libAtomVM/term.h @@ -123,6 +123,7 @@ extern "C" { #define TERM_IMMED2_CATCH 0x1B #define TERM_NIL 0x3B +#define TERM_MAX_CATCH_ID ((unsigned int) ((~((term) 0)) >> TERM_IMMED2_TAG_SIZE)) #define TERM_UNUSED 0x2B #define TERM_RESERVED_MARKER(x) ((x << 6) | TERM_UNUSED) @@ -1129,10 +1130,22 @@ static inline uint8_t term_to_uint8(term t) return ((uint16_t) t) >> 4; } -static inline int term_to_catch_label_and_module(term t, int *module_index) +/** + * @brief Gets the catch id of a catch term + * + * @details A catch term encodes a single catch id, which identifies both the + * module and the label of the exception handler. C.f. + * globalcontext_get_module_by_catch_id() to resolve it back. Packing a module + * index and a label in separate bit fields instead would cap both, as a 32-bit + * term only leaves the bits below TERM_MAX_CATCH_ID for the two of them. + * @param t the catch term, term type is not checked. + * @return the catch id of the exception handler. + */ +static inline unsigned int term_to_catch_id(term t) { - *module_index = t >> 24; - return (t >> 6) & 0x3FFFF; + TERM_DEBUG_ASSERT(term_is_catch_label(t)); + + return (unsigned int) (t >> TERM_IMMED2_TAG_SIZE); } /** @@ -1864,9 +1877,20 @@ static inline uint64_t term_to_uint64(term t) } } -static inline term term_from_catch_label(unsigned int module_index, unsigned int label) +/** + * @brief Term from a catch id + * + * @details Builds the term that `try`/`catch` stores in a stack slot to install + * an exception handler. C.f. term_to_catch_id(). + * @param catch_id the catch id of the exception handler, which must not be + * greater than TERM_MAX_CATCH_ID. + * @return a term that encapsulates the catch id. + */ +static inline term term_from_catch_id(unsigned int catch_id) { - return (term) ((module_index << 24) | (label << 6) | TERM_IMMED2_CATCH); + TERM_DEBUG_ASSERT(catch_id <= TERM_MAX_CATCH_ID); + + return (term) (((term) catch_id << TERM_IMMED2_TAG_SIZE) | TERM_IMMED2_CATCH); } /** diff --git a/src/libAtomVM/term_typedef.h b/src/libAtomVM/term_typedef.h index be4ce9bcca..b0f97119d9 100644 --- a/src/libAtomVM/term_typedef.h +++ b/src/libAtomVM/term_typedef.h @@ -45,6 +45,17 @@ typedef uintptr_t term; #define TERM_U_FMT PRIuPTR #define TERM_X_FMT PRIXPTR +/** + * A continuation pointer (return address). It packs a module identifier and an + * instruction/code offset. It is always 64 bits wide so that on 32-bit `term` + * platforms it can hold a full Module pointer plus the offset; on such platforms + * it occupies two stack slots, on 64-bit platforms a single one. See the cp + * helpers (make_cp, STORE_CP, LOAD_CP, CP_SIZE_IN_TERMS) in module.h. + */ +typedef uint64_t cp_t; + +#define CP_X_FMT PRIX64 + #if ((UINT32_MAX != 4294967295ULL) || (UINT64_MAX != 18446744073709551615ULL) \ || (INT32_MAX != 2147483647LL) || (INT64_MAX != 9223372036854775807LL)) #error "limits.h or preprocessor is not sane." diff --git a/tests/erlang_tests/CMakeLists.txt b/tests/erlang_tests/CMakeLists.txt index 96fc78d8fb..bbaf5ca5da 100644 --- a/tests/erlang_tests/CMakeLists.txt +++ b/tests/erlang_tests/CMakeLists.txt @@ -611,6 +611,7 @@ compile_erlang(test_crypto) compile_erlang(test_code_all_available_loaded) compile_erlang(test_code_load_binary DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/code_load/export_test_module_data.hrl) +compile_erlang(test_many_modules DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/code_load/export_test_module_data.hrl) compile_erlang(test_code_load_abs) compile_erlang(test_code_ensure_loaded) compile_erlang(test_add_avm_pack_binary DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/code_load/code_load_pack_data.hrl) @@ -1178,6 +1179,7 @@ set(erlang_test_beams test_code_all_available_loaded.beam test_code_load_binary.beam + test_many_modules.beam test_code_load_abs.beam test_code_ensure_loaded.beam test_add_avm_pack_binary.beam diff --git a/tests/erlang_tests/code_load/export_test_module.erl b/tests/erlang_tests/code_load/export_test_module.erl index affc33815d..95e66dd767 100644 --- a/tests/erlang_tests/code_load/export_test_module.erl +++ b/tests/erlang_tests/code_load/export_test_module.erl @@ -20,9 +20,33 @@ -module(export_test_module). --export([exported_func/1]). +-export([exported_func/1, catching_func/2, tracing_func/1, raising_func/1, erroring_func/1]). exported_func(0) -> 1; exported_func(N) -> ?MODULE:exported_func(N - 1) * N. + +catching_func(Other, N) -> + try Other:raising_func(N) of + R -> {?MODULE, unexpected, R} + catch + throw:{thrown, M, X} -> {?MODULE, M, X * 2} + end. + +raising_func(N) -> + X = id(N) * 2, + throw({thrown, ?MODULE, X}). + +erroring_func(N) -> + X = id(N) * 2, + error({my_error, ?MODULE, X}). + +tracing_func(Other) -> + try Other:erroring_func(1) of + R -> {?MODULE, unexpected, R} + catch + error:{my_error, M, _X}:Stacktrace -> {?MODULE, M, Stacktrace} + end. + +id(X) -> X. diff --git a/tests/erlang_tests/test_many_modules.erl b/tests/erlang_tests/test_many_modules.erl new file mode 100644 index 0000000000..7417fea1e3 --- /dev/null +++ b/tests/erlang_tests/test_many_modules.erl @@ -0,0 +1,92 @@ +% +% This file is part of AtomVM. +% +% Copyright 2026 Paul Guyot +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +-module(test_many_modules). + +-export([start/0]). + +-include("code_load/export_test_module_data.hrl"). + +-define(MODULE_COUNT, 300). +-define(BASE_NAME, <<"export_test_module">>). + +start() -> + Base = ?EXPORT_TEST_MODULE_DATA, + %% Load MODULE_COUNT (> 256) distinct copies of the same module under + %% different names and call each one. + ok = load_and_call_all(Base, 1), + ok = catch_all(1), + ok = trace_all(1), + 0. + +%% 18-char name matching the length of "export_test_module": +%% "export_test_md_" (15 chars) ++ 3-digit zero-padded index (1..999). +mod_name(I) when I < 10 -> + list_to_atom("export_test_md_00" ++ integer_to_list(I)); +mod_name(I) when I < 100 -> + list_to_atom("export_test_md_0" ++ integer_to_list(I)); +mod_name(I) -> + list_to_atom("export_test_md_" ++ integer_to_list(I)). + +load_and_call_all(_Base, I) when I > ?MODULE_COUNT -> + ok; +load_and_call_all(Base, I) -> + Name = mod_name(I), + NameBin = atom_to_binary(Name, latin1), + Bin = binary:replace(Base, ?BASE_NAME, NameBin, [global]), + {module, Name} = code:load_binary(Name, atom_to_list(Name) ++ ".beam", Bin), + 24 = Name:exported_func(4), + load_and_call_all(Base, I + 1). + +catch_all(I) when I > ?MODULE_COUNT -> + ok; +catch_all(I) -> + Catcher = mod_name(I), + Raiser = mod_name(?MODULE_COUNT + 1 - I), + {Catcher, Raiser, 8} = Catcher:catching_func(Raiser, 2), + catch_all(I + 1). + +trace_all(I) when I > ?MODULE_COUNT -> + ok; +trace_all(I) -> + Catcher = mod_name(I), + Raiser = mod_name(?MODULE_COUNT + 1 - I), + case Catcher:tracing_func(Raiser) of + {Catcher, Raiser, undefined} -> + %% Built without AVM_CREATE_STACKTRACES, so stacktrace_build/3 + %% yields 'undefined'. The cross-module raise is still exercised. + ok; + {Catcher, Raiser, Stacktrace} -> + Modules = [M || {M, _F, _A, _L} <- Stacktrace], + true = has_module(Modules, Catcher), + true = has_module(Modules, Raiser), + [] = [M || M <- Modules, is_loaded_copy(M), M =/= Catcher, M =/= Raiser] + end, + trace_all(I + 1). + +has_module([], _Module) -> false; +has_module([Module | _T], Module) -> true; +has_module([_H | T], Module) -> has_module(T, Module). + +is_loaded_copy(Module) -> + is_loaded_copy0(atom_to_list(Module)). + +is_loaded_copy0("export_test_md_" ++ _Rest) -> true; +is_loaded_copy0(_Other) -> false. diff --git a/tests/libs/jit/jit_arm32_tests.erl b/tests/libs/jit/jit_arm32_tests.erl index 5ff9c5749b..4663ae57cd 100644 --- a/tests/libs/jit/jit_arm32_tests.erl +++ b/tests/libs/jit/jit_arm32_tests.erl @@ -400,7 +400,10 @@ move_to_cp_test() -> << " 0: e590a014 ldr sl, [r0, #20]\n" " 4: e59ab000 ldr fp, [sl]\n" - " 8: e580b05c str fp, [r0, #92] @ 0x5c" + " 8: e580b05c str fp, [r0, #92] @ 0x5c\n" + " c: e590a014 ldr sl, [r0, #20]\n" + " 10: e59ab004 ldr fp, [sl, #4]\n" + " 14: e580b060 str fp, [r0, #96] @ 0x60" >>, ?assertStream(arm32, Dump, Stream). @@ -874,9 +877,9 @@ set_bs_test() -> Dump = << " 0: e590b018 ldr fp, [r0, #24]\n" - " 4: e580b064 str fp, [r0, #100] @ 0x64\n" + " 4: e580b068 str fp, [r0, #104] @ 0x68\n" " 8: e3a0a000 mov sl, #0\n" - " c: e580a068 str sl, [r0, #104] @ 0x68" + " c: e580a06c str sl, [r0, #108] @ 0x6c" >>, ?assertStream(arm32, Dump, Stream). @@ -896,51 +899,49 @@ call_or_schedule_next_test() -> Dump = << " 0: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 4: ea000026 b 0xa4\n" + " 4: ea000024 b 0x9c\n" " 8: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" " c: ea000003 b 0x20\n" " 10: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 14: ea000018 b 0x7c\n" + " 14: ea000016 b 0x74\n" " 18: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 1c: ea00001b b 0x90\n" + " 1c: ea000019 b 0x88\n" " 20: e59da000 ldr sl, [sp]\n" " 24: e59ab000 ldr fp, [sl]\n" - " 28: e59bb000 ldr fp, [fp]\n" - " 2c: e1a0bc0b lsl fp, fp, #24\n" - " 30: e3a0ae1e mov sl, #480 @ 0x1e0\n" - " 34: e18bb00a orr fp, fp, sl\n" - " 38: e580b05c str fp, [r0, #92] @ 0x5c\n" - " 3c: e59da000 ldr sl, [sp]\n" - " 40: e59ab008 ldr fp, [sl, #8]\n" - " 44: e25bb001 subs fp, fp, #1\n" - " 48: e58ab008 str fp, [sl, #8]\n" - " 4c: 1a00000a bne 0x7c\n" - " 50: e1a0b00f mov fp, pc\n" - " 54: e3e0a047 mvn sl, #71 @ 0x47\n" - " 58: e08aa00b add sl, sl, fp\n" - " 5c: e59db000 ldr fp, [sp]\n" - " 60: e58ba004 str sl, [fp, #4]\n" - " 64: e592b008 ldr fp, [r2, #8]\n" - " 68: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " 6c: e58db024 str fp, [sp, #36] @ 0x24\n" - " 70: e1a0e007 mov lr, r7\n" - " 74: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" - " 78: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 7c: e592b000 ldr fp, [r2]\n" - " 80: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " 84: e58db024 str fp, [sp, #36] @ 0x24\n" - " 88: e1a0e007 mov lr, r7\n" - " 8c: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" - " 90: e592b004 ldr fp, [r2, #4]\n" - " 94: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " 98: e58db024 str fp, [sp, #36] @ 0x24\n" - " 9c: e1a0e007 mov lr, r7\n" - " a0: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" - " a4: e592b004 ldr fp, [r2, #4]\n" - " a8: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " ac: e58db024 str fp, [sp, #36] @ 0x24\n" - " b0: e1a0e007 mov lr, r7\n" - " b4: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}" + " 28: e580b060 str fp, [r0, #96] @ 0x60\n" + " 2c: e3a0bd07 mov fp, #448 @ 0x1c0\n" + " 30: e580b05c str fp, [r0, #92] @ 0x5c\n" + " 34: e59da000 ldr sl, [sp]\n" + " 38: e59ab008 ldr fp, [sl, #8]\n" + " 3c: e25bb001 subs fp, fp, #1\n" + " 40: e58ab008 str fp, [sl, #8]\n" + " 44: 1a00000a bne 0x74\n" + " 48: e1a0b00f mov fp, pc\n" + " 4c: e3e0a03f mvn sl, #63 @ 0x3f\n" + " 50: e08aa00b add sl, sl, fp\n" + " 54: e59db000 ldr fp, [sp]\n" + " 58: e58ba004 str sl, [fp, #4]\n" + " 5c: e592b008 ldr fp, [r2, #8]\n" + " 60: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " 64: e58db024 str fp, [sp, #36] @ 0x24\n" + " 68: e1a0e007 mov lr, r7\n" + " 6c: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" + " 70: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" + " 74: e592b000 ldr fp, [r2]\n" + " 78: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " 7c: e58db024 str fp, [sp, #36] @ 0x24\n" + " 80: e1a0e007 mov lr, r7\n" + " 84: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" + " 88: e592b004 ldr fp, [r2, #4]\n" + " 8c: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " 90: e58db024 str fp, [sp, #36] @ 0x24\n" + " 94: e1a0e007 mov lr, r7\n" + " 98: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" + " 9c: e592b004 ldr fp, [r2, #4]\n" + " a0: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " a4: e58db024 str fp, [sp, #36] @ 0x24\n" + " a8: e1a0e007 mov lr, r7\n" + " ac: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}" >>, ?assertStream(arm32, Dump, Stream). diff --git a/tests/libs/jit/jit_armv6m_tests.erl b/tests/libs/jit/jit_armv6m_tests.erl index 88d1e9e795..fb3c099e46 100644 --- a/tests/libs/jit/jit_armv6m_tests.erl +++ b/tests/libs/jit/jit_armv6m_tests.erl @@ -443,7 +443,10 @@ move_to_cp_test() -> << " 0: 6946 ldr r6, [r0, #20]\n" " 2: 6837 ldr r7, [r6, #0]\n" - " 4: 65c7 str r7, [r0, #92] ; 0x5c" + " 4: 65c7 str r7, [r0, #92] @ 0x5c\n" + " 6: 6946 ldr r6, [r0, #20]\n" + " 8: 6877 ldr r7, [r6, #4]\n" + " a: 6607 str r7, [r0, #96] @ 0x60" >>, ?assertStream(arm, Dump, Stream). @@ -2341,7 +2344,7 @@ call_ext_test() -> " 4: 3f01 subs r7, #1\n" " 6: 60b7 str r7, [r6, #8]\n" " 8: d109 bne.n 0x1e\n" - " a: a704 add r7, pc, #16 ; (adr r7, 0x1c)\n" + " a: a704 add r7, pc, #16 @ (adr r7, 0x1c)\n" " c: 3701 adds r7, #1\n" " e: 6077 str r7, [r6, #4]\n" " 10: 6897 ldr r7, [r2, #8]\n" @@ -2349,30 +2352,28 @@ call_ext_test() -> " 14: 9705 str r7, [sp, #20]\n" " 16: 46b6 mov lr, r6\n" " 18: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " 1a: 46c0 nop ; (mov r8, r8)\n" + " 1a: 46c0 nop @ (mov r8, r8)\n" " 1c: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 1e: 9e00 ldr r6, [sp, #0]\n" " 20: 6837 ldr r7, [r6, #0]\n" - " 22: 683f ldr r7, [r7, #0]\n" - " 24: 063f lsls r7, r7, #24\n" - " 26: 4e07 ldr r6, [pc, #28] ; (0x44)\n" - " 28: 4337 orrs r7, r6\n" - " 2a: 65c7 str r7, [r0, #92] ; 0x5c\n" - " 2c: 6917 ldr r7, [r2, #16]\n" - " 2e: b082 sub sp, #8\n" - " 30: 2601 movs r6, #1\n" - " 32: 4276 negs r6, r6\n" - " 34: 9600 str r6, [sp, #0]\n" - " 36: 9902 ldr r1, [sp, #8]\n" - " 38: 2202 movs r2, #2\n" - " 3a: 2305 movs r3, #5\n" - " 3c: 47b8 blx r7\n" - " 3e: b002 add sp, #8\n" - " 40: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 22: 6607 str r7, [r0, #96] @ 0x60\n" + " 24: 4f06 ldr r7, [pc, #24] @ (0x40)\n" + " 26: 65c7 str r7, [r0, #92] @ 0x5c\n" + " 28: 6917 ldr r7, [r2, #16]\n" + " 2a: b082 sub sp, #8\n" + " 2c: 2601 movs r6, #1\n" + " 2e: 4276 negs r6, r6\n" + " 30: 9600 str r6, [sp, #0]\n" + " 32: 9902 ldr r1, [sp, #8]\n" + " 34: 2202 movs r2, #2\n" + " 36: 2305 movs r3, #5\n" + " 38: 47b8 blx r7\n" + " 3a: b002 add sp, #8\n" + " 3c: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 3e: 0000 movs r0, r0\n" + " 40: 0110 lsls r0, r2, #4\n" " 42: 0000 movs r0, r0\n" - " 44: 0120 lsls r0, r4, #4\n" - " 46: 0000 movs r0, r0\n" - " 48: b5f2 push {r1, r4, r5, r6, r7, lr}" + " 44: b5f2 push {r1, r4, r5, r6, r7, lr}" >>, ?assertStream(arm, Dump, Stream). @@ -2411,7 +2412,7 @@ call_fun_test() -> " 4: 3f01 subs r7, #1\n" " 6: 60b7 str r7, [r6, #8]\n" " 8: d109 bne.n 0x1e\n" - " a: a704 add r7, pc, #16 ; (adr r7, 0x1c)\n" + " a: a704 add r7, pc, #16 @ (adr r7, 0x1c)\n" " c: 3701 adds r7, #1\n" " e: 6077 str r7, [r6, #4]\n" " 10: 6897 ldr r7, [r2, #8]\n" @@ -2419,7 +2420,7 @@ call_fun_test() -> " 14: 9705 str r7, [sp, #20]\n" " 16: 46b6 mov lr, r6\n" " 18: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " 1a: 46c0 nop ; (mov r8, r8)\n" + " 1a: 46c0 nop @ (mov r8, r8)\n" " 1c: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 1e: 6987 ldr r7, [r0, #24]\n" " 20: 463e mov r6, r7\n" @@ -2428,13 +2429,13 @@ call_fun_test() -> " 26: 4025 ands r5, r4\n" " 28: 2d02 cmp r5, #2\n" " 2a: d009 beq.n 0x40\n" - " 2c: 6cd7 ldr r7, [r2, #76] ; 0x4c\n" + " 2c: 6cd7 ldr r7, [r2, #76] @ 0x4c\n" " 2e: b082 sub sp, #8\n" " 30: 9600 str r6, [sp, #0]\n" " 32: 9902 ldr r1, [sp, #8]\n" - " 34: 222e movs r2, #46 ; 0x2e\n" - " 36: 23ff movs r3, #255 ; 0xff\n" - " 38: 338c adds r3, #140 ; 0x8c\n" + " 34: 222e movs r2, #46 @ 0x2e\n" + " 36: 23ff movs r3, #255 @ 0xff\n" + " 38: 338c adds r3, #140 @ 0x8c\n" " 3a: 47b8 blx r7\n" " 3c: b002 add sp, #8\n" " 3e: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" @@ -2442,39 +2443,37 @@ call_fun_test() -> " 42: 43ae bics r6, r5\n" " 44: 6836 ldr r6, [r6, #0]\n" " 46: 4635 mov r5, r6\n" - " 48: 243f movs r4, #63 ; 0x3f\n" + " 48: 243f movs r4, #63 @ 0x3f\n" " 4a: 4025 ands r5, r4\n" " 4c: 2d14 cmp r5, #20\n" " 4e: d009 beq.n 0x64\n" - " 50: 6cd7 ldr r7, [r2, #76] ; 0x4c\n" + " 50: 6cd7 ldr r7, [r2, #76] @ 0x4c\n" " 52: b082 sub sp, #8\n" " 54: 9600 str r6, [sp, #0]\n" " 56: 9902 ldr r1, [sp, #8]\n" - " 58: 2252 movs r2, #82 ; 0x52\n" - " 5a: 23ff movs r3, #255 ; 0xff\n" - " 5c: 338c adds r3, #140 ; 0x8c\n" + " 58: 2252 movs r2, #82 @ 0x52\n" + " 5a: 23ff movs r3, #255 @ 0xff\n" + " 5c: 338c adds r3, #140 @ 0x8c\n" " 5e: 47b8 blx r7\n" " 60: b002 add sp, #8\n" " 62: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" " 64: 9d00 ldr r5, [sp, #0]\n" " 66: 682e ldr r6, [r5, #0]\n" - " 68: 6836 ldr r6, [r6, #0]\n" - " 6a: 0636 lsls r6, r6, #24\n" - " 6c: 4d05 ldr r5, [pc, #20] ; (0x84)\n" - " 6e: 432e orrs r6, r5\n" - " 70: 65c6 str r6, [r0, #92] ; 0x5c\n" - " 72: 2680 movs r6, #128 ; 0x80\n" - " 74: 5996 ldr r6, [r2, r6]\n" - " 76: 463a mov r2, r7\n" - " 78: 2300 movs r3, #0\n" - " 7a: 9f05 ldr r7, [sp, #20]\n" - " 7c: 9605 str r6, [sp, #20]\n" - " 7e: 46be mov lr, r7\n" - " 80: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 68: 6606 str r6, [r0, #96] @ 0x60\n" + " 6a: 4e05 ldr r6, [pc, #20] @ (0x80)\n" + " 6c: 65c6 str r6, [r0, #92] @ 0x5c\n" + " 6e: 2680 movs r6, #128 @ 0x80\n" + " 70: 5996 ldr r6, [r2, r6]\n" + " 72: 463a mov r2, r7\n" + " 74: 2300 movs r3, #0\n" + " 76: 9f05 ldr r7, [sp, #20]\n" + " 78: 9605 str r6, [sp, #20]\n" + " 7a: 46be mov lr, r7\n" + " 7c: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 7e: 0000 movs r0, r0\n" + " 80: 0210 lsls r0, r2, #8\n" " 82: 0000 movs r0, r0\n" - " 84: 0220 lsls r0, r4, #8\n" - " 86: 0000 movs r0, r0\n" - " 88: b5f2 push {r1, r4, r5, r6, r7, lr}" + " 84: b5f2 push {r1, r4, r5, r6, r7, lr}" >>, ?assertStream(arm, Dump, Stream). @@ -3217,7 +3216,7 @@ move_to_native_register_test_() -> Stream = ?BACKEND:stream(State2), Dump = << " 0: 6987 ldr r7, [r0, #24]\n" - " 2: 6e06 ldr r6, [r0, #96] ; 0x60\n" + " 2: 6e46 ldr r6, [r0, #100] @ 0x64\n" " 4: 687d ldr r5, [r7, #4]\n" " 6: 61b5 str r5, [r6, #24]\n" " 8: 68bd ldr r5, [r7, #8]\n" @@ -3966,51 +3965,46 @@ add_beam_test() -> Stream = ?BACKEND:stream(State15), Dump = << - % jump table - " 0: 4b01 ldr r3, [pc, #4] ; (0x8)\n" + " 0: 4b01 ldr r3, [pc, #4] @ (0x8)\n" " 2: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 4: 449f add pc, r3\n" - " 6: 46c0 nop ; (mov r8, r8)\n" - " 8: 00d9 lsls r1, r3, #3\n" + " 6: 46c0 nop @ (mov r8, r8)\n" + " 8: 00d5 lsls r5, r2, #3\n" " a: 0000 movs r0, r0\n" - " c: 4b01 ldr r3, [pc, #4] ; (0x14)\n" + " c: 4b01 ldr r3, [pc, #4] @ (0x14)\n" " e: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 10: 449f add pc, r3\n" - " 12: 46c0 nop ; (mov r8, r8)\n" + " 12: 46c0 nop @ (mov r8, r8)\n" " 14: 001d movs r5, r3\n" " 16: 0000 movs r0, r0\n" - " 18: 4b01 ldr r3, [pc, #4] ; (0x20)\n" + " 18: 4b01 ldr r3, [pc, #4] @ (0x20)\n" " 1a: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 1c: 449f add pc, r3\n" - " 1e: 46c0 nop ; (mov r8, r8)\n" + " 1e: 46c0 nop @ (mov r8, r8)\n" " 20: 0045 lsls r5, r0, #1\n" " 22: 0000 movs r0, r0\n" - " 24: 4b01 ldr r3, [pc, #4] ; (0x2c)\n" + " 24: 4b01 ldr r3, [pc, #4] @ (0x2c)\n" " 26: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 28: 449f add pc, r3\n" - " 2a: 46c0 nop ; (mov r8, r8)\n" - " 2c: 00a9 lsls r1, r5, #2\n" + " 2a: 46c0 nop @ (mov r8, r8)\n" + " 2c: 00a5 lsls r5, r4, #2\n" " 2e: 0000 movs r0, r0\n" - % label 1 - % {move,{integer,9},{x,1}}. - " 30: 279f movs r7, #159 ; 0x9f\n" + " 30: 279f movs r7, #159 @ 0x9f\n" " 32: 61c7 str r7, [r0, #28]\n" - % {move,{integer,8},{x,0}} - " 34: 278f movs r7, #143 ; 0x8f\n" + " 34: 278f movs r7, #143 @ 0x8f\n" " 36: 6187 str r7, [r0, #24]\n" - % {call_only,2,{f,2}}. " 38: 9e00 ldr r6, [sp, #0]\n" " 3a: 68b7 ldr r7, [r6, #8]\n" " 3c: 3f01 subs r7, #1\n" " 3e: 60b7 str r7, [r6, #8]\n" " 40: d004 beq.n 0x4c\n" " 42: e00f b.n 0x64\n" - " 44: 46c0 nop ; (mov r8, r8)\n" - " 46: 46c0 nop ; (mov r8, r8)\n" - " 48: 46c0 nop ; (mov r8, r8)\n" - " 4a: 46c0 nop ; (mov r8, r8)\n" - " 4c: a700 add r7, pc, #0 ; (adr r7, 0x50)\n" - " 4e: 2637 movs r6, #55 ; 0x37\n" + " 44: 46c0 nop @ (mov r8, r8)\n" + " 46: 46c0 nop @ (mov r8, r8)\n" + " 48: 46c0 nop @ (mov r8, r8)\n" + " 4a: 46c0 nop @ (mov r8, r8)\n" + " 4c: a700 add r7, pc, #0 @ (adr r7, 0x50)\n" + " 4e: 2637 movs r6, #55 @ 0x37\n" " 50: 4276 negs r6, r6\n" " 52: 19f6 adds r6, r6, r7\n" " 54: 9f00 ldr r7, [sp, #0]\n" @@ -4020,9 +4014,7 @@ add_beam_test() -> " 5c: 9705 str r7, [sp, #20]\n" " 5e: 46b6 mov lr, r6\n" " 60: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " 62: 46c0 nop ; (mov r8, r8)\n" - % label 2 - % {allocate,1,1}. + " 62: 46c0 nop @ (mov r8, r8)\n" " 64: 6957 ldr r7, [r2, #20]\n" " 66: b405 push {r0, r2}\n" " 68: b082 sub sp, #8\n" @@ -4038,65 +4030,56 @@ add_beam_test() -> " 7c: 07fe lsls r6, r7, #31\n" " 7e: d405 bmi.n 0x8c\n" " 80: 6997 ldr r7, [r2, #24]\n" - " 82: 2282 movs r2, #130 ; 0x82\n" + " 82: 2282 movs r2, #130 @ 0x82\n" " 84: 9e05 ldr r6, [sp, #20]\n" " 86: 9705 str r7, [sp, #20]\n" " 88: 46b6 mov lr, r6\n" " 8a: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - % {init_yregs,{list,[{y,0}]}}. - %% move_to_vm_register(State8, ?TERM_NIL, {y_reg, 0}), - " 8c: 263b movs r6, #59 ; 0x3b\n" + " 8c: 263b movs r6, #59 @ 0x3b\n" " 8e: 6947 ldr r7, [r0, #20]\n" " 90: 603e str r6, [r7, #0]\n" - % {call,1,{f,3}} - %% call_or_schedule_next(State9, 3), " 92: 9e00 ldr r6, [sp, #0]\n" " 94: 6837 ldr r7, [r6, #0]\n" - " 96: 683f ldr r7, [r7, #0]\n" - " 98: 063f lsls r7, r7, #24\n" - " 9a: 4e0c ldr r6, [pc, #48] ; (0xcc)\n" - " 9c: 4337 orrs r7, r6\n" - " 9e: 65c7 str r7, [r0, #92] ; 0x5c\n" - " a0: 9e00 ldr r6, [sp, #0]\n" - " a2: 68b7 ldr r7, [r6, #8]\n" - " a4: 3f01 subs r7, #1\n" - " a6: 60b7 str r7, [r6, #8]\n" - " a8: d004 beq.n 0xb4\n" - " aa: e013 b.n 0xd4\n" - " ac: 46c0 nop ; (mov r8, r8)\n" - " ae: 46c0 nop ; (mov r8, r8)\n" - " b0: 46c0 nop ; (mov r8, r8)\n" - " b2: 46c0 nop ; (mov r8, r8)\n" - " b4: a700 add r7, pc, #0 ; (adr r7, 0xb8)\n" - " b6: 2693 movs r6, #147 ; 0x93\n" - " b8: 4276 negs r6, r6\n" - " ba: 19f6 adds r6, r6, r7\n" - " bc: 9f00 ldr r7, [sp, #0]\n" - " be: 607e str r6, [r7, #4]\n" - " c0: 6897 ldr r7, [r2, #8]\n" - " c2: 9e05 ldr r6, [sp, #20]\n" - " c4: 9705 str r7, [sp, #20]\n" - " c6: 46b6 mov lr, r6\n" - " c8: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 96: 6607 str r7, [r0, #96] @ 0x60\n" + " 98: 4f0b ldr r7, [pc, #44] @ (0xc8)\n" + " 9a: 65c7 str r7, [r0, #92] @ 0x5c\n" + " 9c: 9e00 ldr r6, [sp, #0]\n" + " 9e: 68b7 ldr r7, [r6, #8]\n" + " a0: 3f01 subs r7, #1\n" + " a2: 60b7 str r7, [r6, #8]\n" + " a4: d004 beq.n 0xb0\n" + " a6: e013 b.n 0xd0\n" + " a8: 46c0 nop @ (mov r8, r8)\n" + " aa: 46c0 nop @ (mov r8, r8)\n" + " ac: 46c0 nop @ (mov r8, r8)\n" + " ae: 46c0 nop @ (mov r8, r8)\n" + " b0: a700 add r7, pc, #0 @ (adr r7, 0xb4)\n" + " b2: 268f movs r6, #143 @ 0x8f\n" + " b4: 4276 negs r6, r6\n" + " b6: 19f6 adds r6, r6, r7\n" + " b8: 9f00 ldr r7, [sp, #0]\n" + " ba: 607e str r6, [r7, #4]\n" + " bc: 6897 ldr r7, [r2, #8]\n" + " be: 9e05 ldr r6, [sp, #20]\n" + " c0: 9705 str r7, [sp, #20]\n" + " c2: 46b6 mov lr, r6\n" + " c4: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " c6: 0000 movs r0, r0\n" + " c8: 0330 lsls r0, r6, #12\n" " ca: 0000 movs r0, r0\n" - " cc: 0340 lsls r0, r0, #13\n" - " ce: 0000 movs r0, r0\n" - %% (continuation) - " d0: b5f2 push {r1, r4, r5, r6, r7, lr}\n" - " d2: 46c0 nop ; (mov r8, r8)\n" - % label 3 - " d4: 6857 ldr r7, [r2, #4]\n" - " d6: 9e05 ldr r6, [sp, #20]\n" - " d8: 9705 str r7, [sp, #20]\n" - " da: 46b6 mov lr, r6\n" - " dc: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " de: 46c0 nop ; (mov r8, r8)\n" - % label 0 - " e0: 6857 ldr r7, [r2, #4]\n" - " e2: 9e05 ldr r6, [sp, #20]\n" - " e4: 9705 str r7, [sp, #20]\n" - " e6: 46b6 mov lr, r6\n" - " e8: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " cc: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " ce: 46c0 nop @ (mov r8, r8)\n" + " d0: 6857 ldr r7, [r2, #4]\n" + " d2: 9e05 ldr r6, [sp, #20]\n" + " d4: 9705 str r7, [sp, #20]\n" + " d6: 46b6 mov lr, r6\n" + " d8: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " da: 46c0 nop @ (mov r8, r8)\n" + " dc: 6857 ldr r7, [r2, #4]\n" + " de: 9e05 ldr r6, [sp, #20]\n" + " e0: 9705 str r7, [sp, #20]\n" + " e2: 46b6 mov lr, r6\n" + " e4: bdf2 pop {r1, r4, r5, r6, r7, pc}" >>, ?assertStream(arm, Dump, Stream). diff --git a/tests/libs/jit/jit_riscv32_tests.erl b/tests/libs/jit/jit_riscv32_tests.erl index 00bebc1316..a86aa6ba3c 100644 --- a/tests/libs/jit/jit_riscv32_tests.erl +++ b/tests/libs/jit/jit_riscv32_tests.erl @@ -446,9 +446,12 @@ move_to_cp_test() -> Stream = ?BACKEND:stream(State1), Dump = << - " 0: 01452f03 lw t5,20(a0)\n" - " 4: 000f2f83 lw t6,0(t5)\n" - " 8: 05f52e23 sw t6,92(a0)" + " 0: 01452f03 lw t5,20(a0)\n" + " 4: 000f2f83 lw t6,0(t5)\n" + " 8: 05f52e23 sw t6,92(a0)\n" + " c: 01452f03 lw t5,20(a0)\n" + " 10: 004f2f83 lw t6,4(t5)\n" + " 14: 07f52023 sw t6,96(a0)" >>, ?assertStream(riscv32, Dump, Stream). @@ -1998,28 +2001,26 @@ call_ext_test() -> Stream = ?BACKEND:stream(State2), Dump = << - " 0: 0085af83 lw t6,8(a1)\n" - " 4: 1ffd addi t6,t6,-1\n" - " 6: 01f5a423 sw t6,8(a1)\n" - " a: 000f9b63 bnez t6,0x20\n" - " e: 00000f97 auipc t6,0x0\n" - " 12: 0fc9 addi t6,t6,18 # 0x20\n" - " 14: 0001 nop\n" - " 16: 01f5a223 sw t6,4(a1)\n" - " 1a: 00862f83 lw t6,8(a2)\n" - " 1e: 8f82 jr t6\n" - " 20: 0005af03 lw t5,0(a1)\n" - " 24: 000f2f03 lw t5,0(t5)\n" - " 28: 0f62 slli t5,t5,0x18\n" - " 2a: 11800f93 li t6,280\n" - " 2e: 00000013 nop\n" - " 32: 01ff6f33 or t5,t5,t6\n" - " 36: 05e52e23 sw t5,92(a0)\n" - " 3a: 01062f83 lw t6,16(a2)\n" - " 3e: 4609 li a2,2\n" - " 40: 4695 li a3,5\n" - " 42: 577d li a4,-1\n" - " 44: 8f82 jr t6" + " 0: 0085af83 lw t6,8(a1)\n" + " 4: 1ffd addi t6,t6,-1\n" + " 6: 01f5a423 sw t6,8(a1)\n" + " a: 000f9b63 bnez t6,0x20\n" + " e: 00000f97 auipc t6,0x0\n" + " 12: 0fc9 addi t6,t6,18 # 0x20\n" + " 14: 0001 nop\n" + " 16: 01f5a223 sw t6,4(a1)\n" + " 1a: 00862f83 lw t6,8(a2)\n" + " 1e: 8f82 jr t6\n" + " 20: 0005af83 lw t6,0(a1)\n" + " 24: 07f52023 sw t6,96(a0)\n" + " 28: 10000f93 li t6,256\n" + " 2c: 00000013 nop\n" + " 30: 05f52e23 sw t6,92(a0)\n" + " 34: 01062f83 lw t6,16(a2)\n" + " 38: 4609 li a2,2\n" + " 3a: 4695 li a3,5\n" + " 3c: 577d li a4,-1\n" + " 3e: 8f82 jr t6" >>, ?assertStream(riscv32, Dump, Stream). @@ -2054,51 +2055,49 @@ call_fun_test() -> Stream = ?BACKEND:stream(State9), Dump = << - " 0: 0085af83 lw t6,8(a1)\n" - " 4: 1ffd addi t6,t6,-1\n" - " 6: 01f5a423 sw t6,8(a1)\n" - " a: 000f9b63 bnez t6,0x20\n" - " e: 00000f97 auipc t6,0x0\n" - " 12: 0fc9 addi t6,t6,18 # 0x20\n" - " 14: 0001 nop\n" - " 16: 01f5a223 sw t6,4(a1)\n" - " 1a: 00862f83 lw t6,8(a2)\n" - " 1e: 8f82 jr t6\n" - " 20: 01852f83 lw t6,24(a0)\n" - " 24: 8f7e mv t5,t6\n" - " 26: 8efa mv t4,t5\n" - " 28: 003efe93 andi t4,t4,3\n" - " 2c: 4e09 li t3,2\n" - " 2e: 01ce8a63 beq t4,t3,0x42\n" - " 32: 04c62f83 lw t6,76(a2)\n" - " 36: 03600613 li a2,54\n" - " 3a: 18b00693 li a3,395\n" - " 3e: 877a mv a4,t5\n" - " 40: 8f82 jr t6\n" - " 42: ffcf7f13 andi t5,t5,-4\n" - " 46: 000f2f03 lw t5,0(t5)\n" - " 4a: 8efa mv t4,t5\n" - " 4c: 03fefe93 andi t4,t4,63\n" - " 50: 4e51 li t3,20\n" - " 52: 01ce8a63 beq t4,t3,0x66\n" - " 56: 04c62f83 lw t6,76(a2)\n" - " 5a: 05a00613 li a2,90\n" - " 5e: 18b00693 li a3,395\n" - " 62: 877a mv a4,t5\n" - " 64: 8f82 jr t6\n" - " 66: 0005ae83 lw t4,0(a1)\n" - " 6a: 000eae83 lw t4,0(t4)\n" - " 6e: 0ee2 slli t4,t4,0x18\n" - " 70: 24000f13 li t5,576\n" - " 74: 00000013 nop\n" - " 78: 01eeeeb3 or t4,t4,t5\n" - " 7c: 05d52e23 sw t4,92(a0)\n" - " 80: 08000f13 li t5,128\n" - " 84: 9f32 add t5,t5,a2\n" - " 86: 000f2f03 lw t5,0(t5)\n" - " 8a: 867e mv a2,t6\n" - " 8c: 4681 li a3,0\n" - " 8e: 8f02 jr t5" + " 0: 0085af83 lw t6,8(a1)\n" + " 4: 1ffd addi t6,t6,-1\n" + " 6: 01f5a423 sw t6,8(a1)\n" + " a: 000f9b63 bnez t6,0x20\n" + " e: 00000f97 auipc t6,0x0\n" + " 12: 0fc9 addi t6,t6,18 # 0x20\n" + " 14: 0001 nop\n" + " 16: 01f5a223 sw t6,4(a1)\n" + " 1a: 00862f83 lw t6,8(a2)\n" + " 1e: 8f82 jr t6\n" + " 20: 01852f83 lw t6,24(a0)\n" + " 24: 8f7e mv t5,t6\n" + " 26: 8efa mv t4,t5\n" + " 28: 003efe93 andi t4,t4,3\n" + " 2c: 4e09 li t3,2\n" + " 2e: 01ce8a63 beq t4,t3,0x42\n" + " 32: 04c62f83 lw t6,76(a2)\n" + " 36: 03600613 li a2,54\n" + " 3a: 18b00693 li a3,395\n" + " 3e: 877a mv a4,t5\n" + " 40: 8f82 jr t6\n" + " 42: ffcf7f13 andi t5,t5,-4\n" + " 46: 000f2f03 lw t5,0(t5)\n" + " 4a: 8efa mv t4,t5\n" + " 4c: 03fefe93 andi t4,t4,63\n" + " 50: 4e51 li t3,20\n" + " 52: 01ce8a63 beq t4,t3,0x66\n" + " 56: 04c62f83 lw t6,76(a2)\n" + " 5a: 05a00613 li a2,90\n" + " 5e: 18b00693 li a3,395\n" + " 62: 877a mv a4,t5\n" + " 64: 8f82 jr t6\n" + " 66: 0005af03 lw t5,0(a1)\n" + " 6a: 07e52023 sw t5,96(a0)\n" + " 6e: 22800f13 li t5,552\n" + " 72: 00000013 nop\n" + " 76: 05e52e23 sw t5,92(a0)\n" + " 7a: 08000f13 li t5,128\n" + " 7e: 9f32 add t5,t5,a2\n" + " 80: 000f2f03 lw t5,0(t5)\n" + " 84: 867e mv a2,t6\n" + " 86: 4681 li a3,0\n" + " 88: 8f02 jr t5" >>, ?assertStream(riscv32, Dump, Stream). @@ -2756,12 +2755,12 @@ move_to_native_register_test_() -> ), Stream = ?BACKEND:stream(State2), Dump = << - " 0: 01852f83 lw t6,24(a0)\n" - " 4: 06052f03 lw t5,96(a0)\n" - " 8: 004fae83 lw t4,4(t6)\n" - " c: 01df2c23 sw t4,24(t5)\n" - " 10: 008fae83 lw t4,8(t6)\n" - " 14: 01df2e23 sw t4,28(t5)" + " 0: 01852f83 lw t6,24(a0)\n" + " 4: 06452f03 lw t5,100(a0)\n" + " 8: 004fae83 lw t4,4(t6)\n" + " c: 01df2c23 sw t4,24(t5)\n" + " 10: 008fae83 lw t4,8(t6)\n" + " 14: 01df2e23 sw t4,28(t5)" >>, ?assertStream(riscv32, Dump, Stream) end) @@ -3522,93 +3521,77 @@ add_beam_test() -> Stream = ?BACKEND:stream(State15), Dump = << - % jump table (new 8-byte format) - " 0: 00000697 auipc a3,0x0\n" - " 4: 0e068067 jr 224(a3) # 0xe0\n" - " 8: 00000697 auipc a3,0x0\n" - " c: 01868067 jr 24(a3) # 0x20\n" - " 10: 00000697 auipc a3,0x0\n" - " 14: 04868067 jr 72(a3) # 0x58\n" - " 18: 00000697 auipc a3,0x0\n" - " 1c: 0c268067 jr 194(a3) # 0xda\n" - % label 1 - % {move,{integer,9},{x,1}}. - " 20: 09f00f93 li t6,159\n" - " 24: 01f52e23 sw t6,28(a0)\n" - % {move,{integer,8},{x,0}} - " 28: 08f00f93 li t6,143\n" - " 2c: 01f52c23 sw t6,24(a0)\n" - % {call_only,2,{f,2}}. - " 30: 0085af83 lw t6,8(a1)\n" - " 34: 1ffd addi t6,t6,-1\n" - " 36: 01f5a423 sw t6,8(a1)\n" - " 3a: 000f8663 beqz t6,0x46\n" - " 3e: a829 j 0x58\n" - " 40: 0001 nop\n" - " 42: 00000013 nop\n" - " 46: 00000f97 auipc t6,0x0\n" - " 4a: 0fc9 addi t6,t6,18 # 0x58\n" - " 4c: 0001 nop\n" - " 4e: 01f5a223 sw t6,4(a1)\n" - " 52: 00862f83 lw t6,8(a2)\n" - " 56: 8f82 jr t6\n" - % label 2 - % {allocate,1,1}. - " 58: 01462f83 lw t6,20(a2)\n" - " 5c: 1141 addi sp,sp,-16\n" - " 5e: c006 sw ra,0(sp)\n" - " 60: c22a sw a0,4(sp)\n" - " 62: c42e sw a1,8(sp)\n" - " 64: c632 sw a2,12(sp)\n" - " 66: 4605 li a2,1\n" - " 68: 4681 li a3,0\n" - " 6a: 4705 li a4,1\n" - " 6c: 9f82 jalr t6\n" - " 6e: 8faa mv t6,a0\n" - " 70: 4082 lw ra,0(sp)\n" - " 72: 4512 lw a0,4(sp)\n" - " 74: 45a2 lw a1,8(sp)\n" - " 76: 4632 lw a2,12(sp)\n" - " 78: 0141 addi sp,sp,16\n" - " 7a: 01ff9f13 slli t5,t6,0x1f\n" - " 7e: 000f4763 bltz t5,0x8c\n" - " 82: 01862f83 lw t6,24(a2)\n" - " 86: 08600613 li a2,134\n" - " 8a: 8f82 jr t6\n" - % {init_yregs,{list,[{y,0}]}}. - %% move_to_vm_register(State8, ?TERM_NIL, {y_reg, 0}), - " 8c: 03b00f13 li t5,59\n" - " 90: 01452f83 lw t6,20(a0)\n" - " 94: 01efa023 sw t5,0(t6)\n" - % {call,1,{f,3}} - %% call_or_schedule_next(State9, 3), - " 98: 0005af03 lw t5,0(a1)\n" - " 9c: 000f2f03 lw t5,0(t5)\n" - " a0: 0f62 slli t5,t5,0x18\n" - " a2: 36800f93 li t6,872\n" - " a6: 00000013 nop\n" - " aa: 01ff6f33 or t5,t5,t6\n" - " ae: 05e52e23 sw t5,92(a0)\n" - " b2: 0085af83 lw t6,8(a1)\n" - " b6: 1ffd addi t6,t6,-1\n" - " b8: 01f5a423 sw t6,8(a1)\n" - " bc: 000f8663 beqz t6,0xc8\n" - " c0: a829 j 0xda\n" - " c2: 0001 nop\n" - " c4: 00000013 nop\n" - " c8: 00000f97 auipc t6,0x0\n" - " cc: 0fc9 addi t6,t6,18 # 0xda\n" - " ce: 0001 nop\n" - " d0: 01f5a223 sw t6,4(a1)\n" - " d4: 00862f83 lw t6,8(a2)\n" - " d8: 8f82 jr t6\n" - %% (continuation) - % label 3 - " da: 00462f83 lw t6,4(a2)\n" - " de: 8f82 jr t6\n" - % label 0 - " e0: 00462f83 lw t6,4(a2)\n" - " e4: 8f82 jr t6" + " 0: 00000697 auipc a3,0x0\n" + " 4: 0da68067 jr 218(a3) # 0xda\n" + " 8: 00000697 auipc a3,0x0\n" + " c: 01868067 jr 24(a3) # 0x20\n" + " 10: 00000697 auipc a3,0x0\n" + " 14: 04868067 jr 72(a3) # 0x58\n" + " 18: 00000697 auipc a3,0x0\n" + " 1c: 0bc68067 jr 188(a3) # 0xd4\n" + " 20: 09f00f93 li t6,159\n" + " 24: 01f52e23 sw t6,28(a0)\n" + " 28: 08f00f93 li t6,143\n" + " 2c: 01f52c23 sw t6,24(a0)\n" + " 30: 0085af83 lw t6,8(a1)\n" + " 34: 1ffd addi t6,t6,-1\n" + " 36: 01f5a423 sw t6,8(a1)\n" + " 3a: 000f8663 beqz t6,0x46\n" + " 3e: a829 j 0x58\n" + " 40: 0001 nop\n" + " 42: 00000013 nop\n" + " 46: 00000f97 auipc t6,0x0\n" + " 4a: 0fc9 addi t6,t6,18 # 0x58\n" + " 4c: 0001 nop\n" + " 4e: 01f5a223 sw t6,4(a1)\n" + " 52: 00862f83 lw t6,8(a2)\n" + " 56: 8f82 jr t6\n" + " 58: 01462f83 lw t6,20(a2)\n" + " 5c: 1141 addi sp,sp,-16\n" + " 5e: c006 sw ra,0(sp)\n" + " 60: c22a sw a0,4(sp)\n" + " 62: c42e sw a1,8(sp)\n" + " 64: c632 sw a2,12(sp)\n" + " 66: 4605 li a2,1\n" + " 68: 4681 li a3,0\n" + " 6a: 4705 li a4,1\n" + " 6c: 9f82 jalr t6\n" + " 6e: 8faa mv t6,a0\n" + " 70: 4082 lw ra,0(sp)\n" + " 72: 4512 lw a0,4(sp)\n" + " 74: 45a2 lw a1,8(sp)\n" + " 76: 4632 lw a2,12(sp)\n" + " 78: 0141 addi sp,sp,16\n" + " 7a: 01ff9f13 slli t5,t6,0x1f\n" + " 7e: 000f4763 bltz t5,0x8c\n" + " 82: 01862f83 lw t6,24(a2)\n" + " 86: 08600613 li a2,134\n" + " 8a: 8f82 jr t6\n" + " 8c: 03b00f13 li t5,59\n" + " 90: 01452f83 lw t6,20(a0)\n" + " 94: 01efa023 sw t5,0(t6)\n" + " 98: 0005af83 lw t6,0(a1)\n" + " 9c: 07f52023 sw t6,96(a0)\n" + " a0: 35000f93 li t6,848\n" + " a4: 00000013 nop\n" + " a8: 05f52e23 sw t6,92(a0)\n" + " ac: 0085af83 lw t6,8(a1)\n" + " b0: 1ffd addi t6,t6,-1\n" + " b2: 01f5a423 sw t6,8(a1)\n" + " b6: 000f8663 beqz t6,0xc2\n" + " ba: a829 j 0xd4\n" + " bc: 0001 nop\n" + " be: 00000013 nop\n" + " c2: 00000f97 auipc t6,0x0\n" + " c6: 0fc9 addi t6,t6,18 # 0xd4\n" + " c8: 0001 nop\n" + " ca: 01f5a223 sw t6,4(a1)\n" + " ce: 00862f83 lw t6,8(a2)\n" + " d2: 8f82 jr t6\n" + " d4: 00462f83 lw t6,4(a2)\n" + " d8: 8f82 jr t6\n" + " da: 00462f83 lw t6,4(a2)\n" + " de: 8f82 jr t6" >>, ?assertStream(riscv32, Dump, Stream). diff --git a/tests/libs/jit/jit_tests_common.erl b/tests/libs/jit/jit_tests_common.erl index 21783251be..70bd75304b 100644 --- a/tests/libs/jit/jit_tests_common.erl +++ b/tests/libs/jit/jit_tests_common.erl @@ -154,6 +154,7 @@ toolchain_prefixes(xtensa) -> %% whose objdump displays raw bytes rather than instruction words, %% breaking hex_to_bin/3's little-endian conversion. ["xtensa-esp32-elf", "xtensa-esp32s2-elf", "xtensa-esp32s3-elf"] ++ + ["xtensa-lx106-elf"] ++ ["xtensa" ++ V || V <- ["-unknown-elf", "-elf", "-linux-gnu"]] ++ ["xtensa-lx6-linux-gnu"]; toolchain_prefixes(Arch) -> diff --git a/tests/libs/jit/jit_wasm32_tests.erl b/tests/libs/jit/jit_wasm32_tests.erl index 07c986c5fa..0698dde60a 100644 --- a/tests/libs/jit/jit_wasm32_tests.erl +++ b/tests/libs/jit/jit_wasm32_tests.erl @@ -473,22 +473,22 @@ set_bs_test() -> State5 = ?BACKEND:return_labels_and_lines(State4, []), Stream = ?BACKEND:stream(State5), Dump = << - " 0000e0: 08 7f local[3..10] type=i32\n" - " 0000e2: 20 00 local.get 0\n" - " 0000e4: 28 02 18 i32.load 2 24\n" - " 0000e7: 21 03 local.set 3\n" - " 0000e9: 20 00 local.get 0\n" - " 0000eb: 20 03 local.get 3\n" - " 0000ed: 36 02 64 i32.store 2 100\n" - " 0000f0: 20 00 local.get 0\n" - " 0000f2: 41 00 i32.const 0\n" - " 0000f4: 36 02 68 i32.store 2 104\n" - " 0000f7: 20 00 local.get 0\n" - " 0000f9: 0f return\n" - " 0000fa: 0b end\n" - " 0000fd: 08 7f local[3..10] type=i32\n" - " 0000ff: 20 00 local.get 0\n" - " 000101: 0b end" + " 0000b3: 08 7f local[3..10] type=i32\n" + " 0000b5: 20 00 local.get 0\n" + " 0000b7: 28 02 18 i32.load 2 24\n" + " 0000ba: 21 03 local.set 3\n" + " 0000bc: 20 00 local.get 0\n" + " 0000be: 20 03 local.get 3\n" + " 0000c0: 36 02 68 i32.store 2 104\n" + " 0000c3: 20 00 local.get 0\n" + " 0000c5: 41 00 i32.const 0\n" + " 0000c7: 36 02 6c i32.store 2 108\n" + " 0000ca: 20 00 local.get 0\n" + " 0000cc: 0f return\n" + " 0000cd: 0b end\n" + " 0000d0: 08 7f local[3..10] type=i32\n" + " 0000d2: 20 00 local.get 0\n" + " 0000d4: 0b end" >>, ?assertStream(wasm32, Dump, Stream). @@ -1409,66 +1409,58 @@ call_or_schedule_next_test() -> State4 = ?BACKEND:return_labels_and_lines(State3, []), Stream = ?BACKEND:stream(State4), Dump = << - " 0000f2: 08 7f local[3..10] type=i32\n" - " 0000f4: 20 01 local.get 1\n" - " 0000f6: 28 02 00 i32.load 2 0\n" - " 0000f9: 28 02 00 i32.load 2 0\n" - " 0000fc: 21 03 local.set 3\n" - " 0000fe: 20 03 local.get 3\n" - " 000100: 41 18 i32.const 24\n" - " 000102: 74 i32.shl\n" - " 000103: 21 03 local.set 3\n" - " 000105: 41 c0 00 i32.const 64\n" - " 000108: 21 04 local.set 4\n" - " 00010a: 20 03 local.get 3\n" - " 00010c: 20 04 local.get 4\n" - " 00010e: 72 i32.or\n" - " 00010f: 21 03 local.set 3\n" - " 000111: 20 00 local.get 0\n" - " 000113: 20 03 local.get 3\n" - " 000115: 36 02 5c i32.store 2 92\n" - " 000118: 20 01 local.get 1\n" - " 00011a: 28 02 08 i32.load 2 8\n" - " 00011d: 41 01 i32.const 1\n" - " 00011f: 6b i32.sub\n" - " 000120: 21 03 local.set 3\n" - " 000122: 20 01 local.get 1\n" - " 000124: 20 03 local.get 3\n" - " 000126: 36 02 08 i32.store 2 8\n" - " 000129: 20 03 local.get 3\n" - " 00012b: 45 i32.eqz\n" - " 00012c: 04 40 if\n" - " 00012e: 20 01 local.get 1\n" - " 000130: 41 02 i32.const 2\n" - " 000132: 36 02 04 i32.store 2 4\n" - " 000135: 20 00 local.get 0\n" - " 000137: 20 01 local.get 1\n" - " 000139: 20 02 local.get 2\n" - " 00013b: 28 02 08 i32.load 2 8\n" - " 00013e: 11 01 00 call_indirect 0 (type 1)\n" - " 000141: 0f return\n" - " 000142: 0b end\n" - " 000143: 20 01 local.get 1\n" - " 000145: 41 02 i32.const 2\n" - " 000147: 36 02 04 i32.store 2 4\n" - " 00014a: 20 00 local.get 0\n" - " 00014c: 0f return\n" - " 00014d: 20 00 local.get 0\n" - " 00014f: 0f return\n" - " 000150: 0b end\n" - " 000153: 08 7f local[3..10] type=i32\n" - " 000155: 20 00 local.get 0\n" - " 000157: 0b end\n" - " 00015a: 08 7f local[3..10] type=i32\n" - " 00015c: 20 00 local.get 0\n" - " 00015e: 0b end\n" - " 000161: 08 7f local[3..10] type=i32\n" - " 000163: 20 00 local.get 0\n" - " 000165: 0b end\n" - " 000168: 08 7f local[3..10] type=i32\n" - " 00016a: 20 00 local.get 0\n" - " 00016c: 0f return\n" - " 00016d: 0b end" + " 0000c5: 08 7f local[3..10] type=i32\n" + " 0000c7: 20 01 local.get 1\n" + " 0000c9: 28 02 00 i32.load 2 0\n" + " 0000cc: 21 03 local.set 3\n" + " 0000ce: 20 00 local.get 0\n" + " 0000d0: 41 c0 00 i32.const 64\n" + " 0000d3: 36 02 5c i32.store 2 92\n" + " 0000d6: 20 00 local.get 0\n" + " 0000d8: 20 03 local.get 3\n" + " 0000da: 36 02 60 i32.store 2 96\n" + " 0000dd: 20 01 local.get 1\n" + " 0000df: 28 02 08 i32.load 2 8\n" + " 0000e2: 41 01 i32.const 1\n" + " 0000e4: 6b i32.sub\n" + " 0000e5: 21 03 local.set 3\n" + " 0000e7: 20 01 local.get 1\n" + " 0000e9: 20 03 local.get 3\n" + " 0000eb: 36 02 08 i32.store 2 8\n" + " 0000ee: 20 03 local.get 3\n" + " 0000f0: 45 i32.eqz\n" + " 0000f1: 04 40 if\n" + " 0000f3: 20 01 local.get 1\n" + " 0000f5: 41 02 i32.const 2\n" + " 0000f7: 36 02 04 i32.store 2 4\n" + " 0000fa: 20 00 local.get 0\n" + " 0000fc: 20 01 local.get 1\n" + " 0000fe: 20 02 local.get 2\n" + " 000100: 28 02 08 i32.load 2 8\n" + " 000103: 11 01 00 call_indirect 0 (type 1)\n" + " 000106: 0f return\n" + " 000107: 0b end\n" + " 000108: 20 01 local.get 1\n" + " 00010a: 41 02 i32.const 2\n" + " 00010c: 36 02 04 i32.store 2 4\n" + " 00010f: 20 00 local.get 0\n" + " 000111: 0f return\n" + " 000112: 20 00 local.get 0\n" + " 000114: 0f return\n" + " 000115: 0b end\n" + " 000118: 08 7f local[3..10] type=i32\n" + " 00011a: 20 00 local.get 0\n" + " 00011c: 0b end\n" + " 00011f: 08 7f local[3..10] type=i32\n" + " 000121: 20 00 local.get 0\n" + " 000123: 0b end\n" + " 000126: 08 7f local[3..10] type=i32\n" + " 000128: 20 00 local.get 0\n" + " 00012a: 0b end\n" + " 00012d: 08 7f local[3..10] type=i32\n" + " 00012f: 20 00 local.get 0\n" + " 000131: 0f return\n" + " 000132: 0b end" >>, ?assertStream(wasm32, Dump, Stream). @@ -1534,22 +1526,28 @@ move_to_cp_test() -> State4 = ?BACKEND:return_labels_and_lines(State3, []), Stream = ?BACKEND:stream(State4), Dump = << - " 0000e0: 08 7f local[3..10] type=i32\n" - " 0000e2: 20 00 local.get 0\n" - " 0000e4: 28 02 14 i32.load 2 20\n" - " 0000e7: 21 03 local.set 3\n" - " 0000e9: 20 03 local.get 3\n" - " 0000eb: 28 02 00 i32.load 2 0\n" - " 0000ee: 21 03 local.set 3\n" - " 0000f0: 20 00 local.get 0\n" - " 0000f2: 20 03 local.get 3\n" - " 0000f4: 36 02 5c i32.store 2 92\n" - " 0000f7: 20 00 local.get 0\n" - " 0000f9: 0f return\n" - " 0000fa: 0b end\n" - " 0000fd: 08 7f local[3..10] type=i32\n" - " 0000ff: 20 00 local.get 0\n" - " 000101: 0b end" + " 0000b3: 08 7f local[3..10] type=i32\n" + " 0000b5: 20 00 local.get 0\n" + " 0000b7: 28 02 14 i32.load 2 20\n" + " 0000ba: 21 03 local.set 3\n" + " 0000bc: 20 03 local.get 3\n" + " 0000be: 28 02 00 i32.load 2 0\n" + " 0000c1: 21 04 local.set 4\n" + " 0000c3: 20 00 local.get 0\n" + " 0000c5: 20 04 local.get 4\n" + " 0000c7: 36 02 5c i32.store 2 92\n" + " 0000ca: 20 03 local.get 3\n" + " 0000cc: 28 02 04 i32.load 2 4\n" + " 0000cf: 21 04 local.set 4\n" + " 0000d1: 20 00 local.get 0\n" + " 0000d3: 20 04 local.get 4\n" + " 0000d5: 36 02 60 i32.store 2 96\n" + " 0000d8: 20 00 local.get 0\n" + " 0000da: 0f return\n" + " 0000db: 0b end\n" + " 0000de: 08 7f local[3..10] type=i32\n" + " 0000e0: 20 00 local.get 0\n" + " 0000e2: 0b end" >>, ?assertStream(wasm32, Dump, Stream). @@ -2011,40 +2009,32 @@ call_primitive_with_cp_test() -> State4 = ?BACKEND:return_labels_and_lines(State3, []), Stream = ?BACKEND:stream(State4), Dump = << - " 0000e6: 08 7f local[3..10] type=i32\n" - " 0000e8: 20 01 local.get 1\n" - " 0000ea: 28 02 00 i32.load 2 0\n" - " 0000ed: 28 02 00 i32.load 2 0\n" - " 0000f0: 21 03 local.set 3\n" - " 0000f2: 20 03 local.get 3\n" - " 0000f4: 41 18 i32.const 24\n" - " 0000f6: 74 i32.shl\n" - " 0000f7: 21 03 local.set 3\n" - " 0000f9: 41 20 i32.const 32\n" - " 0000fb: 21 04 local.set 4\n" - " 0000fd: 20 03 local.get 3\n" - " 0000ff: 20 04 local.get 4\n" - " 000101: 72 i32.or\n" - " 000102: 21 03 local.set 3\n" - " 000104: 20 00 local.get 0\n" - " 000106: 20 03 local.get 3\n" - " 000108: 36 02 5c i32.store 2 92\n" - " 00010b: 20 00 local.get 0\n" - " 00010d: 20 01 local.get 1\n" - " 00010f: 20 02 local.get 2\n" - " 000111: 28 02 00 i32.load 2 0\n" - " 000114: 11 01 00 call_indirect 0 (type 1)\n" - " 000117: 0f return\n" - " 000118: 20 00 local.get 0\n" - " 00011a: 0f return\n" - " 00011b: 0b end\n" - " 00011e: 08 7f local[3..10] type=i32\n" - " 000120: 20 00 local.get 0\n" - " 000122: 0b end\n" - " 000125: 08 7f local[3..10] type=i32\n" - " 000127: 20 00 local.get 0\n" - " 000129: 0f return\n" - " 00012a: 0b end" + " 0000b9: 08 7f local[3..10] type=i32\n" + " 0000bb: 20 01 local.get 1\n" + " 0000bd: 28 02 00 i32.load 2 0\n" + " 0000c0: 21 03 local.set 3\n" + " 0000c2: 20 00 local.get 0\n" + " 0000c4: 41 20 i32.const 32\n" + " 0000c6: 36 02 5c i32.store 2 92\n" + " 0000c9: 20 00 local.get 0\n" + " 0000cb: 20 03 local.get 3\n" + " 0000cd: 36 02 60 i32.store 2 96\n" + " 0000d0: 20 00 local.get 0\n" + " 0000d2: 20 01 local.get 1\n" + " 0000d4: 20 02 local.get 2\n" + " 0000d6: 28 02 00 i32.load 2 0\n" + " 0000d9: 11 01 00 call_indirect 0 (type 1)\n" + " 0000dc: 0f return\n" + " 0000dd: 20 00 local.get 0\n" + " 0000df: 0f return\n" + " 0000e0: 0b end\n" + " 0000e3: 08 7f local[3..10] type=i32\n" + " 0000e5: 20 00 local.get 0\n" + " 0000e7: 0b end\n" + " 0000ea: 08 7f local[3..10] type=i32\n" + " 0000ec: 20 00 local.get 0\n" + " 0000ee: 0f return\n" + " 0000ef: 0b end" >>, ?assertStream(wasm32, Dump, Stream). diff --git a/tests/libs/jit/jit_x86_64_asm_tests.erl b/tests/libs/jit/jit_x86_64_asm_tests.erl index d3e19d24f1..6c084d3be1 100644 --- a/tests/libs/jit/jit_x86_64_asm_tests.erl +++ b/tests/libs/jit/jit_x86_64_asm_tests.erl @@ -300,6 +300,43 @@ movl_test_() -> ), ?_assertAsmEqual( <<16#45, 16#8B, 16#01>>, "movl (%r9),%r8d", jit_x86_64_asm:movl({0, r9}, r8) + ), + + % movl({Offset, SrcReg}, DestReg) - memory with displacement to register + ?_assertAsmEqual( + <<16#8B, 16#40, 16#04>>, "movl 0x4(%rax),%eax", jit_x86_64_asm:movl({4, rax}, rax) + ), + ?_assertAsmEqual( + <<16#8B, 16#49, 16#04>>, "movl 0x4(%rcx),%ecx", jit_x86_64_asm:movl({4, rcx}, rcx) + ), + ?_assertAsmEqual( + <<16#8B, 16#48, 16#7F>>, "movl 0x7f(%rax),%ecx", jit_x86_64_asm:movl({127, rax}, rcx) + ), + ?_assertAsmEqual( + <<16#44, 16#8B, 16#40, 16#04>>, + "movl 0x4(%rax),%r8d", + jit_x86_64_asm:movl({4, rax}, r8) + ), + ?_assertAsmEqual( + <<16#41, 16#8B, 16#40, 16#04>>, + "movl 0x4(%r8),%eax", + jit_x86_64_asm:movl({4, r8}, rax) + ), + ?_assertAsmEqual( + <<16#45, 16#8B, 16#49, 16#04>>, + "movl 0x4(%r9),%r9d", + jit_x86_64_asm:movl({4, r9}, r9) + ), + % disp32 as soon as the displacement no longer fits a signed byte + ?_assertAsmEqual( + <<16#8B, 16#80, 16#80, 16#00, 16#00, 16#00>>, + "movl 0x80(%rax),%eax", + jit_x86_64_asm:movl({128, rax}, rax) + ), + ?_assertAsmEqual( + <<16#45, 16#8B, 16#89, 16#78, 16#56, 16#34, 16#12>>, + "movl 0x12345678(%r9),%r9d", + jit_x86_64_asm:movl({16#12345678, r9}, r9) ) ]. diff --git a/tests/libs/jit/jit_xtensa_tests.erl b/tests/libs/jit/jit_xtensa_tests.erl index cc891f47b8..4a85dfbd74 100644 --- a/tests/libs/jit/jit_xtensa_tests.erl +++ b/tests/libs/jit/jit_xtensa_tests.erl @@ -406,7 +406,10 @@ move_to_cp_test() -> Dump = << " 0: 0522e2 l32i a14, a2, 20\n" " 3: 002ef2 l32i a15, a14, 0\n" - " 6: 1762f2 s32i a15, a2, 92" + " 6: 1762f2 s32i a15, a2, 92\n" + " 9: 0522e2 l32i a14, a2, 20\n" + " c: 012ef2 l32i a15, a14, 4\n" + " f: 1862f2 s32i a15, a2, 96" >>, ?assertStream(xtensa, Dump, Stream). diff --git a/tests/test-structs.c b/tests/test-structs.c index 52ce22dcd2..c5dea90453 100644 --- a/tests/test-structs.c +++ b/tests/test-structs.c @@ -19,9 +19,12 @@ */ #include +#include #include #include "atom_table.h" +#include "module.h" +#include "term.h" #include "utils.h" #include "valueshashtable.h" @@ -479,6 +482,72 @@ void test_atom_table(void) atom_table_destroy(table); } +static void test_cp_encoding(void) +{ + // A stack Module is suitably aligned; only module_index is read by make_cp. + Module m; + memset(&m, 0, sizeof(m)); + m.module_index = 300; // >= 256: the case the old 8-bit packing overflowed. + + // Offsets up to the 64-bit packing limit (offset << 2 must fit in 24 bits). + unsigned int offsets[] = { 0, 4, 1000, (1u << 20), (1u << 22) - 4 }; + for (size_t i = 0; i < sizeof(offsets) / sizeof(offsets[0]); i++) { + unsigned int off = offsets[i]; + cp_t cp = make_cp(&m, off); + + // Offset round-trips. + assert(cp_to_offset(cp) == off); + + // Module identity round-trips. +#if TERM_BITS == 64 + // 64-bit: the module index is packed in the high bits. + assert((unsigned int) (cp >> 24) == 300u); +#else + // 32-bit: the Module pointer is stored directly (no index lookup). + assert(cp_to_module(cp, NULL) == &m); +#endif + + // store_cp/load_cp round-trip across the on-stack representation. + term slots[2] = { 0, 0 }; + store_cp(slots, cp); + assert(load_cp(slots) == cp); + + // Every stored slot must be GC-safe (low 2 bits clear => TERM_PRIMARY_CP), + // so the collector skips it instead of following it as a pointer. + for (int s = 0; s < CP_SIZE_IN_TERMS; s++) { + assert((slots[s] & TERM_PRIMARY_MASK) == TERM_PRIMARY_CP); + } + } + + // The process-termination sentinel is recognized; a real cp is not. + assert(cp_is_terminate((cp_t) -1)); + assert(!cp_is_terminate(make_cp(&m, 0))); +} + +static void test_catch_encoding(void) +{ + _Static_assert(offsetof(Module, module_index) == 0, "module_index must be at offset 0"); + _Static_assert(offsetof(Module, catch_labels_base) == 4, "catch_labels_base must be at offset 4"); + + Module m; + memset(&m, 0, sizeof(m)); + m.catch_labels_base = 300u * 512u; + + unsigned int labels[] = { 0, 1, 42, 511 }; + for (size_t i = 0; i < sizeof(labels) / sizeof(labels[0]); i++) { + term catch_term = module_term_from_catch_label(&m, labels[i]); + + assert(term_is_catch_label(catch_term)); + assert(term_to_catch_id(catch_term) == m.catch_labels_base + labels[i]); + + assert(!term_is_cp(catch_term)); + } + + term max_catch = term_from_catch_id(TERM_MAX_CATCH_ID); + assert(term_is_catch_label(max_catch)); + assert(term_to_catch_id(max_catch) == TERM_MAX_CATCH_ID); +} + int main(int argc, char **argv) { UNUSED(argc); @@ -486,6 +555,8 @@ int main(int argc, char **argv) test_valueshashtable(); test_atom_table(); + test_cp_encoding(); + test_catch_encoding(); return EXIT_SUCCESS; } diff --git a/tests/test.c b/tests/test.c index 18ae322d2a..4249eb5e38 100644 --- a/tests/test.c +++ b/tests/test.c @@ -565,6 +565,7 @@ struct Test tests[] = { TEST_CASE(test_code_all_available_loaded), TEST_CASE_EXPECTED(test_code_load_binary, 24), + TEST_CASE(test_many_modules), TEST_CASE_EXPECTED(test_code_load_abs, 24), TEST_CASE(test_code_ensure_loaded), TEST_CASE_ATOMVM_ONLY(test_add_avm_pack_binary, 24),