diff --git a/gulpfile.js b/gulpfile.js index f93b940ad563..808c004ae18c 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -735,6 +735,7 @@ const lint = lintWithEslint const testdecompiler = testTask("decompile-test", "decompilerunner.js"); const testlang = testTask("compile-test", "compilerunner.js"); +const testthumb = testTask("thumb-test", "thumbrunner.js"); const testhelpers = testTask("helpers-test", "helperrunner.js"); const testerr = testTask("errors-test", "errorrunner.js"); const testfmt = testTask("format-test", "formatrunner.js"); @@ -773,6 +774,7 @@ const browserifyBlocksPrep = () => const testAll = gulp.series( testdecompiler, testlang, + testthumb, testhelpers, testerr, testfmt, @@ -886,6 +888,7 @@ exports.updatestrings = updatestrings; exports.lint = lint exports.testdecompiler = testdecompiler; exports.testlang = testlang; +exports.testthumb = testthumb; exports.testerr = testerr; exports.testfmt = testfmt; exports.testembed = testembed; diff --git a/package.json b/package.json index 940536ea44cf..dc59f3fbd925 100644 --- a/package.json +++ b/package.json @@ -183,6 +183,7 @@ "update": "gulp update", "watch-streamer": "cd docs/static/streamer && tsc -t es6 --watch", "prepare": "node ./scripts/npm-prepare.js", - "compile-aw": "gh aw compile --strict" + "compile-aw": "gh aw compile --strict", + "hwab": "node tests/hw-ab/hwab.js" } } diff --git a/tests/compile-test/lang-test0/54conditiontruthiness.ts b/tests/compile-test/lang-test0/54conditiontruthiness.ts new file mode 100644 index 000000000000..319c871f9ba3 --- /dev/null +++ b/tests/compile-test/lang-test0/54conditiontruthiness.ts @@ -0,0 +1,195 @@ +// Truthiness matrix for every construct that lowers to a condition test. +// +// The falsy set here is exactly the set numops::toBool treats as false: +// false, 0 (any representation, including -0 and a boxed double zero), null, +// undefined, NaN and "". Everything else is truthy, including "0", " ", +// "false", [], {} and function values. +// +// Every value reaches its condition through an any-typed identity call so the +// test site sees an opaque operand and cannot be folded at compile time. + +namespace CondTruth { + function opaque(v: any): any { + return v + } + + function opaqueStr(v: string): string { + return v + } + + function opaqueNum(v: number): number { + return v + } + + function opaqueArr(v: number[]): number[] { + return v + } + + class TCase { + v: any + t: boolean + id: string + constructor(v: any, t: boolean, id: string) { + this.v = v + this.t = t + this.id = id + } + } + + // Pushes one value through every construct that lowers to a condition. + function check(c: TCase) { + const v = opaque(c.v) + const want = c.t + + // plain `if` condition + let hit = false + if (v) hit = true + assert(hit == want, "if:" + c.id) + + // `if (!v)` -- the negated condition, which lowers to the inverted jump + let nhit = false + if (!v) nhit = true + assert(nhit == !want, "ifnot:" + c.id) + + // else branch of the same test must be the complement + let branch = 0 + if (v) branch = 1 + else branch = 2 + assert(branch == (want ? 1 : 2), "ifelse:" + c.id) + + // conditional expression + const t = v ? 11 : 22 + assert(t == (want ? 11 : 22), "ternary:" + c.id) + + // `while` header, body entered at most once + let iter = 0 + while (v) { + iter++ + break + } + assert(iter == (want ? 1 : 0), "while:" + c.id) + + // `do..while` runs the body once and only then tests the condition + let dit = 0 + do { + dit++ + } while (v && dit < 2) + assert(dit == (want ? 2 : 1), "dowhile:" + c.id) + + // `for` header condition + let fit = 0 + for (; v;) { + fit++ + break + } + assert(fit == (want ? 1 : 0), "for:" + c.id) + + // condition-position && : truth of the pair, never the operand value + let andHit = false + if (v && true) andHit = true + assert(andHit == want, "and1:" + c.id) + let andZero = false + if (v && false) andZero = true + assert(andZero == false, "and0:" + c.id) + + // condition-position || + let orHit = false + if (v || false) orHit = true + assert(orHit == want, "or0:" + c.id) + let orOne = false + if (v || true) orOne = true + assert(orOne == true, "or1:" + c.id) + + // double negation must yield a real tagged boolean, comparable strictly + const b = !!v + assert(b === want, "bangbang:" + c.id) + assert(typeof b == "boolean", "bangbangtype:" + c.id) + + // single negation is also a real boolean + const nb = !v + assert(nb === !want, "bang:" + c.id) + } + + export function run() { + msg("condition truthiness") + + const zero = opaqueNum(0) + const one = opaqueNum(1) + + const cases: TCase[] = [] + function add(v: any, t: boolean, id: string) { + cases.push(new TCase(v, t, id)) + } + + // --- falsy --- + add(false, false, "false") + add(zero, false, "zero") + add(zero * -1, false, "negzero") + add(null, false, "null") + add(undefined, false, "undefined") + add(zero / zero, false, "nan") + add(opaqueStr(""), false, "emptystr") + + // --- truthy --- + add(true, true, "true") + add(one, true, "one") + add(zero - one, true, "minusone") + // forced past the tagged-int range, so the value is a boxed double + add(0x20000000 * (one + one), true, "bigint") + add(opaqueStr("0"), true, "str0") + add(opaqueStr(" "), true, "space") + add(opaqueStr("false"), true, "strfalse") + add(opaqueStr("00"), true, "str00") + add([], true, "emptyarr") + add([zero], true, "zeroarr") + add({}, true, "emptyobj") + add({ a: zero }, true, "obj") + add(() => zero, true, "fn") + + // Doubles only exist on floating-point targets; a boxed zero and a + // fractional nonzero both go through the boxed side of the falsy test. + if (hasFloat) { + const big = opaqueNum(1e18) + add(big - big, false, "boxedzero") + add(opaqueNum(0.5) - opaqueNum(0.25), true, "fraction") + add(opaqueNum(-0.5), true, "negfraction") + } + + for (const c of cases) check(c) + + assert(cases.length == (hasFloat ? 23 : 20), "casecount") + + // The same values reaching a condition through a non-any local, so the + // condition sees a statically typed operand rather than a boxed any. + const es = opaqueStr("") + assert(!es, "typed:emptystr") + const ns = opaqueStr("x") + assert(!!ns, "typed:str") + const nz = opaqueNum(0) + assert(!nz, "typed:zero") + const nn = opaqueNum(3) + assert(!!nn, "typed:num") + const na = opaqueArr(null) + assert(!na, "typed:nullarr") + const ea = opaqueArr([]) + assert(!!ea, "typed:emptyarr") + + // A boolean-typed local is already 0/1 in the lowered form; it must + // still compare strictly and survive a round trip through a call. + function passBool(b: boolean): boolean { + return b + } + const bt = passBool(!!ns) + const bf = passBool(!!es) + assert(bt === true, "roundtrip:true") + assert(bf === false, "roundtrip:false") + let rhit = 0 + if (bt) rhit++ + if (bf) rhit += 10 + assert(rhit == 1, "roundtrip:cond") + + msg("condition truthiness done") + } +} + +CondTruth.run() diff --git a/tests/compile-test/lang-test0/55conditionlowering.ts b/tests/compile-test/lang-test0/55conditionlowering.ts new file mode 100644 index 000000000000..a32fac828064 --- /dev/null +++ b/tests/compile-test/lang-test0/55conditionlowering.ts @@ -0,0 +1,528 @@ +// Structural semantics of condition lowering: evaluation order and counts for +// &&/||/!, the difference between condition position and value position, every +// syntactic construct that takes a condition, comparisons feeding logical +// operators, freshly allocated operands re-evaluated in loop headers, and +// exceptions raised part way through a condition. + +namespace CondLower { + let lg = "" + + function reset() { + lg = "" + } + + // Each operand records that it ran, then yields the requested truth value, + // so `lg` is the exact left-to-right evaluation trace of a condition. + function A(r: boolean): boolean { + lg += "A" + return r + } + + function B(r: boolean): boolean { + lg += "B" + return r + } + + function C(r: boolean): boolean { + lg += "C" + return r + } + + function D(r: boolean): boolean { + lg += "D" + return r + } + + // --- short-circuit order and operand counts ------------------------- + + function testShortCircuit() { + msg("short circuit") + + // && stops at the first falsy operand + reset() + let r = A(true) && B(false) + assert(lg == "AB", "sc:and:both") + assert(r === false, "sc:and:bothv") + + reset() + r = A(false) && B(true) + assert(lg == "A", "sc:and:short") + assert(r === false, "sc:and:shortv") + + // || stops at the first truthy operand + reset() + r = A(true) || B(true) + assert(lg == "A", "sc:or:short") + assert(r === true, "sc:or:shortv") + + reset() + r = A(false) || B(true) + assert(lg == "AB", "sc:or:both") + assert(r === true, "sc:or:bothv") + + // nested group: the right operand of && is itself a || + reset() + let hit = false + if (A(true) && (B(false) || C(true))) hit = true + assert(lg == "ABC", "sc:group1") + assert(hit, "sc:group1v") + + reset() + hit = false + if (A(false) && (B(true) || C(true))) hit = true + assert(lg == "A", "sc:group2") + assert(!hit, "sc:group2v") + + reset() + hit = false + if (A(true) && (B(true) || C(true))) hit = true + assert(lg == "AB", "sc:group3") + assert(hit, "sc:group3v") + + // negation wrapped around a short-circuit chain + reset() + r = !(A(true) && !B(true)) + assert(lg == "AB", "sc:notand1") + assert(r === true, "sc:notand1v") + + reset() + r = !(A(true) && !B(false)) + assert(lg == "AB", "sc:notand2") + assert(r === false, "sc:notand2v") + + reset() + r = !(A(false) && !B(true)) + assert(lg == "A", "sc:notand3") + assert(r === true, "sc:notand3v") + + // chains of three and four operands + reset() + hit = false + if (A(true) && B(true) && C(false)) hit = true + assert(lg == "ABC", "sc:and3") + assert(!hit, "sc:and3v") + + reset() + hit = false + if (A(true) && B(false) && C(true)) hit = true + assert(lg == "AB", "sc:and3short") + + reset() + hit = false + if (A(true) || B(true) || C(true)) hit = true + assert(lg == "A", "sc:or3short") + assert(hit, "sc:or3shortv") + + reset() + hit = false + if (A(false) || B(false) || C(true)) hit = true + assert(lg == "ABC", "sc:or3") + assert(hit, "sc:or3v") + + reset() + hit = false + if (A(true) && B(true) && C(true) && D(false)) hit = true + assert(lg == "ABCD", "sc:and4") + assert(!hit, "sc:and4v") + + // mixed precedence: && binds tighter than || + reset() + hit = false + if (A(false) && B(true) || C(true)) hit = true + assert(lg == "AC", "sc:mixed1") + assert(hit, "sc:mixed1v") + + reset() + hit = false + if (A(true) && B(true) || C(true)) hit = true + assert(lg == "AB", "sc:mixed2") + assert(hit, "sc:mixed2v") + } + + // --- value position vs condition position --------------------------- + + function testValuePosition() { + msg("value position") + + // In value position || yields the operand itself, not a 0/1 truth. + const fb = "f" + "b" + const empty = "" + "" + let s: string = empty || fb + assert(s == "fb", "vp:orstr") + s = fb || "other" + assert(s == "fb", "vp:orstr2") + + let calls = 0 + function f(): number { + calls++ + return 7 + } + + // && in value position yields the falsy left operand, right not run + const z = 0 + let n: number = z && f() + assert(n === 0, "vp:andnum") + assert(calls == 0, "vp:andnocall") + + // and the right operand when the left is truthy + const five = 5 + n = five && f() + assert(n == 7, "vp:andnum2") + assert(calls == 1, "vp:andcall") + + const nine = 9 + n = five || nine + assert(n == 5, "vp:ornum") + + // ! always yields a real boolean, comparable strictly + let b = !empty + assert(b === true, "vp:bangempty") + b = !fb + assert(b === false, "vp:bangstr") + assert(!!fb === true, "vp:bangbang") + assert(typeof !fb == "boolean", "vp:bangtype") + + // a boolean stored, passed through a call, then used as a condition: + // this round-trips the raw 0/1 form back to a tagged boolean + function takes(v: boolean): boolean { + return v + } + const stored = !empty + const returned = takes(stored) + assert(returned === true, "vp:rt1") + let hit = 0 + if (returned) hit++ + if (takes(!fb)) hit += 10 + assert(hit == 1, "vp:rt2") + + // the same value observed as an any + const anyb: any = returned + assert(anyb === true, "vp:rtany") + assert(!!anyb === true, "vp:rtanybang") + } + + // --- every construct that takes a condition ------------------------- + + function classify(n: number): string { + if (n < 0) return "neg" + else if (n == 0) return "zero" + else if (n < 10) return "small" + else return "big" + } + + function testConstructs() { + msg("condition constructs") + + assert(classify(-3) == "neg", "cc:if1") + assert(classify(0) == "zero", "cc:if2") + assert(classify(5) == "small", "cc:if3") + assert(classify(50) == "big", "cc:if4") + + // while + let i = 0 + let acc = 0 + while (i < 5 && acc < 100) { + acc += i + i++ + } + assert(i == 5 && acc == 10, "cc:while") + + // do..while, body always runs once + let j = 100 + let runs = 0 + do { + runs++ + } while (j < 5) + assert(runs == 1, "cc:dowhile") + + j = 0 + runs = 0 + do { + runs++ + j++ + } while (j < 4 || runs < 2) + assert(j == 4 && runs == 4, "cc:dowhile2") + + // for with a compound header condition + let k = 0 + let sum2 = 0 + for (k = 0; k < 10 && sum2 < 12; k++) sum2 += k + assert(k == 6 && sum2 == 15, "cc:for") + + // nested ternaries + function grade(n: number): string { + return n > 90 ? "a" : n > 80 ? "b" : n > 70 ? "c" : "f" + } + assert(grade(95) == "a", "cc:tern1") + assert(grade(85) == "b", "cc:tern2") + assert(grade(75) == "c", "cc:tern3") + assert(grade(5) == "f", "cc:tern4") + + // ternary whose branches are themselves conditions + const t = (1 < 2) ? (3 > 4) : (5 > 6) + assert(t === false, "cc:tern5") + } + + // --- comparisons feeding logical operators -------------------------- + + function testComparisons() { + msg("comparisons") + + const a = 1 + const b = 2 + const c = 4 + const d = 3 + + assert((a < b && c > d) === true, "cmp:1") + assert((a > b && c > d) === false, "cmp:2") + assert((a > b || c > d) === true, "cmp:3") + assert((a > b || c < d) === false, "cmp:4") + + const x = "x" + "y" + const y = "xy" + const p: number = 1 + const q: number = 2 + assert((x == y || p != q) === true, "cmp:5") + assert((x != y && p == q) === false, "cmp:6") + + // string relational comparisons inside a chain + const s1 = "a" + "" + const s2 = "b" + "" + assert((s1 < s2 && s2 > s1) === true, "cmp:str1") + assert((s1 > s2 || s1 == s2) === false, "cmp:str2") + assert((s1 <= s1 && s2 >= s2) === true, "cmp:str3") + assert((s1 < s2 && s1.length > 0 && s2 != "") === true, "cmp:str4") + + // comparison result stored and re-tested + const cmp = s1 < s2 + assert(cmp === true, "cmp:store") + let hit = 0 + if (cmp && a < b) hit++ + assert(hit == 1, "cmp:retest") + + // arr.length directly as a condition and inside chains + const empty: number[] = [] + const full = [1, 2, 3] + assert(!empty.length, "cmp:len0") + assert(!!full.length, "cmp:len1") + let lh = 0 + if (empty.length) lh += 100 + if (full.length) lh += 1 + if (full.length && !empty.length) lh += 10 + if (empty.length || full.length) lh += 1000 + assert(lh == 1011, "cmp:lenchain") + } + + // --- freshly allocated operands re-evaluated in loop headers --------- + // + // Each iteration allocates the value the condition tests, so an + // imbalanced reference count on the condition path shows up as a GC + // failure or a wrong final state rather than as a wrong first iteration. + + function testFreshOperands() { + msg("fresh operands"); + + let s = "x" + let i = 0 + while ((s + i).length < 8) { + i++ + s = s + "y" + } + assert(s.length == 7, "fresh:concat") + assert(i == 6, "fresh:concatn") + + // array literal built in the loop header + let n = 0 + while ([n, n + 1].length > 0 && n < 200) n++ + assert(n == 200, "fresh:arrlit") + + // slices built in the header and in the body + const base = [1, 2, 3, 4, 5] + let k = 0 + let seen = 0 + while (base.slice(0, (k % 5) + 1).length > 0 && k < 300) { + seen += base.slice(k % 5).length + k++ + } + assert(k == 300, "fresh:slicek") + assert(seen == 900, "fresh:slicelen") + + // fresh object in a condition, in both operand positions of || + let m = 0 + while (({ v: m }).v < 150 || m < 0) m++ + assert(m == 150, "fresh:obj") + + // fresh string compared in a do..while + let c = 0 + do { + c++ + } while (("s" + c) != "s50") + assert(c == 50, "fresh:strcmp") + + // ternary over a freshly built array, repeated + let tsum = 0 + for (let q = 0; q < 200; q++) tsum += [q].length ? 1 : 0 + assert(tsum == 200, "fresh:tern") + } + + // --- exceptions raised part way through a condition ----------------- + + function testExceptionsInConditions() { + msg("exceptions in conditions") + + function t(): boolean { + lg += "t" + return false + } + + function boom(): boolean { + lg += "!" + throw "boom" + } + + // throw from the right operand of || (left already evaluated) + reset() + let caught = "" + try { + if (t() || boom()) lg += "?" + lg += "after" + } catch (e) { + caught = e + lg += "c" + } + assert(lg == "t!c", "exn:right " + lg) + assert(caught == "boom", "exn:rightval") + + // throw from the left operand: the right operand must never run + reset() + caught = "" + try { + if (boom() || t()) lg += "?" + } catch (e) { + caught = e + lg += "c" + } + assert(lg == "!c", "exn:left " + lg) + assert(caught == "boom", "exn:leftval") + + // throw from the right operand of && + reset() + try { + if (A(true) && boom()) lg += "?" + } catch (e) { + lg += "c" + } + assert(lg == "A!c", "exn:and " + lg) + + // && short-circuits before the throwing operand, so nothing is caught + reset() + let ok = false + try { + if (A(false) && boom()) lg += "?" + ok = true + } catch (e) { + lg += "c" + } + assert(lg == "A", "exn:andshort " + lg) + assert(ok, "exn:andshortok") + + // conditions still behave after the stack has been unwound + reset() + let hit = 0 + if (A(true) && B(true)) hit++ + if (A(false) || B(true)) hit += 10 + assert(hit == 11, "exn:after") + assert(lg == "ABAB", "exn:afterlog " + lg) + + // throw from inside a loop header condition + let iters = 0 + caught = "" + try { + while (iters < 10) { + iters++ + if (iters > 3 && boom()) iters += 100 + } + } catch (e) { + caught = e + } + assert(iters == 4, "exn:loop") + assert(caught == "boom", "exn:loopval") + } + + // --- switch over computed scrutinees -------------------------------- + + function testSwitch() { + msg("switch") + + function pick(s: string): number { + switch (s) { + case "a" + "": + case "b": + return 12 + case "c": + return 3 + default: + return 0 + } + } + assert(pick("a") == 12, "sw:s1") + assert(pick("b") == 12, "sw:s2") + assert(pick("c") == 3, "sw:s3") + assert(pick("z") == 0, "sw:s4") + + // fallthrough that accumulates rather than returning + function acc(n: number): string { + let r = "" + switch (n) { + case 0: + r += "0" + case 1: + r += "1" + break + case 2: + r += "2" + default: + r += "d" + } + return r + } + assert(acc(0) == "01", "sw:n0") + assert(acc(1) == "1", "sw:n1") + assert(acc(2) == "2d", "sw:n2") + assert(acc(9) == "d", "sw:n3") + + // computed scrutinee, so the switch cannot be resolved statically + let base = 0 + function bump(): number { + base += 2 + return base + } + assert(acc(bump() - 2) == "01", "sw:c0") + assert(acc(bump() - 3) == "1", "sw:c1") + + // switch whose scrutinee is a condition result + function fromBool(b: boolean): string { + switch (b ? "y" : "n") { + case "y": return "yes" + default: return "no" + } + } + const s1 = "a" + "" + assert(fromBool(!!s1) == "yes", "sw:b1") + assert(fromBool(!s1) == "no", "sw:b2") + } + + export function run() { + msg("condition lowering") + testShortCircuit() + testValuePosition() + testConstructs() + testComparisons() + testFreshOperands() + testExceptionsInConditions() + testSwitch() + msg("condition lowering done") + } +} + +CondLower.run() diff --git a/tests/compile-test/lang-test0/56ifacedispatch.ts b/tests/compile-test/lang-test0/56ifacedispatch.ts new file mode 100644 index 000000000000..ff278e06f373 --- /dev/null +++ b/tests/compile-test/lang-test0/56ifacedispatch.ts @@ -0,0 +1,552 @@ +// Interface dispatch semantics: dynamic member get/set/call through +// interface-typed, structurally typed and any-typed references. +// +// Interface member ids are global by name across the whole program, and some +// dispatch specializations are gated on how many static call sites a member +// has. Every member exercised here therefore carries a `qz` prefix that is +// unique to this file, and the call sites of the threshold-straddling members +// are counted deliberately: `qzLo` and `qzRare` sit below their thresholds, +// `qzHi`, `qzMany` and `qzHot` sit above. Both sides must behave identically. + +namespace IfaceDispatch { + + // --- threshold straddling ------------------------------------------- + + interface QzShape { + qzFew: number + qzMany: number + qzLo(n: number): number + qzHi(n: number): number + } + + class QzThing implements QzShape { + qzFew: number + qzMany: number + constructor(few: number, many: number) { + this.qzFew = few + this.qzMany = many + } + qzLo(n: number): number { + return n + 1 + } + qzHi(n: number): number { + return n + 2 + } + } + + // Within this function: qzLo is dispatched from 2 sites, qzHi from 5; + // qzFew is read at 4 sites and qzMany at 6. All of them go through an + // interface-typed reference, i.e. the checked/dynamic path. + // + // Call-site counts that feed the specialization thresholds are + // PROGRAM-WIDE per member name. qzHi and qzMany have additional sites in + // testAgreement and testDynamicGet, which keeps them above their gates; + // qzLo (2 total) and qzFew (4 total) have no sites outside this function + // and must stay below their gates (3 and 5). When editing this file, + // grep the member name before adding a call or read anywhere. + function testThresholds() { + msg("thresholds") + + const a: QzShape = new QzThing(1, 10) + const b: QzShape = new QzThing(2, 20) + + // qzLo: call site 1 and 2 + assert(a.qzLo(0) == 1, "th:lo1") + assert(b.qzLo(5) == 6, "th:lo2") + + // qzHi: call sites 1..5 + assert(a.qzHi(0) == 2, "th:hi1") + assert(b.qzHi(5) == 7, "th:hi2") + assert(a.qzHi(10) == 12, "th:hi3") + assert(b.qzHi(-2) == 0, "th:hi4") + let hisum = 0 + for (let i = 0; i < 4; i++) hisum += a.qzHi(i) + assert(hisum == 14, "th:hi5") + + // qzFew: read sites 1..4 + assert(a.qzFew == 1, "th:few1") + assert(b.qzFew == 2, "th:few2") + let fsum = a.qzFew + fsum += b.qzFew + assert(fsum == 3, "th:few34") + + // qzMany: read sites 1..6 + assert(a.qzMany == 10, "th:many1") + assert(b.qzMany == 20, "th:many2") + let msum = a.qzMany + b.qzMany + assert(msum == 30, "th:many34") + msum = msum - a.qzMany + assert(msum == 20, "th:many5") + msum = msum - b.qzMany + assert(msum == 0, "th:many6") + + // two receivers reaching the same proc through the same call site + assert(a.qzHi(3) == b.qzHi(3), "th:agree") + } + + // --- the same method through three kinds of reference ---------------- + + interface QzTriIface { + qzTri(n: number): number + } + + class QzTriImpl implements QzTriIface { + base: number + constructor(base: number) { + this.base = base + } + qzTri(n: number): number { + return this.base + n + } + } + + function testThreeRefs() { + msg("three refs") + + const concrete = new QzTriImpl(100) + const iface: QzTriIface = concrete + const dyn: any = concrete + + // concrete (static vtable call), interface (dynamic) and any (dynamic) + assert(concrete.qzTri(1) == 101, "tri:concrete") + assert(iface.qzTri(1) == 101, "tri:iface") + assert(dyn.qzTri(1) == 101, "tri:any") + assert(concrete.qzTri(2) == iface.qzTri(2), "tri:same1") + assert("" + iface.qzTri(3) == "" + dyn.qzTri(3), "tri:same2") + } + + // --- optional and defaulted parameters through an interface ---------- + // + // The emitter fills in a defaulted argument at the call site from the + // statically known signature. Through an interface- or any-typed + // reference there is no such signature, so an omitted argument arrives as + // undefined and the callee's own default does not apply. What must hold + // is that the interface and any paths agree with each other, and that a + // callee written to test for undefined works on every path. + + interface QzOptIface { + qzOpt(a: number, b?: number): number + qzOptSafe(a: number, b?: number): number + } + + class QzOptImpl implements QzOptIface { + qzOpt(a: number, b = 5): number { + return a + b + } + qzOptSafe(a: number, b?: number): number { + if (b === undefined) b = 5 + return a + b + } + } + + function testDefaults() { + msg("defaults") + + const impl = new QzOptImpl() + const iface: QzOptIface = impl + const dyn: any = impl + + // a concrete call gets the default filled in at the call site + assert(impl.qzOpt(1) == 6, "opt:concrete") + + // explicit arguments behave the same on every path + assert(impl.qzOpt(1, 2) == 3, "opt:concrete2") + assert(iface.qzOpt(1, 2) == 3, "opt:iface2") + assert(dyn.qzOpt(1, 2) == 3, "opt:any2") + + // omitting the argument yields the same result on both dynamic paths + assert(("" + iface.qzOpt(1)) == ("" + dyn.qzOpt(1)), "opt:dynagree") + + // a callee that defaults explicitly works everywhere + assert(impl.qzOptSafe(1) == 6, "opt:safe1") + assert(iface.qzOptSafe(1) == 6, "opt:safe2") + assert(dyn.qzOptSafe(1) == 6, "opt:safe3") + assert(iface.qzOptSafe(1, 2) == 3, "opt:safe4") + } + + // --- one member name, two unrelated interfaces, different arities ----- + + interface QzOneArg { + qzShared(a: number): number + } + + interface QzTwoArg { + qzShared(a: number, b: number): number + } + + class QzOneImpl implements QzOneArg { + qzShared(a: number): number { + return a + 1 + } + } + + class QzTwoImpl implements QzTwoArg { + qzShared(a: number, b: number): number { + return a * b + } + } + + function testArityCollision() { + msg("arity collision") + + const one: QzOneArg = new QzOneImpl() + const two: QzTwoArg = new QzTwoImpl() + + assert(one.qzShared(1) == 2, "ar:one1") + assert(one.qzShared(9) == 10, "ar:one2") + assert(two.qzShared(3, 4) == 12, "ar:two1") + assert(two.qzShared(5, 6) == 30, "ar:two2") + + // interleaved, so neither dispatch can be hoisted into the other + let acc = 0 + for (let i = 1; i <= 3; i++) { + acc += one.qzShared(i) + acc += two.qzShared(i, i) + } + assert(acc == 23, "ar:mixed") + } + + // --- one member name used as a property get and as a method ---------- + + interface QzCollGet { + qzColl: number + } + + interface QzCollCall { + qzColl(a: number, b: number): number + } + + class QzCollField implements QzCollGet { + qzColl: number + constructor() { + this.qzColl = 42 + } + } + + class QzCollMethod implements QzCollCall { + qzColl(a: number, b: number): number { + return a * b + } + } + + function testGetCallCollision() { + msg("get/call collision") + + const g: QzCollGet = new QzCollField() + const c: QzCollCall = new QzCollMethod() + + assert(g.qzColl == 42, "gc:get1") + assert(c.qzColl(6, 7) == 42, "gc:call1") + assert(g.qzColl == c.qzColl(2, 21), "gc:agree") + + const ga: any = g + const ca: any = c + assert(ga.qzColl == 42, "gc:get2") + assert(ca.qzColl(3, 14) == 42, "gc:call2") + assert(typeof ca.qzColl == "function", "gc:type") + } + + // --- a method that is both interface-dispatched and used as a value --- + + interface QzTakeIface { + qzTake(n: number): number + } + + class QzTaker implements QzTakeIface { + mul: number + constructor(mul: number) { + this.mul = mul + } + qzTake(n: number): number { + return n * this.mul + } + } + + function testMethodAsValue() { + msg("method as value") + + const t = new QzTaker(3) + const iface: QzTakeIface = t + + // A bare method reference is not a value in this language; wrapping it + // in a lambda is the supported form. + const f = (n: number) => t.qzTake(n) + const g = (n: number) => iface.qzTake(n) + + assert(f(2) == 6, "mv:lambda") + assert(g(2) == 6, "mv:ifacelambda") + assert(iface.qzTake(2) == 6, "mv:direct") + + // the same proc reached through a higher order call + function apply(fn: (n: number) => number, n: number): number { + return fn(n) + } + assert(apply(f, 4) == 12, "mv:apply1") + assert(apply(g, 4) == 12, "mv:apply2") + assert(apply(n => iface.qzTake(n), 5) == 15, "mv:apply3") + } + + // --- polymorphic sites: class instances and object literals ----------- + + interface QzValIface { + qzVal(): number + } + + class QzValClass implements QzValIface { + n: number + constructor(n: number) { + this.n = n + } + qzVal(): number { + return this.n + } + } + + function useQzVal(v: QzValIface): number { + return v.qzVal() + 1 + } + + function testPolymorphic() { + msg("polymorphic"); + + // the same call site sees a class instance and an object literal + assert(useQzVal(new QzValClass(1)) == 2, "poly:class") + assert(useQzVal({ qzVal: () => 5 }) == 6, "poly:literal") + + const mixed: QzValIface[] = [ + new QzValClass(1), + { qzVal: () => 2 }, + new QzValClass(3), + { qzVal: () => 4 } + ] + const expected = [2, 3, 4, 5] + for (let i = 0; i < mixed.length; i++) + assert(useQzVal(mixed[i]) == expected[i], "poly:elem" + i) + + let total = 0 + for (const m of mixed) total += m.qzVal() + assert(total == 10, "poly:total") + } + + // --- toString override ----------------------------------------------- + + interface QzNamed { + toString(): string + } + + class QzLabel implements QzNamed { + id: number + constructor(id: number) { + this.id = id + } + toString(): string { + return "L" + this.id + } + } + + function testToString() { + msg("toString") + + const l = new QzLabel(7) + assert("" + l == "L7", "ts:concat") + assert(`${l}` == "L7", "ts:template") + assert(l.toString() == "L7", "ts:direct") + + const named: QzNamed = l + assert(named.toString() == "L7", "ts:iface") + + const dyn: any = l + assert(dyn.toString() == "L7", "ts:any") + assert("" + dyn == "L7", "ts:anyconcat") + + // in a condition, so the override result feeds a truth test + assert(!!("" + l), "ts:truthy") + assert(("" + l).length == 2, "ts:len") + } + + // --- stores through every dynamic path -------------------------------- + + interface QzRec { + // qzRare is never present in the initializer, so its only stores are + // the two explicit ones below + qzRare?: number + qzHot: number + } + + interface QzPropOnly { + qzProp: number + } + + class QzPropClass implements QzPropOnly { + qzProp: number + constructor() { + this.qzProp = 0 + } + } + + class QzAccClass { + _v: number + constructor() { + this._v = 0 + } + get qzAcc(): number { + return this._v + 1 + } + set qzAcc(v: number) { + this._v = v * 2 + } + } + + interface QzAccIface { + qzAcc: number + } + + class QzBase { + qzInherited: number + constructor() { + this.qzInherited = 1 + } + qzSuper(n: number): number { + return n + 1 + } + } + + class QzDerived extends QzBase { + constructor() { + super() + } + qzSuper(n: number): number { + return super.qzSuper(n) * 10 + } + } + + interface QzInheritedIface { + qzInherited: number + qzSuper(n: number): number + } + + function testStores() { + msg("stores") + + // Object literal STORE sites (the count the store specialization + // gates on): qzRare at 2 (below the gate of 3), qzHot at 4 store + // sites plus 2 initializers (above it). Reads of both keys also occur + // below but do not feed the store count. + const r1: QzRec = { qzHot: 0 } + const r2: QzRec = { qzHot: 0 } + r1.qzRare = 1 + r2.qzRare = 2 + r1.qzHot = 10 + r2.qzHot = 20 + r1.qzHot = r1.qzHot + 1 + r2.qzHot = r2.qzHot + 1 + assert(r1.qzRare == 1 && r2.qzRare == 2, "st:rare") + assert(r1.qzHot == 11 && r2.qzHot == 21, "st:hot") + + // store through an interface that only declares a property signature + const p: QzPropOnly = new QzPropClass() + p.qzProp = 5 + assert(p.qzProp == 5, "st:prop:class") + const pl: QzPropOnly = { qzProp: 0 } + pl.qzProp = 6 + assert(pl.qzProp == 6, "st:prop:literal") + + // accessor get/set reached through an interface-typed reference + const acc = new QzAccClass() + const ai: QzAccIface = acc + ai.qzAcc = 4 + assert(acc._v == 8, "st:acc:set") + assert(ai.qzAcc == 9, "st:acc:get") + ai.qzAcc = ai.qzAcc + 1 + assert(acc._v == 20, "st:acc:rmw") + + // a growing map: every key is built at run time + const grow: any = {} + for (let i = 0; i < 40; i++) grow["qzk" + i] = i * 3 + assert(grow["qzk0"] == 0, "st:grow0") + assert(grow["qzk7"] == 21, "st:grow7") + assert(grow["qzk39"] == 117, "st:grow39") + assert(grow["qzk" + (20 + 1)] == 63, "st:grow21") + let gsum = 0 + for (let i = 0; i < 40; i++) gsum += grow["qzk" + i] + assert(gsum == 2340, "st:growsum") + + // typed string index signature over a plain object literal + const tbl: { [k: string]: number } = {} + tbl["a"] = 1 + tbl["b"] = 2 + tbl["a"] = tbl["a"] + 10 + assert(tbl["a"] == 11 && tbl["b"] == 2, "st:idx:literal") + + // the same index-signature type pointing at a class instance: a store + // of a declared field must reach the field, not trap + const inst = new QzPropClass() + const asIdx = inst as any as { [k: string]: number } + asIdx["qzProp"] = 12 + assert(inst.qzProp == 12, "st:idx:class:set") + assert(asIdx["qzProp"] == 12, "st:idx:class:get") + asIdx["qzProp"] = asIdx["qzProp"] + 1 + assert(inst.qzProp == 13, "st:idx:class:rmw") + + // inherited field, through the subclass and through an interface + const d = new QzDerived() + assert(d.qzInherited == 1, "st:inh:init") + d.qzInherited = 3 + assert(d.qzInherited == 3, "st:inh:set") + const ii: QzInheritedIface = d + ii.qzInherited = 4 + assert(d.qzInherited == 4, "st:inh:ifaceset") + assert(ii.qzInherited == 4, "st:inh:ifaceget") + + // super call, direct and interface-dispatched + assert(d.qzSuper(1) == 20, "st:super:direct") + assert(ii.qzSuper(1) == 20, "st:super:iface") + + // class field store through a plain concrete reference, for contrast + const base = new QzBase() + base.qzInherited = 9 + assert(base.qzInherited == 9, "st:base") + assert(base.qzSuper(1) == 2, "st:base:call") + } + + // --- dynamic member get on an any-typed variable ----------------------- + + function testDynamicGet() { + msg("dynamic get") + + const o: any = { qzDyn: 5, qzOther: "s" } + const key = "qz" + "Dyn" + + assert(o.qzDyn == 5, "dg:dot") + assert(o["qzDyn"] == 5, "dg:literal") + assert(o[key] == 5, "dg:computed") + assert(o["qzMissing"] === undefined, "dg:missing") + assert(o.qzOther == "s", "dg:other") + + // the same object reached as a class instance + const inst: any = new QzThing(1, 2) + // qzMany rather than qzFew: qzFew's read-site count is deliberately + // held below its threshold by the block in testThresholds + assert(inst.qzMany == 2, "dg:inst1") + assert(inst["qzMany"] == 2, "dg:inst2") + assert(inst["qz" + "Many"] == 2, "dg:inst3") + assert(inst.qzHi(1) == 3, "dg:instcall") + } + + export function run() { + msg("iface dispatch") + testThresholds() + testThreeRefs() + testDefaults() + testArityCollision() + testGetCallCollision() + testMethodAsValue() + testPolymorphic() + testToString() + testStores() + testDynamicGet() + msg("iface dispatch done") + } +} + +IfaceDispatch.run() diff --git a/tests/compile-test/lang-test0/57defaultparamdispatch.ts b/tests/compile-test/lang-test0/57defaultparamdispatch.ts new file mode 100644 index 000000000000..1a5825738517 --- /dev/null +++ b/tests/compile-test/lang-test0/57defaultparamdispatch.ts @@ -0,0 +1,57 @@ +// Reproduces a known defect: default parameter values are not applied when a +// method is invoked through dynamic (interface or any-typed) dispatch. +// +// Mechanism: a statically resolved call site fills omitted arguments with the +// declared defaults, because the compiler can see the declaration. A dynamic +// call site pushes only the arguments written at the call, and the arity +// wrapper on the callee fills the missing slots with `undefined` -- the +// parameter initializer (`b = 5` below) never runs. The method then computes +// 1 + undefined = NaN. +// +// The assertions that fail today are commented out (block marked REPRO) so +// the suite stays green. To reproduce: uncomment that block and run +// `gulp testlang` -- qzdp:iface and qzdp:any fail with the calls yielding +// NaN instead of 6. A fix for the defect makes the REPRO block pass; it can +// then be uncommented permanently and the qzdp:agree assertion below retired. +// +// 56ifacedispatch.ts documents the same constraint where it affects the +// dispatch corpus; this file is the minimal standalone repro. + +interface DfShape { + dfOpt(a: number, b?: number): number; +} + +class DfThing implements DfShape { + constructor() { } + dfOpt(a: number, b = 5): number { + return a + b; + } +} + +function testDefaultParamDispatch() { + msg("default params through dynamic dispatch"); + + const direct = new DfThing(); + const viaIface: DfShape = direct; + const viaAny: any = direct; + + // Statically resolved call: the compiler applies the default. Passes. + assert(direct.dfOpt(1) == 6, "qzdp:direct"); + + // Passing the argument explicitly works on every path. + assert(direct.dfOpt(1, 2) == 3, "qzdp:direct2"); + assert(viaIface.dfOpt(1, 2) == 3, "qzdp:iface2"); + assert(viaAny.dfOpt(1, 2) == 3, "qzdp:any2"); + + // REPRO -- uncomment these two lines to surface the defect: + // assert(viaIface.dfOpt(1) == 6, "qzdp:iface"); + // assert(viaAny.dfOpt(1) == 6, "qzdp:any"); + + // Kept active so any CHANGE in the current behavior is noticed: the two + // dynamic paths must at least agree with each other. Compared as strings + // so this holds both before a fix (NaN == NaN fails as numbers) and + // after one. + assert("" + viaIface.dfOpt(1) == "" + viaAny.dfOpt(1), "qzdp:agree"); +} + +testDefaultParamDispatch(); diff --git a/tests/compile-test/lang-test0/README-codegen.md b/tests/compile-test/lang-test0/README-codegen.md new file mode 100644 index 000000000000..1f3118e5b617 --- /dev/null +++ b/tests/compile-test/lang-test0/README-codegen.md @@ -0,0 +1,243 @@ +# Codegen test corpus: conditions and interface dispatch + +Tests for how the compiler lowers boolean conditions and dynamic member access +(interface-typed, structural, `any`). Three layers; each catches what the one +before structurally cannot: + +| layer | run | catches | +| --- | --- | --- | +| JS-executed semantics | `gulp testlang` | wrong answers. The corpus files in this directory are compiled and *run* on the simulator backend -- and condition lowering changes the JS backend too, so this is real coverage, not a stand-in for native | +| native assembly shape | `gulp testthumb` | which sequences codegen chose, unencodable instructions, large size swings. Nothing executes, so no wrong answer is visible here | +| on-device A/B + soak | `npm run hwab` (see `tests/hw-ab/README.md`) | the real allocator, real GC, real panics, leaks over time | + +The coverage matrix below is the index: failure mode -> covering test -> layer +-> what the failure looks like when it fires. + +The corpus is sized for two optimization families: + +- **Boolean condition lowering.** Conditions lower to short-circuit jumps that + yield a raw 0/1 rather than a tagged value materialized and then narrowed, + with Thumb fast-path helpers taking over the common truthiness tests from a + runtime call. +- **Interface dispatch specialization.** Count-gated checked-field-load + helpers, shared interface-call thunks, object-literal store specialization, + vtable wrapper-skip for calls whose arity already matches, and a typed + index-signature store fast path. + +## Coverage matrix + +| Failure mode | Covered by | Layers | How it presents | +| --- | --- | --- | --- | +| Truthiness divergence between a fast path and the runtime's own `toBool` (`-0`, `NaN`, boxed zero, `""` vs `"0"`, `[]`, `{}`, functions) | `54conditiontruthiness.ts`, whole `check()` matrix plus the `--- falsy ---` / `--- truthy ---` case list | testlang, hw-ab | `assertion failed: if:` / `ternary:` / `while:` / `bangbang:`, where `` names the value (`negzero`, `nan`, `boxedzero`, `str0`, `emptyarr`, ...) | +| Statically typed operand and boxed `any` operand disagreeing at the same construct | `54conditiontruthiness.ts`, the `typed:` block; every matrix value additionally arrives through `opaque()` as an `any` | testlang, hw-ab | `typed:emptystr`, `typed:zero`, `typed:nullarr`, `typed:emptyarr` | +| Raw 0/1 form leaking out of a condition instead of round-tripping to a tagged boolean | `54conditiontruthiness.ts` `roundtrip:` block; `55conditionlowering.ts` `testValuePosition` `vp:rt1`, `vp:rt2`, `vp:rtany` | testlang, hw-ab | `roundtrip:true`, `roundtrip:cond`, `vp:rtany`; also `bangbangtype:` and `vp:bangtype` when `typeof` stops saying `boolean` | +| Short-circuit order or operand count changing (`&&`, `\|\|`, `!`, nesting, 3- and 4-operand chains, mixed precedence) | `55conditionlowering.ts` `testShortCircuit`, which records a left-to-right trace in `lg` | testlang, hw-ab | `sc:and:short`, `sc:group1`, `sc:notand1`, `sc:and3short`, `sc:or3short`, `sc:and4`, `sc:mixed1` | +| Value position confused with condition position -- `\|\|`/`&&` yielding a truth value where the operand itself is required | `55conditionlowering.ts` `testValuePosition`; the condition-position complement is in `54conditiontruthiness.ts` (`and1:`, `and0:`, `or0:`, `or1:`) | testlang, hw-ab | `vp:orstr`, `vp:andnum`, `vp:andnocall`, `vp:ornum` | +| Conditions inside constructs other than plain `if` (else-if chains, `while`, `do..while`, `for` header, nested ternaries, switch on a computed scrutinee) | `55conditionlowering.ts` `testConstructs` and `testSwitch` | testlang, hw-ab | `cc:if*`, `cc:while`, `cc:dowhile2`, `cc:for`, `cc:tern*`, `sw:s*`, `sw:n*`, `sw:c*`, `sw:b*` | +| Comparisons feeding logical operators, including string relational comparisons and `length` used directly as a condition | `55conditionlowering.ts` `testComparisons` | testlang, hw-ab | `cmp:1`..`cmp:6`, `cmp:str1`..`cmp:str4`, `cmp:len0`, `cmp:lenchain` | +| Reference imbalance on an operand consumed by a condition (allocation in a loop header, re-evaluated every iteration) | `55conditionlowering.ts` `testFreshOperands` -- string concatenation, array literal, `slice`, object literal and ternary operands all built inside the header | testlang (wrong final state), hw-ab (allocator/GC failure) | `fresh:concat`, `fresh:arrlit`, `fresh:slicelen`, `fresh:obj`, `fresh:strcmp`, `fresh:tern`; on device, a memory panic instead of an assert | +| A condition abandoned part way through by an exception, and conditions used again after the stack unwinds | `55conditionlowering.ts` `testExceptionsInConditions` | testlang, hw-ab | `exn:right`, `exn:left`, `exn:and`, `exn:andshort`, `exn:after`, `exn:loop`; the message carries the actual trace, e.g. `exn:right t!c` | +| Threshold miscounting in a count-gated specialization -- members just below and just above the gate behaving differently | `56ifacedispatch.ts` `testThresholds` (`qzLo` 2 sites vs `qzHi` 5+, `qzFew` 4 reads vs `qzMany` 6+) and `testStores` (`qzRare` 2 stores vs `qzHot` 4) | testlang, hw-ab | `th:lo1`, `th:hi1`..`th:hi5`, `th:few*`, `th:many*`, `th:agree`, `st:rare`, `st:hot` | +| The same miscounting for the checked-field-load gate, which counts per class field rather than per interface member and so is not reachable from the semantic layer | `tests/thumb-test/cases/fieldbaseline.ts`, one class whose `qzTally` is read above the gate and `qzSpare` below it, with `asmchecks.ts` counting the inline checked-load sequences by field offset | testthumb | `expected at least 6 inline checked loads of qzTally, found N`, or `listing unexpectedly contains N checked-field-load thunks (ldfldchk_)` | +| Arity holes in wrapper-skip -- one member name declared at two arities by two unrelated interfaces | `56ifacedispatch.ts` `testArityCollision`, with the two dispatches interleaved in a loop so neither can be hoisted | testlang, hw-ab | `ar:one1`, `ar:two1`, `ar:mixed` | +| A missing trailing argument on a dynamic call (the documented default-parameter constraint) | `56ifacedispatch.ts` `testDefaults` | testlang, hw-ab | `opt:concrete`, `opt:dynagree`, `opt:safe1`..`opt:safe4`, `opt:iface2`, `opt:any2` | +| Get and call sharing a dispatch bucket -- one member name that is a field on one type and a method on another | `56ifacedispatch.ts` `testGetCallCollision`, through both interface-typed and `any`-typed references | testlang, hw-ab | `gc:get1`, `gc:call1`, `gc:agree`, `gc:get2`, `gc:call2`, `gc:type` | +| `toString` fixed-slot violation -- an override not reached through concatenation, templates, direct call, interface or `any` | `56ifacedispatch.ts` `testToString`; also present in `tests/thumb-test/cases/ifacebaseline.ts` | testlang, testthumb (shape), hw-ab | `ts:concat`, `ts:template`, `ts:iface`, `ts:any`, `ts:anyconcat` | +| Polymorphic call site confused between class instances and object literals (maps) | `56ifacedispatch.ts` `testPolymorphic` -- one call site `useQzVal` fed both, plus a mixed array iterated twice | testlang, hw-ab | `poly:class`, `poly:literal`, `poly:elem0`..`poly:elem3`, `poly:total` | +| Index-signature store fast path misrouting -- a declared field reached through `{ [k: string]: number }`, versus a real map | `56ifacedispatch.ts` `testStores` (`st:idx:*` against a class instance, `st:grow*` against a run-time-keyed map); also in `ifacebaseline.ts` | testlang, testthumb (shape), hw-ab | `st:idx:literal`, `st:idx:class:set`, `st:idx:class:rmw`, `st:grow0`, `st:growsum` | +| Stores and accessor calls through interface-typed references, including inherited fields and `super` | `56ifacedispatch.ts` `testStores` | testlang, hw-ab | `st:prop:class`, `st:acc:set`, `st:acc:rmw`, `st:inh:ifaceset`, `st:super:iface`, `st:base:call` | +| Dynamic member get on an `any`, by dot, by string literal and by computed key | `56ifacedispatch.ts` `testDynamicGet` | testlang, hw-ab | `dg:dot`, `dg:literal`, `dg:computed`, `dg:missing`, `dg:inst1`..`dg:inst3`, `dg:instcall` | +| Silent optimization loss -- an expected helper, thunk or dispatch sequence quietly stops being emitted, or an unexpected one appears | `tests/thumb-test/cases/boolbaseline.ts`, `ifacebaseline.ts` and `fieldbaseline.ts`, whose entries in `asmchecks.ts` name the exact helpers that must and must not appear | testthumb | chai failure from `assertAtLeast` / `assertAbsent` / `assertNoMatch`, e.g. `expected at least 12 calls to numops::toBoolDecr, found 0` or `listing unexpectedly mentions _pxt_map_set_by_string` | +| Large silent swing in generated code size | `tests/thumb-test/cases/sizebaseline.ts` via `codeSize` and `assertWithin` | testthumb | `generated code size is N, outside the band lo..hi` | +| Invalid Thumb emission -- an unencodable instruction or a stack imbalance | Every thumb-test case, plus every semantic file registered in `externalCases` in `asmchecks.ts`, compiled natively with the lang-test0 prelude | testthumb | `native compile of failed:` followed by the code-9200 diagnostics from the in-process assembler | + +## Per-file summaries + +### `54conditiontruthiness.ts` + +A value-by-value truthiness matrix. `check()` pushes a single value through +every construct that lowers to a condition -- `if`, `if (!v)`, if/else, the +conditional expression, `while`, `do..while`, a `for` header, condition-position +`&&` and `||`, and single and double negation -- and asserts the outcome of +each. The falsy set is pinned to exactly what the runtime treats as false: +`false`, zero in any representation (including `-0` and a boxed double zero), +`null`, `undefined`, `NaN` and `""`; everything else is truthy, including +`"0"`, `" "`, `"false"`, `[]`, `{}` and function values. Every value reaches its +test site through an `any`-typed identity call, so the operand is opaque and no +condition can be folded at compile time. Floating-point-only values (boxed +zero, fractions) are gated on `hasFloat`, and a `casecount` assert pins the +matrix size per target so a case cannot silently disappear. A closing block +re-runs the same values through statically typed locals and round-trips a +boolean through a call, which is where a raw 0/1 form that failed to become a +tagged boolean shows up. + +### `55conditionlowering.ts` + +Structural semantics rather than truth values. Operand functions `A`..`D` +append to a module-level trace string, so each assertion pins the exact +left-to-right evaluation order and operand count of a condition -- including +nested groups, negation wrapped around a chain, three- and four-operand chains, +and `&&` binding tighter than `||`. A separate section contrasts condition +position with value position, where `&&` and `||` must yield an operand rather +than a truth value. `testFreshOperands` allocates the operand inside the loop +header on every iteration (string concatenation, array literals, `slice`, +object literals) and runs the loops for hundreds of iterations, so a +reference-count imbalance on the condition path surfaces as a wrong final state +or an allocator failure rather than as a wrong first iteration. +`testExceptionsInConditions` throws from either operand of `&&` and `||` and +from inside a loop header, asserting both the partial trace and that conditions +still behave once the stack has unwound. `testSwitch` uses computed scrutinees +and a condition result as the scrutinee so nothing resolves statically. + +### `56ifacedispatch.ts` + +Dynamic member get, set and call through interface-typed, structurally typed +and `any`-typed references. Interface member ids are global by name across a +whole program and some specializations are gated on how many static sites a +member has, so every member here carries a `qz` prefix that is unique to this +file, and the threshold-straddling members are counted deliberately: `qzLo`, +`qzFew` and `qzRare` sit below their gates while `qzHi`, `qzMany` and `qzHot` +sit above, and both sides must behave identically. The remaining sections +isolate one dispatch hazard each: the same method through concrete, interface +and `any` references; one member name at two arities in two unrelated +interfaces; one member name that is a field on one type and a method on +another; a method reached through a lambda and through a higher-order call; a +single call site fed both class instances and object literals; a `toString` +override reached five different ways; stores through property signatures, +accessors, inherited fields, `super`, a run-time-keyed map and a typed index +signature over both a literal and a class instance; and dynamic gets by dot, by +string literal and by computed key. One constraint is documented in the file +rather than asserted away: the emitter fills a defaulted argument in at the +call site from the statically known signature, so through an interface- or +`any`-typed reference there is no signature and the callee's own default does +not apply. `testDefaults` therefore asserts that the interface and `any` paths +agree with each other, and that a callee written to test for `undefined` works +on every path. + +`57defaultparamdispatch.ts` is the minimal standalone repro of that defect: a +two-line block marked REPRO is commented out so the suite stays green, and +uncommenting it makes `gulp testlang` fail with `qzdp:iface` -- a ready-made +red test for whoever picks the fix up. The file's active assertions pin the +parts that must hold either way. + +### `tests/thumb-test/cases/boolbaseline.ts` + +A small program that drives conditions in `if` and `while` headers, `&&`, `||` +and `!` over numbers, booleans, strings and arrays, at enough distinct sites to +make a count assertion meaningful. Every value feeds a module-level accumulator +so no condition site can be dropped as unused. Its entry in `asmchecks.ts` +asserts on the shape of the resulting listing: how many test sites narrow their +operand through a `numops::toBoolDecr` call, and which condition-lowering +helpers appear in the listing. Nothing executes -- this is purely about which +sequence codegen chose. + +### `tests/thumb-test/cases/ifacebaseline.ts` + +The dispatch counterpart. Two classes implement one interface; the program does +repeated interface-typed reads of a single field, repeated interface-dispatched +calls of a single method, repeated object-literal stores of a single key, an +overridden `toString`, and a typed index signature -- each repeated enough times +to cross a plausible count gate. Everything feeds a module-level accumulator. +Its `asmchecks.ts` entry asserts both on the generic map runtime entries the +program reaches and on the presence or absence of the specialized forms: +checked-field-load thunks, interface-call thunks, specialized map-store thunks, +`_iface` proc labels and the by-string map-set helper. + +### `tests/thumb-test/cases/fieldbaseline.ts` + +The field-access counterpart, and the only case that reaches the checked field +load path at all: `ifacebaseline.ts`'s field reads have interface-typed +receivers, so they lower to interface dispatch instead. A field read is checked +whenever its receiver is not `this`, so this program keeps every receiver in a +class-typed variable or parameter and reads `qzTally` at six static sites and +`qzSpare` at four, straddling the count gate that decides whether the checked +sequence is hoisted into a per-field thunk. Both fields live on one class that +implements no interface and has no subclass -- an overridden field is treated as +slow and routed through interface dispatch, which would take the reads out of +the path this case exists to pin. Its `asmchecks.ts` entry counts the inline +validate-then-load sequences separately per field, by the load offset, and +asserts the thunk form is absent. + +### `tests/thumb-test/cases/sizebaseline.ts` + +A fixed dispatch-heavy program -- an interface with three implementations +driven through a loop, string building, a string-keyed counter map and an array +sort -- whose emitted code size is tracked with a wide percentage band via +`codeSize` and `assertWithin`. The band is deliberately wide: it is there to +catch large silent swings in either direction, not to turn every codegen tweak +into a test edit. This program must stay stable, since changing it invalidates +the recorded baseline. + +## How to run + +The layer table at the top: `gulp testlang` and `gulp testthumb` (both in +`gulp test`), and `tests/hw-ab/README.md` for hardware. + +## Extending this corpus + +Where a new test belongs: + +- **A behavior that can be observed by running the program.** A new file in + this directory, named `NNname.ts`. It is picked up automatically by + `compilerunner.ts`, which compiles it as `main.ts` together with the + `lang-test0.ts` prelude and runs it. Use `assert(cond, "prefix:id")` with a + prefix unique to the section, since the assert id is the only thing a failure + report carries. +- **A codegen shape that a running program cannot observe.** A new program in + `tests/thumb-test/cases/` plus an entry keyed by its file name in + `asmChecks`. A case with no entry fails, so an expectation is never + accidentally omitted. Feed every value into a module-level accumulator or the + optimizer will drop the code under test. +- **A semantic file that should also survive the native emitter.** Add its + repo-relative path to `externalCases` in `asmchecks.ts`. Those entries are + compiled with the lang-test0 prelude prepended but are not executed, so keep + the expectation light -- the semantics are already covered by `gulp testlang`. +- **Anything that needs the real allocator, the real GC or a real panic.** The + hardware layer; see `tests/hw-ab/README.md`. + +Two rules that are easy to violate: + +- **One program per case file.** Each file compiles as its own `main.ts`, and + its top-level declarations share a scope with the prelude's globals (for + lang-test0 files) or with the target's libraries (everywhere). Wrap the + entire test in a namespace and call its `run()` at the end of the file, as + `54`, `55` and `56` do. Do not merge two case programs into one file, and do + not add bare top-level names. +- **Member-name call-site counts are program-wide.** Interface member ids are + keyed by name across the whole program, target libraries included, so a + member named `size` or `value` already has an unknown number of sites before + your test adds any. A test that depends on sitting on a particular side of a + count gate must use a name that occurs nowhere else -- hence the `qz` prefix + in `56ifacedispatch.ts`. Adding one more use of such a name anywhere in the + same file changes its count and can move it across the gate. + +## Known gaps + +Deliberately not covered by any layer in this corpus: + +- **Asm shape for the semantic programs.** The semantic files are registered in + `externalCases` only to prove they compile and assemble; the expectation is + `codeSize(asm) > 0`. Their condition and dispatch shape is not pinned -- that + is what the purpose-built baseline cases are for. +- **The VM / stack-machine backend.** `tests/thumb-test` pins + `target.nativeType` to Thumb. Nothing asserts on the `backvm.ts` output for + any of these programs. +- **Device-side GC stress.** `testFreshOperands` allocates in loop headers, but + nothing forces a collection, applies memory pressure, or inspects heap growth. + A reference imbalance is only visible indirectly, as a wrong final state or as + an allocator failure that happens to occur. +- **Runtime speed.** These are performance optimizations, and no layer asserts a + speedup. Generated code size (`sizebaseline.ts`) is the only quantitative + signal in the offline layers. +- **Event handlers and concurrency.** The thumb-test cases must terminate and + avoid event loops, and the lang-test0 files are straight-line programs. + Conditions and dispatch inside event handlers or across fibers are not + exercised offline. +- **Error paths on dynamic dispatch.** `dg:missing` reads an absent key and + expects `undefined`, but nothing calls a member that does not exist, or + stores through a reference of the wrong shape, to pin the failure behavior. +- **Accessors through an `any`-typed reference.** `56ifacedispatch.ts` reaches + a getter/setter pair through an interface-typed reference (`st:acc:*`) but + not through an `any`. +- **Multi-variant packaging.** The thumb layer produces single-variant output; + the universal-hex combiner is not loaded. See `tests/thumb-test/README.md`. diff --git a/tests/hw-ab/.gitignore b/tests/hw-ab/.gitignore new file mode 100644 index 000000000000..89f9ac04aac6 --- /dev/null +++ b/tests/hw-ab/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/tests/hw-ab/README.md b/tests/hw-ab/README.md new file mode 100644 index 000000000000..0729ddd5abd8 --- /dev/null +++ b/tests/hw-ab/README.md @@ -0,0 +1,358 @@ +# hw-ab: hardware A/B comparison on micro:bit + +Build a candidate and a reference compiler's hexes from one checkout, run both +on a real micro:bit, and diff what the board prints. One board, three commands: + + npm run hwab -- build --v2-only --ref-switches "" + npm run hwab -- ab truthiness # also: ab lowering, ab iface + npm run hwab -- capture soak candidate --soak 30 # GC/leak watch + +Needs a micro:bit on USB and the prerequisites below. + +The programs it runs are the codegen corpus files from +`tests/compile-test/lang-test0/` named in `gen-project.js`'s `CASES` table +(coverage map: `README-codegen.md` there). + +## Contents + +| file | what it does | +| --- | --- | +| `hwab.js` | one entry point for the whole workflow; the normal interface | +| `gen-project.js` | writes one device project per case (its `CASES` table, plus soak) under `/projects/hw-ab/` | +| `build-ab.js` | builds candidate and reference hexes into `out/` | +| `run-capture.js` | flashes one hex and captures its serial output | +| `diff-ab.js` | normalizes and diffs two captures | + +`out/` is generated and gitignored. + +To add a case: add one entry to the `CASES` table in `gen-project.js`. +Everything downstream -- the build, `hwab`, the `cases` listing -- reads the +generated `cases.txt`, so nothing else needs editing. + +## Commands + +Every artifact is at a derived path -- case `C` on side `S` is +`out//.hex` and `out//.log` -- so `hwab.js` derives them and only +the case name is ever typed. A case may be named exactly or by any unique +substring of its name: `truthiness` and `iface` resolve, while an ambiguous +substring like `condition` is rejected with the candidate list. + +| command | what it does | +| --- | --- | +| `build [args...]` | builds the hexes; every argument passes through to `build-ab.js` | +| `capture [side] [--timeout ] [--soak ]` | flashes one hex and captures its serial output; side is `candidate` (default) or `reference`; `--soak` is for (and required by) the soak case | +| `diff ` | diffs the case's candidate and reference captures | +| `ab [--timeout ]` | capture candidate, capture reference, diff -- the whole per-case loop on one board | +| `cases` | lists the known cases and which hexes and logs exist right now | +| `help` | the usage summary | + +Two invocation forms: + + npm run hwab -- cases + node tests/hw-ab/hwab.js cases # cwd-independent + +Exit codes are the underlying script's, verbatim; `ab` reports the first +failing stage's. + +The underlying scripts remain directly invocable as `node tests/hw-ab/.js` +for scripting and debugging -- the wrapper only assembles their arguments, and +each script's own `--help` documents its full contract. + +## Prerequisites + +- `pxt` and `pxt-microbit` sibling checkouts, npm-linked, `pxt` CLI on PATH. +- A current `npm run build` in `pxt` -- rebuild after compiler changes. +- A populated `pxt-microbit/built/hexcache/`, so builds need no toolchain or + build service. +- One micro:bit on USB, MICROBIT drive mounted (see Platforms). + +## Two reference strategies + +**Switch-based.** One checkout, two builds: the candidate with no compile +switches, the reference with the opt-out switches that turn the change under +test off, named explicitly via `--ref-switches` (there is no default -- the +right names depend on which change is being tested). Fast, and the two hexes +are guaranteed to differ in nothing else. + +Two constraints: + +1. The compiler must actually implement the switches. PXT parses + `PXT_COMPILE_SWITCHES` name-agnostically -- an unknown switch name is + accepted silently and then never read. A misspelled or not-yet-implemented + switch therefore produces a "reference" that is a second candidate build. + `build-ab.js` compares the two hexes of every case and refuses to continue + when all of them are byte-identical, rather than letting you diff a build + against itself. Some cases identical is not that failure -- a switch only + changes the programs that use what it gates -- so those are reported as a + note and the build succeeds. +2. It only covers changes that are gated behind those switches. An ungated + change is in both builds and the comparison cannot see it. + +**Git-based.** Build the reference from a reference commit. Slower (a full +`npm run build` of pxt per side) but it covers every change, gated or not. This +is the gold standard; use it whenever ungated changes are in play. + + # reference side + git -C checkout + (cd && npm run build) + npm run hwab -- build --reference-only --v2-only + + # candidate side + git -C checkout + (cd && npm run build) + npm run hwab -- build --candidate-only --v2-only + +The two runs fill `out//reference.hex` and `candidate.hex` side by side +-- `out/` is preserved between runs -- and `ab ` / `diff ` then +work as usual. The preservation cuts both ways: a stale hex from an earlier +run pairs silently with a fresh one, so check `sizes.txt` if in doubt. + +## Full A/B walkthrough + +Switch-based, V2/CODAL only, naming the opt-out switches of the change under +test: + + cd + npm run hwab -- build --v2-only --ref-switches "" + +That generates the projects, builds both sides of every case, and writes +`out//{candidate,reference}.hex` plus `out/sizes.txt` (byte size and +sha256 per hex). Then, for each semantic case: + + npm run hwab -- ab truthiness + npm run hwab -- ab lowering + npm run hwab -- ab iface + +One `ab` run is the whole loop for one case on one board: it flashes the +candidate hex and captures until the board reports its verdict, then flashes the +reference hex and captures again, then diffs the two logs. It stops at the first +stage that fails and names that stage. The stages are also available one at a +time -- `capture candidate`, `capture reference`, `diff ` -- +which is what to reach for when the two sides are run on different boards or at +different times. + +`run-capture.js` exits 0 on `HWAB PASS`, 1 on `ASSERT` (printing the failing +assertion id), 2 on timeout, 3 on a host/board problem. A timeout with no +banner at all means the program never ran, so it re-flashes once before +reporting. `diff-ab.js` exits nonzero on divergence and prints a unified diff of +the normalized lines; normalization drops what belongs to the capture rather +than to the build -- a final line cut short when the capture stopped, and the +previous program's output arriving out of DAPLink's buffer, which can include +that program's entire run. The expected case therefore comes from `--case` +(passed by `hwab`) or the log's directory, never from log content. + +A pass on both sides is not the whole story, which is why `ab` diffs even when +both captures passed: the trace lines between the banners can differ while the +final verdict does not. + +### What the device prints + + HWAB START once, after a 10 s startup delay + from the prelude's msg() + HWAB PASS repeated every 2 s for about a minute + +The 10 second delay and the repeating PASS both exist for the same reason: +serial capture cannot begin until the board re-enumerates after flashing, so a +program that printed its verdict once and immediately would race the capture. +Only that first minute can be raced, so the heartbeat stops there and the board +idles quietly rather than streaming serial until it is unplugged. + +Verdicts are read only from the case's own `HWAB START` banner onward, and +`hwab` passes the expected case name through: DAPLink buffers serial while no +host is reading, so the first bytes after opening the port can belong to the +previously flashed program -- including its PASS banner. + +On failure the generated `assert` prints `ASSERT ` and then panics 45. The +host prelude's `assert` throws instead, and an uncaught throw on device is just +panic 999, which does not say which assertion failed -- `gen-project.js` +rewrites the prelude's assert at generation time to get a legible failure. The +rewrite is bracketed by `---- hw-ab generated replacement ----` comments in the +generated `main.ts`. + +## Soak procedure + +The `soak` project loops forever over the allocation shapes the corpus +exercises -- strings built in condition position, a map grown by computed key, +polymorphic interface dispatch over mixed class/literal receivers -- and every +five seconds forces a collection and prints the collector's own numbers: + + HWAB SOAK free= min= total= numgc= + +Those come from `control.gcStats()` (declared in +`pxt-common-packages/libs/base/gcstats.ts`, backed by `getGCStats` in +`gc.cpp`), read after an explicit `control.gc()` so that `lastFreeBytes` is +current rather than left over from whenever the runtime last collected. +`free` is `lastFreeBytes`, `min` is `minFreeBytes` (the low-water mark since +boot), `total` is the total heap in bytes, `numgc` is the collection count. + +Run both sides for at least 30 minutes each: + + npm run hwab -- capture soak candidate --soak 30 + npm run hwab -- capture soak reference --soak 30 + npm run hwab -- diff soak + +`--soak` mode exits 0 if `HWAB SOAK` lines were still arriving at the end and +1 if output stalled. The soak program has no pass/fail verdict, so `hwab` +refuses a plain capture of it (and `ab soak`) rather than running a capture +that can only time out. + +What a leak looks like: + +- `free` trends down across the run while `numgc` keeps climbing. A healthy run + oscillates around a stable `free` -- the workload allocates and the collector + reclaims it -- and `min` settles rather than falling monotonically. +- Eventually the board stops printing. That is an out-of-memory panic: the LED + matrix shows a sad face with 020 or 021. +- Compare time-to-failure, not just pass/fail. `HWAB SOAK` lines carry elapsed + milliseconds precisely so the two builds can be compared on how long they + lasted. + +`diff-ab.js` masks the elapsed and heap numbers before diffing, since they vary +legitimately between two runs of the same build; it prints the final soak line +from each log side by side for eyeballing. Heap trend analysis is out of scope +for the diff -- read the logs directly for that. + +### PXT_GC_STRESS (advanced) + +`pxt-common-packages/libs/base/gc.cpp` has a commented-out +`//#define PXT_GC_STRESS 1` near line 47. Enabling it makes the runtime collect +far more aggressively, which surfaces missing GC roots and premature frees in +minutes instead of hours -- a value that survives normal execution because a +collection never happened to run at the wrong moment will be collected out from +under the code almost immediately. + +This changes C++, not TypeScript, so it does not take effect through the +hexcache path used above. It requires a real runtime rebuild: either a local +CODAL toolchain (`pxt buildtarget` / `pxt build --local` with arm-none-eabi-gcc +and the CODAL sources fetched) or a cloud rebuild against a modified +common-packages. Budget the setup time before reaching for it, and remember to +revert the define -- a stress build's timing is not comparable with a normal +build's. + +## Interpreting device panics + +| number | meaning | +| --- | --- | +| 45 | an `assert` in the generated program failed; the id is on serial | +| 999 | unhandled thrown value (a `throw` that reached the top) | +| 020 / 021 | out of memory | +| 521 | the flash itself failed (DAPLink could not decode the hex); the program never ran -- re-flash | + +A panic 999 in a generated case project means something threw that was not an +assertion -- the case files raise exceptions deliberately in a few places, so a +999 points at an exception escaping a `try` that should have caught it. + +## Troubleshooting + +**Sad face with 521 right after flashing** -- DAPLink rejected the incoming +hex (a flash-time checksum failure). The program never ran, so the capture +times out with no banner, which is the case `run-capture.js` re-flashes once by +itself; it also prints `FAIL.TXT` from the MICROBIT drive when DAPLink left one +there. Reaching the report means both attempts were rejected: unplug and replug +the board before retrying. + +**`no MICROBIT volume within 30s`** -- the drive is not mounted. Check the +cable is a data cable, not charge-only. If a `MAINTENANCE` drive appears +instead, DAPLink is in bootloader mode and its interface firmware needs +reflashing. See the Platforms section for where each platform looks and how to +point the script somewhere else. + +**`no found`** -- mass storage came up but the serial port did +not. Unplug and replug; if it persists the DAPLink interface firmware is likely +out of date. Note that no pxt CLI command can substitute here: `pxt console` +and `hidserial` speak HF2, and the micro:bit is DAPLink, so the capture has to +read the serial device directly. + +**`more than one `** -- another USB serial device is attached +and the capture cannot tell which board is which. Unplug the others, or name +the right one with `MICROBIT_SERIAL`. + +**`candidate and reference hexes are byte-identical for every case`** -- the +reference build differs from the candidate only by compile switches, and those +switches changed nothing anywhere. Almost always this means the compiler on +this branch does not implement them: an unrecognised switch name is accepted +silently and never read. Check the spelling against what the compiler reads, or +switch to the git-based reference strategy. The per-case +`no codegen difference for ` note is the benign version of the same +observation and does not stop the build. + +**Empty capture / timeout** -- the capture attached after the program had +already printed, or the board reset. The repeating PASS banner normally covers +this; an empty log with a sad face on the matrix means the program panicked +before reaching its verdict. + +**Timeout with bytes but no lines (garbage in the log)** -- the port was read +with the wrong line settings, so every byte decodes as noise. The script guards +against the usual cause on each platform (see the ordering paragraph under +Platforms). If it still happens, something else has the port open and is +reconfiguring it, or the device is not the micro:bit. + +**Build fails with `Package not installed: `** -- the generated +`pxt.json` stamps the current target version so pxt-microbit's upgrade rules do +not inject extra dependencies. If the stamp is missing the project reads as +version 0.0.0 and those rules fire. Regenerate with `gen-project.js`. + +## Platforms + +Only `run-capture.js` touches the hardware; it detects the platform itself. +Everything else is platform-independent node: + +| | macOS | Linux | Windows | +| --- | --- | --- | --- | +| MICROBIT volume | `/Volumes/MICROBIT` | `/media/$USER/MICROBIT`, `/run/media/$USER/MICROBIT`, `/media/MICROBIT` | the drive letter whose volume label is `MICROBIT`, from `Get-CimInstance Win32_LogicalDisk` | +| serial device | `/dev/cu.usbmodem*` | `/dev/ttyACM*` | the `COM` port whose name matches mbed/DAPLink/USB Serial, from `Get-CimInstance Win32_SerialPort`; opened as `\\.\COM` | +| line settings | `stty -f 115200 raw -echo` | `stty -F 115200 raw -echo` | `mode COM: BAUD=115200 PARITY=n DATA=8 STOP=1 to=off xon=off dtr=on rts=on` | + +Two environment variables override the discovery when it needs help: +`MICROBIT_VOLUME` (path or drive of the mounted drive, e.g. `/Volumes/MICROBIT` +or `E:`) and `MICROBIT_SERIAL` (serial device to read, e.g. `/dev/ttyACM0` or +`COM5`). Both accept exactly what the platform's own tools print. + +Ordering differs by platform and matters. On POSIX the device is opened first +and the settings are applied second: terminal settings reset when the last open +of a tty closes, and `stty` opens and closes the device itself, so the held fd +is what makes 115200/raw survive to the reader. On Windows the configuration +belongs to the port rather than to a handle, so `mode` runs before the handle is +opened. `dtr=on` is not optional there -- a CDC device may transmit nothing +until DTR is asserted. + +Linux permissions: opening `/dev/ttyACM*` usually requires membership in the +`dialout` group (`sudo usermod -a -G dialout $USER`, then log in again). + +### Windows troubleshooting + +**No MICROBIT drive letter.** Check what Windows sees: + + powershell -NoProfile -Command "Get-CimInstance Win32_LogicalDisk | Select-Object DeviceID,VolumeName" + +If no volume is labelled `MICROBIT` the board is not in mass-storage mode -- the +cable may be charge-only, or a `MAINTENANCE` label means DAPLink is in +bootloader mode and needs its interface firmware reflashed. If the drive is +there under a different label, point the script at it with `MICROBIT_VOLUME=E:`. + +**No COM port.** Check what the port is called: + + powershell -NoProfile -Command "Get-CimInstance Win32_SerialPort | Select-Object DeviceID,Name" + +Device Manager lists the same port under Ports (COM & LPT). If the port exists +but its name does not mention mbed, DAPLink or USB Serial, name it directly with +`MICROBIT_SERIAL=COM5`. If there is no port at all, the DAPLink interface +firmware or its Windows serial driver is missing. + +**`mode` fails.** Another program is holding the port -- the MakeCode editor's +serial view, PuTTY, Tera Term, the Arduino IDE. Close it and rerun; only one +program may configure and read the port at a time. + +WSL2 also works, through the Linux path: pass the board through with +[usbipd-win](https://github.com/dorssel/usbipd-win) so the board appears as +`/dev/ttyACM*`, and mount the Windows-side MICROBIT drive into WSL +(`sudo mount -t drvfs E: /mnt/microbit`, then `MICROBIT_VOLUME=/mnt/microbit`). +The native Windows path needs none of that. + +**Manual (no setup, any platform).** Copy the hex onto the MICROBIT drive in +Explorer or Finder, then watch the board's serial port at 115200 8N1 in any +serial terminal (PuTTY, or the MakeCode editor's serial view). The verdict is +readable by eye: the program prints `HWAB START `, then either +`HWAB PASS ` repeatedly or `ASSERT ` followed by a sad face and 45 on +the LED matrix; the soak build prints `HWAB SOAK` lines whose `free=` value +should stay flat. diff --git a/tests/hw-ab/build-ab.js b/tests/hw-ab/build-ab.js new file mode 100644 index 000000000000..37261edc57b3 --- /dev/null +++ b/tests/hw-ab/build-ab.js @@ -0,0 +1,320 @@ +/* + * Builds candidate and reference hexes for every hw-ab device project. + * + * The candidate is this checkout's compiler with no switches. The reference is + * the same compiler with opt-out switches set, which is only meaningful when + * the compiler actually implements them; see the identical-hash guard below. + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const crypto = require("crypto"); +const { spawnSync } = require("child_process"); + +const isWindows = process.platform === "win32"; + +const hwabDir = __dirname; +const pxtDir = path.resolve(hwabDir, "..", ".."); +const targetSpec = process.env.PXT_TARGET_DIR || path.join(pxtDir, "..", "pxt-microbit"); +const outDir = path.join(hwabDir, "out"); + +const USAGE = [ + "Usage: node tests/hw-ab/build-ab.js [options]", + "", + "Builds the hw-ab device projects twice -- once as the candidate (this", + "checkout's compiler, no switches) and once as the reference -- and collects the", + "hexes into tests/hw-ab/out//{candidate,reference}.hex.", + "", + "Options:", + " --ref-switches \"\" compile switches for the reference build; required", + " when both sides build in one run. Name the opt-out", + " switches of the codegen change under test -- there", + " is no default, because a wrong or missing switch", + " name silently builds a second candidate (see", + " below).", + " --v2-only build only the V2/CODAL variant, by adding the", + " csv---mbcodal variant selector to BOTH builds", + " --candidate-only build only the candidate", + " --reference-only build only the reference; without --ref-switches", + " this builds the current checkout as-is and files", + " it under reference.hex (the git-based strategy)", + " -h, --help this text", + "", + "Environment:", + " PXT_TARGET_DIR target checkout (default: sibling ../pxt-microbit)", + "", + "Two reference strategies", + "------------------------", + "Switch-based (--ref-switches): one checkout, two builds, fast. It only", + "works if the compiler on this branch implements the named opt-out switches.", + "PXT parses PXT_COMPILE_SWITCHES name-agnostically, so an unknown switch is", + "accepted silently and never read -- the \"reference\" is then byte-identical to", + "the candidate for every case, which this script detects and fails on. Identical", + "output for only some cases is normal (the switch gates a feature those programs", + "do not use) and is reported as a note.", + "", + "It also only covers changes that are gated behind those switches. Any ungated", + "change is present in both builds and invisible to the comparison.", + "", + "Git-based (gold standard): build the reference from a reference commit.", + "", + " # reference side", + " git -C checkout && npm run build", + " node tests/hw-ab/build-ab.js --reference-only --v2-only", + "", + " # candidate side", + " git -C checkout && npm run build", + " node tests/hw-ab/build-ab.js --candidate-only --v2-only", + "", + "The two runs fill out//reference.hex and candidate.hex side by side;", + "out/ is preserved between runs. That also means a stale hex from an earlier", + "run will happily pair with a fresh one -- check sizes.txt if in doubt.", + "", + "Exit codes: 0 built, 1 usage or build error, 3 identical-hash guard tripped." +].join("\n"); + +function usage(stream) { + stream.write(USAGE + "\n"); +} + +function fail(msg) { + process.stderr.write("build-ab: " + msg + "\n"); + process.exit(1); +} + +let refSwitches = ""; +let v2Only = false; +let doCandidate = true; +let doReference = true; + +const argv = process.argv.slice(2); +for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--ref-switches") { + if (i + 1 >= argv.length) fail("--ref-switches needs a value"); + refSwitches = argv[++i]; + } else if (arg === "--v2-only") { + v2Only = true; + } else if (arg === "--candidate-only") { + doReference = false; + } else if (arg === "--reference-only") { + doCandidate = false; + } else if (arg === "-h" || arg === "--help") { + usage(process.stdout); + process.exit(0); + } else { + usage(process.stderr); + fail("unknown argument: " + arg); + } +} + +if (!doCandidate && !doReference) + fail("--candidate-only and --reference-only are mutually exclusive"); + +// A switch-less reference is only a hazard when the candidate builds in the +// same run -- it would be a second candidate and the A/B would compare a +// build with itself. Reference-only with no switches is the git-based +// strategy: the current checkout IS the reference. +if (doReference && doCandidate && !refSwitches) + fail("building both sides in one run needs --ref-switches \"\" " + + "naming the opt-out switches of the change under test, or use the " + + "git-based strategy (--reference-only / --candidate-only per " + + "checkout; see --help)"); + +let targetDir = ""; +try { + const resolved = fs.realpathSync(targetSpec); + if (fs.statSync(resolved).isDirectory()) targetDir = resolved; +} catch (e) { + targetDir = ""; +} +if (!targetDir || !fs.existsSync(path.join(targetDir, "pxtarget.json"))) + fail("no pxtarget.json in '" + targetSpec + "' (set PXT_TARGET_DIR)"); + +// A PATH lookup, not a trial run: `pxt --version` exits nonzero when it is not +// invoked inside a target checkout, so its exit status cannot answer "is the +// CLI installed". +function onPath(cmd) { + const dirs = (process.env.PATH || "").split(path.delimiter).filter(Boolean); + const exts = isWindows + ? (process.env.PATHEXT || ".COM;.EXE;.BAT;.CMD").split(";").filter(Boolean) + : [""]; + for (const dir of dirs) { + for (const ext of exts) { + try { + fs.accessSync(path.join(dir, cmd + ext), fs.constants.X_OK); + return true; + } catch (e) { /* keep looking */ } + } + } + return false; +} + +if (!onPath("pxt")) + fail("no 'pxt' on PATH -- install the pxt CLI (npm i -g pxt)"); +if (!fs.existsSync(path.join(pxtDir, "built", "pxt.js"))) + fail("missing " + path.join(pxtDir, "built", "pxt.js") + + " -- run 'npm run build' in " + pxtDir + " first"); + +const projRoot = path.join(targetDir, "projects", "hw-ab"); + +function sha256(file) { + return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +function nowSeconds() { + return Math.floor(Date.now() / 1000); +} + +function pad(s, width) { + return s.length >= width ? s : s + " ".repeat(width - s.length); +} + +// The variant selector is not a compile switch; it picks the V2/CODAL-only +// build instead of the universal V1+V2 hex. It must be identical on both sides. +const variantSw = v2Only ? "csv---mbcodal" : ""; + +function joinSwitches(a, b) { + return a && b ? a + "," + b : a + b; +} + +process.stdout.write("build-ab: pxt " + pxtDir + "\n"); +process.stdout.write("build-ab: target " + targetDir + "\n"); +process.stdout.write("build-ab: variant " + (variantSw || "") + "\n"); +process.stdout.write("build-ab: ref sw " + + (!doReference ? "" : refSwitches || "") + "\n"); +process.stdout.write("\n"); + +const gen = spawnSync(process.execPath, [path.join(hwabDir, "gen-project.js")], + { stdio: "inherit" }); +if (gen.error) fail("could not run gen-project.js: " + gen.error.message); +if (gen.status !== 0) process.exit(gen.status === null ? 1 : gen.status); +const casesFile = path.join(projRoot, "cases.txt"); +if (!fs.existsSync(casesFile)) fail("gen-project.js produced no cases.txt"); +process.stdout.write("\n"); + +function buildOne(caseName, mode, switches) { + const proj = path.join(projRoot, caseName); + const dest = path.join(outDir, caseName, mode + ".hex"); + const logPath = path.join(outDir, caseName, mode + ".build.log"); + + if (!fs.existsSync(proj)) fail("missing project " + proj); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + + // A stale built/ can hand back the previous mode's output. + fs.rmSync(path.join(proj, "built"), { recursive: true, force: true }); + + process.stdout.write("build-ab: " + pad(caseName, 22) + " " + pad(mode, 10) + + " switches=" + (switches || "") + "\n"); + const t0 = nowSeconds(); + + const env = Object.assign({}, process.env); + if (switches) env.PXT_COMPILE_SWITCHES = switches; + else delete env.PXT_COMPILE_SWITCHES; + + // shell:true on Windows so the pxt.cmd shim resolves; the log fd is handed + // to both stdout and stderr so the two interleave as they did on a console. + const logFd = fs.openSync(logPath, "w"); + let res; + try { + res = spawnSync("pxt", ["build"], { + cwd: proj, + env: env, + stdio: ["ignore", logFd, logFd], + shell: isWindows + }); + } finally { + fs.closeSync(logFd); + } + + if (res.error || res.status !== 0) { + const log = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf8") : ""; + const lines = log.split("\n"); + if (lines.length && lines[lines.length - 1] === "") lines.pop(); + for (const line of lines) process.stderr.write(" | " + line + "\n"); + if (res.error) process.stderr.write(" | " + res.error.message + "\n"); + fail(caseName + "/" + mode + " build failed (log above)"); + } + const t1 = nowSeconds(); + + const binary = path.join(proj, "built", "binary.hex"); + if (!fs.existsSync(binary)) fail(caseName + "/" + mode + " built no binary.hex"); + fs.copyFileSync(binary, dest); + process.stdout.write("build-ab: -> " + dest + " (" + + fs.statSync(dest).size + " bytes, " + (t1 - t0) + "s)\n"); +} + +fs.mkdirSync(outDir, { recursive: true }); +const sizesFile = path.join(outDir, "sizes.txt"); +fs.writeFileSync(sizesFile, [ + "# hw-ab build " + new Date().toISOString().replace(/\.[0-9]{3}Z$/, "Z"), + "# variant: " + (variantSw || "universal"), + "# reference switches: " + refSwitches, + "# case mode bytes sha256", + "" +].join("\n")); + +const identical = []; +let pairs = 0; +const cases = fs.readFileSync(casesFile, "utf8").split("\n") + .map(l => l.replace(/\r$/, "")) + .filter(l => l.length > 0); + +for (const caseName of cases) { + fs.mkdirSync(path.join(outDir, caseName), { recursive: true }); + + if (doCandidate) buildOne(caseName, "candidate", variantSw); + if (doReference) buildOne(caseName, "reference", joinSwitches(variantSw, refSwitches)); + + for (const mode of ["candidate", "reference"]) { + const hex = path.join(outDir, caseName, mode + ".hex"); + if (!fs.existsSync(hex)) continue; + fs.appendFileSync(sizesFile, caseName + " " + mode + " " + + fs.statSync(hex).size + " " + sha256(hex) + "\n"); + } + + if (doCandidate && doReference) { + pairs++; + if (sha256(path.join(outDir, caseName, "candidate.hex")) === + sha256(path.join(outDir, caseName, "reference.hex"))) { + identical.push(caseName); + } + } + process.stdout.write("\n"); +} + +process.stdout.write("build-ab: sizes -> " + sizesFile + "\n"); + +// Some pairs identical is expected: a switch only changes the programs that use +// the feature it gates. Every pair identical is the failure mode worth catching +// -- the switch names buy nothing, so the A/B would compare a build with itself. +if (identical.length && identical.length === pairs) { + process.stderr.write([ + "", + "build-ab: ERROR -- candidate and reference hexes are byte-identical for every", + "build-ab: case:", + "build-ab: " + identical.join(" "), + "", + "The reference build differs from the candidate only by the compile switches", + "\"" + refSwitches + "\". Identical output everywhere means those switches changed", + "nothing at all. The likely cause is that the compiler on this branch does not", + "implement them: PXT_COMPILE_SWITCHES is parsed name-agnostically, so an", + "unrecognised switch name is accepted silently and then never read, and the", + "\"reference\" build is just a second candidate build.", + "", + "Check that the switch names are spelled as the compiler reads them, or use the", + "git-based reference strategy instead (build-ab.js --help). An A/B run against", + "an identical hex would compare a build with itself and always report \"same\".", + "" + ].join("\n")); + process.exit(3); +} + +for (const caseName of identical) { + process.stdout.write("build-ab: note -- no codegen difference for " + caseName + + " under these switches\n"); +} + +process.stdout.write("build-ab: done\n"); diff --git a/tests/hw-ab/diff-ab.js b/tests/hw-ab/diff-ab.js new file mode 100644 index 000000000000..9819d73a6690 --- /dev/null +++ b/tests/hw-ab/diff-ab.js @@ -0,0 +1,288 @@ +/* + * Compares two hw-ab serial captures. + * + * Only the lines that carry a verdict or a trace are compared: HWAB banners, + * ASSERT failures, and the prelude's own msg() progress lines. Everything a + * capture picks up incidentally is dropped, and the parts of a line that vary + * legitimately between two runs of the same build -- elapsed milliseconds and + * heap byte counts on soak lines -- are masked before the comparison. + * + * Two capture artifacts are also dropped, since both are properties of the + * capture rather than of the build: a final line the capture cut short, and a + * banner belonging to the previously flashed program. + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const USAGE = [ + "Usage: node tests/hw-ab/diff-ab.js [--case ]", + "", + "Normalizes two run-capture.js logs and diffs them.", + "", + "Normalization:", + " - CR stripped, blank lines dropped", + " - the expected case comes from --case, else from the log's parent", + " directory (the out//.log layout); everything before its", + " \"HWAB START \" banner is discarded, including a stale prefix glued", + " to the banner line. DAPLink's buffer can hold a previous program's", + " entire run, START banner included, so content cannot name its own case.", + " With no --case and no such banner, the first START of any case is the", + " sync point.", + " - an unterminated final line is dropped -- it means the capture stopped", + " mid-line, which is not a divergence", + " - HWAB lines naming a different case are the previous program's, buffered", + " by DAPLink, and are dropped", + " - HWAB banners, ASSERT failures and the lang-test0 msg() trace lines are all", + " compared; a leading \"[12345]\" or \"123ms\" timestamp is stripped", + " - \"HWAB SOAK free=.. min=.. total=.. numgc=..\" collapses to", + " \"HWAB SOAK \" -- soak timing and heap numbers vary between runs", + " and are not a divergence", + " - repeated identical lines collapse to one (the PASS banner and the START", + " banner both repeat by design)", + "", + "Soak heap trend is out of scope. The final heap line from each log is printed", + "side by side so the two can be eyeballed, but it does not affect the exit code.", + "", + "Exit codes: 0 identical, 1 divergence (a unified diff is printed), 2 usage." +].join("\n"); + +function usage(stream) { + stream.write(USAGE + "\n"); +} + +// Splits file text into lines the way the line tools do: a trailing newline +// terminates the last line rather than starting an empty one, while a final +// unterminated fragment is still a line. +function splitLines(text) { + const lines = text.split("\n"); + if (lines.length && lines[lines.length - 1] === "") lines.pop(); + return lines; +} + +// Case name carried by an "HWAB START " or "HWAB PASS " line, or +// "" for any other line (HWAB SOAK carries no case name). +function hwabCase(line) { + const m = /HWAB (?:START|PASS)[ \t]+([^ \t]+)/.exec(line); + return m ? m[1] : ""; +} + +function normalize(file, expectedCase) { + const text = fs.readFileSync(file, "utf8").replace(/\r/g, ""); + const raw = splitLines(text); + + // A capture stopped mid-line leaves an unterminated fragment; where the + // capture happened to stop is not a divergence. + if (raw.length && !/\n$/.test(text)) raw.pop(); + + // Serial capture can attach part way through a line, and DAPLink's buffer + // can hold a previous program's entire run, START banner included -- so + // when the expected case is known, the sync point is ITS banner, and a + // stale fragment glued to the front of that line is cut off. Only with no + // expected case (or a log that never prints its banner) does the first + // START of any case stand in. A log with no banner at all keeps all of + // its lines except the first, which is the one that can be a fragment. + let start = 0; + let caseName = ""; + const wanted = expectedCase ? "HWAB START " + expectedCase : null; + for (let i = 0; i < raw.length; i++) { + const at = wanted ? raw[i].indexOf(wanted) : -1; + if (at >= 0) { + raw[i] = raw[i].substr(at); + start = i + 1; + caseName = expectedCase; + break; + } + if (!wanted && raw[i].indexOf("HWAB START") >= 0) { + start = i + 1; + caseName = hwabCase(raw[i]); + break; + } + } + if (!start && wanted) { + // Expected banner never seen: fall back to any-case sync so a log from + // outside the out// layout still normalizes. + for (let i = 0; i < raw.length; i++) { + if (raw[i].indexOf("HWAB START") >= 0) { + start = i + 1; + caseName = hwabCase(raw[i]); + break; + } + } + } + const from = start ? start : 2; + + const out = []; + let prev = null; + for (let i = from - 1; i < raw.length; i++) { + const line = raw[i] + .replace(/^\[[0-9]+\][ \t]*/, "") + .replace(/^[0-9]+ms[ \t]+/, "") + .replace(/^HWAB SOAK [0-9]+ .*$/, "HWAB SOAK "); + // Blank lines are dropped without resetting the duplicate filter. + if (!/[^ \t]/.test(line)) continue; + // A whole banner from the previously flashed program can survive the + // discard above, buffered by DAPLink and emitted after this program's + // own START. It names the other case, which is how it is recognised. + const lineCase = hwabCase(line); + if (caseName && lineCase && lineCase !== caseName) continue; + if (line === prev) continue; + out.push(line); + prev = line; + } + return out; +} + +function finalHeap(file) { + const lines = splitLines(fs.readFileSync(file, "utf8").replace(/\r/g, "")); + let last = ""; + for (const line of lines) { + if (line.indexOf("HWAB SOAK ") >= 0) last = line; + } + return last || "(no soak lines)"; +} + +// Longest common subsequence over whole lines; the logs compared here are a few +// hundred lines at most, so the quadratic table is not worth avoiding. +function lcsTable(a, b) { + const rows = a.length + 1; + const cols = b.length + 1; + const table = new Uint32Array(rows * cols); + for (let i = a.length - 1; i >= 0; i--) { + for (let j = b.length - 1; j >= 0; j--) { + table[i * cols + j] = a[i] === b[j] + ? table[(i + 1) * cols + (j + 1)] + 1 + : Math.max(table[(i + 1) * cols + j], table[i * cols + (j + 1)]); + } + } + return table; +} + +// Returns the edit script as {op, line} entries, op in " ", "-", "+". +function diffLines(a, b) { + const cols = b.length + 1; + const table = lcsTable(a, b); + const script = []; + let i = 0, j = 0; + while (i < a.length && j < b.length) { + if (a[i] === b[j]) { + script.push({ op: " ", line: a[i] }); i++; j++; + } else if (table[(i + 1) * cols + j] >= table[i * cols + (j + 1)]) { + script.push({ op: "-", line: a[i] }); i++; + } else { + script.push({ op: "+", line: b[j] }); j++; + } + } + while (i < a.length) script.push({ op: "-", line: a[i++] }); + while (j < b.length) script.push({ op: "+", line: b[j++] }); + return script; +} + +// A range of zero lines is reported against the line before it, and a range of +// one line omits the count -- the conventions unified diff readers expect. +function range(start, len) { + if (len === 0) return (start - 1) + ",0"; + if (len === 1) return String(start); + return start + "," + len; +} + +function unifiedDiff(a, b, labelA, labelB, context) { + const script = diffLines(a, b); + if (!script.some(e => e.op !== " ")) return null; + + // Group the script into hunks: every changed entry plus `context` unchanged + // entries on each side, merged when their context windows touch. + const changed = []; + script.forEach((e, idx) => { if (e.op !== " ") changed.push(idx); }); + const hunks = []; + let k = 0; + while (k < changed.length) { + let lo = Math.max(0, changed[k] - context); + let hi = Math.min(script.length - 1, changed[k] + context); + k++; + while (k < changed.length && changed[k] - context <= hi + 1) { + hi = Math.min(script.length - 1, changed[k] + context); + k++; + } + hunks.push([lo, hi]); + } + + // Lines of each file consumed before each script entry, so a hunk header + // can name the 1-based line each side starts at. + const aBefore = new Array(script.length); + const bBefore = new Array(script.length); + let an = 0, bn = 0; + script.forEach((e, idx) => { + aBefore[idx] = an; + bBefore[idx] = bn; + if (e.op !== "+") an++; + if (e.op !== "-") bn++; + }); + + const out = ["--- " + labelA, "+++ " + labelB]; + for (const [lo, hi] of hunks) { + let aLen = 0, bLen = 0; + for (let idx = lo; idx <= hi; idx++) { + if (script[idx].op !== "+") aLen++; + if (script[idx].op !== "-") bLen++; + } + out.push("@@ -" + range(aBefore[lo] + 1, aLen) + + " +" + range(bBefore[lo] + 1, bLen) + " @@"); + for (let idx = lo; idx <= hi; idx++) { + out.push(script[idx].op + script[idx].line); + } + } + return out.join("\n") + "\n"; +} + +function main(argv) { + let expectedCase = ""; + const files = []; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === "--case") { + if (i + 1 >= argv.length) { usage(process.stderr); return 2; } + expectedCase = argv[++i]; + } else { + files.push(argv[i]); + } + } + if (files.length !== 2) { usage(process.stderr); return 2; } + for (const f of files) { + let ok = false; + try { ok = fs.statSync(f).isFile(); } catch (e) { ok = false; } + if (!ok) { + process.stderr.write("diff-ab: no such log: " + f + "\n"); + return 2; + } + } + + const a = files[0]; + const b = files[1]; + // Without --case, the out//.log layout names the case; a log + // from elsewhere falls back to content sync inside normalize(). + if (!expectedCase) expectedCase = path.basename(path.dirname(a)); + const na = normalize(a, expectedCase); + const nb = normalize(b, expectedCase); + + if (na.concat(nb).some(l => l.indexOf("HWAB SOAK") >= 0)) { + process.stdout.write("diff-ab: final soak line\n"); + process.stdout.write(" A (" + a + "): " + finalHeap(a) + "\n"); + process.stdout.write(" B (" + b + "): " + finalHeap(b) + "\n"); + process.stdout.write("\n"); + } + + const d = unifiedDiff(na, nb, "A " + a, "B " + b, 3); + if (!d) { + process.stdout.write( + "diff-ab: identical (" + na.length + " normalized line(s))\n"); + return 0; + } + + process.stderr.write("diff-ab: DIVERGENCE\n"); + process.stderr.write(d); + return 1; +} + +process.exitCode = main(process.argv.slice(2)); diff --git a/tests/hw-ab/gen-project.js b/tests/hw-ab/gen-project.js new file mode 100644 index 000000000000..0b354611f8d7 --- /dev/null +++ b/tests/hw-ab/gen-project.js @@ -0,0 +1,305 @@ +/* + * Generates micro:bit device projects for the hardware A/B comparison. + * + * Each project is a standalone pxt project under + * /projects/hw-ab// that can be built with `pxt build` and + * flashed to a board. One project is generated per semantic case file in + * tests/compile-test/lang-test0 (the case files cannot be merged: their + * top-level names collide), plus a `soak` project used for leak detection. + * + * A generated main.ts is: + * + * header banner + startup delay so a serial capture can attach + * prelude tests/compile-test/lang-test0/lang-test0.ts, with its throwing + * `assert` textually replaced by one that prints "ASSERT " to + * serial and then panics 45 (the lang-test1 convention). A thrown + * string on device surfaces only as panic 999, which carries no + * identity; the rewrite is what makes a device failure legible. + * body the case file text, verbatim + * footer prints "HWAB PASS " for a bounded window, then idles + * without resetting + * + * The trailing PASS loop and the startup delay both exist because serial + * capture cannot begin until the board has re-enumerated after flashing: a + * program that printed its result once, early, would race the capture. The + * window is bounded because only that first minute can be raced. + * + * Usage: + * node tests/hw-ab/gen-project.js + * + * Environment: + * PXT_TARGET_DIR target checkout to generate into + * (default: sibling ../pxt-microbit) + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const pxtDir = path.resolve(__dirname, "..", ".."); +const targetDir = path.resolve( + process.env.PXT_TARGET_DIR || path.join(pxtDir, "..", "pxt-microbit")); +const langTest0Dir = path.join(pxtDir, "tests", "compile-test", "lang-test0"); +const outRoot = path.join(targetDir, "projects", "hw-ab"); + +// Milliseconds the device idles before running anything, so that the host has +// time to reopen the serial device after the post-flash reset. +const START_DELAY_MS = 10000; +// Interval of the trailing PASS heartbeat and of the soak progress line. +const PASS_INTERVAL_MS = 2000; +const SOAK_INTERVAL_MS = 5000; +// How long the PASS heartbeat runs before the program goes quiet. It only has +// to outlast the post-flash re-enumeration the capture waits for; streaming +// past that is noise, and a board transmitting indefinitely is suspected of +// aggravating flash failures. +const PASS_WINDOW_MS = 60000; +const PASS_REPEATS = Math.round(PASS_WINDOW_MS / PASS_INTERVAL_MS); + +// case name -> lang-test0 source file +const CASES = { + "conditiontruthiness": "54conditiontruthiness.ts", + "conditionlowering": "55conditionlowering.ts", + "ifacedispatch": "56ifacedispatch.ts" +}; + +// Exact text of the host prelude's assert. Matching it exactly rather than by +// regex is deliberate: if the prelude changes shape, generation fails loudly +// instead of silently emitting a program whose failures are invisible. +const HOST_ASSERT = + "function assert(cond: boolean, m?: string) {\n" + + " if (!cond) {\n" + + " throw `assertion failed: ${m || \"\"}`;\n" + + " }\n" + + "}\n"; + +const DEVICE_ASSERT = + "// ---- hw-ab generated replacement -------------------------------------\n" + + "// The host prelude's assert throws. An uncaught throw on device is just\n" + + "// panic 999, which does not say which assertion failed, so the device\n" + + "// build prints the assertion id to serial first and then panics 45.\n" + + "//% shim=pxtrt::panic\n" + + "function panic(code2: number): void { }\n" + + "\n" + + "function assert(cond: boolean, m?: string) {\n" + + " if (!cond) {\n" + + " console.log(\"ASSERT \" + (m || \"?\"))\n" + + " control.dmesg(\"ASSERT \" + (m || \"?\"))\n" + + " // Panic does not flush serial; without this drain pause the tail\n" + + " // of the assert id is lost, and the id is the failure report.\n" + + " basic.pause(250)\n" + + " panic(45)\n" + + " }\n" + + "}\n" + + "// ---- end hw-ab generated replacement ----------------------------------\n"; + +function fail(msg) { + console.error("gen-project: " + msg); + process.exit(1); +} + +function targetVersion() { + // Stamping the current target version keeps the target's "upgrades" rules + // from injecting extra dependencies (they are all gated on older + // versions). Without the stamp the project reads as version 0.0.0, + // pxt-microbit's missingPackage rules match it, and the build dies with an + // error like "Package not installed: microphone". + const pkg = path.join(targetDir, "package.json"); + return JSON.parse(fs.readFileSync(pkg, "utf8")).version; +} + +function devicePrelude() { + const src = fs.readFileSync(path.join(langTest0Dir, "lang-test0.ts"), "utf8"); + if (src.indexOf(HOST_ASSERT) < 0) + fail("lang-test0.ts prelude no longer contains the expected throwing " + + "assert -- update HOST_ASSERT in gen-project.js"); + return src.replace(HOST_ASSERT, DEVICE_ASSERT); +} + +function header(name) { + return [ + "// Generated by tests/hw-ab/gen-project.js -- do not edit.", + "// Case: " + name, + "", + "// Serial capture cannot start until the board re-enumerates after", + "// flashing, so nothing is printed for the first " + START_DELAY_MS + " ms.", + "basic.pause(" + START_DELAY_MS + ")", + "console.log(\"HWAB START " + name + "\")", + "control.dmesg(\"HWAB START " + name + "\")", + "" + ].join("\n"); +} + +function footer(name) { + return [ + "", + "// Reaching here means every assertion held. Repeat the verdict for", + "// " + Math.round(PASS_WINDOW_MS / 1000) + "s so a capture that could only attach after the board", + "// re-enumerated still sees it, then go quiet without resetting.", + "for (let hwabI = 0; hwabI < " + PASS_REPEATS + "; hwabI++) {", + " console.log(\"HWAB PASS " + name + "\")", + " control.dmesg(\"HWAB PASS " + name + "\")", + " basic.pause(" + PASS_INTERVAL_MS + ")", + "}", + "while (true) {", + " basic.pause(1000)", + "}", + "" + ].join("\n"); +} + +// The soak body is written here rather than imported from the case files: it +// needs the allocation shapes those files exercise, not their assertions, and +// it has to run forever. The three shapes are the ones whose codegen the A/B +// is about -- string building in condition position, growth of a dynamically +// keyed map, and polymorphic interface dispatch over mixed receivers. +function soakBody() { + return [ + "namespace Soak {", + " interface SoakIface {", + " soakVal(): number", + " }", + "", + " class SoakClass implements SoakIface {", + " n: number", + " constructor(n: number) {", + " this.n = n", + " }", + " soakVal(): number {", + " return this.n", + " }", + " }", + "", + " function useSoak(v: SoakIface): number {", + " return v.soakVal() + 1", + " }", + "", + " // Freshly built strings and arrays evaluated in condition position.", + " function churnConditions() {", + " let s = \"x\"", + " let i = 0", + " while ((s + i).length < 24) {", + " i++", + " s = s + \"y\"", + " }", + " const base = [1, 2, 3, 4, 5]", + " let k = 0", + " let seen = 0", + " while (base.slice(0, (k % 5) + 1).length > 0 && k < 200) {", + " seen += base.slice(k % 5).length", + " k++", + " }", + " assert(seen == 600, \"soak:slice\")", + " let m = 0", + " while (({ v: m }).v < 150 || m < 0) m++", + " assert(m == 150, \"soak:obj\")", + " }", + "", + " // A map grown by computed key, then discarded.", + " function churnMap() {", + " const o: any = {}", + " for (let i = 0; i < 60; i++) o[\"soakKey\" + i] = i", + " let tot = 0", + " for (let i = 0; i < 60; i++) tot += o[\"soakKey\" + i]", + " assert(tot == 1770, \"soak:map\")", + " }", + "", + " // The same call site over class instances and object literals.", + " // The literal's lambda captures a call parameter rather than the", + " // loop variable: a closure over a `let` loop variable sees the", + " // loop's final value in static TypeScript, unlike standard", + " // TypeScript.", + " function mkSoakLit(v: number): SoakIface {", + " return { soakVal: () => v }", + " }", + "", + " function churnDispatch() {", + " const mixed: SoakIface[] = []", + " for (let i = 0; i < 40; i++) {", + " if (i % 2 == 0) mixed.push(new SoakClass(i))", + " else mixed.push(mkSoakLit(i))", + " }", + " let tot = 0", + " for (const mm of mixed) tot += useSoak(mm)", + " assert(tot == 820, \"soak:dispatch\")", + " }", + "", + " // Free bytes reported after a forced collection. Undefined until", + " // the first GC has run, hence the explicit control.gc() call.", + " function heapMetric(): string {", + " control.gc()", + " const st = control.gcStats()", + " if (!st) return \"free=na\"", + " return \"free=\" + st.lastFreeBytes +", + " \" min=\" + st.minFreeBytes +", + " \" total=\" + st.totalBytes +", + " \" numgc=\" + st.numGC", + " }", + "", + " export function run() {", + " const t0 = control.millis()", + " let next = 0", + " while (true) {", + " churnConditions()", + " churnMap()", + " churnDispatch()", + " const el = control.millis() - t0", + " if (el >= next) {", + " const line = \"HWAB SOAK \" + el + \" \" + heapMetric()", + " console.log(line)", + " control.dmesg(line)", + " next = el + " + SOAK_INTERVAL_MS, + " }", + " }", + " }", + "}", + "", + "Soak.run()", + "" + ].join("\n"); +} + +function writeProject(name, mainTs, version) { + const dir = path.join(outRoot, name); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "pxt.json"), JSON.stringify({ + name: "hw-ab-" + name, + description: "hw-ab device project (generated) -- " + name, + dependencies: { core: "file:../../../libs/core" }, + files: ["main.ts"], + targetVersions: { target: version }, + supportedTargets: ["microbit"] + }, null, 4) + "\n"); + fs.writeFileSync(path.join(dir, "main.ts"), mainTs); + console.log("gen-project: " + dir + " (" + mainTs.length + " bytes main.ts)"); + return dir; +} + +function main() { + if (!fs.existsSync(path.join(targetDir, "pxtarget.json"))) + fail("no pxtarget.json in " + targetDir + " (set PXT_TARGET_DIR)"); + + const version = targetVersion(); + const prelude = devicePrelude(); + const names = []; + + for (const name of Object.keys(CASES)) { + const caseFile = path.join(langTest0Dir, CASES[name]); + if (!fs.existsSync(caseFile)) + fail("missing case file " + caseFile); + const body = fs.readFileSync(caseFile, "utf8"); + writeProject(name, header(name) + prelude + "\n" + body + "\n" + footer(name), + version); + names.push(name); + } + + // The soak program never passes or fails; it prints progress until the + // operator stops it or the heap gives out. + writeProject("soak", header("soak") + prelude + "\n" + soakBody(), version); + names.push("soak"); + + fs.writeFileSync(path.join(outRoot, "cases.txt"), names.join("\n") + "\n"); + console.log("gen-project: " + names.length + " projects in " + outRoot); +} + +main(); diff --git a/tests/hw-ab/hwab.js b/tests/hw-ab/hwab.js new file mode 100644 index 000000000000..cc9c5d0c5330 --- /dev/null +++ b/tests/hw-ab/hwab.js @@ -0,0 +1,372 @@ +/* + * One entry point for the hw-ab workflow. + * + * The underlying scripts take full paths, which are long enough to mistype and + * are fully derivable from a case name and a side: every artifact of case C on + * side S lives at tests/hw-ab/out//.{hex,log}. This wrapper does that + * derivation, resolves a case name from any unique substring of it, and hands + * the result to build-ab.js / run-capture.js / diff-ab.js unchanged. + * + * It adds no behaviour of its own beyond the `ab` sequence: each command is a + * single child process whose exit code is passed through verbatim, so the exit + * codes documented for those scripts are the exit codes seen here. + * + * Paths are resolved from this file's location, never from the working + * directory, so the command works from anywhere. + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const { spawnSync } = require("child_process"); + +const hwabDir = __dirname; +const pxtDir = path.resolve(hwabDir, "..", ".."); +const outDir = path.join(hwabDir, "out"); + +// Same resolution as gen-project.js and build-ab.js: PXT_TARGET_DIR, else the +// sibling pxt-microbit checkout. +const targetSpec = process.env.PXT_TARGET_DIR || path.join(pxtDir, "..", "pxt-microbit"); + +const SIDES = ["candidate", "reference"]; + +// The soak program (generated by gen-project.js alongside the CASES table) +// prints HWAB SOAK heap telemetry forever and never a HWAB PASS verdict, so a +// plain capture of it +// can only time out, and a soak-mode capture of any other case can only +// report a stall. Both mismatches are refused up front. +const SOAK_CASE = "soak"; + +const USAGE = [ + "Usage: node tests/hw-ab/hwab.js [args]", + " npm run hwab -- [args]", + "", + "One entry point for the hw-ab hardware A/B workflow. Hex and log paths are", + "derived from the case name and the side, so only the case is ever typed:", + "case C on side S is tests/hw-ab/out//.{hex,log}.", + "", + "Commands:", + " build [args...] build the hexes; every argument is passed to", + " build-ab.js unchanged (--help lists them)", + " capture [side] flash one hex and capture its serial output;", + " [--timeout ] side is candidate (default) or reference.", + " [--soak ] --soak is for the soak case, which prints", + " heap telemetry instead of a verdict", + " diff diff the case's candidate and reference logs", + " ab [--timeout ] the whole per-case loop on one board: capture", + " candidate, capture reference, then diff", + " cases list the known cases and which hexes and logs", + " exist right now", + " help this text", + "", + "A case may be named exactly or by any unique substring of its name, so", + "\"truthiness\" and \"iface\" resolve; an ambiguous substring like \"condition\"", + "is rejected with the candidate list rather than guessed.", + "", + "Examples:", + " npm run hwab -- build --v2-only --ref-switches \"\"", + " npm run hwab -- ab truthiness", + " node tests/hw-ab/hwab.js ab truthiness", + "", + "Exit codes: the invoked script's own code, verbatim; `ab` reports the first", + "failing stage's. 0 for help, 2 for an unknown command or bad arguments." +].join("\n"); + +function usage(stream) { + stream.write(USAGE + "\n"); +} + +// Exit 2 is this wrapper's own "you asked for something I cannot turn into a +// command" code; it never overlaps with a child's, because no child runs. +function fail(msg) { + process.stderr.write("hwab: " + msg + "\n"); + process.exit(2); +} + +function say(msg) { + process.stdout.write("hwab: " + msg + "\n"); +} + +// ---- case names ----------------------------------------------------------- + +function targetDir() { + try { + const resolved = fs.realpathSync(targetSpec); + if (fs.statSync(resolved).isDirectory()) return resolved; + } catch (e) { /* no target checkout */ } + return ""; +} + +function isDir(p) { + try { return fs.statSync(p).isDirectory(); } catch (e) { return false; } +} + +// The generated cases.txt is authoritative -- it is what build-ab.js iterates. +// out/ is the fallback for a checkout with no target beside it, and is enough +// for anything that only reads already-built artifacts. +function knownCases() { + const target = targetDir(); + if (target) { + const casesFile = path.join(target, "projects", "hw-ab", "cases.txt"); + if (fs.existsSync(casesFile)) { + const names = fs.readFileSync(casesFile, "utf8").split("\n") + .map(l => l.replace(/\r$/, "")) + .filter(l => l.length > 0); + if (names.length) return { names: names, source: casesFile }; + } + } + if (isDir(outDir)) { + const names = fs.readdirSync(outDir) + .filter(e => isDir(path.join(outDir, e))).sort(); + if (names.length) return { names: names, source: outDir }; + } + fail("no cases known yet -- neither " + + path.join(targetSpec, "projects", "hw-ab", "cases.txt") + " nor " + + outDir + " lists any.\n" + + " Run the build first: npm run hwab -- build --candidate-only --v2-only"); +} + +function resolveCase(name) { + const known = knownCases(); + if (known.names.indexOf(name) >= 0) return name; + const hits = known.names.filter(n => n.indexOf(name) >= 0); + if (hits.length === 1) return hits[0]; + if (hits.length > 1) + fail("'" + name + "' matches more than one case: " + hits.join(" ") + "\n" + + " Name one of them exactly, or use a substring that only one contains."); + fail("no case matches '" + name + "'. Known cases (from " + known.source + "):\n" + + " " + known.names.join(" ")); +} + +function resolveSide(name) { + if (SIDES.indexOf(name) >= 0) return name; + fail("'" + name + "' is not a side -- expected " + SIDES.join(" or ")); +} + +function hexPath(caseName, side) { + return path.join(outDir, caseName, side + ".hex"); +} + +function logPath(caseName, side) { + return path.join(outDir, caseName, side + ".log"); +} + +function buildHint(side) { + return side === "reference" + ? "npm run hwab -- build --v2-only --ref-switches \"\"" + : "npm run hwab -- build --candidate-only --v2-only"; +} + +function requireHex(caseName, side) { + const hex = hexPath(caseName, side); + let ok = false; + try { ok = fs.statSync(hex).isFile(); } catch (e) { ok = false; } + if (!ok) + fail("no " + side + " hex for " + caseName + " at " + hex + "\n" + + " Build it first: " + buildHint(side)); + return hex; +} + +// ---- child processes ------------------------------------------------------ + +// Never shell:true: the child is always node running a script whose path this +// process computed, and a path with a space in it must not be re-parsed by a +// shell. spawnSync with an argument array is also the form that behaves the +// same on Windows as it does elsewhere. +function run(script, args) { + const res = spawnSync(process.execPath, + [path.join(hwabDir, script)].concat(args), { stdio: "inherit" }); + if (res.error) { + process.stderr.write("hwab: could not run " + script + ": " + + res.error.message + "\n"); + return 1; + } + return res.status === null ? 1 : res.status; +} + +// ---- commands ------------------------------------------------------------- + +function cmdBuild(argv) { + return run("build-ab.js", argv); +} + +function cmdCapture(argv) { + let caseArg = null; + let side = null; + const passthrough = []; + + let soakMinutes = null; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--timeout") { + if (i + 1 >= argv.length) fail(arg + " needs a value"); + passthrough.push(arg, argv[++i]); + } else if (arg === "--soak") { + if (i + 1 >= argv.length) fail(arg + " needs a value"); + soakMinutes = argv[++i]; + if (!/^[1-9][0-9]*$/.test(soakMinutes)) + fail("--soak needs a positive whole number of minutes"); + passthrough.push(arg, soakMinutes); + } else if (arg.charAt(0) === "-") { + fail("unknown option: " + arg); + } else if (caseArg === null) { + caseArg = arg; + } else if (side === null) { + side = arg; + } else { + fail("unexpected argument: " + arg); + } + } + + if (caseArg === null) fail("capture needs a case name (see: hwab.js cases)"); + const caseName = resolveCase(caseArg); + side = side === null ? "candidate" : resolveSide(side); + + if (caseName === SOAK_CASE && soakMinutes === null) + fail("the soak program prints no verdict, so a plain capture of it can " + + "only time out.\n Give a duration: npm run hwab -- capture soak " + + side + " --soak "); + if (caseName !== SOAK_CASE && soakMinutes !== null) + fail("--soak is for the soak case; " + caseName + " prints HWAB PASS, " + + "not HWAB SOAK -- use a plain capture"); + + const hex = requireHex(caseName, side); + const log = logPath(caseName, side); + say("capture " + caseName + " " + side); + return run("run-capture.js", + [hex, log, "--expect", caseName].concat(passthrough)); +} + +function cmdDiff(argv) { + if (argv.length < 1) fail("diff needs a case name (see: hwab.js cases)"); + if (argv.length > 1) fail("unexpected argument: " + argv[1]); + const caseName = resolveCase(argv[0]); + + const missing = SIDES.filter(s => !fs.existsSync(logPath(caseName, s))); + if (missing.length) + fail("no " + missing.join(" and ") + " capture for " + caseName + ":\n" + + missing.map(s => " " + logPath(caseName, s)).join("\n") + "\n" + + " Capture it first: " + + missing.map(s => "npm run hwab -- capture " + caseName + " " + s).join(" ; ")); + + return run("diff-ab.js", [logPath(caseName, "candidate"), + logPath(caseName, "reference"), + "--case", caseName]); +} + +function cmdAb(argv) { + let caseArg = null; + const passthrough = []; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--timeout") { + if (i + 1 >= argv.length) fail("--timeout needs a value"); + passthrough.push(arg, argv[++i]); + } else if (arg.charAt(0) === "-") { + fail("unknown option: " + arg); + } else if (caseArg === null) { + caseArg = arg; + } else { + fail("unexpected argument: " + arg); + } + } + + if (caseArg === null) fail("ab needs a case name (see: hwab.js cases)"); + const caseName = resolveCase(caseArg); + if (caseName === SOAK_CASE) + fail("the soak program prints no pass/fail verdict, so there is " + + "nothing for ab to sequence.\n" + + " Run each side for a duration, then compare:\n" + + " npm run hwab -- capture soak candidate --soak \n" + + " npm run hwab -- capture soak reference --soak \n" + + " npm run hwab -- diff soak"); + // Both hexes are checked up front: the run flashes the board twice, and + // discovering the second hex is missing after the first flash wastes the + // whole capture. + for (const side of SIDES) requireHex(caseName, side); + + // One board, so the sides run in turn; run-capture.js does the flash and + // waits for the board to come back each time. + const stages = SIDES.map(side => ({ + name: "capture " + side, + run: () => run("run-capture.js", + [hexPath(caseName, side), logPath(caseName, side), + "--expect", caseName].concat(passthrough)) + })); + stages.push({ + name: "diff", + run: () => run("diff-ab.js", + [logPath(caseName, "candidate"), logPath(caseName, "reference"), + "--case", caseName]) + }); + + for (const stage of stages) { + say("ab " + caseName + ": " + stage.name); + const code = stage.run(); + if (code !== 0) { + process.stderr.write("hwab: ab " + caseName + " FAILED at stage '" + + stage.name + "' (exit " + code + ")\n"); + return code; + } + } + say("ab " + caseName + ": OK -- both sides passed and the captures agree"); + return 0; +} + +function pad(s, width) { + return s.length >= width ? s : s + " ".repeat(width - s.length); +} + +function cmdCases() { + const known = knownCases(); + const headers = ["case", "cand.hex", "cand.log", "ref.hex", "ref.log"]; + const rows = known.names.map(name => [ + name, + fs.existsSync(hexPath(name, "candidate")) ? "yes" : "-", + fs.existsSync(logPath(name, "candidate")) ? "yes" : "-", + fs.existsSync(hexPath(name, "reference")) ? "yes" : "-", + fs.existsSync(logPath(name, "reference")) ? "yes" : "-" + ]); + + const widths = headers.map((h, i) => + rows.reduce((w, r) => Math.max(w, r[i].length), h.length)); + const line = cells => + cells.map((c, i) => pad(c, widths[i])).join(" ").replace(/ +$/, ""); + + process.stdout.write("hwab: cases from " + known.source + "\n"); + process.stdout.write("hwab: artifacts in " + outDir + "\n"); + process.stdout.write("\n"); + process.stdout.write(line(headers) + "\n"); + process.stdout.write(widths.map(w => "-".repeat(w)).join(" ") + "\n"); + for (const row of rows) process.stdout.write(line(row) + "\n"); + return 0; +} + +// ---- dispatch ------------------------------------------------------------- + +function main(argv) { + const cmd = argv[0]; + const rest = argv.slice(1); + + if (!cmd || cmd === "help" || cmd === "-h" || cmd === "--help") { + usage(process.stdout); + return 0; + } + + switch (cmd) { + case "build": return cmdBuild(rest); + case "capture": return cmdCapture(rest); + case "diff": return cmdDiff(rest); + case "ab": return cmdAb(rest); + case "cases": return cmdCases(); + } + + usage(process.stderr); + process.stderr.write("\nhwab: unknown command: " + cmd + "\n"); + return 2; +} + +process.exitCode = main(process.argv.slice(2)); diff --git a/tests/hw-ab/run-capture.js b/tests/hw-ab/run-capture.js new file mode 100644 index 000000000000..d6a0b2f48663 --- /dev/null +++ b/tests/hw-ab/run-capture.js @@ -0,0 +1,652 @@ +/* + * Flashes a hex to an attached micro:bit and captures its serial output. + * + * micro:bit boards speak USB-CDC through DAPLink. No pxt CLI command can attach + * to that (pxt console / hidserial are HF2-only), so the capture is done by + * setting the line discipline with the platform's own tool (stty / mode) and + * reading the device directly. + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const { spawnSync } = require("child_process"); + +const VOLUME_WAIT = 30; // seconds to wait for the board to appear +const REMOUNT_WAIT = 60; // seconds to wait for the board to come back after flashing +const UNMOUNT_GRACE = 15; // seconds to wait for the drive to go away while programming + +const USAGE = [ + "Usage: node tests/hw-ab/run-capture.js [--timeout ]", + " [--soak ] [--expect ]", + "", + "Copies to the attached micro:bit, waits for it to re-enumerate, opens its", + "USB-CDC serial device at 115200 8N1 raw, and appends everything it prints to", + ".", + "", + "Verdicts are read only from the first \"HWAB START\" banner onward: DAPLink", + "buffers serial while no host is reading, so the first bytes after opening the", + "port can belong to the previously flashed program -- including its PASS", + "banner. --expect additionally requires the banner (and the verdict) to", + "name that case.", + "", + "Normal mode (default, --timeout 120):", + " stops at the first \"HWAB PASS\" or \"ASSERT\" line, or at the timeout. A", + " timeout with no banner at all means the program never ran -- usually a flash", + " DAPLink rejected -- so the whole flash and capture is retried once, and only", + " the second result is reported. Budget twice the timeout for that case.", + " exit 0 HWAB PASS seen", + " exit 1 ASSERT seen (the failing line is printed)", + " exit 2 timeout with neither", + " exit 3 no board, no serial device, or more than one serial device", + "", + "Soak mode (--soak ):", + " captures for the whole duration and then checks whether HWAB SOAK lines were", + " still arriving at the end.", + " exit 0 still printing", + " exit 1 output stalled (the program panicked or hung)", + "", + "Platforms: macOS, Linux and Windows are detected automatically (where the drive", + "mounts, how the serial device is named, which tool sets the line settings).", + "When discovery needs help -- an unusual mount point, several serial devices --", + "override it:", + " MICROBIT_VOLUME path or drive of the mounted MICROBIT drive", + " MICROBIT_SERIAL serial device to read (/dev/cu.usbmodem1102, COM5, ...)", + "See the Platforms section of tests/hw-ab/README.md." +].join("\n"); + +function usage(stream) { + stream.write(USAGE + "\n"); +} + +function fail(msg) { + process.stderr.write("run-capture: " + msg + "\n"); + process.exit(3); +} + +// ---- arguments ------------------------------------------------------------ + +const argv = process.argv.slice(2); +if (argv.indexOf("-h") >= 0 || argv.indexOf("--help") >= 0) { + usage(process.stdout); + process.exit(0); +} +if (argv.length < 2) { + usage(process.stderr); + process.exit(3); +} + +const hex = argv[0]; +const logFile = argv[1]; +let timeout = 120; +let soakMin = 0; +let expectCase = ""; + +for (let i = 2; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--timeout") { + if (i + 1 >= argv.length) fail("--timeout needs a value"); + timeout = argv[++i]; + } else if (arg === "--soak") { + if (i + 1 >= argv.length) fail("--soak needs a value"); + soakMin = argv[++i]; + } else if (arg === "--expect") { + if (i + 1 >= argv.length) fail("--expect needs a value"); + expectCase = argv[++i]; + } else { + usage(process.stderr); + fail("unknown argument: " + arg); + } +} + +let hexIsFile = false; +try { hexIsFile = fs.statSync(hex).isFile(); } catch (e) { hexIsFile = false; } +if (!hexIsFile) fail("no such hex: " + hex); +if (!/^[0-9]+$/.test(String(timeout))) fail("--timeout must be an integer"); +if (!/^[0-9]+$/.test(String(soakMin))) fail("--soak must be an integer"); +timeout = parseInt(timeout, 10); +soakMin = parseInt(soakMin, 10); +if (soakMin > 0) timeout = soakMin * 60; + +// ---- platform ------------------------------------------------------------- + +// Platform differences live here: where the mass-storage volume appears, how +// the USB-CDC serial device is named, and which tool sets the line settings. +// MICROBIT_VOLUME / MICROBIT_SERIAL override the discovery. +const platform = process.platform; +const isWindows = platform === "win32"; + +const WIN_VOLUME_PS = + "(Get-CimInstance Win32_LogicalDisk | " + + "Where-Object {$_.VolumeName -eq 'MICROBIT'}).DeviceID"; +const WIN_SERIAL_PS = + "(Get-CimInstance Win32_SerialPort | " + + "Where-Object {$_.Name -match 'mbed|DAPLink|USB Serial'}) | " + + "ForEach-Object DeviceID"; + +let staticVolumes = null; // fixed candidate list, or null when discovery is dynamic +let volumeDesc = ""; // what the "no volume" message says was looked at +let devDesc = ""; // what the serial messages call the device +let sttyFlag = ""; + +if (platform === "darwin") { + staticVolumes = ["/Volumes/MICROBIT"]; + devDesc = "/dev/cu.usbmodem* device"; + sttyFlag = "-f"; +} else if (platform === "linux") { + const user = process.env.USER || "root"; + staticVolumes = ["/media/" + user + "/MICROBIT", + "/run/media/" + user + "/MICROBIT", + "/media/MICROBIT"]; + devDesc = "/dev/ttyACM* device"; + sttyFlag = "-F"; +} else if (isWindows) { + devDesc = "mbed/DAPLink/USB Serial COM port"; +} else { + fail("unsupported platform '" + platform + + "' -- see the Platforms section of tests/hw-ab/README.md"); +} + +function powershellLines(script) { + const res = spawnSync("powershell", ["-NoProfile", "-Command", script], + { encoding: "utf8" }); + if (res.error || res.status !== 0) return []; + return String(res.stdout || "").split(/\r?\n/) + .map(s => s.trim()).filter(s => s.length > 0); +} + +// "E:" from Windows tools names a drive plus its current directory; "E:\" is +// the root, which is what a path join and an existence test need. +function driveRoot(v) { + return /^[A-Za-z]:$/.test(v) ? v + "\\" : v; +} + +if (process.env.MICROBIT_VOLUME) { + staticVolumes = [isWindows + ? driveRoot(process.env.MICROBIT_VOLUME) + : process.env.MICROBIT_VOLUME]; +} +volumeDesc = staticVolumes + ? staticVolumes.join(" ") + : "the drive whose volume label is MICROBIT"; + +function volumeCandidates() { + if (staticVolumes) return staticVolumes; + return powershellLines(WIN_VOLUME_PS).map(driveRoot); +} + +function isDir(p) { + try { return fs.statSync(p).isDirectory(); } catch (e) { return false; } +} + +function findVolume() { + for (const v of volumeCandidates()) { + if (isDir(v)) return v; + } + return null; +} + +// ---- small helpers -------------------------------------------------------- + +function nowSeconds() { + return Math.floor(Date.now() / 1000); +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function waitForVolume(seconds) { + const deadline = nowSeconds() + seconds; + while (nowSeconds() < deadline) { + const v = findVolume(); + if (v) return v; + await sleep(1000); + } + return null; +} + +// The log is read back as latin1 so that undecodable serial noise survives a +// round trip and is reported byte for byte. +function readLog() { + try { return fs.readFileSync(logFile, "latin1"); } catch (e) { return ""; } +} + +function splitLines(text) { + const lines = text.split("\n"); + if (lines.length && lines[lines.length - 1] === "") lines.pop(); + return lines; +} + +function firstMatch(text, needle) { + for (const line of splitLines(text)) { + if (line.indexOf(needle) >= 0) return line; + } + return null; +} + +function countMatches(text, needle) { + let n = 0; + for (const line of splitLines(text)) { + if (line.indexOf(needle) >= 0) n++; + } + return n; +} + +// Verdict lines are only meaningful from the flashed program's own START +// banner onward: the port can open onto buffered output from the previously +// flashed program (DAPLink keeps transmitting into its buffer while no host +// reads), and that output can contain a PASS banner. Returns null until the +// banner has been seen. +function startNeedle() { + return "HWAB START" + (expectCase ? " " + expectCase : ""); +} + +function verdictBody(content) { + const lines = splitLines(content); + for (let i = 0; i < lines.length; i++) { + if (lines[i].indexOf(startNeedle()) >= 0) + return lines.slice(i).join("\n"); + } + return null; +} + +function firstHwabLine(text) { + return firstMatch(text, "HWAB "); +} + +function countLines(text) { + let n = 0; + for (let i = 0; i < text.length; i++) if (text[i] === "\n") n++; + return n; +} + +// The trailing bytes of the last n lines, exactly as `tail -n ` emits them. +function tailBytes(text, n) { + let i = text.length - 1; + if (i >= 0 && text[i] === "\n") i--; + let count = 0; + for (; i >= 0; i--) { + if (text[i] === "\n") { + count++; + if (count === n) return text.slice(i + 1); + } + } + return text; +} + +function writeRaw(stream, text) { + stream.write(Buffer.from(text, "latin1")); +} + +// ---- capture -------------------------------------------------------------- + +let devFd = null; +let logFd = null; +// Serial arriving in the first moments after the port opens is buffered +// history, not the flashed program (which idles for 10s post-reset; the port +// opens within ~6s of it). 3s of discard ends safely inside that quiet window. +const STALE_DISCARD_MS = 3000; +let discardUntil = 0; +let pumpTimer = null; +let stopped = false; +let volumePath = null; // volume the current attempt flashed through + +// DAPLink writes FAIL.TXT to the volume when it rejects an incoming hex (a +// decode failure surfaces as panic 521), and the program then never runs. +// Returns the file's text, or null when there is none or the volume has gone. +function readFailTxt() { + if (!volumePath) return null; + let name = null; + try { + for (const entry of fs.readdirSync(volumePath)) { + if (/^fail\.txt$/i.test(entry)) { name = entry; break; } + } + } catch (e) { + return null; // unmounted, or not readable + } + if (!name) return null; + try { + return fs.readFileSync(path.join(volumePath, name), "latin1"); + } catch (e) { + return null; + } +} + +function cleanup() { + stopped = true; + if (pumpTimer !== null) { clearInterval(pumpTimer); pumpTimer = null; } + if (logFd !== null) { try { fs.closeSync(logFd); } catch (e) { /* closed */ } logFd = null; } + if (devFd !== null) { try { fs.closeSync(devFd); } catch (e) { /* closed */ } devFd = null; } +} + +const READ_BUF = Buffer.allocUnsafe(4096); + +// Drains whatever the device has buffered into the log, so that a poll of the +// log file sees bytes as they arrive. The read must not block: an outstanding +// blocking read holds a worker thread that node waits for on the way out, which +// would leave the script hanging past its own verdict. On POSIX the fd carries +// O_NONBLOCK and a quiet port raises EAGAIN; on Windows the mode-configured +// handle returns what it has rather than waiting for a full buffer. +function pump() { + while (!stopped && devFd !== null && logFd !== null) { + let bytes; + try { + bytes = fs.readSync(devFd, READ_BUF, 0, READ_BUF.length, null); + } catch (e) { + return; // EAGAIN while the port is quiet, or the device went away + } + if (bytes <= 0) return; + // DAPLink flushes serial it buffered while no host was reading, and + // that history can include a previous run of the SAME case -- banners, + // asserts and all -- which no content check can tell apart. But the + // generated programs are silent for their first 10s after the + // post-flash reset, and the port opens well inside that window, so + // anything arriving this early is necessarily history. + if (Date.now() < discardUntil) continue; + try { fs.writeSync(logFd, READ_BUF, 0, bytes); } catch (e) { return; } + } +} + +function startReader() { + // Frequently enough that a burst at 115200 baud cannot outrun the tty's own + // input buffer between polls. Clearing `stopped` is what lets a second + // attempt read after the first one's cleanup. + stopped = false; + discardUntil = Date.now() + STALE_DISCARD_MS; + pumpTimer = setInterval(pump, 50); +} + +// One full flash-and-capture cycle: copy the hex, wait for the board to come +// back, open its serial device and read until a verdict or the timeout. Returns +// the outcome for report(). Host and board problems still exit through fail() +// here -- a second attempt at a missing board finds the same missing board. +async function attempt() { + volumePath = await waitForVolume(VOLUME_WAIT); + if (!volumePath) { + process.stderr.write( + "run-capture: no MICROBIT volume within " + VOLUME_WAIT + "s (looked at:\n" + + volumeDesc + ").\n" + + "\n" + + "Plug a micro:bit into USB and wait for the MICROBIT drive to mount. If the\n" + + "board is plugged in and the drive is absent, the cable may be charge-only, or\n" + + "the board may be in a bad state -- unplug it, hold the reset button while\n" + + "plugging it back in, and look for a MAINTENANCE drive (that means DAPLink is\n" + + "in bootloader mode and needs its firmware reflashed). If the drive is mounted\n" + + "somewhere this script did not look, point it there with MICROBIT_VOLUME.\n"); + process.exit(3); + } + + process.stdout.write("run-capture: flashing " + hex + " -> " + volumePath + "\n"); + try { + fs.copyFileSync(hex, path.join(volumePath, path.basename(hex))); + } catch (e) { + fail("copy to " + volumePath + " failed"); + } + if (!isWindows) { + // Best effort: push the copy out of the page cache before the drive + // goes away. Windows has no equivalent that is worth spawning. + spawnSync("sync", [], { stdio: "ignore" }); + } + + // DAPLink unmounts the drive while it programs, then remounts and resets the + // board. Wait for the drive to go away (best effort -- flashing a small hex + // can be quicker than this poll) and then for it to come back. + const graceDeadline = nowSeconds() + UNMOUNT_GRACE; + while (nowSeconds() < graceDeadline && isDir(volumePath)) await sleep(1000); + if (!await waitForVolume(REMOUNT_WAIT)) + fail(volumePath + " did not remount within " + REMOUNT_WAIT + "s after flashing"); + process.stdout.write("run-capture: flashed, board remounted\n"); + + // The generated programs idle for ~10s before printing, which is the window + // this discovery and the line-settings setup have to fit into. + let devs; + if (process.env.MICROBIT_SERIAL) { + devs = [process.env.MICROBIT_SERIAL]; + // A COM port is not a filesystem entry, so it can only be checked by + // opening it, which happens below. + if (!isWindows && !fs.existsSync(devs[0])) + fail("MICROBIT_SERIAL=" + process.env.MICROBIT_SERIAL + " does not exist"); + } else if (isWindows) { + devs = powershellLines(WIN_SERIAL_PS); + } else { + const prefix = platform === "darwin" ? "cu.usbmodem" : "ttyACM"; + let entries = []; + try { entries = fs.readdirSync("/dev"); } catch (e) { entries = []; } + devs = entries.filter(e => e.indexOf(prefix) === 0).sort() + .map(e => "/dev/" + e); + } + + if (devs.length === 0) { + process.stderr.write( + "run-capture: no " + devDesc + " found.\n" + + "\n" + + "The MICROBIT drive mounted, so DAPLink's mass storage is up but its serial\n" + + "port is not. Unplug and replug the board; if that does not help the DAPLink\n" + + "interface firmware is likely out of date and needs updating. On Linux, also\n" + + "check that your user may open serial devices (typically the dialout group).\n"); + process.exit(3); + } + if (devs.length > 1) { + process.stderr.write("run-capture: more than one " + devDesc + ":\n"); + for (const d of devs) process.stderr.write(" " + d + "\n"); + process.stderr.write( + "\n" + + "Only one board may be attached, otherwise the capture could read the wrong one.\n" + + "Unplug the other USB serial devices and rerun, or name the right device with\n" + + "MICROBIT_SERIAL.\n"); + process.exit(3); + } + + const dev = devs[0]; + process.stdout.write("run-capture: serial " + dev + "\n"); + + if (isWindows) { + // Windows keeps the port configuration with the port, so `mode` is + // applied before the handle is opened. The \\.\ prefix is required for + // COM10 and above and harmless below it. + const port = dev.replace(/^\\\\\.\\/, ""); + const cmd = "mode " + port + + ": BAUD=115200 PARITY=n DATA=8 STOP=1 to=off xon=off dtr=on rts=on"; + const res = spawnSync(cmd, { shell: true, stdio: "ignore" }); + if (res.error || res.status !== 0) + fail("mode failed on " + dev + " (is another program holding the port?)"); + try { + devFd = fs.openSync("\\\\.\\" + port, "r"); + } catch (e) { + devFd = null; + fail("cannot open " + dev); + } + } else { + // Hold the device open for the whole capture. Terminal settings reset + // when the last open of a tty closes, and stty performs its own + // open/close -- so without this held fd the 115200/raw settings would be + // gone by the time the reader opens the port, and it would read garbage + // at the default rate. The fd is opened first so the settings apply to + // the held-open port, and every byte is read from this same fd. + try { + devFd = fs.openSync(dev, fs.constants.O_RDONLY | fs.constants.O_NONBLOCK); + } catch (e) { + devFd = null; + fail("cannot open " + dev); + } + const res = spawnSync("stty", [sttyFlag, dev, "115200", "raw", "-echo"], + { stdio: "ignore" }); + if (res.error || res.status !== 0) + fail("stty failed on " + dev + " (is another program holding the port?)"); + } + + fs.mkdirSync(path.dirname(path.resolve(logFile)), { recursive: true }); + logFd = fs.openSync(logFile, "w"); + startReader(); + + const deadline = nowSeconds() + timeout; + + if (soakMin > 0) { + process.stdout.write( + "run-capture: soak for " + soakMin + " minute(s) -> " + logFile + "\n"); + // Snapshot the SOAK line count 60s before the end; if it has not grown + // by the deadline the program stopped printing, which is how a leak ends. + let checkpoint = deadline - 60; + if (checkpoint <= nowSeconds()) checkpoint = deadline - Math.floor(timeout / 4); + // The program's banner appears within seconds of the startup delay. A + // soak that has printed nothing by this grace deadline never started + // (a failed flash), which must not consume the whole soak window -- + // returning a no-banner timeout here hands it to the re-flash retry. + const bannerGrace = nowSeconds() + 90; + let sawBanner = false; + let before = null; + while (nowSeconds() < deadline) { + if (!sawBanner) { + sawBanner = verdictBody(readLog()) !== null; + if (!sawBanner && nowSeconds() >= bannerGrace) { + cleanup(); + const content = readLog(); + return { kind: "timeout", content: content, noBanner: true }; + } + } + if (before === null && nowSeconds() >= checkpoint) + before = countMatches(verdictBody(readLog()) || "", "HWAB SOAK"); + await sleep(5000); + } + cleanup(); + const content = verdictBody(readLog()) || ""; + return { + kind: "soak", + before: before === null ? 0 : before, + after: countMatches(content, "HWAB SOAK"), + content: content + }; + } + + process.stdout.write( + "run-capture: capturing up to " + timeout + "s -> " + logFile + "\n"); + const passNeedle = "HWAB PASS" + (expectCase ? " " + expectCase : ""); + while (nowSeconds() < deadline) { + const body = verdictBody(readLog()); + if (body !== null) { + const assertLine = firstMatch(body, "ASSERT "); + if (assertLine !== null) { + cleanup(); + return { kind: "assert", line: assertLine }; + } + const passLine = firstMatch(body, passNeedle); + if (passLine !== null) { + cleanup(); + return { kind: "pass", line: passLine }; + } + } + await sleep(1000); + } + + cleanup(); + const timedOut = readLog(); + return { + kind: "timeout", + content: timedOut, + noBanner: verdictBody(timedOut) === null + }; +} + +function reportSoak(result) { + const content = result.content; + process.stdout.write("run-capture: HWAB SOAK lines: " + result.before + + " at checkpoint, " + result.after + " at end\n"); + const assertLine = firstMatch(content, "ASSERT "); + if (assertLine !== null) { + process.stderr.write("run-capture: FAIL -- assertion during soak:\n"); + writeRaw(process.stderr, assertLine + "\n"); + process.exit(1); + } + if (result.after > result.before) { + process.stdout.write( + "run-capture: PASS -- still printing after " + soakMin + " minute(s)\n"); + writeRaw(process.stdout, tailBytes(content, 1)); + process.exit(0); + } + process.stderr.write( + "run-capture: FAIL -- output stalled before the end of the soak.\n" + + "The board stopped printing HWAB SOAK lines, which is what an out-of-memory\n" + + "panic looks like from the host side (the LED matrix will show a sad face and a\n" + + "number; 020/021 are the memory panics). Last captured lines:\n"); + writeRaw(process.stderr, tailBytes(content, 5)); + process.exit(1); +} + +// Turns an attempt's outcome into the script's output and exit code. +function report(result) { + if (result.kind === "soak") reportSoak(result); + if (result.kind === "assert") { + process.stderr.write("run-capture: FAIL -- assertion failed on device:\n"); + writeRaw(process.stderr, result.line + "\n"); + process.stderr.write("run-capture: the board shows a sad face and 45.\n"); + process.exit(1); + } + if (result.kind === "pass") { + writeRaw(process.stdout, + "run-capture: PASS -- " + result.line.replace(/\r/g, "") + "\n"); + process.exit(0); + } + + const content = result.content; + const logLines = countLines(content); + const logBytes = content.length; + process.stderr.write( + "run-capture: TIMEOUT -- no HWAB PASS and no ASSERT within " + timeout + "s.\n" + + "Captured " + logLines + " line(s), " + logBytes + " byte(s). Last lines:\n"); + writeRaw(process.stderr, tailBytes(content, 5)); + if (result.noBanner) { + process.stderr.write( + "\nNo \"" + startNeedle() + "\" banner was seen, so no verdict could be read.\n"); + const failText = readFailTxt(); + if (failText !== null) { + process.stderr.write( + "DAPLink left FAIL.TXT on " + volumePath + ": it rejected the hex, so the\n" + + "program never ran.\n"); + writeRaw(process.stderr, failText.replace(/[\r\n]+$/, "") + "\n"); + } + const stray = firstHwabLine(content); + if (stray !== null) { + process.stderr.write( + "HWAB output from another program was ignored (stale serial buffer from\n" + + "the previously flashed program, or the flash did not take). First\n" + + "ignored line:\n"); + writeRaw(process.stderr, stray + "\n"); + } + } + if (logLines === 0 && logBytes > 0) { + process.stderr.write( + "\n" + + "Bytes arrived but formed no complete line -- usually undecodable garbage from\n" + + "wrong serial line settings (baud or framing). Check that nothing else has the\n" + + "port open and reconfigures it, and that the device really is the micro:bit.\n"); + } + process.stderr.write( + "\n" + + "If the log is empty the capture attached after the program had already run, or\n" + + "the board reset. If the board shows a sad face the program panicked before\n" + + "reaching its verdict; read the number off the matrix (999 = unhandled throw,\n" + + "020/021 = out of memory).\n"); + process.exit(2); +} + +async function main() { + let result = await attempt(); + // A timeout with no banner means the program never spoke, and the usual + // cause is a rejected flash (DAPLink error 521), which is intermittent and + // clears on a re-flash. A timeout after a banner is a real verdict about a + // program that did run, so it is reported as it stands. + if (result.kind === "timeout" && result.noBanner) { + process.stderr.write("run-capture: no banner -- re-flashing once\n"); + result = await attempt(); + } + report(result); +} + +process.on("exit", cleanup); + +main().catch(err => { + cleanup(); + fail(err && err.message ? err.message : String(err)); +}); diff --git a/tests/thumb-test/README.md b/tests/thumb-test/README.md new file mode 100644 index 000000000000..11f96be3f02a --- /dev/null +++ b/tests/thumb-test/README.md @@ -0,0 +1,105 @@ +# thumb-test + +Compiles PXT TypeScript programs to native ARM Thumb in-process and asserts on +the generated assembly listing. + + gulp testthumb # also part of gulp test + +Runs offline, no C++ toolchain needed: the whole native path -- emitter, +register allocation, peephole, Thumb assembler -- runs in-process. + +Catches: which helpers, thunks and dispatch sequences codegen chose; invalid +instruction streams (the assembler fails the compile); silent code-size +swings. Cannot catch: runtime behavior -- nothing executes here, that is +`tests/hw-ab/` -- and multi-variant packaging (the universal-hex combiner is +not loaded; output is single-variant). + +## How it works + +Native compilation needs `opts.extinfo.hexinfo`; here it comes from a fixture: +`fixtures/microbit-mbcodal.json.gz` is a gzipped `CompileOptions` environment +captured from a real native micro:bit build (the mbcodal / V2 variant), holding +`target`, `extinfo` (with `hexinfo`), `fileSystem` (all dependency TS sources), +`sourceFiles` and `jres`. + +For each case program the runner parses a fresh copy of the fixture, +substitutes the case text as `main.ts` in `fileSystem`, forces +`target.isNative`, `target.nativeType = "thumb"` and `target.switches.size`, +then calls `pxtc.compile`. The result must succeed, and the listing in +`res.outfiles["binary.asm"]` is handed to the case's expectation function. + +Compile success is itself a meaningful assertion: the in-process assembler +turns invalid instructions and stack imbalances into code-9200 diagnostics. + +## Adding a case + +1. Drop a `.ts` program in `cases/`. Use core language constructs only -- + classes, interfaces, strings, arrays, maps, arithmetic. Avoid event loops and + device APIs; the program must terminate. Feed every value into a module-level + accumulator so the optimizer cannot drop the code under test. +2. Add an entry keyed by the file name to `asmChecks` in `asmchecks.ts`. A case + with no entry fails, so an expectation is never accidentally omitted. + +Helpers available to expectations: `hasLabel`, `mentions`, `countMatches`, +`codeSize`, `hexSize`, `assertAbsent`, `assertNoMatch`, `assertAtLeast`, +`assertWithin`. + +Size assertions use `codeSize`, which reads the generated-code byte count from +the size stats header. That number tracks codegen; `hexSize` is dominated by the +fixed runtime image and moves very little, so it is only useful as a +sanity check. + +## Regenerating the fixture + +The fixture is a self-consistent snapshot; it does not track pxt-microbit +releases, and a target version bump alone does not require regeneration. +Regenerate only when: + +- the compiler emits calls to a runtime function the snapshot does not carry + (fails loudly at hex setup with the missing function name) +- the shape of `CompileOptions`/`extinfo` consumed by `pxtc.compile` changes + (fails loudly when the replayed options are rejected) +- a test case needs core APIs or runtime behavior newer than the snapshot + +Codegen changes in this repo never require it: the current compiler always +runs against the fixture, so compiler changes are exercised regardless of the +fixture's age. The `meta` block inside the fixture records the capture date, +target version and extinfo sha for diagnosis. + +Requires a sibling `pxt-microbit` checkout that is npm-linked to this one +(`node_modules/pxt-core -> ../../pxt`) and has a populated `built/hexcache/`. + + npm run build + node tests/thumb-test/scripts/capture-fixture.js + +Set `PXT_TARGET_DIR` to capture from a checkout that is not the sibling +`../pxt-microbit`. + +If the capture build fails with `Package not installed: `, the target's +`upgrades` rules injected a dependency into the scratch project. The script +guards against this by stamping the current target version into the scratch +`pxt.json` (`targetVersions.target`), which gates off rules aimed at older +projects; a rule not gated on version would need a matching dependency added to +`makeScratchProject`. + +The script creates a scratch project under +`/projects/thumb-fixture`, loads the compiled CLI bundle +(`built/pxt.js`), patches `ts.pxtc.compile` to intercept the options the CLI +assembles, and drives the CLI's own `build` command with +`PXT_COMPILE_SWITCHES=csv---mbcodal` (a variant selector, not a real switch) +to pin the single mbcodal variant. The hex runtime is resolved from +`built/hexcache/`, so nothing is downloaded or compiled in C++. The script +prints the fixture path and size, and drops `extinfo.compileData`, +`generatedFiles`, `extensionFiles` and `otherMultiVariants`, none of which +`pxtc.compile` reads. + +## Coverage + +This layer is the middle of three that cover condition compilation and +interface dispatch. The coverage map for all of them -- which failure mode is +caught where, what each case program and each semantic test file exercises, and +where a new test belongs -- is +`tests/compile-test/lang-test0/README-codegen.md`. Read it before adding a case +here: some things belong in the JS-executed layer (`gulp testlang`) or on +hardware (`tests/hw-ab/`) instead, and it also lists what none of the layers +cover. diff --git a/tests/thumb-test/asmchecks.ts b/tests/thumb-test/asmchecks.ts new file mode 100644 index 000000000000..c5716b42f0cb --- /dev/null +++ b/tests/thumb-test/asmchecks.ts @@ -0,0 +1,191 @@ +/// + +import * as chai from "chai"; + +/** + * Expectations over the ARM Thumb assembly listing produced for each case + * program in tests/thumb-test/cases. + * + * A case file named .ts is checked by the entry keyed ".ts" in + * `asmChecks` below. A case with no entry fails, so that adding a program + * without an expectation is not silently a no-op. + */ + +export type AsmCheck = (asm: string, res: pxtc.CompileResult) => void; + +// --- helpers ------------------------------------------------------------- + +/** True when `name` appears as a label definition, e.g. "_main___P1:". */ +export function hasLabel(asm: string, name: string): boolean { + return new RegExp("^\\s*" + escapeRegExp(name) + ":", "m").test(asm); +} + +/** True when `name` appears anywhere in the listing as a whole token. */ +export function mentions(asm: string, name: string): boolean { + return new RegExp("(^|[^A-Za-z0-9_])" + escapeRegExp(name) + "([^A-Za-z0-9_]|$)").test(asm); +} + +/** Number of matches of a global regex in the listing. */ +export function countMatches(asm: string, re: RegExp): number { + const g = re.global ? re : new RegExp(re.source, re.flags + "g"); + const m = asm.match(g); + return m ? m.length : 0; +} + +/** + * Bytes of code generated for the program, from the size stats header that + * target.switches.size adds to the listing. This excludes the fixed runtime + * image, so it tracks codegen rather than the size of the hex template. + */ +export function codeSize(asm: string): number { + const m = /^; generated code sizes \(bytes\): (\d+)/m.exec(asm); + chai.assert(!!m, "no code size stats in listing (is target.switches.size set?)"); + return parseInt(m[1], 10); +} + +/** Bytes of the emitted Intel hex text. */ +export function hexSize(res: pxtc.CompileResult): number { + const hex = res.outfiles[pxtc.BINARY_HEX]; + chai.assert(!!hex, "no " + pxtc.BINARY_HEX + " in compile result"); + return hex.length; +} + +/** Fails if any of `names` occurs in the listing. */ +export function assertAbsent(asm: string, names: string[]) { + for (const n of names) + chai.assert(!mentions(asm, n), "listing unexpectedly mentions " + n); +} + +/** Fails if `re` matches the listing. */ +export function assertNoMatch(asm: string, re: RegExp, what: string) { + const n = countMatches(asm, re); + chai.assert(n === 0, "listing unexpectedly contains " + n + " " + what); +} + +/** Fails unless `re` matches the listing at least `min` times. */ +export function assertAtLeast(asm: string, re: RegExp, min: number, what: string) { + const n = countMatches(asm, re); + chai.assert(n >= min, "expected at least " + min + " " + what + ", found " + n); +} + +/** + * Fails unless `actual` is within `pct` percent of `expected`. Bands are wide + * on purpose: they catch silent large regressions in either direction without + * turning every codegen tweak into a test edit. + */ +export function assertWithin(actual: number, expected: number, pct: number, what: string) { + const lo = Math.floor(expected * (1 - pct / 100)); + const hi = Math.ceil(expected * (1 + pct / 100)); + chai.assert(actual >= lo && actual <= hi, + what + " is " + actual + ", outside the band " + lo + ".." + hi + + " (baseline " + expected + ", +/-" + pct + "%)"); +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +// --- per-case expectations ----------------------------------------------- +// +// The *baseline cases pin how the compiler in this tree lowers specific +// constructs, partly by asserting which specialized helpers are ABSENT from +// the listing. They are the reference half of a differential pair: work that +// specializes those constructs flips or extends these expectations alongside +// its codegen change, and a mismatch in either direction is the signal (the +// specialization silently stopped firing, or it appeared where it was not +// expected). The helper names and label patterns asserted absent here are +// therefore a naming contract with that work -- renaming an emitted helper or +// thunk label means updating the patterns below in the same change. + +export const asmChecks: pxt.Map = { + + "boolbaseline.ts": (asm) => { + // Conditions are materialized as tagged values and narrowed by a call + // into the runtime at each test site. + assertAtLeast(asm, /bl numops::toBoolDecr/g, 12, "calls to numops::toBoolDecr"); + + // Helpers introduced by boolean condition lowering. Absent here. + assertAbsent(asm, [ + "_numops_toBool", + "_numops_toBoolDecr", + "_pxt_fromBool", + "_pxt_boolean_bang", + ]); + }, + + "ifacebaseline.ts": (asm) => { + // The program does reach the generic interface/map runtime entries. + assertAtLeast(asm, /bl pxtrt::(mapGet|mapSet|lookupMapKey)/g, 4, + "generic map runtime calls"); + + // Specializations introduced by interface dispatch specialization. + // Absent here: every access goes through the generic runtime path. + assertAbsent(asm, [ + "_pxt_map_set_by_string", + ]); + assertNoMatch(asm, /ldfldchk_/g, "checked-field-load thunks (ldfldchk_)"); + assertNoMatch(asm, /ifacecall\d+_.*_i\d+/g, "interface call thunks"); + assertNoMatch(asm, /mapset_i/g, "specialized map-store thunks (mapset_i)"); + assertNoMatch(asm, /^\s*\S*_iface:/gm, "_iface: proc labels"); + }, + + "fieldbaseline.ts": (asm) => { + // Checked field loads are emitted inline here: a call into the class's + // validate helper immediately followed by the load. QzCell declares two + // fields, so their offsets are fixed at #4 (qzTally) and #8 (qzSpare), + // and the offset in the sequence is what tells the two apart. The + // floors are the exact read-site counts in the case program -- 6 above + // the count gate and 4 below it -- so a read that stops taking the + // checked path fails here instead of quietly making the case vacuous. + assertAtLeast(asm, /bl _inst_QzCell\S*_validate\S*\n\s*ldr r0, \[r0, #4\]/g, 6, + "inline checked loads of qzTally"); + assertAtLeast(asm, /bl _inst_QzCell\S*_validate\S*\n\s*ldr r0, \[r0, #8\]/g, 4, + "inline checked loads of qzSpare"); + + // Count-gated checked-field-load specialization. Absent here: qzTally + // is calibrated to sit above the gate, so a thunk of any kind means the + // specialization started firing in this tree. + assertNoMatch(asm, /ldfldchk_/g, "checked-field-load thunks (ldfldchk_)"); + }, + + "sizebaseline.ts": (asm, res) => { + chai.assert(hexSize(res) > 0, "empty hex output"); + // Baseline measured on this fixture; see assertWithin on why the band + // is wide. + assertWithin(codeSize(asm), 5592, 15, "generated code size"); + }, +}; + +// --- external case programs ---------------------------------------------- + +/** + * Semantic case programs that live outside tests/thumb-test/cases. These are + * lang-test0 files that the compile-test suite executes on the JS backend; + * compiling them here proves the same source also survives the native emitter + * and the thumb assembler, and makes them available as an on-device payload. + * + * Keys are paths relative to the repository root. The runner prepends the + * lang-test0 prelude, which supplies assert() and msg(). + * + * The expectations are deliberately light. These programs assert language + * semantics rather than codegen shape, so the check here is only that the + * emitter produced real code for them. + */ +export const externalCases: pxt.Map = { + + "tests/compile-test/lang-test0/54conditiontruthiness.ts": (asm) => { + chai.assert(codeSize(asm) > 0, "no code generated"); + }, + + "tests/compile-test/lang-test0/55conditionlowering.ts": (asm) => { + chai.assert(codeSize(asm) > 0, "no code generated"); + }, + + "tests/compile-test/lang-test0/56ifacedispatch.ts": (asm) => { + chai.assert(codeSize(asm) > 0, "no code generated"); + }, + + "tests/compile-test/lang-test0/57defaultparamdispatch.ts": (asm) => { + chai.assert(codeSize(asm) > 0, "no code generated"); + }, +}; diff --git a/tests/thumb-test/cases/boolbaseline.ts b/tests/thumb-test/cases/boolbaseline.ts new file mode 100644 index 000000000000..80e496819500 --- /dev/null +++ b/tests/thumb-test/cases/boolbaseline.ts @@ -0,0 +1,28 @@ +// Exercises conditions in if/while, &&, ||, and ! over numbers, booleans, +// strings and arrays. Everything feeds a module-level accumulator so no +// condition site can be dropped as unused. + +let acc = 0 +let flag = true +let text = "ab" +let n = 3 +let arr = [1, 2] + +function classify(v: number, on: boolean, txt: string): number { + let r = 0 + if (v > 2 && on) r += 1 + if (!on || v < 0) r += 2 + while (v > 0) { + r += v + v -= 1 + } + if (txt) r += 4 + if (!txt) r += 8 + return r +} + +acc += classify(n, flag, text) +acc += classify(0, !flag, "") +if (arr.length > 1) acc += 16 +if (flag && arr.length > 0 && text.length > 1) acc += 32 +if (!flag || !text) acc += 64 diff --git a/tests/thumb-test/cases/fieldbaseline.ts b/tests/thumb-test/cases/fieldbaseline.ts new file mode 100644 index 000000000000..5b038316a295 --- /dev/null +++ b/tests/thumb-test/cases/fieldbaseline.ts @@ -0,0 +1,48 @@ +// Exercises checked field loads through class-typed receivers. A field read +// takes the checked path whenever its receiver is not `this`, and the +// count-gated checked-field-load specialization keys on how many static read +// sites one class field has, so `qzTally` sits above a plausible gate and +// `qzSpare` below it. Everything feeds a module-level accumulator so no read +// site can be dropped as unused. +// +// Constraints this program must keep, or the case stops testing what it names: +// +// - The class implements no interface and has no subclass, so neither field is +// treated as overridden. An overridden field routes its reads through +// interface dispatch instead of the checked field path -- that is the shape +// `ifacebaseline.ts` covers, and it would make this case vacuous. +// - No receiver is `this`. A `this`-receiver read is unchecked and is not +// counted, so moving a read into a method silently drops it from the count. +// - The field names are unique to this program. Adding or removing one read of +// either name moves it relative to the gate. + +class QzCell { + qzTally: number + qzSpare: number + constructor(tally: number, spare: number) { + this.qzTally = tally + this.qzSpare = spare + } +} + +let acc = 0 +let cellA = new QzCell(1, 2) +let cellB = new QzCell(3, 4) +let cellC = new QzCell(5, 6) + +function addTally(cell: QzCell) { + acc += cell.qzTally +} + +// qzTally: 6 checked read sites, above the gate. +acc += cellA.qzTally +acc += cellB.qzTally +acc += cellC.qzTally +acc += cellA.qzTally + cellB.qzTally +addTally(cellC) + +// qzSpare: 4 checked read sites, below the gate. +acc += cellA.qzSpare +acc += cellB.qzSpare +acc += cellC.qzSpare +acc += cellA.qzSpare diff --git a/tests/thumb-test/cases/ifacebaseline.ts b/tests/thumb-test/cases/ifacebaseline.ts new file mode 100644 index 000000000000..a5a14823c72f --- /dev/null +++ b/tests/thumb-test/cases/ifacebaseline.ts @@ -0,0 +1,71 @@ +// Exercises interface-typed field reads, interface-dispatched method calls, +// repeated object-literal keys, an overridden toString, and a typed index +// signature. Everything feeds a module-level accumulator so no access site can +// be dropped as unused. + +interface Shape { + size: number + scale(by: number): number + toString(): string +} + +class Box implements Shape { + size: number + label: string + constructor(size: number, label: string) { + this.size = size + this.label = label + } + scale(by: number): number { + this.size = this.size * by + return this.size + } + toString(): string { + return this.label + this.size + } +} + +class Dot implements Shape { + size: number + constructor() { + this.size = 1 + } + scale(by: number): number { + this.size = this.size + by + return this.size + } + toString(): string { + return "dot" + } +} + +let acc = 0 +let shapes: Shape[] = [new Box(2, "b"), new Dot(), new Box(5, "c"), new Dot()] + +// checked field loads of one field +acc += shapes[0].size +acc += shapes[1].size +acc += shapes[2].size +acc += shapes[3].size +acc += shapes[0].size + shapes[1].size +acc += shapes[2].size + shapes[3].size + +// interface-dispatched calls of one method +acc += shapes[0].scale(2) +acc += shapes[1].scale(3) +acc += shapes[2].scale(4) +acc += shapes[3].scale(5) + +// object-literal stores of one key +let recs = [{ size: 1 }, { size: 2 }, { size: 3 }, { size: 4 }] +for (let r of recs) acc += r.size + +// overridden toString +acc += shapes[0].toString().length +acc += shapes[1].toString().length + +// typed index signature +let table: { [k: string]: number } = {} +table["a"] = acc +table["b"] = acc + 1 +acc += table["a"] + table["b"] diff --git a/tests/thumb-test/cases/sizebaseline.ts b/tests/thumb-test/cases/sizebaseline.ts new file mode 100644 index 000000000000..ae65b45fa2bb --- /dev/null +++ b/tests/thumb-test/cases/sizebaseline.ts @@ -0,0 +1,45 @@ +// A fixed dispatch-heavy program whose emitted code size is tracked. Keep this +// program stable: changing it invalidates the size baseline in asmchecks.ts. + +interface Op { + apply(v: number): number +} + +class AddOp implements Op { + constructor(public k: number) { } + apply(v: number) { return v + this.k } +} + +class MulOp implements Op { + constructor(public k: number) { } + apply(v: number) { return v * this.k } +} + +class ClampOp implements Op { + constructor(public lo: number, public hi: number) { } + apply(v: number) { return v < this.lo ? this.lo : (v > this.hi ? this.hi : v) } +} + +let acc = 0 +const ops: Op[] = [new AddOp(3), new MulOp(2), new ClampOp(0, 100), new AddOp(-1), new MulOp(5)] + +function runAll(v: number): number { + for (const o of ops) v = o.apply(v) + return v +} + +for (let i = 0; i < 5; ++i) acc += runAll(i) + +const names = ["add", "mul", "clamp"] +let joined = "" +for (const n of names) joined = joined + n + ":" +acc += joined.length + +const counts: { [k: string]: number } = {} +for (const n of names) counts[n] = (counts[n] || 0) + acc +for (const n of names) acc += counts[n] + +const nums = [5, 3, 9, 1, 7] +nums.sort((a, b) => a - b) +for (const v of nums) acc = acc + v * 2 +acc += nums.indexOf(9) diff --git a/tests/thumb-test/fixtures/microbit-mbcodal.json.gz b/tests/thumb-test/fixtures/microbit-mbcodal.json.gz new file mode 100644 index 000000000000..9aa554368b78 Binary files /dev/null and b/tests/thumb-test/fixtures/microbit-mbcodal.json.gz differ diff --git a/tests/thumb-test/scripts/capture-fixture.js b/tests/thumb-test/scripts/capture-fixture.js new file mode 100644 index 000000000000..a2509e8bc2c1 --- /dev/null +++ b/tests/thumb-test/scripts/capture-fixture.js @@ -0,0 +1,164 @@ +/* + * Captures the CompileOptions environment needed to compile PXT programs to + * native ARM Thumb entirely in-process, and writes it to + * tests/thumb-test/fixtures/.json.gz. + * + * Strategy: load the compiled pxt CLI bundle (built/pxt.js), monkey-patch + * ts.pxtc.compile, then drive the CLI's own "build" command inside a scratch + * project in the pxt-microbit checkout. The CLI assembles the full + * CompileOptions -- including extinfo.hexinfo, which is the piece that native + * compilation cannot proceed without -- and the patch intercepts it before any + * code is emitted. The hex runtime comes from pxt-microbit/built/hexcache, so + * no network and no C++ toolchain are involved. + * + * Usage: + * node tests/thumb-test/scripts/capture-fixture.js + * + * Environment: + * PXT_TARGET_DIR target checkout to capture from + * (default: sibling ../pxt-microbit) + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const zlib = require("zlib"); + +const pxtDir = path.resolve(__dirname, "..", "..", ".."); +const targetDir = path.resolve( + process.env.PXT_TARGET_DIR || path.join(pxtDir, "..", "pxt-microbit")); +const variant = "mbcodal"; +const fixtureName = "microbit-" + variant; +const fixturePath = path.join( + pxtDir, "tests", "thumb-test", "fixtures", fixtureName + ".json.gz"); +const projectDir = path.join(targetDir, "projects", "thumb-fixture"); + +// Fields of extinfo that are only needed by the C++ build service. They are +// large and never read by pxtc.compile. +const EXTINFO_DROP = ["compileData", "generatedFiles", "extensionFiles"]; + +function fail(msg) { + console.error("capture-fixture: " + msg); + process.exit(1); +} + +function makeScratchProject() { + // Stamping the current target version keeps the target's "upgrades" rules + // from injecting extra dependencies (they are all gated on older versions). + // Without the stamp the project reads as version 0.0.0, pxt-microbit's + // missingPackage rules match it, and the build dies with an error like + // "Package not installed: microphone". + const targetVersion = + JSON.parse(fs.readFileSync(path.join(targetDir, "package.json"), "utf8")).version; + + fs.mkdirSync(projectDir, { recursive: true }); + fs.writeFileSync(path.join(projectDir, "pxt.json"), JSON.stringify({ + name: "thumb-fixture", + description: "Scratch project used to capture the thumb-test fixture.", + dependencies: { core: "file:../../libs/core" }, + files: ["main.ts"], + targetVersions: { target: targetVersion }, + supportedTargets: ["microbit"] + }, null, 4) + "\n"); + // Deliberately trivial: the captured program text is replaced per test case. + fs.writeFileSync(path.join(projectDir, "main.ts"), "let x = 1\n"); +} + +function stripOptions(opts) { + const out = { + target: opts.target, + extinfo: opts.extinfo, + fileSystem: opts.fileSystem, + sourceFiles: opts.sourceFiles, + jres: opts.jres, + name: opts.name, + bannedCategories: opts.bannedCategories, + embedMeta: opts.embedMeta, + embedBlob: opts.embedBlob, + hexinfo: undefined + }; + delete out.hexinfo; + + if (!out.extinfo || !out.extinfo.hexinfo || !out.extinfo.hexinfo.hex) + fail("captured options have no extinfo.hexinfo -- native build did not " + + "resolve a hex runtime (is built/hexcache populated?)"); + + out.extinfo = Object.assign({}, out.extinfo); + for (const k of EXTINFO_DROP) + delete out.extinfo[k]; + + // Other variants carry a second full hex image and are not exercised here. + delete out.otherMultiVariants; + + return out; +} + +function writeFixture(opts) { + const captured = stripOptions(opts); + const meta = { + capturedAt: new Date().toISOString(), + target: (global.pxt && global.pxt.appTarget && global.pxt.appTarget.id) || "unknown", + targetVersion: (global.pxt && global.pxt.appTarget && global.pxt.appTarget.versions + && global.pxt.appTarget.versions.target) || "unknown", + variant: variant, + sha: captured.extinfo.sha + }; + const json = JSON.stringify({ meta: meta, options: captured }); + const gz = zlib.gzipSync(Buffer.from(json, "utf8"), { level: 9 }); + fs.mkdirSync(path.dirname(fixturePath), { recursive: true }); + fs.writeFileSync(fixturePath, gz); + + console.log(""); + console.log("capture-fixture: wrote " + fixturePath); + console.log(" raw json: " + json.length + " bytes"); + console.log(" gzipped: " + gz.length + " bytes (" + + (gz.length / 1024 / 1024).toFixed(2) + " MB)"); + console.log(" sha: " + meta.sha); + console.log(" variant: " + meta.variant); + console.log(" hex lines: " + captured.extinfo.hexinfo.hex.length); + console.log(" fs entries: " + Object.keys(captured.fileSystem).length); + console.log(" isNative: " + captured.target.isNative + + " nativeType=" + captured.target.nativeType); +} + +function main() { + if (!fs.existsSync(path.join(targetDir, "pxtarget.json"))) + fail("no pxtarget.json in " + targetDir + " (set PXT_TARGET_DIR)"); + if (!fs.existsSync(path.join(pxtDir, "built", "pxt.js"))) + fail("missing " + path.join(pxtDir, "built", "pxt.js") + " -- run npm run build first"); + + makeScratchProject(); + process.chdir(projectDir); + + // Selects the V2/CODAL-only variant. This is a variant selector rather than + // a real compile switch; see pxt.setCompileSwitch. + process.env.PXT_COMPILE_SWITCHES = "csv---" + variant; + + const cli = require(path.join(pxtDir, "built", "pxt.js")); + if (!cli || typeof cli.mainCli !== "function") + fail("built/pxt.js did not export mainCli"); + + const pxtc = global.ts.pxtc; + const origCompile = pxtc.compile; + let captured = false; + + pxtc.compile = function (opts) { + if (!captured && opts && opts.target && opts.target.isNative) { + captured = true; + writeFixture(opts); + // The emitted binary is of no interest; stop before the assembler runs. + process.exit(0); + } + return origCompile.apply(this, arguments); + }; + + cli.mainCli(targetDir, ["build"]).then(() => { + if (!captured) + fail("build finished without a native compile -- nothing captured"); + }, err => { + fail("build failed: " + (err && err.stack || err)); + }); +} + +main(); diff --git a/tests/thumb-test/thumbrunner.ts b/tests/thumb-test/thumbrunner.ts new file mode 100644 index 000000000000..67727325a604 --- /dev/null +++ b/tests/thumb-test/thumbrunner.ts @@ -0,0 +1,137 @@ +/// +/// + +import * as fs from 'fs'; +import * as path from 'path'; +import * as zlib from 'zlib'; + +import "mocha"; +import * as chai from "chai"; + +import { asmChecks, externalCases } from "./asmchecks"; + +const testDir = path.join(process.cwd(), "tests", "thumb-test"); +const casesDir = path.join(testDir, "cases"); +const fixturePath = path.join(testDir, "fixtures", "microbit-mbcodal.json.gz"); +const langTestPreludePath = path.join(process.cwd(), "tests", "compile-test", + "lang-test0", "lang-test0.ts"); + +// Native compilation runs the whole emitter plus the in-process assembler. +const TIMEOUT_MS = 30000; + +function initGlobals() { + let g = global as any + g.pxt = pxt; + g.ts = ts; + g.pxtc = pxtc; + g.btoa = (str: string) => Buffer.from(str, "binary").toString("base64"); + g.atob = (str: string) => Buffer.from(str, "base64").toString("binary"); +} + +initGlobals(); + +/** + * The fixture holds a CompileOptions environment captured from a real native + * build of the micro:bit target (see scripts/capture-fixture.js). It is kept as + * text so each case can parse its own copy and mutate it freely; the compiler + * writes back into the options it is given. + */ +function loadFixtureText(): string { + chai.assert(fs.existsSync(fixturePath), + "missing fixture " + fixturePath + " -- see tests/thumb-test/README.md"); + return zlib.gunzipSync(fs.readFileSync(fixturePath)).toString("utf8"); +} + +function optionsFor(fixtureText: string, programText: string): pxtc.CompileOptions { + const opts: pxtc.CompileOptions = JSON.parse(fixtureText).options; + + opts.fileSystem[pxt.MAIN_TS] = programText; + + opts.target.isNative = true; + opts.target.nativeType = pxtc.NATIVE_TYPE_THUMB; + opts.target.switches = opts.target.switches || {}; + // Emits the code-size stats header at the top of the listing. + opts.target.switches.size = true; + + return opts; +} + +/** + * The lang-test0 case files are written against the helpers in lang-test0.ts: + * assert(), msg() and a few shared globals. That file is small and uses only + * console.log, control.dmesg and core types, all of which the micro:bit + * fixture provides, so it is prepended verbatim instead of being restated + * here -- a hand-written stand-in would drift from the prelude the JS-side + * suite actually runs against. + */ +function langTestPrelude(): string { + chai.assert(fs.existsSync(langTestPreludePath), + "missing lang-test0 prelude " + langTestPreludePath); + return fs.readFileSync(langTestPreludePath, "utf8"); +} + +function describeDiagnostics(res: pxtc.CompileResult): string { + return res.diagnostics.map(d => { + const where = d.fileName ? d.fileName + "(" + (d.line + 1) + "," + (d.column + 1) + "): " : ""; + return where + "TS" + d.code + ": " + ts.flattenDiagnosticMessageText(d.messageText, "\n"); + }).join("\n"); +} + +describe("thumb codegen", () => { + const fixtureText = loadFixtureText(); + + const caseFiles = fs.readdirSync(casesDir) + .filter(f => f[0] !== "." && f.substr(-3) === ".ts") + .sort(); + + chai.assert(caseFiles.length > 0, "no case programs in " + casesDir); + + caseFiles.forEach(caseFile => { + it("compiles " + caseFile + " to thumb", function () { + this.timeout(TIMEOUT_MS); + + const check = asmChecks[caseFile]; + chai.assert(!!check, + "no expectation for " + caseFile + " -- add an entry to asmchecks.ts"); + + const programText = fs.readFileSync(path.join(casesDir, caseFile), "utf8"); + const opts = optionsFor(fixtureText, programText); + + const res = pxtc.compile(opts); + chai.assert(res.success, + "native compile of " + caseFile + " failed:\n" + describeDiagnostics(res)); + + const asm = res.outfiles[pxtc.BINARY_ASM]; + chai.assert(!!asm, "no " + pxtc.BINARY_ASM + " in compile result"); + + check(asm, res); + }); + }); + + // Semantic case programs kept elsewhere in the tree; see externalCases. + // They are compiled and assembled here, not executed. + const externalPaths = Object.keys(externalCases).sort(); + + externalPaths.forEach(relPath => { + it("compiles " + relPath + " to thumb", function () { + this.timeout(TIMEOUT_MS); + + const fullPath = path.join(process.cwd(), relPath); + chai.assert(fs.existsSync(fullPath), + "missing external case " + fullPath); + + const programText = langTestPrelude() + "\n" + + fs.readFileSync(fullPath, "utf8"); + const opts = optionsFor(fixtureText, programText); + + const res = pxtc.compile(opts); + chai.assert(res.success, + "native compile of " + relPath + " failed:\n" + describeDiagnostics(res)); + + const asm = res.outfiles[pxtc.BINARY_ASM]; + chai.assert(!!asm, "no " + pxtc.BINARY_ASM + " in compile result"); + + externalCases[relPath](asm, res); + }); + }); +}); diff --git a/tests/thumb-test/tsconfig.json b/tests/thumb-test/tsconfig.json new file mode 100644 index 000000000000..877a13256b5b --- /dev/null +++ b/tests/thumb-test/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "es2017", + "noImplicitAny": true, + "noImplicitReturns": true, + "declaration": true, + // rootDir keeps the emit at built/tests/thumb-test/, which is where the + // gulp test task looks for the runner. + "rootDir": "..", + "outDir": "../../built/tests", + "newLine": "LF", + "module": "commonjs", + "lib": [ + "dom", + "dom.iterable", + "es2017", + "ES2018.Promise" + ], + "types": [ + "chai", + "mocha", + "node" + ], + "sourceMap": false, + "skipLibCheck": true + }, + "files": [ + "thumbrunner.ts", + "asmchecks.ts" + ] +}