未验证 提交 6316b332 编写于 作者: N Nathaniel Wesley Filardo 提交者: GitHub

More NTest prep work for eventual test harness (#3353)

* Rename to tests/README.md

* Expand tests/README.md a bit

* NTest: remove report() in favor of named fields

Use a metatable to provide defaults which can be shadowed by the calling
code.

* NTest: remove old interface flag

I think we have few enough tests that we can verify not needing this
alert for ourselves.

* NTest tests: new standard prelude

Allow for NTest constructor to be passed in to the test itself.
The test harness can use this to provide a wrapper that will
pre-configure NTest itself.

* NTest output handler for TAP messages

* expect tests: core library functions

* expect tests: file xfer TCL module

* expect tests: add TAP-based test runner

* Begin documenting TCL goo

* Add .gitattributes to make sure lineends are correct ...

... if checked out under windows and executed under linux (say docker)

* tests/README: enumerate dependencies

* tests: more README.md
Co-authored-by: NGregor Hartmann <HHHartmann@users.noreply.github.com>
上级 c3dd27cf
# Enforce Unix newlines
*.css text eol=lf
*.html text eol=lf
*.js text eol=lf
*.json text eol=lf
*.less text eol=lf
*.md text eol=lf
*.svg text eol=lf
*.yml text eol=lf
*.py text eol=lf
*.sh text eol=lf
*.tcl text eol=lf
*.expect text eol=lf
......@@ -171,22 +171,26 @@ local function fail(handler, name, func, expected, msg)
handler('pass', name, msg)
end
local function NTest(testrunname, failoldinterface)
local nmt = {
env = _G,
outputhandler = TERMINAL_HANDLER
}
nmt.__index = nmt
if failoldinterface then error("The interface has changed. Please see documentstion.") end
return function(testrunname)
local pendingtests = {}
local env = _G
local outputhandler = TERMINAL_HANDLER
local started
local N = setmetatable({}, nmt)
local function runpending()
if pendingtests[1] ~= nil then
node.task.post(node.task.LOW_PRIORITY, function()
pendingtests[1](runpending)
end)
else
outputhandler('finish', testrunname)
N.outputhandler('finish', testrunname)
end
end
......@@ -202,9 +206,9 @@ local function NTest(testrunname, failoldinterface)
local testfn = function(next)
local prev = {}
copyenv(prev, env)
copyenv(prev, N.env)
local handler = outputhandler
local handler = N.outputhandler
local restore = function(err)
if err then
......@@ -214,8 +218,8 @@ local function NTest(testrunname, failoldinterface)
end
end
if node then node.setonerror() end
copyenv(env, prev)
outputhandler('end', name)
copyenv(N.env, prev)
handler('end', name)
table.remove(pendingtests, 1)
collectgarbage()
if next then next() end
......@@ -233,6 +237,7 @@ local function NTest(testrunname, failoldinterface)
restore()
end
local env = N.env
env.eq = deepeq
env.spy = spy
env.ok = function (cond, msg) wrap(assertok, false, cond, msg) end
......@@ -258,7 +263,7 @@ local function NTest(testrunname, failoldinterface)
end
if not started then
outputhandler('start', testrunname)
N.outputhandler('start', testrunname)
started = true
end
......@@ -270,25 +275,20 @@ local function NTest(testrunname, failoldinterface)
end
end
local function test(name, f)
function N.test(name, f)
testimpl(name, f)
end
local function testasync(name, f)
function N.testasync(name, f)
testimpl(name, f, true)
end
local function report(f, envP)
outputhandler = f or outputhandler
env = envP or env
end
local currentCoName
local function testco(name, func)
function N.testco(name, func)
-- local t = tmr.create();
local co
testasync(name, function(Next)
N.testasync(name, function(Next)
currentCoName = name
local function getCB(cbName)
......@@ -299,7 +299,7 @@ local function NTest(testrunname, failoldinterface)
currentCoName = nil
Next(err)
else
outputhandler('fail', name, "Found stray Callback '"..cbName.."' from test '"..name.."'")
N.outputhandler('fail', name, "Found stray Callback '"..cbName.."' from test '"..name.."'")
end
elseif coroutine.status(co) == "dead" then
currentCoName = nil
......@@ -327,9 +327,5 @@ local function NTest(testrunname, failoldinterface)
end)
end
return {test = test, testasync = testasync, testco = testco, report = report}
return N
end
return NTest
......@@ -145,7 +145,7 @@ ok(f.errors[3] ~= nil)
## Reports
Another useful feature is that you can customize test reports as you need. The default `reports` just more or less prints out a basic report. You can easily override this behavior as well as add any other information you need (number of passed/failed assertions, time the test took etc):
Another useful feature is that you can customize test reports as you need. The default `outputhandler` just more or less prints out a basic report. You can easily override (or augment by wrapping, e.g.) this behavior as well as add any other information you need (number of passed/failed assertions, time the test took etc):
Events are:
`start` when testing starts
......@@ -161,7 +161,7 @@ Events are:
local passed = 0
local failed = 0
tests.report(function(event, testfunc, msg)
tests.outputhandler = function(event, testfunc, msg)
if event == 'begin' then
print('Started test', testfunc)
passed = 0
......@@ -176,7 +176,7 @@ tests.report(function(event, testfunc, msg)
elseif event == 'except' then
print('ERROR', testfunc, msg)
end
end)
end
```
Additionally, you can pass a different environment to keep `_G` unpolluted:
......@@ -184,8 +184,7 @@ You need to set it, so the helper functions mentioned above can be added before
``` Lua
local myenv = {}
tests.report(function() ... end, myenv)
tests.env = myenv
tests.test('Some test', function()
myenv.ok(myenv.eq(...))
......@@ -193,7 +192,7 @@ tests.test('Some test', function()
end)
```
You can set any of the parameters to `nil` to leave the value unchanged.
You can restore `env` or `outputhandler` to their defaults by setting their values to `nil`.
## Appendix
......
......@@ -434,7 +434,7 @@ end
local pass
-- Set meta test handler
N.report(function(e, test, msg, errormsg)
N.outputhandler = function(e, test, msg, errormsg)
local function consumemsg(msg, area) -- luacheck: ignore
if not expected[1][area][1] then
print("--- FAIL "..expected[1].name..' ('..area..'ed): unexpected "'..
......@@ -487,7 +487,7 @@ N.report(function(e, test, msg, errormsg)
else
print("Extra output: ", e, test, msg, errormsg)
end
end)
end
local async_queue = {}
async = function(f) table.insert(async_queue, cbWrap(f)) end
......
-- Walk the ADC through a stepped triangle wave using the attached voltage
-- divider and I2C GPIO expander.
local N = require('NTest')("adc-env")
local N = ...
N = (N or require "NTest")("adc-env")
-- TODO: Preflight test that we are in the correct environment with an I2C
-- expander in the right place with the right connections.
......
local N = require('NTest')("file")
local N = ...
N = (N or require "NTest")("file")
local function cleanup()
file.remove("testfile")
......
......@@ -3,7 +3,8 @@
-- Node GPIO 13 (index 7) is connected to I2C expander channel B6; node OUT
-- Node GPIO 15 (index 8) is connected to I2C expander channel B7; node IN
local N = require('NTest')("gpio-env")
local N = ...
N = (N or require "NTest")("gpio-env")
-- TODO: Preflight test that we are in the correct environment with an I2C
-- expander in the right place with the right connections.
......
local N = require('NTest')("tmr")
local N = ...
N = (N or require "NTest")("tmr")
N.testasync('SINGLE alarm', function(next)
local t = tmr.create();
......
NodeMCU Testing Environment
===========================
# Introduction
Welcome to the NodeMCU self-test suite. Here you will find our growing effort
to ensure that our software behaves as we think it should and that we do not
regress against earlier versions.
Our tests are written using [NTest](./NTest/NTest.md), a lightweight yet
featureful framework for specifying unit tests.
# Building and Running Test Software on NodeMCU Devices
Naturally, to test NodeMCU on its intended hardware, you will need one or more
NodeMCU-capable boards. At present, the test environment is specified using
two ESP8266 Devices Under Test (DUTs), but we envision expanding this to mixed
ESP8266/ESP32 environments as well.
Test programs live beside this file. While many test programs run on the
NodeMCU DUTs, but there is reason to want to orchestrate DUTs and the
environment using the host. Files matching the glob `NTest_*.lua` are intended
for on-DUT execution.
## Manual Test Invocation
At the moment, the testing regime and host-based orchestration is still in
development, and so things are a little more manual than perhaps desired. The
`NTest`-based test programs all assume that they can `require "NTest"`, and so
the easiest route to success is to
* build an LFS image containing
* [package.loader support for LFS](../lua_examples/lfs/_init.lua)
* [NTest itself](./NTest/NTest.lua)
* Any additional Lua support modules required (e.g., [mcp23017
support](../lua_modules/mcp23017/mcp23017.lua) )
* build a firmware with the appropriate C modules
* program the board with your firmware and LFS images
* ensure that `package.loader` is patched appropriately on startup
* transfer the `NTest_foo` program you wish to run to the device SPIFFS
(or have included it in the LFS).
* at the interpreter prompt, say `dofile("NTest_foo.lua")` (or
`node.LFS.get("NTest_foo")()`) to run the `foo` test program.
## Experimental Host Orchestration
Enthusiastic testers are encouraged to try using our very new, very
experimental host test runner, [tap-driver.expect](./tap-driver.expect). To
use this program, in addition to the above, the LFS environment should contain
[NTestTapOut](./tests/utils/NTestTapOut.lua), an output adapter for `NTest`,
making it speak a slight variant of the [Test Anything
Protocol](https://testanything.org/). This structured output is scanned for
by the script on the host.
You'll need `expect` and TCL and some TCL libraries available; on Debian, that
amounts to
apt install tcl tcllib tclx8.4 expect
This program should be invoked from beside this file with something like
TCLLIBPATH=./expectnmcu ./tap-driver.expect -serial /dev/ttyUSB3 -lfs ./lfs.img NTest_file.lua
This will...
* transfer and install the specified LFS module (and reboot the device to load LFS)
* transfer the test program
* run the test program with `NTest` shimmed to use the `NTestTapOut` output
handler
* summarize the results
* return 0 if and only if all tests have passed
This tool is quite flexible and takes a number of other options and flags
controlling aspects of its behavior:
* Additional files, Lua or otherwise, may be transferred by specifing them
before the test to run (e.g., `./tap-driver.expect a.lua b.lua
NTest_foo.lua`); dually, a `-noxfer` flag will suppress transferring even the
last file. All transferred files are moved byte-for-byte to the DUT's
SPIFFS with names, but not directory components, preserved.
* The `-lfs LFS.img` option need not be specified and, if not given, any
existing `LFS` image will remain on the device for use by the test.
* A `-nontestshim` flag will skip attempting to shim the given test program
with `NTestTapOut`; the test program is expected to provide its own TAP
output. The `-tpfx` argument can be used to override the leading `TAP: `
sigil used by the `NTestTapOut` output handler.
* A `-runfunc` option indicates that the last argument is not a file to
transfer but rather a function to be run. It will be invoked at the REPL
with a single argument, the shimmed `NTest` constructor, unless `-nontestshim`
is given, in which case the argument will be `nil`.
* A `-notests` option suppresses running tests (making the tool merely another
option for loading files to the device).
Transfers will be significantly faster if
[pipeutils](../lua_examples/pipeutils.lua) is available to `require` on the
DUT, but a fallback strategy exists if not. We suggest either including
`pipeutils` in LFS images, in SPIFFS, or as the first file to be transferred.
# NodeMCU Testing Environment
Herein we define the environment our testing framework expects to see
when it runs. It is composed of two ESP8266 devices, each capable of
......@@ -14,8 +124,7 @@ devices, as found on almost all ESP8266 boards with USB to UART
adapters, but the host does not necessarily need to use USB to connect,
so long as TXD, RXD, DTR, and RTS are wired across.
Peripherals
-----------
## Peripherals
### I2C Bus
......
namespace eval expectnmcu::core {
set panicre "powered by Lua \[0-9.\]+ on SDK \[0-9.\]+"
set promptstr "\n> "
namespace export reboot waitboot connect
namespace export send_exp_prompt send_exp_res_prompt send_exp_prompt_c
}
package require cmdline
# Use DTR/RTS signaling to reboot the device
## I'm not sure why we have to keep resetting the mode, but so it goes.
proc ::expectnmcu::core::reboot { dev } {
set victimfd [open ${dev} ]
set mode [fconfigure ${victimfd} -mode ]
fconfigure ${victimfd} -mode ${mode} -ttycontrol {DTR 0 RTS 1}
sleep 0.1
fconfigure ${victimfd} -mode ${mode} -ttycontrol {DTR 0 RTS 0}
close ${victimfd}
}
proc ::expectnmcu::core::waitboot { victim } {
expect {
-i ${victim} "Formatting file system" {
set timeout 120
exp_continue
}
-i ${victim} "powered by Lua" { }
timeout { return -code error "Timeout" }
}
# Catch nwf's system bootup, in case we're testing an existing system,
# rather than a blank firmware.
expect {
-i ${victim} -re "Reset delay!.*${::expectnmcu::core::promptstr}" {
send -i ${victim} "stop(true)\n"
expect -i ${victim} -ex ${::expectnmcu::core::promptstr}
}
-i ${victim} -ex ${::expectnmcu::core::promptstr} { }
timeout { return -code error "Timeout" }
}
# Do a little more active synchronization with the DUT: send it a command
# and wait for the side-effect of that command to happen, thereby ensuring
# that the next prompt we see is after this point in the input.
send -i ${victim} "print(\"a\",\"z\")\n"
expect {
-i ${victim} -ex "a\tz" { }
}
expect {
-i ${victim} -ex ${::expectnmcu::core::promptstr} { }
timeout { return -code error "Timeout" }
}
}
# Establish a serial connection to the device via socat. Takes
# -baud=N, -reboot=0/1/dontwait, -waitboot=0/1 optional parameters
proc ::expectnmcu::core::connect { dev args } {
set opts {
{ baud.arg 115200 }
{ reboot.arg 1 }
}
array set arg [::cmdline::getoptions args $opts]
spawn "socat" "STDIO" "${dev},b${arg(baud)},raw,crnl"
close -onexec 1 -i ${spawn_id}
set victim ${spawn_id}
# XXX?
set victimfd [open ${dev} ]
set mode [fconfigure ${victimfd} -mode ${arg(baud)},n,8,1 ]
if { ${arg(reboot)} != 0 } {
::expectnmcu::core::reboot ${dev}
if { ${arg(reboot)} != "dontwait" } {
::expectnmcu::core::waitboot ${victim}
}
}
close ${victimfd}
return ${victim}
}
# This one is somewhat "for experts only" -- it expects that you have either
# consumed whatever command you flung at the node or that you have some reason
# to not be concerned with its echo (and return)
proc ::expectnmcu::core::exp_prompt { sid } {
expect {
-i ${sid} -ex ${::expectnmcu::core::promptstr} { }
-i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
timeout { return -code error "Timeout" }
}
}
proc ::expectnmcu::core::send_exp_prompt { sid cmd } {
send -i ${sid} -- "${cmd}\n"
expect {
-i ${sid} -ex "${cmd}" { }
timeout { return -code error "Timeout" }
}
expect {
-i ${sid} -ex ${::expectnmcu::core::promptstr} { }
-i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
timeout { return -code error "Timeout" }
}
}
proc ::expectnmcu::core::send_exp_res_prompt { sid cmd res } {
send -i ${sid} -- "${cmd}\n"
expect {
-i ${sid} -ex "${cmd}" { }
timeout { return -code error "Timeout" }
}
expect {
-i ${sid} -re "${res}.*${::expectnmcu::core::promptstr}" { }
-i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
-i ${sid} -ex ${::expectnmcu::core::promptstr} { return -code error "Prompt before expected response" }
timeout { return -code error "Timeout" }
}
}
proc ::expectnmcu::core::send_exp_prompt_c { sid cmd } {
send -i ${sid} -- "${cmd}\n"
expect {
-i ${sid} -ex "${cmd}" { }
timeout { return -code error "Timeout" }
}
expect {
-i ${sid} -ex "\n>> " { }
-i ${sid} -ex ${::expectnmcu::core::promptstr} { return -code error "Non-continuation prompt" }
-i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
timeout { return -code error "Timeout" }
}
}
package provide expectnmcu::core 1.0
# Tcl package index file, version 1.1
# This file is generated by the "pkg_mkIndex" command
# and sourced either when an application starts up or
# by a "package unknown" script. It invokes the
# "package ifneeded" command to set up package-related
# information so that packages will be loaded automatically
# in response to "package require" commands. When this
# script is sourced, the variable $dir must contain the
# full path name of this file's directory.
package ifneeded expectnmcu::core 1.0 [list source [file join $dir core.tcl]]
package ifneeded expectnmcu::xfer 1.0 [list source [file join $dir xfer.tcl]]
namespace eval expectnmcu::xfer {
}
package require expectnmcu::core
# Open remote file `which` on `dev` in `mode` as Lua object `dfh`
proc ::expectnmcu::xfer::open { dev dfh which mode } {
::expectnmcu::core::send_exp_prompt ${dev} "${dfh} = nil"
::expectnmcu::core::send_exp_prompt ${dev} "${dfh} = file.open(\"${which}\",\"${mode}\")"
::expectnmcu::core::send_exp_res_prompt ${dev} "=type(${dfh})" "userdata"
}
# Close Lua file object `dfh` on `dev`
proc ::expectnmcu::xfer::close { dev dfh } {
::expectnmcu::core::send_exp_prompt ${dev} "${dfh}:close()"
}
# Write to `dfh` on `dev` at `where` `what`, using base64 as transport
#
# This does not split lines; write only short amounts of data.
proc ::expectnmcu::xfer::pwrite { dev dfh where what } {
send -i ${dev} -- [string cat \
"do local d,e = encoder.fromBase64(\"[binary encode base64 -maxlen 0 ${what}]\");" \
"${dfh}:seek(\"set\",${where});" \
"print(${dfh}:write(d));" \
"end\n" \
]
expect {
-i ${dev} -re "true\[\r\n\]+> " { }
-i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
-i ${dev} -ex "\n> " { return -code error "Bad result from pwrite" }
timeout { return -code error "Timeout while waiting for pwrite" }
}
}
# Read `howmuch` byetes from `dfh` on `dev` at `where`, using base64
# as transport. This buffers the whole data and its base64 encoding
# in device RAM; read only short strings.
proc ::expectnmcu::xfer::pread { dev dfh where howmuch } {
send -i ${dev} -- "${dfh}:seek(\"set\",${where}); print(encoder.toBase64(${dfh}:read(${howmuch})))\n"
expect {
-i ${dev} -re "\\)\\)\\)\[\r\n\]+(\[^\r\n\]+)\[\r\n\]+> " {
return [binary decode base64 ${expect_out(1,string)}]
}
-i ${dev} -ex "\n> " { return -code error "No reply to pread" }
-i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
timeout { return -code error "Timeout while pread-ing" }
}
}
# Check for pipeutils on the target device
proc ::expectnmcu::xfer::haspipeutils { dev } {
send -i ${dev} -- "local ok, pu = pcall(require, \"pipeutils\"); print(ok and type(pu) == \"table\" and pu.chunker and pu.debase64 and true or false)\n"
expect {
-i ${dev} -re "\[\r\n\]+false\[\r\n\]+> " { return 0 }
-i ${dev} -re "\[\r\n\]+true\[\r\n\]+> " { return 1 }
-i ${dev} -ex "\n> " { return -code error "No reply to pipeutils probe" }
-i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
timeout { return -code error "Timeout while probing for pipeutils" }
}
}
# Send local file `lfn` to the remote filesystem on `dev` and name it `rfn`.
# Use `dfo` as the Lua handle to the remote file for the duration of writing,
# (and `nil` it out afterwards)
proc ::expectnmcu::xfer::sendfile { dev lfn rfn {dfo "xfo"} } {
package require sha256
set has_pipeutils [::expectnmcu::xfer::haspipeutils ${dev} ]
set ltf [::open ${lfn} ]
fconfigure ${ltf} -translation binary
file stat ${lfn} lfstat
::expectnmcu::xfer::open ${dev} ${dfo} "${rfn}.sf" "w+"
if { ${has_pipeutils} } {
# Send over a loader program
::expectnmcu::core::send_exp_prompt_c ${dev} "do"
::expectnmcu::core::send_exp_prompt_c ${dev} " local pu = require \"pipeutils\""
::expectnmcu::core::send_exp_prompt_c ${dev} " local ch = pu.chunker(function(d) ${dfo}:write(d) end, 256)"
::expectnmcu::core::send_exp_prompt_c ${dev} " local db = pu.debase64(ch.write, function(ed,ee)"
::expectnmcu::core::send_exp_prompt_c ${dev} " if ed:match(\"^%.\[\\r\\n\]*$\") then ch.flush() print(\"F I N\")"
::expectnmcu::core::send_exp_prompt_c ${dev} " else print(\"ABORT\", ee, ed) end"
::expectnmcu::core::send_exp_prompt_c ${dev} " uart.on(\"data\") end)"
# TODO: make echo use CRC not full string; probably best add to crypto module
::expectnmcu::core::send_exp_prompt_c ${dev} " uart.on(\"data\", \"\\n\", function(x) db.write(x); uart.write(0, \"OK: \", x) end, 0)"
::expectnmcu::core::send_exp_prompt ${dev} "end"
set xln 90
} else {
set xln 48
}
set lho [sha2::SHA256Init]
set fpos 0
while { 1 } {
send_user ">> xfer ${fpos} of ${lfstat(size)}\n"
set data [read ${ltf} ${xln}]
sha2::SHA256Update ${lho} ${data}
if { ${has_pipeutils} } {
set estr [binary encode base64 -maxlen 0 ${data}]
send -i ${dev} -- "${estr}\n"
expect {
-i ${dev} -ex "OK: ${estr}" { expect -i ${dev} -re "\[\r\n\]+" {} }
-i ${dev} -ex "\n> " { return -code error "Prompt while sending data" }
-i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
timeout { return -code error "Timeout while sending data" }
}
} else {
::expectnmcu::xfer::pwrite ${dev} ${dfo} ${fpos} ${data}
}
set fpos [expr $fpos + ${xln}]
if { [string length ${data}] != ${xln} } { break }
}
if { ${has_pipeutils} } {
send -i ${dev} -- ".\n"
expect {
-i ${dev} -re "F I N\[\r\n\]+" { }
-i ${dev} -ex "\n> " { return -code error "Prompt while awaiting acknowledgement" }
-i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
timeout { return -code error "Timeout while awaiting acknowledgement" }
}
}
::close ${ltf}
::expectnmcu::xfer::close ${dev} ${dfo}
::expectnmcu::core::send_exp_prompt ${dev} "${dfo} = nil"
set exphash [sha2::Hex [sha2::SHA256Final ${lho}]]
send -i ${dev} "=encoder.toHex(crypto.fhash(\"sha256\",\"${rfn}.sf\"))\n"
expect {
-i ${dev} -re "\[\r\n\]+(\[a-f0-9\]+)\[\r\n\]+> " {
if { ${expect_out(1,string)} != ${exphash} } {
return -code error \
"Sendfile checksum mismatch: ${expect_out(1,string)} != ${exphash}"
}
}