diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8ab4162e4a..fa915872d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -160,3 +160,11 @@ jobs: with: name: ${{ inputs.name && inputs.name != 'infix' && format('xpath-coverage-report-{0}', inputs.name) || 'xpath-coverage-report' }} path: output/images/xpath-coverage-report.pdf + + - name: Upload DUT Syslog as Artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.name && inputs.name != 'infix' && format('dut-syslog-{0}', inputs.name) || 'dut-syslog' }} + path: ${{ env.TEST_PATH }}/.log/last/logs.tar.gz + if-no-files-found: ignore diff --git a/board/common/rootfs/usr/libexec/infix/init.d/30-test-mode b/board/common/rootfs/usr/libexec/infix/init.d/30-test-mode new file mode 100755 index 0000000000..9662774451 --- /dev/null +++ b/board/common/rootfs/usr/libexec/infix/init.d/30-test-mode @@ -0,0 +1,28 @@ +#!/bin/sh +# In test mode, forward all logs to the test controller's syslog sink. +# +# The test mode settings are a JSON object, from qemu (fw_cfg) or the +# test-mode file on aux, which is an empty marker on older setups: +# +# { "management-interface": "e1", "physical-name": "dut1" } + +ident=$(basename "$0") +conf=/etc/syslog.d/test-mode.conf +group=ff02::161 + +[ -f /mnt/aux/test-mode ] || exit 0 + +# Qemu settings (for testing) take precedence over the test-mode file +file=/sys/firmware/qemu_fw_cfg/by_name/opt/test-mode/raw +[ -f "$file" ] || file=/mnt/aux/test-mode + +iface=$(jq -r '."management-interface" // empty' "$file" 2>/dev/null) +[ -n "$iface" ] || exit 0 +name=$(jq -r '."physical-name" // empty' "$file" 2>/dev/null) + +opts="RFC5424,iface=$iface" +[ -n "$name" ] && opts="$opts,sd=[device@32473 name=\"$name\"]" +echo "*.* @[$group] ;$opts" > "$conf" + +logger -k -p user.notice -t "$ident" "Forwarding logs to $group on $iface" +kill -HUP "$(cat /run/syslogd.pid)" diff --git a/doc/test-arch.md b/doc/test-arch.md index a0cceb5064..6f6e3248b6 100644 --- a/doc/test-arch.md +++ b/doc/test-arch.md @@ -262,8 +262,53 @@ Infamy: once, instead of the `test-config` (used by tests that verify save-and-reboot behaviour) -Test mode is activated by the presence of an empty file named -`test-mode` on the device's `aux` partition (`/mnt/aux/test-mode`). +Test mode is activated by the presence of a file named `test-mode` on +the device's `aux` partition (`/mnt/aux/test-mode`). The file may be +empty, or hold the device's test mode settings as a JSON object: + +```json +{ "management-interface": "e1", "physical-name": "dut1" } +``` + +### DUT Logs + +When a test fails in CI, the DUTs' logs are often the only way to find +out why, but by then the DUTs are gone. So every DUT in test mode +forwards all its logs to the test controller, and the test environment +saves them. + +With `management-interface` set in the test mode settings, the DUT +sends all its logs, in RFC 5424 format, to the multicast group +`ff02::161` on that interface. This is set up at boot, outside the +configuration, so it keeps going when a test configures syslog, and +starts again after a reboot. With `physical-name` set, each message +also carries the structured data `[device@32473 name="dut1"]`, since +tests change the hostname to the DUT's role in the test. + +`test/env` always starts a syslog sink, `test/syslog-sink`, which +listens to the group on each of the controller's management ports and +saves the logs of all DUTs in one file, `test/.log/syslog-