From b6054c0206609f3755f71d819643644d250288b0 Mon Sep 17 00:00:00 2001 From: Stefan Kerkmann Date: Mon, 22 Nov 2021 22:15:04 +0100 Subject: [Tests] Increase QMK test coverage (#13789) * Add per-test keymaps * Add better trace and info logs for failed unit-tests * Add layer state assertion with tracing message * Use individual test binaries configuration options * Add basic qmk functionality tests * Add tap hold configurations tests * Add auto shift tests * `qmk format-c * Fix tests Co-authored-by: Nick Brassel --- tests/test_common/build.mk | 16 ++++ tests/test_common/keyboard_report_util.cpp | 21 +++-- tests/test_common/keymap.c | 33 +++++++ tests/test_common/test_common.h | 4 + tests/test_common/test_driver.cpp | 5 +- tests/test_common/test_driver.hpp | 29 +++--- tests/test_common/test_fixture.cpp | 142 ++++++++++++++++++++++++++--- tests/test_common/test_fixture.hpp | 21 ++++- tests/test_common/test_keymap_key.cpp | 30 ++++++ tests/test_common/test_keymap_key.hpp | 46 ++++++++++ tests/test_common/test_logger.cpp | 39 ++++++++ tests/test_common/test_logger.hpp | 35 +++++++ 12 files changed, 382 insertions(+), 39 deletions(-) create mode 100644 tests/test_common/build.mk create mode 100644 tests/test_common/keymap.c create mode 100644 tests/test_common/test_common.h create mode 100644 tests/test_common/test_keymap_key.cpp create mode 100644 tests/test_common/test_keymap_key.hpp create mode 100644 tests/test_common/test_logger.cpp create mode 100644 tests/test_common/test_logger.hpp (limited to 'tests/test_common') diff --git a/tests/test_common/build.mk b/tests/test_common/build.mk new file mode 100644 index 0000000000..aeb3057854 --- /dev/null +++ b/tests/test_common/build.mk @@ -0,0 +1,16 @@ +# Copyright 2021 Stefan Kerkmann +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +CUSTOM_MATRIX=yes \ No newline at end of file diff --git a/tests/test_common/keyboard_report_util.cpp b/tests/test_common/keyboard_report_util.cpp index f73cf239e0..e148c76beb 100644 --- a/tests/test_common/keyboard_report_util.cpp +++ b/tests/test_common/keyboard_report_util.cpp @@ -44,16 +44,21 @@ bool operator==(const report_keyboard_t& lhs, const report_keyboard_t& rhs) { return lhs.mods == rhs.mods && lhskeys == rhskeys; } -std::ostream& operator<<(std::ostream& stream, const report_keyboard_t& value) { - stream << "Keyboard report:" << std::endl; - stream << "Mods: " << (uint32_t)value.mods << std::endl; - stream << "Keys: "; +std::ostream& operator<<(std::ostream& stream, const report_keyboard_t& report) { + auto keys = get_keys(report); + // TODO: This should probably print friendly names for the keys - for (uint32_t k : get_keys(value)) { - stream << k << " "; + stream << "Keyboard Report: Mods (" << (uint32_t)report.mods << ") Keys ("; + + for (auto key = keys.cbegin(); key != keys.cend();) { + stream << +(*key); + key++; + if (key != keys.cend()) { + stream << ","; + } } - stream << std::endl; - return stream; + + return stream << ")" << std::endl; } KeyboardReportMatcher::KeyboardReportMatcher(const std::vector& keys) { diff --git a/tests/test_common/keymap.c b/tests/test_common/keymap.c new file mode 100644 index 0000000000..fc3a56a007 --- /dev/null +++ b/tests/test_common/keymap.c @@ -0,0 +1,33 @@ +/* Copyright 2017 Fred Sundvik + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "quantum.h" + +// clang-format off + +const uint16_t PROGMEM + keymaps[][MATRIX_ROWS][MATRIX_COLS] = + { + [0] = + { + {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO}, + {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO}, + {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO}, + {KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO, KC_NO}, + }, +}; + +// clang-format on diff --git a/tests/test_common/test_common.h b/tests/test_common/test_common.h new file mode 100644 index 0000000000..19ffcddcea --- /dev/null +++ b/tests/test_common/test_common.h @@ -0,0 +1,4 @@ +#pragma once + +#define MATRIX_ROWS 4 +#define MATRIX_COLS 10 \ No newline at end of file diff --git a/tests/test_common/test_driver.cpp b/tests/test_common/test_driver.cpp index 84e249d838..2fa2b6a2e7 100644 --- a/tests/test_common/test_driver.cpp +++ b/tests/test_common/test_driver.cpp @@ -27,7 +27,10 @@ TestDriver::~TestDriver() { m_this = nullptr; } uint8_t TestDriver::keyboard_leds(void) { return m_this->m_leds; } -void TestDriver::send_keyboard(report_keyboard_t* report) { m_this->send_keyboard_mock(*report); } +void TestDriver::send_keyboard(report_keyboard_t* report) { + test_logger.trace() << *report; + m_this->send_keyboard_mock(*report); +} void TestDriver::send_mouse(report_mouse_t* report) { m_this->send_mouse_mock(*report); } diff --git a/tests/test_common/test_driver.hpp b/tests/test_common/test_driver.hpp index f86308df95..f9197b3634 100644 --- a/tests/test_common/test_driver.hpp +++ b/tests/test_common/test_driver.hpp @@ -20,25 +20,26 @@ #include #include "host.h" #include "keyboard_report_util.hpp" - +#include "test_logger.hpp" class TestDriver { -public: + public: TestDriver(); ~TestDriver(); void set_leds(uint8_t leds) { m_leds = leds; } - MOCK_METHOD1(send_keyboard_mock, void (report_keyboard_t&)); - MOCK_METHOD1(send_mouse_mock, void (report_mouse_t&)); - MOCK_METHOD1(send_system_mock, void (uint16_t)); - MOCK_METHOD1(send_consumer_mock, void (uint16_t)); -private: - static uint8_t keyboard_leds(void); - static void send_keyboard(report_keyboard_t *report); - static void send_mouse(report_mouse_t* report); - static void send_system(uint16_t data); - static void send_consumer(uint16_t data); - host_driver_t m_driver; - uint8_t m_leds = 0; + MOCK_METHOD1(send_keyboard_mock, void(report_keyboard_t&)); + MOCK_METHOD1(send_mouse_mock, void(report_mouse_t&)); + MOCK_METHOD1(send_system_mock, void(uint16_t)); + MOCK_METHOD1(send_consumer_mock, void(uint16_t)); + + private: + static uint8_t keyboard_leds(void); + static void send_keyboard(report_keyboard_t* report); + static void send_mouse(report_mouse_t* report); + static void send_system(uint16_t data); + static void send_consumer(uint16_t data); + host_driver_t m_driver; + uint8_t m_leds = 0; static TestDriver* m_this; }; diff --git a/tests/test_common/test_fixture.cpp b/tests/test_common/test_fixture.cpp index e041df7128..0601b17191 100644 --- a/tests/test_common/test_fixture.cpp +++ b/tests/test_common/test_fixture.cpp @@ -1,26 +1,48 @@ #include "test_fixture.hpp" +#include +#include +#include +#include +#include "gmock/gmock-cardinalities.h" #include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "keyboard_report_util.hpp" +#include "keycode.h" #include "test_driver.hpp" +#include "test_logger.hpp" #include "test_matrix.h" -#include "keyboard.h" -#include "action.h" -#include "action_tapping.h" +#include "test_keymap_key.hpp" extern "C" { +#include "action.h" +#include "action_tapping.h" +#include "action_util.h" +#include "action_layer.h" #include "debug.h" #include "eeconfig.h" -#include "action_layer.h" +#include "keyboard.h" +#include "keymap.h" void set_time(uint32_t t); void advance_time(uint32_t ms); } using testing::_; -using testing::AnyNumber; -using testing::Between; -using testing::Return; + +/* This is used for dynamic dispatching keymap_key_to_keycode calls to the current active test_fixture. */ +TestFixture* TestFixture::m_this = nullptr; + +/* Override weak QMK function to allow the usage of isolated per-test keymaps in unit-tests. + * The actual call is dynamicaly dispatched to the current active test fixture, which in turn has it's own keymap. */ +extern "C" uint16_t keymap_key_to_keycode(uint8_t layer, keypos_t position) { + uint16_t keycode; + TestFixture::m_this->get_keycode(layer, position, &keycode); + return keycode; +} void TestFixture::SetUpTestCase() { + test_logger.info() << "TestFixture setup-up start." << std::endl; + // The following is enough to bootstrap the values set in main eeconfig_init_quantum(); eeconfig_update_debug(debug_config.raw); @@ -28,23 +50,99 @@ void TestFixture::SetUpTestCase() { TestDriver driver; EXPECT_CALL(driver, send_keyboard_mock(_)); keyboard_init(); + + test_logger.info() << "TestFixture setup-up end." << std::endl; } void TestFixture::TearDownTestCase() {} -TestFixture::TestFixture() {} +TestFixture::TestFixture() { m_this = this; } TestFixture::~TestFixture() { + test_logger.info() << "TestFixture clean-up start." << std::endl; TestDriver driver; - // Run for a while to make sure all keys are completely released - EXPECT_CALL(driver, send_keyboard_mock(_)).Times(AnyNumber()); - layer_clear(); + + EXPECT_CALL(driver, send_keyboard_mock(KeyboardReport())).Times(2); + + /* Reset keyboard state. */ clear_all_keys(); - idle_for(TAPPING_TERM + 10); + + clear_keyboard(); + + clear_oneshot_mods(); + clear_oneshot_locked_mods(); + reset_oneshot_layer(); + + layer_clear(); + +#if defined(SWAP_HANDS_ENABLE) + clear_oneshot_swaphands(); +#endif + + idle_for(TAPPING_TERM * 10); + testing::Mock::VerifyAndClearExpectations(&driver); + + /* Verify that the matrix really is cleared */ + EXPECT_CALL(driver, send_keyboard_mock(_)).Times(0); + idle_for(TAPPING_TERM * 10); testing::Mock::VerifyAndClearExpectations(&driver); - // Verify that the matrix really is cleared - EXPECT_CALL(driver, send_keyboard_mock(KeyboardReport())).Times(0); - idle_for(TAPPING_TERM + 10); + + m_this = nullptr; + + test_logger.info() << "TestFixture clean-up end." << std::endl; + + print_test_log(); +} + +void TestFixture::add_key(KeymapKey key) { + if (this->find_key(key.layer, key.position)) { + FAIL() << "Key is already mapped for layer " << +key.layer << " and (column,row) (" << +key.position.col << "," << +key.position.row << ")"; + } + + this->keymap.push_back(key); +} + +void TestFixture::set_keymap(std::initializer_list keys) { + this->keymap.clear(); + for (auto& key : keys) { + add_key(key); + } +} + +const KeymapKey* TestFixture::find_key(layer_t layer, keypos_t position) const { + auto keymap_key_predicate = [&](KeymapKey candidate) { return candidate.layer == layer && candidate.position.col == position.col && candidate.position.row == position.row; }; + + auto result = std::find_if(this->keymap.begin(), this->keymap.end(), keymap_key_predicate); + + if (result != std::end(this->keymap)) { + return &(*result); + } + return nullptr; +} + +void TestFixture::get_keycode(const layer_t layer, const keypos_t position, uint16_t* result) const { + bool key_is_out_of_bounds = position.col >= MATRIX_COLS && position.row >= MATRIX_ROWS; + + if (key_is_out_of_bounds) { + /* See if this is done in hardware as well, because this is 100% out of bounds reads on all QMK keebs out there. */ + auto msg = [&]() { + std::stringstream msg; + msg << "Keycode for position (" << +position.col << "," << +position.row << ") requested! This is out of bounds." << std::endl; + return msg.str(); + }(); + + *result = KC_NO; + test_logger.error() << msg; + EXPECT_FALSE(key_is_out_of_bounds) << msg; + return; + } + + if (auto key = this->find_key(layer, position)) { + *result = key->code; + return; + } + + FAIL() << "No key is mapped for layer " << +layer << " and (column,row) " << +position.col << "," << +position.row << ")"; } void TestFixture::run_one_scan_loop() { @@ -57,3 +155,17 @@ void TestFixture::idle_for(unsigned time) { run_one_scan_loop(); } } + +void TestFixture::print_test_log() const { + const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + if (HasFailure()) { + std::cerr << test_info->test_case_name() << "." << test_info->name() << " failed!" << std::endl; + test_logger.print_log(); + } + test_logger.reset(); +} + +void TestFixture::expect_layer_state(layer_t layer_state) const { + test_logger.trace() << "Layer state: (" << +layer_state << ") Highest layer bit: (" << +get_highest_layer(layer_state) << ")" << std::endl; + EXPECT_TRUE(layer_state_is(layer_state)); +} diff --git a/tests/test_common/test_fixture.hpp b/tests/test_common/test_fixture.hpp index 340503665b..73b5d8d3e8 100644 --- a/tests/test_common/test_fixture.hpp +++ b/tests/test_common/test_fixture.hpp @@ -16,15 +16,34 @@ #pragma once +#include +#include +#include #include "gtest/gtest.h" +#include "keyboard.h" +#include "test_keymap_key.hpp" class TestFixture : public testing::Test { -public: + public: + static TestFixture* m_this; + TestFixture(); ~TestFixture(); static void SetUpTestCase(); static void TearDownTestCase(); + void set_keymap(std::initializer_list keycodes); + void add_key(const KeymapKey key); + + const KeymapKey* find_key(const layer_t layer_t, const keypos_t position) const; + void get_keycode(const layer_t layer, const keypos_t position, uint16_t* result) const; + void run_one_scan_loop(); void idle_for(unsigned ms); + + void expect_layer_state(layer_t layer) const; + + protected: + void print_test_log() const; + std::vector keymap; }; diff --git a/tests/test_common/test_keymap_key.cpp b/tests/test_common/test_keymap_key.cpp new file mode 100644 index 0000000000..878ae097bf --- /dev/null +++ b/tests/test_common/test_keymap_key.cpp @@ -0,0 +1,30 @@ +/* Copyright 2021 Stefan Kerkmann + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "test_keymap_key.hpp" +#include "test_logger.hpp" +#include "gtest/gtest-message.h" +#include "gtest/gtest.h" + +void KeymapKey::press() { + test_logger.trace() << "Key pressed: (" << +this->position.col << "," << +this->position.row << ")" << std::endl; + press_key(this->position.col, this->position.row); +} + +void KeymapKey::release() { + test_logger.trace() << "Key released: (" << +this->position.col << "," << +this->position.row << ")" << std::endl; + release_key(this->position.col, this->position.row); +} \ No newline at end of file diff --git a/tests/test_common/test_keymap_key.hpp b/tests/test_common/test_keymap_key.hpp new file mode 100644 index 0000000000..7861cb4a32 --- /dev/null +++ b/tests/test_common/test_keymap_key.hpp @@ -0,0 +1,46 @@ +/* Copyright 2021 Stefan Kerkmann + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +extern "C" { +#include "keyboard.h" +#include "test_matrix.h" +} + +#include + +typedef uint8_t layer_t; + +struct KeymapKey { + KeymapKey(layer_t layer, uint8_t col, uint8_t row, uint16_t keycode) : layer(layer), position({.col = col, .row = row}), code(keycode), report_code(keycode) { validate(); } + KeymapKey(layer_t layer, uint8_t col, uint8_t row, uint16_t keycode, uint16_t report_code) : layer(layer), position({.col = col, .row = row}), code(keycode), report_code(report_code) { validate(); } + + void press(); + void release(); + + const layer_t layer; + const keypos_t position; + const uint16_t code; + /* Sometimes the keycode does not match the code that is send in the usb report, so we provide it here. */ + const uint16_t report_code; + + private: + void validate() { + assert(position.col <= MATRIX_COLS); + assert(position.row <= MATRIX_ROWS); + } +}; \ No newline at end of file diff --git a/tests/test_common/test_logger.cpp b/tests/test_common/test_logger.cpp new file mode 100644 index 0000000000..959fdde5ec --- /dev/null +++ b/tests/test_common/test_logger.cpp @@ -0,0 +1,39 @@ +/* Copyright 2021 Stefan Kerkmann + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include "test_logger.hpp" + +TestLogger test_logger; + +TestLogger& TestLogger::info() { + *this << "[ INFO ] "; + return *this; +} + +TestLogger& TestLogger::trace() { + *this << "[ TRACE ] "; + return *this; +} + +TestLogger& TestLogger::error() { + *this << "[ ERROR ] "; + return *this; +} + +void TestLogger::reset() { this->m_log.str(""); }; + +void TestLogger::print_log() { std::cerr << this->m_log.str(); } diff --git a/tests/test_common/test_logger.hpp b/tests/test_common/test_logger.hpp new file mode 100644 index 0000000000..348af7fab8 --- /dev/null +++ b/tests/test_common/test_logger.hpp @@ -0,0 +1,35 @@ +/* Copyright 2021 Stefan Kerkmann + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include +#include + +class TestLogger : public std::ostream { + public: + TestLogger() : std::ostream(&m_log){}; + TestLogger& info(); + TestLogger& trace(); + TestLogger& error(); + void print_log(); + void reset(); + + private: + std::stringbuf m_log; +}; + +extern TestLogger test_logger; \ No newline at end of file -- cgit v1.2.1