New command: qmk console (#12828)

* stash poc

* stash

* tidy up implementation

* Tidy up slightly for review

* Tidy up slightly for review

* Bodge environment to make tests pass

* Refactor away from asyncio due to windows issues

* Filter devices

* align vid/pid printing

* Add hidapi to the installers

* start preparing for multiple hid_listeners

* udev rules for hid_listen

* refactor to move closer to end state

* very basic implementation of the threaded model

* refactor how vid/pid/index are supplied and parsed

* windows improvements

* read the report directly when usage page isn't available

* add per-device colors, the choice to show names or numbers, and refactor

* add timestamps

* Add support for showing bootloaders

* tweak the color for bootloaders

* Align bootloader disconnect with connect color

* add support for showing all bootloaders

* fix the pyusb check

* tweaks

* fix exception

* hide a stack trace behind -v

* add --no-bootloaders option

* add documentation for qmk console

* Apply suggestions from code review

Co-authored-by: Ryan <fauxpark@gmail.com>

* pyformat

* clean up and flesh out KNOWN_BOOTLOADERS

Co-authored-by: zvecr <git@zvecr.com>
Co-authored-by: Ryan <fauxpark@gmail.com>
master
Zach White 2021-05-08 20:56:07 -07:00 committed by GitHub
parent d0a3bca9ec
commit 7a25dcacff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 378 additions and 22 deletions

View File

@ -23,6 +23,6 @@ jobs:
with: with:
submodules: recursive submodules: recursive
- name: Install dependencies - name: Install dependencies
run: pip3 install -r requirements.txt run: pip3 install -r requirements-dev.txt
- name: Run tests - name: Run tests
run: bin/qmk pytest run: bin/qmk pytest

View File

@ -33,6 +33,8 @@ def _check_modules(requirements):
# Not every module is importable by its own name. # Not every module is importable by its own name.
if module['name'] == "pep8-naming": if module['name'] == "pep8-naming":
module['import'] = "pep8ext_naming" module['import'] = "pep8ext_naming"
elif module['name'] == 'pyusb':
module['import'] = 'usb.core'
if not find_spec(module['import']): if not find_spec(module['import']):
print('Could not find module %s!' % module['name']) print('Could not find module %s!' % module['name'])

View File

@ -107,6 +107,54 @@ This command lets you configure the behavior of QMK. For the full `qmk config` d
qmk config [-ro] [config_token1] [config_token2] [...] [config_tokenN] qmk config [-ro] [config_token1] [config_token2] [...] [config_tokenN]
``` ```
## `qmk console`
This command lets you connect to keyboard consoles to get debugging messages. It only works if your keyboard firmware has been compiled with `CONSOLE_ENABLED=yes`.
**Usage**:
```
qmk console [-d <pid>:<vid>[:<index>]] [-l] [-n] [-t] [-w <seconds>]
```
**Examples**:
Connect to all available keyboards and show their console messages:
```
qmk console
```
List all devices:
```
qmk console -l
```
Show only messages from clueboard/66/rev3 keyboards:
```
qmk console -d C1ED:2370
```
Show only messages from the second clueboard/66/rev3:
```
qmk console -d C1ED:2370:2
```
Show timestamps and VID:PID instead of names:
```
qmk console -n -t
```
Disable bootloader messages:
```
qmk console --no-bootloaders
```
## `qmk doctor` ## `qmk doctor`
This command examines your environment and alerts you to potential build or flash problems. It can fix many of them if you want it to. This command examines your environment and alerts you to potential build or flash problems. It can fix many of them if you want it to.

View File

@ -12,6 +12,7 @@ from . import chibios
from . import clean from . import clean
from . import compile from . import compile
from . import config from . import config
from . import console
from . import docs from . import docs
from . import doctor from . import doctor
from . import fileformat from . import fileformat

View File

@ -0,0 +1,302 @@
"""Acquire debugging information from usb hid devices
cli implementation of https://www.pjrc.com/teensy/hid_listen.html
"""
from pathlib import Path
from threading import Thread
from time import sleep, strftime
import hid
import usb.core
from milc import cli
LOG_COLOR = {
'next': 0,
'colors': [
'{fg_blue}',
'{fg_cyan}',
'{fg_green}',
'{fg_magenta}',
'{fg_red}',
'{fg_yellow}',
],
}
KNOWN_BOOTLOADERS = {
# VID , PID
('03EB', '2FEF'): 'atmel-dfu: ATmega16U2',
('03EB', '2FF0'): 'atmel-dfu: ATmega32U2',
('03EB', '2FF3'): 'atmel-dfu: ATmega16U4',
('03EB', '2FF4'): 'atmel-dfu: ATmega32U4',
('03EB', '2FF9'): 'atmel-dfu: AT90USB64',
('03EB', '2FFA'): 'atmel-dfu: AT90USB162',
('03EB', '2FFB'): 'atmel-dfu: AT90USB128',
('03EB', '6124'): 'Microchip SAM-BA',
('0483', 'DF11'): 'stm32-dfu: STM32 BOOTLOADER',
('16C0', '05DC'): 'USBasp: USBaspLoader',
('16C0', '05DF'): 'bootloadHID: HIDBoot',
('16C0', '0478'): 'halfkay: Teensy Halfkay',
('1B4F', '9203'): 'caterina: Pro Micro 3.3V',
('1B4F', '9205'): 'caterina: Pro Micro 5V',
('1B4F', '9207'): 'caterina: LilyPadUSB',
('1C11', 'B007'): 'kiibohd: Kiibohd DFU Bootloader',
('1EAF', '0003'): 'stm32duino: Maple 003',
('1FFB', '0101'): 'caterina: Polou A-Star 32U4 Bootloader',
('2341', '0036'): 'caterina: Arduino Leonardo',
('2341', '0037'): 'caterina: Arduino Micro',
('239A', '000C'): 'caterina: Adafruit Feather 32U4',
('239A', '000D'): 'caterina: Adafruit ItsyBitsy 32U4 3v',
('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v',
('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v',
('2A03', '0036'): 'caterina: Arduino Leonardo',
('2A03', '0037'): 'caterina: Arduino Micro',
('314B', '0106'): 'apm32-dfu: APM32 DFU ISP Mode'
}
class MonitorDevice(object):
def __init__(self, hid_device, numeric):
self.hid_device = hid_device
self.numeric = numeric
self.device = hid.Device(path=hid_device['path'])
self.current_line = ''
cli.log.info('Console Connected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', hid_device)
def read(self, size, encoding='ascii', timeout=1):
"""Read size bytes from the device.
"""
return self.device.read(size, timeout).decode(encoding)
def read_line(self):
"""Read from the device's console until we get a \n.
"""
while '\n' not in self.current_line:
self.current_line += self.read(32).replace('\x00', '')
lines = self.current_line.split('\n', 1)
self.current_line = lines[1]
return lines[0]
def run_forever(self):
while True:
try:
message = {**self.hid_device, 'text': self.read_line()}
identifier = (int2hex(message['vendor_id']), int2hex(message['product_id'])) if self.numeric else (message['manufacturer_string'], message['product_string'])
message['identifier'] = ':'.join(identifier)
message['ts'] = '{style_dim}{fg_green}%s{style_reset_all} ' % (strftime(cli.config.general.datetime_fmt),) if cli.args.timestamp else ''
cli.echo('%(ts)s%(color)s%(identifier)s:%(index)d{style_reset_all}: %(text)s' % message)
except hid.HIDException:
break
class FindDevices(object):
def __init__(self, vid, pid, index, numeric):
self.vid = vid
self.pid = pid
self.index = index
self.numeric = numeric
def run_forever(self):
"""Process messages from our queue in a loop.
"""
live_devices = {}
live_bootloaders = {}
while True:
try:
for device in list(live_devices):
if not live_devices[device]['thread'].is_alive():
cli.log.info('Console Disconnected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', live_devices[device])
del live_devices[device]
for device in self.find_devices():
if device['path'] not in live_devices:
device['color'] = LOG_COLOR['colors'][LOG_COLOR['next']]
LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors'])
live_devices[device['path']] = device
try:
monitor = MonitorDevice(device, self.numeric)
device['thread'] = Thread(target=monitor.run_forever, daemon=True)
device['thread'].start()
except Exception as e:
device['e'] = e
device['e_name'] = e.__class__.__name__
cli.log.error("Could not connect to %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s:%(vendor_id)04X:%(product_id)04X:%(index)d): %(e_name)s: %(e)s", device)
if cli.config.general.verbose:
cli.log.exception(e)
del live_devices[device['path']]
if cli.args.bootloaders:
for device in self.find_bootloaders():
if device.address in live_bootloaders:
live_bootloaders[device.address]._qmk_found = True
else:
name = KNOWN_BOOTLOADERS[(int2hex(device.idVendor), int2hex(device.idProduct))]
cli.log.info('Bootloader Connected: {style_bright}{fg_magenta}%s', name)
device._qmk_found = True
live_bootloaders[device.address] = device
for device in list(live_bootloaders):
if live_bootloaders[device]._qmk_found:
live_bootloaders[device]._qmk_found = False
else:
name = KNOWN_BOOTLOADERS[(int2hex(live_bootloaders[device].idVendor), int2hex(live_bootloaders[device].idProduct))]
cli.log.info('Bootloader Disconnected: {style_bright}{fg_magenta}%s', name)
del live_bootloaders[device]
sleep(.1)
except KeyboardInterrupt:
break
def is_bootloader(self, hid_device):
"""Returns true if the device in question matches a known bootloader vid/pid.
"""
return (int2hex(hid_device.idVendor), int2hex(hid_device.idProduct)) in KNOWN_BOOTLOADERS
def is_console_hid(self, hid_device):
"""Returns true when the usage page indicates it's a teensy-style console.
"""
return hid_device['usage_page'] == 0xFF31 and hid_device['usage'] == 0x0074
def is_filtered_device(self, hid_device):
"""Returns True if the device should be included in the list of available consoles.
"""
return int2hex(hid_device['vendor_id']) == self.vid and int2hex(hid_device['product_id']) == self.pid
def find_devices_by_report(self, hid_devices):
"""Returns a list of available teensy-style consoles by doing a brute-force search.
Some versions of linux don't report usage and usage_page. In that case we fallback to reading the report (possibly inaccurately) ourselves.
"""
devices = []
for device in hid_devices:
path = device['path'].decode('utf-8')
if path.startswith('/dev/hidraw'):
number = path[11:]
report = Path(f'/sys/class/hidraw/hidraw{number}/device/report_descriptor')
if report.exists():
rp = report.read_bytes()
if rp[1] == 0x31 and rp[3] == 0x09:
devices.append(device)
return devices
def find_bootloaders(self):
"""Returns a list of available bootloader devices.
"""
return list(filter(self.is_bootloader, usb.core.find(find_all=True)))
def find_devices(self):
"""Returns a list of available teensy-style consoles.
"""
hid_devices = hid.enumerate()
devices = list(filter(self.is_console_hid, hid_devices))
if not devices:
devices = self.find_devices_by_report(hid_devices)
if self.vid and self.pid:
devices = list(filter(self.is_filtered_device, devices))
# Add index numbers
device_index = {}
for device in devices:
id = ':'.join((int2hex(device['vendor_id']), int2hex(device['product_id'])))
if id not in device_index:
device_index[id] = 0
device_index[id] += 1
device['index'] = device_index[id]
return devices
def int2hex(number):
"""Returns a string representation of the number as hex.
"""
return "%04X" % number
def list_devices(device_finder):
"""Show the user a nicely formatted list of devices.
"""
devices = device_finder.find_devices()
if devices:
cli.log.info('Available devices:')
for dev in devices:
color = LOG_COLOR['colors'][LOG_COLOR['next']]
LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors'])
cli.log.info("\t%s%s:%s:%d{style_reset_all}\t%s %s", color, int2hex(dev['vendor_id']), int2hex(dev['product_id']), dev['index'], dev['manufacturer_string'], dev['product_string'])
if cli.args.bootloaders:
bootloaders = device_finder.find_bootloaders()
if bootloaders:
cli.log.info('Available Bootloaders:')
for dev in bootloaders:
cli.log.info("\t%s:%s\t%s", int2hex(dev.idVendor), int2hex(dev.idProduct), KNOWN_BOOTLOADERS[(int2hex(dev.idVendor), int2hex(dev.idProduct))])
@cli.argument('--bootloaders', arg_only=True, default=True, action='store_boolean', help='displaying bootloaders.')
@cli.argument('-d', '--device', help='Device to select - uses format <pid>:<vid>[:<index>].')
@cli.argument('-l', '--list', arg_only=True, action='store_true', help='List available hid_listen devices.')
@cli.argument('-n', '--numeric', arg_only=True, action='store_true', help='Show VID/PID instead of names.')
@cli.argument('-t', '--timestamp', arg_only=True, action='store_true', help='Print the timestamp for received messages as well.')
@cli.argument('-w', '--wait', type=int, default=1, help="How many seconds to wait between checks (Default: 1)")
@cli.subcommand('Acquire debugging information from usb hid devices.', hidden=False if cli.config.user.developer else True)
def console(cli):
"""Acquire debugging information from usb hid devices
"""
vid = None
pid = None
index = 1
if cli.config.console.device:
device = cli.config.console.device.split(':')
if len(device) == 2:
vid, pid = device
elif len(device) == 3:
vid, pid, index = device
if not index.isdigit():
cli.log.error('Device index must be a number! Got "%s" instead.', index)
exit(1)
index = int(index)
if index < 1:
cli.log.error('Device index must be greater than 0! Got %s', index)
exit(1)
else:
cli.log.error('Invalid format for device, expected "<pid>:<vid>[:<index>]" but got "%s".', cli.config.console.device)
cli.print_help()
exit(1)
vid = vid.upper()
pid = pid.upper()
device_finder = FindDevices(vid, pid, index, cli.args.numeric)
if cli.args.list:
return list_devices(device_finder)
print('Looking for devices...', flush=True)
device_finder.run_forever()

View File

@ -4,5 +4,7 @@
# Python development requirements # Python development requirements
nose2 nose2
flake8 flake8
hid
pep8-naming pep8-naming
pyusb
yapf yapf

View File

@ -4,13 +4,13 @@ _qmk_install() {
echo "Installing dependencies" echo "Installing dependencies"
sudo pacman --needed --noconfirm -S \ sudo pacman --needed --noconfirm -S \
base-devel clang diffutils gcc git unzip wget zip \ base-devel clang diffutils gcc git unzip wget zip python-pip \
python-pip \ avr-binutils arm-none-eabi-binutils arm-none-eabi-gcc \
avr-binutils \ arm-none-eabi-newlib avrdude dfu-programmer dfu-util
arm-none-eabi-binutils arm-none-eabi-gcc arm-none-eabi-newlib \
avrdude dfu-programmer dfu-util
sudo pacman --needed --noconfirm -U https://archive.archlinux.org/packages/a/avr-gcc/avr-gcc-8.3.0-1-x86_64.pkg.tar.xz sudo pacman --needed --noconfirm -U https://archive.archlinux.org/packages/a/avr-gcc/avr-gcc-8.3.0-1-x86_64.pkg.tar.xz
sudo pacman --needed --noconfirm -S avr-libc # Must be installed after the above, or it will bring in the latest avr-gcc instead sudo pacman --needed --noconfirm -S avr-libc # Must be installed after the above, or it will bring in the latest avr-gcc instead
sudo pacman --needed --noconfirm -S hidapi # This will fail if the community repo isn't enabled
python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt
} }

View File

@ -13,10 +13,9 @@ _qmk_install() {
sudo apt-get -yq install \ sudo apt-get -yq install \
build-essential clang-format diffutils gcc git unzip wget zip \ build-essential clang-format diffutils gcc git unzip wget zip \
python3-pip \ python3-pip binutils-avr gcc-avr avr-libc binutils-arm-none-eabi \
binutils-avr gcc-avr avr-libc \ gcc-arm-none-eabi libnewlib-arm-none-eabi avrdude dfu-programmer \
binutils-arm-none-eabi gcc-arm-none-eabi libnewlib-arm-none-eabi \ dfu-util teensy-loader-cli libhidapi-hidraw0
avrdude dfu-programmer dfu-util teensy-loader-cli libusb-dev
python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt
} }

View File

@ -5,11 +5,10 @@ _qmk_install() {
# TODO: Check whether devel/headers packages are really needed # TODO: Check whether devel/headers packages are really needed
sudo dnf -y install \ sudo dnf -y install \
clang diffutils git gcc glibc-headers kernel-devel kernel-headers make unzip wget zip \ clang diffutils git gcc glibc-headers kernel-devel kernel-headers \
python3 \ make unzip wget zip python3 avr-binutils avr-gcc avr-libc \
avr-binutils avr-gcc avr-libc \
arm-none-eabi-binutils-cs arm-none-eabi-gcc-cs arm-none-eabi-newlib \ arm-none-eabi-binutils-cs arm-none-eabi-gcc-cs arm-none-eabi-newlib \
avrdude dfu-programmer dfu-util libusb-devel avrdude dfu-programmer dfu-util hidapi
python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt
} }

View File

@ -22,9 +22,10 @@ _qmk_install() {
echo "sys-devel/gcc multilib" | sudo tee --append /etc/portage/package.use/qmkfirmware >/dev/null echo "sys-devel/gcc multilib" | sudo tee --append /etc/portage/package.use/qmkfirmware >/dev/null
sudo emerge -auN sys-devel/gcc sudo emerge -auN sys-devel/gcc
sudo emerge -au --noreplace \ sudo emerge -au --noreplace \
app-arch/unzip app-arch/zip net-misc/wget sys-devel/clang sys-devel/crossdev \ app-arch/unzip app-arch/zip net-misc/wget sys-devel/clang \
\>=dev-lang/python-3.7 \ sys-devel/crossdev \>=dev-lang/python-3.7 dev-embedded/avrdude \
dev-embedded/avrdude dev-embedded/dfu-programmer app-mobilephone/dfu-util dev-embedded/dfu-programmer app-mobilephone/dfu-util sys-apps/hwloc \
dev-libs/hidapi
sudo crossdev -s4 --stable --g \<9 --portage --verbose --target avr sudo crossdev -s4 --stable --g \<9 --portage --verbose --target avr
sudo crossdev -s4 --stable --g \<9 --portage --verbose --target arm-none-eabi sudo crossdev -s4 --stable --g \<9 --portage --verbose --target arm-none-eabi

View File

@ -9,11 +9,10 @@ _qmk_install() {
pacman --needed --noconfirm --disable-download-timeout -S pactoys-git pacman --needed --noconfirm --disable-download-timeout -S pactoys-git
pacboy sync --needed --noconfirm --disable-download-timeout \ pacboy sync --needed --noconfirm --disable-download-timeout \
base-devel: toolchain:x clang:x git: unzip: \ base-devel: toolchain:x clang:x git: unzip: python3-pip:x \
python3-pip:x \ avr-binutils:x avr-gcc:x avr-libc:x arm-none-eabi-binutils:x \
avr-binutils:x avr-gcc:x avr-libc:x \ arm-none-eabi-gcc:x arm-none-eabi-newlib:x avrdude:x bootloadhid:x \
arm-none-eabi-binutils:x arm-none-eabi-gcc:x arm-none-eabi-newlib:x \ dfu-programmer:x dfu-util:x teensy-loader-cli:x hidapi:x
avrdude:x bootloadhid:x dfu-programmer:x dfu-util:x teensy-loader-cli:x
_qmk_install_drivers _qmk_install_drivers

View File

@ -60,3 +60,6 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="239a", ATTRS{idProduct}=="000e", TAG+="uacc
SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="0036", TAG+="uaccess", ENV{ID_MM_DEVICE_IGNORE}="1" SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="0036", TAG+="uaccess", ENV{ID_MM_DEVICE_IGNORE}="1"
### Micro ### Micro
SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="0037", TAG+="uaccess", ENV{ID_MM_DEVICE_IGNORE}="1" SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="0037", TAG+="uaccess", ENV{ID_MM_DEVICE_IGNORE}="1"
# hid_listen
KERNEL=="hidraw*", MODE="0660", GROUP="plugdev", TAG+="uaccess", TAG+="udev-acl"