Yakut v0.10 with significant usability improvements (#49)

* Improve usability of joystick

* Refactor data type loading and implement service discovery for client; fixes #42 fixes #38

* Make subscription metadata disabled by default

* Fix resource management issues at exit; fixes #40

* Monitor: improve compatibility with small terminals and automatically truncate text to prevent wrapping & scrolling

* Introduce explicit aliases to improve shorthand syntax

---

Close #5 
Close #38 
Close #39 
Presumably fix #40 (hard to reproduce, will reopen later if necessary)
Close #42 (version numbers are no longer required)
Close #45 (automatic type discovery)

.appveyor.yml

@@ -85,5 +85,10 @@ for:
 
       - git push --tags
 
-artifacts:
-  - path: '.nox/*/*/*.log'
+# https://github.com/appveyor/ci/issues/401#issuecomment-301649481
+# https://www.appveyor.com/docs/packaging-artifacts/#pushing-artifacts-from-scripts
+on_finish:
+- ps: >
+    $root = Resolve-Path .nox;
+    [IO.Directory]::GetFiles($root.Path, '*.log', 'AllDirectories') |
+    % { Push-AppveyorArtifact $_ -FileName $_.Substring($root.Path.Length + 1) -DeploymentName to-publish }

noxfile.py

@@ -72,7 +72,13 @@ def test(session):
         "PYTHONPATH": str(DEPS_DIR),
         "PATH": os.pathsep.join([session.env["PATH"], str(DEPS_DIR)]),
     }
-    session.run("pytest", *map(str, src_dirs), env=env)
+    session.run(
+        "pytest",
+        *map(str, src_dirs),
+        # Log format cannot be specified in setup.cfg https://github.com/pytest-dev/pytest/issues/3062
+        r"--log-file-format=%(asctime)s %(levelname)-3.3s %(name)s: %(message)s",
+        env=env,
+    )
 
     # The coverage threshold is intentionally set low for interactive runs because when running locally
     # in a reused virtualenv the DSDL compiler run may be skipped to save time, resulting in a reduced coverage.

setup.cfg

@@ -2,7 +2,7 @@
 name            = yakut
 version         = file: yakut/VERSION
 author          = OpenCyphal
-author_email    = consortium@opencyphal.org
+author_email    = maintainers@opencyphal.org
 url             = https://opencyphal.org
 description     = Simple CLI tool for diagnostics and debugging of Cyphal networks.
 long_description = file: README.md
@@ -48,7 +48,7 @@ zip_safe             = False
 include_package_data = True
 packages             = find:
 install_requires =
-    pycyphal[transport-udp,transport-serial,transport-can-pythoncan] ~= 1.6
+    pycyphal[transport-udp,transport-serial,transport-can-pythoncan] ~= 1.8
     ruamel.yaml     <  0.18
     requests        ~= 2.27
     simplejson      ~= 3.17
@@ -93,7 +93,6 @@ testpaths        = yakut tests
 python_files     = *.py
 python_classes   = _UnitTest
 python_functions = _unittest_
-# Verbose logging is required to ensure full coverage of conditional logging branches.
 log_level        = DEBUG
 log_cli_level    = WARNING
 log_cli          = true

tests/cmd/call.py

@@ -4,16 +4,19 @@
 
 from __future__ import annotations
 import json
+import logging
 import asyncio
 import typing
-import pycyphal
 import pytest
 from tests.subprocess import Subprocess, execute_cli
 from tests.dsdl import OUTPUT_DIR
 from tests.transport import TransportFactory
 from yakut.param.transport import construct_transport
 
 
+_logger = logging.getLogger(__name__)
+
+
 @pytest.mark.asyncio
 async def _unittest_call_custom(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None:
     asyncio.get_running_loop().slow_callback_duration = 5.0
@@ -22,14 +25,23 @@ async def _unittest_call_custom(transport_factory: TransportFactory, compiled_ds
     env = {
         "YAKUT_TRANSPORT": transport_factory(88).expression,
         "YAKUT_PATH": str(OUTPUT_DIR),
+        "PYCYPHAL_LOGLEVEL": "INFO",  # We don't want too much output in the logs.
     }
 
+    import pycyphal.application
+    import uavcan.node
     from sirius_cyber_corp import PerformLinearLeastSquaresFit_1_0
 
     # Set up the server that we will be testing the client against.
-    server_transport = construct_transport(transport_factory(22).expression)
-    server_presentation = pycyphal.presentation.Presentation(server_transport)
-    server = server_presentation.get_server(PerformLinearLeastSquaresFit_1_0, 222)
+    server_node = pycyphal.application.make_node(
+        uavcan.node.GetInfo_1.Response(),
+        transport=construct_transport(transport_factory(22).expression),
+    )
+    server_node.start()
+    server_node.registry["uavcan.srv.least_squares.id"] = pycyphal.application.register.ValueProxy(
+        pycyphal.application.register.Natural16([222])
+    )
+    server = server_node.get_server(PerformLinearLeastSquaresFit_1_0, "least_squares")
     last_metadata: typing.Optional[pycyphal.presentation.ServiceRequestMetadata] = None
 
     async def handle_request(
@@ -50,30 +62,26 @@ async def handle_request(
         print("RESPONSE OBJECT:", response)
         return response
 
-    # Invoke the service and then run the server for a few seconds to let it process the request.
-    proc = Subprocess.cli(
-        "-v",
+    # Invoke the service without discovery and then run the server for a few seconds to let it process the request.
+    proc = Subprocess.cli(  # Windows compat: -v blocks stderr pipe on Windows.
         "--format=json",
         "call",
         "22",
-        "222:sirius_cyber_corp.PerformLinearLeastSquaresFit.1.0",
+        "222:sirius_cyber_corp.PerformLinearLeastSquaresFit",
         "points: [{x: 10, y: 1}, {x: 20, y: 2}]",
         "--priority=SLOW",
         "--with-metadata",
         environment_variables=env,
     )
-    await server.serve_for(handle_request, 3.0)
+    _logger.info("Checkpoint A")
+    await server.serve_for(handle_request, 5.0)
+    _logger.info("Checkpoint B")
     result, stdout, _ = proc.wait(5.0)
+    _logger.info("Checkpoint C")
     assert result == 0
     assert last_metadata is not None
     assert last_metadata.priority == pycyphal.transport.Priority.SLOW
     assert last_metadata.client_node_id == 88
-
-    # Finalize to avoid warnings in the output.
-    server.close()
-    server_presentation.close()
-    await asyncio.sleep(1.0)
-
     # Parse the output and validate it.
     parsed = json.loads(stdout)
     print("PARSED RESPONSE:", parsed)
@@ -82,13 +90,76 @@ async def handle_request(
     assert parsed["222"]["slope"] == pytest.approx(0.1)
     assert parsed["222"]["y_intercept"] == pytest.approx(0.0)
 
+    # Invoke the service with ID discovery and static type.
+    last_metadata = None
+    proc = Subprocess.cli(  # Windows compat: -v blocks stderr pipe on Windows.
+        "--format=json",
+        "call",
+        "22",
+        "least_squares:sirius_cyber_corp.PerformLinearLeastSquaresFit",
+        "points: [{x: 0, y: 0}, {x: 10, y: 3}]",
+        "--priority=FAST",
+        "--with-metadata",
+        "--timeout=5",
+        environment_variables=env,
+    )
+    _logger.info("Checkpoint A")
+    await server.serve_for(handle_request, 10.0)  # The tested process may take a few seconds to start (see logs).
+    _logger.info("Checkpoint B")
+    result, stdout, _ = proc.wait(10.0)
+    _logger.info("Checkpoint C")
+    assert result == 0
+    assert last_metadata is not None
+    assert last_metadata.priority == pycyphal.transport.Priority.FAST
+    assert last_metadata.client_node_id == 88
+    # Parse the output and validate it.
+    parsed = json.loads(stdout)
+    print("PARSED RESPONSE:", parsed)
+    assert parsed["222"]["_metadata_"]["priority"] == "fast"
+    assert parsed["222"]["_metadata_"]["source_node_id"] == 22
+    assert parsed["222"]["slope"] == pytest.approx(0.3)
+    assert parsed["222"]["y_intercept"] == pytest.approx(0.0)
+
+    # Invoke the service with full discovery.
+    last_metadata = None
+    proc = Subprocess.cli(  # Windows compat: -v blocks stderr pipe on Windows.
+        "--format=json",
+        "call",
+        "22",
+        "least_squares",  # Type not specified -- discovered.
+        "points: [{x: 0, y: 0}, {x: 10, y: 4}]",
+        "--with-metadata",
+        "--timeout=5",
+        environment_variables=env,
+    )
+    _logger.info("Checkpoint A")
+    await server.serve_for(handle_request, 10.0)  # The tested process may take a few seconds to start (see logs).
+    _logger.info("Checkpoint B")
+    result, stdout, _ = proc.wait(10.0)
+    _logger.info("Checkpoint C")
+    assert result == 0
+    assert last_metadata is not None
+    assert last_metadata.priority == pycyphal.transport.Priority.NOMINAL
+    assert last_metadata.client_node_id == 88
+    # Parse the output and validate it.
+    parsed = json.loads(stdout)
+    print("PARSED RESPONSE:", parsed)
+    assert parsed["222"]["_metadata_"]["priority"] == "nominal"
+    assert parsed["222"]["_metadata_"]["source_node_id"] == 22
+    assert parsed["222"]["slope"] == pytest.approx(0.4)
+    assert parsed["222"]["y_intercept"] == pytest.approx(0.0)
+
+    # Finalize to avoid warnings in the output.
+    server.close()
+    server_node.close()
+    await asyncio.sleep(1.0)
+
     # Timed-out request.
     result, stdout, stderr = execute_cli(
         "call",
         "--timeout=0.1",
         "22",
-        "222:sirius_cyber_corp.PerformLinearLeastSquaresFit.1.0",
-        "points: [{x: 10, y: 1}, {x: 20, y: 2}]",
+        "222:sirius_cyber_corp.PerformLinearLeastSquaresFit",
         environment_variables=env,
         ensure_success=False,
         log=False,
@@ -97,6 +168,61 @@ async def handle_request(
     assert stdout == ""
     assert "timed out" in stderr
 
+    # Timed out discovery.
+    result, stdout, stderr = execute_cli(
+        "call",
+        "--timeout=0.1",
+        "22",
+        "least_squares",
+        environment_variables=env,
+        ensure_success=False,
+        log=False,
+    )
+    assert result == 1
+    assert stdout == ""
+    assert "resolve service" in stderr
+
+
+@pytest.mark.asyncio
+async def _unittest_call_fixed(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None:
+    asyncio.get_running_loop().slow_callback_duration = 5.0
+
+    _ = compiled_dsdl
+    env = {
+        "YAKUT_TRANSPORT": transport_factory(88).expression,
+        "YAKUT_PATH": str(OUTPUT_DIR),
+        "PYCYPHAL_LOGLEVEL": "INFO",  # We don't want too much output in the logs.
+    }
+
+    import pycyphal.application
+    import uavcan.node
+
+    server_node = pycyphal.application.make_node(
+        uavcan.node.GetInfo_1.Response(),
+        transport=construct_transport(transport_factory(22).expression),
+    )
+    server_node.start()
+
+    # Invoke a fixed port-ID service.
+    proc = Subprocess.cli(  # Windows compat: -v blocks stderr pipe on Windows.
+        "--format=json",
+        "call",
+        "22",
+        "uavcan.node.GetInfo",
+        "--timeout=5.0",
+        environment_variables=env,
+    )
+    await asyncio.sleep(10.0)  # The tested process may take a few seconds to start (see logs).
+    result, stdout, _ = proc.wait(10.0)
+    assert 0 == result
+    parsed = json.loads(stdout)
+    print("PARSED RESPONSE:", parsed)
+    assert parsed["430"]
+
+    # Finalize to avoid warnings in the output.
+    server_node.close()
+    await asyncio.sleep(1.0)
+
 
 def _unittest_call_errors(compiled_dsdl: typing.Any) -> None:
     _ = compiled_dsdl
@@ -108,7 +234,7 @@ def _unittest_call_errors(compiled_dsdl: typing.Any) -> None:
     result, stdout, stderr = execute_cli(
         "call",
         "22",
-        "222:sirius_cyber_corp.PointXY.1.0",
+        "222:sirius_cyber_corp.PointXY",
         environment_variables=env,
         ensure_success=False,
         log=False,
@@ -121,7 +247,7 @@ def _unittest_call_errors(compiled_dsdl: typing.Any) -> None:
     result, stdout, stderr = execute_cli(
         "call",
         "22",
-        "222:sirius_cyber_corp.PointXY.1.0",
+        "222:sirius_cyber_corp.PointXY",
         ensure_success=False,
         log=False,
     )
@@ -134,7 +260,7 @@ def _unittest_call_errors(compiled_dsdl: typing.Any) -> None:
         f"--path={OUTPUT_DIR}",
         "call",
         "22",
-        "222:sirius_cyber_corp.PerformLinearLeastSquaresFit.1.0",
+        "222:sirius_cyber_corp.PerformLinearLeastSquaresFit.1",
         ": }",
         ensure_success=False,
         log=False,

tests/cmd/main.py

@@ -14,12 +14,9 @@ def _unittest_help() -> None:
     execute_cli("--help", timeout=10.0, log=False)
     for cmd in dir(yakut.cmd):
         if not cmd.startswith("_") and cmd not in ("pycyphal", "sys"):
-            execute_cli(cmd, "--help", timeout=3.0, log=False)
+            execute_cli(cmd.replace("_", "-"), "--help", timeout=3.0, log=False)
 
 
 def _unittest_error() -> None:
     with pytest.raises(CalledProcessError):
         execute_cli("invalid-command", timeout=2.0, log=False)
-
-    with pytest.raises(CalledProcessError):  # Ambiguous abbreviation.
-        execute_cli("c", timeout=2.0, log=False)

tests/cmd/monitor.py

@@ -130,7 +130,7 @@ async def _unittest_monitor_errors(compiled_dsdl: Any, serial_broker: str) -> No
     assert cells[5][0] == "3333"
 
     # Error counter
-    assert cells[-1][4] == "1"
+    assert cells[-1][2] == "1"
 
     await asyncio.sleep(3.0)