Adding support for file-based reads and externalized type dependencies (

#103)

This is non-breaking change that adds a new public method:

read_files - a file-oriented entry point to the front end. This takes a
list of target DSDL files allowing the user to maintain an explicit list
instead of depending on globular filesystem discovery. Furthermore this
method returns a set which is the transitive closure of types depended
on by the target list of types. This allows consumers to track
dependencies and compiler back ends to generate .d files and otherwise
support incremental builds.

The new method may increase performance for systems with large pools of
messages when generating code for a small sub-set as it only considers
the target and dependencies of the target when parsing dsdl files.

---------

Co-authored-by: Pavel Kirienko <[email protected]>

.devcontainer/devcontainer.json

@@ -0,0 +1,24 @@
+{
+    "name": "Python dev environment",
+    "image": "ghcr.io/opencyphal/toxic:tx22.4.2",
+    "workspaceFolder": "/workspace",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=delegated",
+    "mounts": [
+        "source=root-vscode-server,target=/root/.vscode-server/extensions,type=volume",
+        "source=pydsdl-tox,target=/workspace/.nox,type=volume"
+    ],
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "uavcan.dsdl",
+                "wholroyd.jinja",
+                "streetsidesoftware.code-spell-checker",
+                "ms-python.python",
+                "ms-python.mypy-type-checker",
+                "ms-python.black-formatter",
+                "ms-python.pylint"
+            ]
+        }
+    },
+    "postCreateCommand": "git clone --depth 1 [email protected]:OpenCyphal/public_regulated_data_types.git .dsdl-test && nox -e test-3.12"
+}

.github/workflows/test-and-release.yml

@@ -1,5 +1,5 @@
 name: 'Test and Release PyDSDL'
-on: push
+on: [ push, pull_request ]
 
 # Ensures that only one workflow is running at a time
 concurrency:
@@ -9,11 +9,14 @@ concurrency:
 jobs:
   pydsdl-test:
     name: Test PyDSDL
+    # Run on push OR on 3rd-party PR.
+    # https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=edited#pull_request
+    if: (github.event_name == 'push') || github.event.pull_request.head.repo.fork
     strategy:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest ]
-        python: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
+        python: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
         include:
           - os: windows-2019
             python: '3.10'
@@ -22,10 +25,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Check out pydsdl
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check out public_regulated_data_types
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: OpenCyphal/public_regulated_data_types
           path: .dsdl-test
@@ -50,27 +53,28 @@ jobs:
       - name: Run build and test
         run: |
           if [ "$RUNNER_OS" == "Linux" ]; then
-            nox --non-interactive --error-on-missing-interpreters --session test test_eol pristine lint --python ${{ matrix.python }}
+            nox --non-interactive --error-on-missing-interpreters --session test pristine lint --python ${{ matrix.python }}
             nox --non-interactive --session docs
           elif [ "$RUNNER_OS" == "Windows" ]; then
-            nox --forcecolor --non-interactive --error-on-missing-interpreters --session test test_eol pristine lint
+            nox --forcecolor --non-interactive --error-on-missing-interpreters --session test pristine lint
           elif [ "$RUNNER_OS" == "macOS" ]; then
-            nox --non-interactive --error-on-missing-interpreters --session test test_eol pristine lint --python ${{ matrix.python }}
+            nox --non-interactive --error-on-missing-interpreters --session test pristine lint --python ${{ matrix.python }}
           else
             echo "${{ runner.os }} not supported"
             exit 1
           fi
-          python -c "import pydsdl; pydsdl.read_namespace('.dsdl-test/uavcan', [])"
         shell: bash
 
   pydsdl-release:
     name: Release PyDSDL
-    if: contains(github.event.head_commit.message, '#release') || contains(github.ref, '/master')
+    if: >
+      (github.event_name == 'push') &&
+      (contains(github.event.head_commit.message, '#release') || contains(github.ref, '/master'))
     needs: pydsdl-test
     runs-on: ubuntu-latest
     steps:
       - name: Check out
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build distribution
         run: |

.readthedocs.yml

@@ -2,6 +2,13 @@
 
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+  apt_packages:
+    - graphviz
+
 sphinx:
   configuration: docs/conf.py
   fail_on_warning: true

conftest.py

@@ -0,0 +1,63 @@
+#
+# Copyright (C) OpenCyphal Development Team  <opencyphal.org>
+# Copyright Amazon.com Inc. or its affiliates.
+# SPDX-License-Identifier: MIT
+#
+"""
+Configuration for pytest tests including fixtures and hooks.
+"""
+
+import tempfile
+from pathlib import Path
+from typing import Any, Optional
+
+import pytest
+
+
+# +-------------------------------------------------------------------------------------------------------------------+
+# | TEST FIXTURES
+# +-------------------------------------------------------------------------------------------------------------------+
+class TemporaryDsdlContext:
+    """
+    Powers the temp_dsdl_factory test fixture.
+    """
+    def __init__(self) -> None:
+        self._base_dir: Optional[Any] = None
+
+    def new_file(self, file_path: Path, text: Optional[str] = None) -> Path:
+        if file_path.is_absolute():
+            raise ValueError(f"{file_path} is an absolute path. The test fixture requires relative paths to work.")
+        file = self.base_dir / file_path
+        file.parent.mkdir(parents=True, exist_ok=True)
+        if text is not None:
+            file.write_text(text)
+        return file
+
+    @property
+    def base_dir(self) -> Path:
+        if self._base_dir is None:
+            self._base_dir = tempfile.TemporaryDirectory()
+        return Path(self._base_dir.name).resolve()
+
+    def _test_path_finalizer(self) -> None:
+        """
+        Finalizer to clean up any temporary directories created during the test.
+        """
+        if self._base_dir is not None:
+            self._base_dir.cleanup()
+            del self._base_dir
+            self._base_dir = None
+
+@pytest.fixture(scope="function")
+def temp_dsdl_factory(request: pytest.FixtureRequest) -> Any:  # pylint: disable=unused-argument
+    """
+    Fixture for pydsdl tests that have to create files as part of the test. This object stays in-scope for a given
+    test method and does not requires a context manager in the test itself.
+
+    Call `new_file(path)` to create a new file path in the fixture's temporary directory. This will create all
+    uncreated parent directories but will _not_ create the file unless text is provided: `new_file(path, "hello")`
+    """
+    f = TemporaryDsdlContext()
+    request.addfinalizer(f._test_path_finalizer)  # pylint: disable=protected-access
+    return f
+

docs/index.rst

@@ -15,6 +15,7 @@ Contents
 --------
 
 .. toctree::
+   :maxdepth: 2
 
    pages/installation
    pages/pydsdl

docs/pages/pydsdl.rst

@@ -12,10 +12,11 @@ You can find a practical usage example in the Nunavut code generation library th
    :local:
 
 
-The main function
-+++++++++++++++++
+The main functions
+++++++++++++++++++
 
 .. autofunction:: pydsdl.read_namespace
+.. autofunction:: pydsdl.read_files
 
 
 Type model

docs/requirements.txt

@@ -1,3 +1,3 @@
-sphinx           == 4.4.0
-sphinx_rtd_theme == 1.0.0
+sphinx           == 7.1.2 # this is the last version that supports Python 3.8
+sphinx_rtd_theme == 2.0.0
 sphinx-computron >= 0.2, < 2.0

noxfile.py

@@ -10,7 +10,7 @@
 import nox
 
 
-PYTHONS = ["3.8", "3.9", "3.10", "3.11"]
+PYTHONS = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 """The newest supported Python shall be listed LAST."""
 
 nox.options.error_on_external_run = True
@@ -33,7 +33,6 @@ def clean(session):
         "*.log",
         "*.tmp",
         ".nox",
-        ".dsdl-test",
     ]
     for w in wildcards:
         for f in Path.cwd().glob(w):
@@ -49,9 +48,9 @@ def test(session):
     session.log("Using the newest supported Python: %s", is_latest_python(session))
     session.install("-e", ".")
     session.install(
-        "pytest          ~= 7.3",
-        "pytest-randomly ~= 3.12",
-        "coverage        ~= 7.2",
+        "pytest          ~= 8.1",
+        "pytest-randomly ~= 3.15",
+        "coverage        ~= 7.5",
     )
     session.run("coverage", "run", "-m", "pytest")
     session.run("coverage", "report", "--fail-under=95")
@@ -61,14 +60,6 @@ def test(session):
         session.log(f"OPEN IN WEB BROWSER: file://{report_file}")
 
 
-@nox.session(python=["3.7"])
-def test_eol(session):
-    """This is a minimal test session for those old Pythons that have EOLed."""
-    session.install("-e", ".")
-    session.install("pytest")
-    session.run("pytest")
-
-
 @nox.session(python=PYTHONS)
 def pristine(session):
     """
@@ -85,8 +76,9 @@ def pristine(session):
 def lint(session):
     session.log("Using the newest supported Python: %s", is_latest_python(session))
     session.install(
-        "mypy   ~= 1.2.0",
-        "pylint ~= 2.17.2",
+        "mypy   ~= 1.10",
+        "types-parsimonious",
+        "pylint ~= 3.2",
     )
     session.run(
         "mypy",
@@ -105,7 +97,8 @@ def lint(session):
         },
     )
     if is_latest_python(session):
-        session.install("black ~= 23.3")
+        # we run black only on the newest Python version to ensure that the code is formatted with the latest version
+        session.install("black ~= 24.4")
         session.run("black", "--check", ".")
 
 

pydsdl/__init__.py

@@ -7,7 +7,7 @@
 import sys as _sys
 from pathlib import Path as _Path
 
-__version__ = "1.20.1"
+__version__ = "1.21.0"
 __version_info__ = tuple(map(int, __version__.split(".")[:3]))
 __license__ = "MIT"
 __author__ = "OpenCyphal"
@@ -25,8 +25,9 @@
 _sys.path = [str(_Path(__file__).parent / "third_party")] + _sys.path
 
 # Never import anything that is not available here - API stability guarantees are only provided for the exposed items.
+from ._dsdl import PrintOutputHandler as PrintOutputHandler
 from ._namespace import read_namespace as read_namespace
-from ._namespace import PrintOutputHandler as PrintOutputHandler
+from ._namespace import read_files as read_files
 
 # Error model.
 from ._error import FrontendError as FrontendError

pydsdl/_bit_length_set/_symbolic_test.py

@@ -2,7 +2,6 @@
 # This software is distributed under the terms of the MIT License.
 # Author: Pavel Kirienko <[email protected]>
 
-import typing
 import random
 import itertools
 from ._symbolic import NullaryOperator, validate_numerically
@@ -140,7 +139,7 @@ def _unittest_repetition() -> None:
     )
     assert op.min == 7 * 3
     assert op.max == 17 * 3
-    assert set(op.expand()) == set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))  # type: ignore
+    assert set(op.expand()) == set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))
     assert set(op.expand()) == {21, 25, 29, 31, 33, 35, 39, 41, 45, 51}
     assert set(op.modulo(7)) == {0, 1, 2, 3, 4, 5, 6}
     assert set(op.modulo(8)) == {1, 3, 5, 7}
@@ -149,15 +148,15 @@ def _unittest_repetition() -> None:
     for _ in range(1):
         child = NullaryOperator(random.randint(0, 100) for _ in range(random.randint(1, 10)))
         k = random.randint(0, 10)
-        ref = set(map(sum, itertools.combinations_with_replacement(child.expand(), k)))  # type: ignore
+        ref = set(map(sum, itertools.combinations_with_replacement(child.expand(), k)))
         op = RepetitionOperator(child, k)
         assert set(op.expand()) == ref
 
         assert op.min == min(child.expand()) * k
         assert op.max == max(child.expand()) * k
 
         div = random.randint(1, 64)
-        assert set(op.modulo(div)) == {typing.cast(int, x) % div for x in ref}
+        assert set(op.modulo(div)) == {x % div for x in ref}
 
         validate_numerically(op)
 
@@ -173,9 +172,9 @@ def _unittest_range_repetition() -> None:
     assert op.max == 17 * 3
     assert set(op.expand()) == (
         {0}
-        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 1)))  # type: ignore
-        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 2)))  # type: ignore
-        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))  # type: ignore
+        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 1)))
+        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 2)))
+        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))
     )
     assert set(op.expand()) == {0, 7, 11, 14, 17, 18, 21, 22, 24, 25, 28, 29, 31, 33, 34, 35, 39, 41, 45, 51}
     assert set(op.modulo(7)) == {0, 1, 2, 3, 4, 5, 6}
@@ -197,10 +196,7 @@ def _unittest_range_repetition() -> None:
         k_max = random.randint(0, 10)
         ref = set(
             itertools.chain(
-                *(
-                    map(sum, itertools.combinations_with_replacement(child.expand(), k))  # type: ignore
-                    for k in range(k_max + 1)
-                )
+                *(map(sum, itertools.combinations_with_replacement(child.expand(), k)) for k in range(k_max + 1))
             )
         )
         op = RangeRepetitionOperator(child, k_max)
@@ -210,7 +206,7 @@ def _unittest_range_repetition() -> None:
         assert op.max == max(child.expand()) * k_max
 
         div = random.randint(1, 64)
-        assert set(op.modulo(div)) == {typing.cast(int, x) % div for x in ref}
+        assert set(op.modulo(div)) == {x % div for x in ref}
 
         validate_numerically(op)