diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 11b29b37..77116542 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -26,54 +26,71 @@ on: # As of January 2021, no YAML anchors :-( env: ubuntu_base_deps: doxygen make default-jre graphviz cmake ninja-build + # The single Python version used to produce the published HTML, picked + # out of the matrix below to avoid duplicate deploy artifacts. + publish_python: '3.13' jobs: supported-reqs: - name: 'Supported scripts/requirements.txt' - runs-on: ubuntu-22.04 + name: 'Supported build (Python ${{ matrix.python-version }})' + runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + # Build against the lockfile on more than one Python so that a pinned + # dependency dropping support for a given interpreter is caught here + # rather than by a contributor months later. + strategy: + fail-fast: false + matrix: + python-version: ['3.11', '3.13'] - # FIXME: remove this time consuming step once github stops - # providing a broken package index, see - # https://github.com/actions/virtual-environments/issues/1757#issuecomment-777700920 - - name: apt-get update github broken package index - run: sudo apt-get update + steps: + - uses: actions/checkout@v4 - - name: apt-get install - run: sudo apt-get -y install $ubuntu_base_deps + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: PATH += .local/bin - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: apt-get install base dependencies + run: | + sudo apt-get update + sudo apt-get -y install $ubuntu_base_deps - - name: 'pip install -r scripts/requirements.txt' - run: pip install -r scripts/requirements.txt + # Reproducible install: requirements.txt is the top-level list, + # constraints.txt pins the full transitive tree to validated versions. + - name: 'pip install -r requirements.txt -c constraints.txt' + run: pip install -r scripts/requirements.txt -c scripts/constraints.txt - # No YAML anchors; this is copy/paste - - name: configure SOF doc + - name: configure and build SOF API docs (Doxygen) run: | git clone https://github.com/thesofproject/sof cmake -GNinja -S sof/doc -B _build_doxy - - # TODO: change the (bad) default value for SOF_DOC_BUILD in - # sof-docs/Makefile and remove this command line override + # Build the API XML up front. The html build below is strict + # (-W), so any doc/source drift -- e.g. a doxygengroup that no + # longer exists in sof -- fails the PR here instead of silently + # dropping API sections. + ninja -C _build_doxy doc + + # SOF_DOC_BUILD overrides the Makefile default (../sof/build_doxygen, + # the documented sibling layout) because CI checks sof out *inside* + # the sof-docs workspace rather than alongside it. - name: build run: | make html VERBOSE=1 SOF_DOC_BUILD=_build_doxy du -shc _build*/* + # Publish only from the canonical Python version so the matrix does + # not upload the "html" artifact twice. - name: prepare file for deploy - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/publish' }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/publish' && matrix.python-version == env.publish_python }} run: ./.github/actions/create-publish-folder.sh # store the build result to artifact, used for later deploy or # download for debug # https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts - name: upload HTML for deploy - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/publish' }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/publish' && matrix.python-version == env.publish_python }} uses: actions/upload-artifact@v4 with: name: html @@ -81,13 +98,13 @@ jobs: deploy: needs: supported-reqs - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/publish' }} steps: # download the build result from the same workflow # https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts - name: download HTML - uses: actions/download-artifact@v4.1.7 + uses: actions/download-artifact@v4 with: name: html path: html @@ -101,48 +118,37 @@ jobs: external_repository: thesofproject/thesofproject.github.io lax: - name: "PIP_IGNORE_INSTALLED=0 requirements-lax.txt" - runs-on: ubuntu-22.04 + name: "Lax requirements (unpinned)" + runs-on: ubuntu-latest # Makefile downgrades the Sphinx warnings, they are not errors any more env: {LAX: 1} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - # FIXME: remove this time consuming step once github stops - # providing a broken package index, see - # https://github.com/actions/virtual-environments/issues/1757#issuecomment-777700920 - - name: apt-get update github broken package index - run: sudo apt-get update + - uses: actions/setup-python@v5 + with: + python-version: '3.13' - name: apt-get install base dependencies - run: sudo apt-get -y install $ubuntu_base_deps - - - name: apt-get instead of pip - run: sudo apt-get -y install - python3-pip python3-setuptools python3-wheel - python3-sphinx python3-breathe python3-docutils - python3-sphinx-rtd-theme python3-sphinxcontrib.plantuml - - - name: PATH += .local/bin - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + run: | + sudo apt-get update + sudo apt-get -y install $ubuntu_base_deps - # should be a no-op + # Unpinned "best effort" install of the loose requirements, the path + # a drive-by contributor would take. No lockfile on purpose. - name: 'pip install -r scripts/requirements-lax.txt' run: pip install -r scripts/requirements-lax.txt - name: config tweaks run: | - # sphinx-build < 1.7 doesn't support "auto" and we're not sure - # all default modules are "thread-safe" - sed -i -e '/SPHINXBUILD/ s/-j *auto/-j 1/' Makefile # We don't want plantUML to raise the contribution bar sed -i -e 's/^\(plantuml_output_format *=\).*/\1 "none"/' conf.py - # No YAML anchors; this is copy/paste - - name: configure SOF doc + - name: configure and build SOF API docs (Doxygen) run: | git clone https://github.com/thesofproject/sof cmake -GNinja -S sof/doc -B _build_doxy + ninja -C _build_doxy doc - name: build run: | diff --git a/architectures/firmware/sof-zephyr/mpp_layer/images/mpp_scheduling/edf_scheduling.diag b/architectures/firmware/sof-zephyr/mpp_layer/images/mpp_scheduling/edf_scheduling.diag deleted file mode 100644 index b9680b22..00000000 --- a/architectures/firmware/sof-zephyr/mpp_layer/images/mpp_scheduling/edf_scheduling.diag +++ /dev/null @@ -1,42 +0,0 @@ -// FIXME: blockdiag is orphaned and not compatible with Pillow anymore: -// https://github.com/blockdiag/blockdiag/pull/171 - -blockdiag edf_scheduling { - - node_width = 250; - node_height = 120; - default_fontsize = 16; - - Comp_1 -> Comp_2 - comment_1 -> Comp_2 [style=dashed] - Comp_2 -> Comp_3 - comment_2 -> Comp_3 [style=dashed] - Comp_3 -> Comp_4 - comment_3 -> Comp_4 [style=dashed] - Comp_4 -> sink - comment_4 -> sink [style=dashed] - - Comp_1 [label="DP component 1\n - *processing period\n - *compute requirement"] - Comp_2 [label="DP component 2\n - *processing period\n - *compute requirement"] - Comp_3 [label="DP component 3\n - *processing period\n - *compute requirement"] - Comp_4 [label="DP component 4\n - *processing period\n - *compute requirement"] - - sink [label="real time sink", shape=endpoint, fontsize = 16] - - comment_1 [label="DP1 to deliver data let\n - DP2 meet its objective"] - comment_2 [label="DP2 to deliver data let\n - DP3 meet its objective"] - comment_3 [label="DP3 to deliver data let\n - DP4 meet its objective"] - comment_4 [label="DP4 to deliver data\n - to real time-sink"] -} diff --git a/architectures/firmware/sof-zephyr/mpp_layer/images/mpp_scheduling/edf_scheduling.dot b/architectures/firmware/sof-zephyr/mpp_layer/images/mpp_scheduling/edf_scheduling.dot new file mode 100644 index 00000000..91447f91 --- /dev/null +++ b/architectures/firmware/sof-zephyr/mpp_layer/images/mpp_scheduling/edf_scheduling.dot @@ -0,0 +1,23 @@ +digraph edf_scheduling { + rankdir = LR; + node [shape = box, fontsize = 12]; + + Comp_1 [label = "DP component 1\n* processing period\n* compute requirement"]; + Comp_2 [label = "DP component 2\n* processing period\n* compute requirement"]; + Comp_3 [label = "DP component 3\n* processing period\n* compute requirement"]; + Comp_4 [label = "DP component 4\n* processing period\n* compute requirement"]; + + sink [label = "real time sink", shape = doublecircle]; + + comment_1 [label = "DP1 to deliver data let\nDP2 meet its objective", shape = note]; + comment_2 [label = "DP2 to deliver data let\nDP3 meet its objective", shape = note]; + comment_3 [label = "DP3 to deliver data let\nDP4 meet its objective", shape = note]; + comment_4 [label = "DP4 to deliver data\nto real time-sink", shape = note]; + + Comp_1 -> Comp_2 -> Comp_3 -> Comp_4 -> sink; + + comment_1 -> Comp_2 [style = dashed]; + comment_2 -> Comp_3 [style = dashed]; + comment_3 -> Comp_4 [style = dashed]; + comment_4 -> sink [style = dashed]; +} diff --git a/architectures/firmware/sof-zephyr/mpp_layer/mpp_scheduling.rst b/architectures/firmware/sof-zephyr/mpp_layer/mpp_scheduling.rst index c4ff5631..20b3df84 100644 --- a/architectures/firmware/sof-zephyr/mpp_layer/mpp_scheduling.rst +++ b/architectures/firmware/sof-zephyr/mpp_layer/mpp_scheduling.rst @@ -128,7 +128,7 @@ deadline for data delivery: EDF scheduling example -.. blockdiag:: images/mpp_scheduling/edf_scheduling.diag +.. graphviz:: images/mpp_scheduling/edf_scheduling.dot The capture pipelines operate in the same way. diff --git a/conf.py b/conf.py index c64f3493..65ba72db 100755 --- a/conf.py +++ b/conf.py @@ -32,10 +32,8 @@ # ones. -# FIXME: blockdiag is orphaned and not compatible with Pillow anymore: -# https://github.com/thesofproject/sof-docs/issues/472 extensions = ['breathe', 'sphinx.ext.graphviz', 'sphinxcontrib.plantuml', - 'sphinx.ext.todo', 'sphinx.ext.extlinks', 'sphinxcontrib.blockdiag', + 'sphinx.ext.todo', 'sphinx.ext.extlinks', 'sphinxcontrib.jquery' ] @@ -104,7 +102,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build','.tox' ] +# Note: a virtualenv created inside this source tree (.venv, venv, env, ...) +# would otherwise be scanned by Sphinx and flood the build with warnings +# about .rst files shipped in installed packages. +exclude_patterns = ['_build', '.tox', '.venv*', 'venv', 'env'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' diff --git a/contribute/process/docbuild.rst b/contribute/process/docbuild.rst index 06570e03..98bb3859 100644 --- a/contribute/process/docbuild.rst +++ b/contribute/process/docbuild.rst @@ -151,30 +151,21 @@ tools: .. code-block:: bash cd ~/thesofproject/sof-docs - pip3 install --user -r scripts/requirements.txt + pip3 install --user -r scripts/requirements.txt -c scripts/constraints.txt -.. note:: The :git-sof-docs-mainline:`scripts/requirements.txt` file hardcodes - versions using ``==``, which may not be compatible with your other - projects. In that case you can either setup a Python ``virtualenv`` or - try the unsupported :git-sof-docs-mainline:`scripts/requirements-lax.txt` - (more details inside this file): +The :git-sof-docs-mainline:`scripts/constraints.txt` lockfile pins the full +dependency tree to exact, validated versions for a reproducible build. It +targets a modern Python (3.11 or newer); use a ``virtualenv`` if those pinned +versions conflict with your other projects. - .. code-block:: bash - - PIP_IGNORE_INSTALLED=0 pip3 install --user -r scripts/requirements-lax.txt - - The hardcoded package versions might need additional libraries installed - in order to compile them. For example, to resolve the following error: +.. note:: For a quick "best effort" install that is not pinned (for example a + drive-by typo fix), you can skip the lockfile and use the unsupported + :git-sof-docs-mainline:`scripts/requirements-lax.txt` instead (more details + inside this file): .. code-block:: bash - ERROR: Could not build wheels for pillow, which is required to install pyproject.toml-based projects - - you should install: - - .. code-block:: bash - - sudo apt install libjpeg-dev zlib1g-dev + PIP_IGNORE_INSTALLED=0 pip3 install --user -r scripts/requirements-lax.txt For Windows, install the needed tools manually: diff --git a/scripts/constraints.txt b/scripts/constraints.txt new file mode 100644 index 00000000..c06a6a2e --- /dev/null +++ b/scripts/constraints.txt @@ -0,0 +1,43 @@ +# scripts/constraints.txt — lockfile for the *supported* documentation build. +# +# This pins the full transitive dependency tree to exact, known-good +# versions so that the supported build is reproducible. Install with: +# +# pip install -r scripts/requirements.txt -c scripts/constraints.txt +# +# requirements.txt lists the top-level packages we depend on; this file +# locks every resolved version (direct and indirect). +# +# Validated on Python 3.14 (requires Python >= 3.11, the Sphinx 9 floor). +# +# To regenerate after intentionally bumping a dependency: +# python3 -m venv /tmp/lock && . /tmp/lock/bin/activate +# pip install -r scripts/requirements.txt +# pip freeze > scripts/constraints.txt # then re-add this header + +alabaster==1.0.0 +babel==2.18.0 +breathe==4.36.0 +certifi==2026.5.20 +charset-normalizer==3.4.7 +docutils==0.22.4 +idna==3.18 +imagesize==2.0.0 +Jinja2==3.1.6 +MarkupSafe==3.0.3 +packaging==26.2 +Pygments==2.20.0 +requests==2.34.2 +roman-numerals==4.1.0 +snowballstemmer==3.1.1 +Sphinx==9.1.0 +sphinx_rtd_theme==3.1.0 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jquery==4.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-plantuml==0.31 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +urllib3==2.7.0 diff --git a/scripts/docker_build/Dockerfile b/scripts/docker_build/Dockerfile index 070f7209..dd07a31a 100644 --- a/scripts/docker_build/Dockerfile +++ b/scripts/docker_build/Dockerfile @@ -5,14 +5,14 @@ # # Usage: # create parent directory for sof and sof-docs repository (e.g. thesofproject) -# clone sof repository to thesofproject\sof folder -# clone sof-docs repository to thesofproject\sof-docs folder -# build docker image from parent directory .\thesofproject: +# clone sof repository to thesofproject/sof folder +# clone sof-docs repository to thesofproject/sof-docs folder +# build docker image from parent directory ./thesofproject: # > docker build -t ubuntu-sofdocs -f ./sof-docs/scripts/docker_build/Dockerfile ./ # run the image container: # > docker run -d --name sofdocs_container ubuntu-sofdocs sleep infinity # copy build output from container to host: -# > docker cp sofdocs_container:/home/thesofproject/sof/doc ./sof/ +# > docker cp sofdocs_container:/home/thesofproject/sof/build_doxygen ./sof/ # > docker cp sofdocs_container:/home/thesofproject/sof-docs/_build ./sof-docs/ # stop the container: # docker stop sofdocs_container @@ -21,33 +21,40 @@ # but each next one will repeat only copy and build steps. # -FROM dokken/ubuntu-22.04 +FROM ubuntu:24.04 # Set image working directory WORKDIR /home/thesofproject -RUN apt-get update - -# Install sof-docs build tools -RUN apt-get install -y python3.6 -RUN apt-get install -y doxygen python3-pip python3-wheel make \ - default-jre graphviz cmake ninja-build - -# Copy sof-docs file with dependency tools list -COPY ./sof-docs/scripts/requirements.txt /home/thesofproject/sof-docs/scripts/requirements.txt - -# Install sof-docs requirements tools -RUN pip3 install --user -r /home/thesofproject/sof-docs/scripts/requirements.txt - -# Directly install sphinx to add 'sphinx-build' to the system -RUN apt-get install -y python3-sphinx +# Install sof-docs build tools. Ubuntu 24.04 ships Python 3.12, which +# satisfies the supported lockfile (Sphinx 9 requires Python >= 3.11). +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-pip \ + doxygen make default-jre graphviz cmake ninja-build \ + && rm -rf /var/lib/apt/lists/* + +# Install the Python tools into a venv (avoids Ubuntu's PEP 668 +# "externally managed environment" restriction) and put it first on PATH. +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv "$VIRTUAL_ENV" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Install the pinned documentation toolchain from the lockfile. Copying +# only these two files first lets Docker cache the install layer across +# source changes. +COPY ./sof-docs/scripts/requirements.txt ./sof-docs/scripts/constraints.txt \ + /home/thesofproject/sof-docs/scripts/ +RUN pip install --no-cache-dir \ + -r sof-docs/scripts/requirements.txt -c sof-docs/scripts/constraints.txt # Copy sof source code from host to image COPY ./sof/ /home/thesofproject/sof/ -# Build API documentation from SOF source (Doxygen) -RUN cmake -S sof/doc -B sof/doc -GNinja -RUN ninja -C sof/doc -v doc +# Build API documentation from SOF source (Doxygen). Build out of source +# into sof/build_doxygen, which is where the sof-docs Makefile expects it +# (its SOF_DOC_BUILD default is ../sof/build_doxygen). +RUN cmake -GNinja -S sof/doc -B sof/build_doxygen +RUN ninja -C sof/build_doxygen -v doc # Copy sof-docs source code from host to image COPY ./sof-docs/ /home/thesofproject/sof-docs/ diff --git a/scripts/docker_build/docker-build.sh b/scripts/docker_build/docker-build.sh index e16e7507..dc8f4555 100644 --- a/scripts/docker_build/docker-build.sh +++ b/scripts/docker_build/docker-build.sh @@ -16,8 +16,8 @@ docker build -t ubuntu-sofdocs -f ./sof-docs/scripts/docker_build/Dockerfile ./ docker run -d --rm --name sofdocs_container ubuntu-sofdocs sleep infinity if [ $build_docs_only = "FALSE" ]; then - echo "Copy SOF Doxygen generated documentation from container to host ./sof/doc/" - docker cp sofdocs_container:/home/thesofproject/sof/doc ./sof/ + echo "Copy SOF Doxygen generated documentation from container to host ./sof/build_doxygen/" + docker cp sofdocs_container:/home/thesofproject/sof/build_doxygen ./sof/ fi echo "Copy SOF-DOCS generated documentation from container to host ./sof-docs/_build .." diff --git a/scripts/requirements-lax.txt b/scripts/requirements-lax.txt index 42077b41..ae8a4076 100644 --- a/scripts/requirements-lax.txt +++ b/scripts/requirements-lax.txt @@ -13,7 +13,6 @@ breathe>=4.29.2 sphinx>=4.5.0 docutils>=0.17.1 sphinx_rtd_theme>=0.2.4 -sphinxcontrib-blockdiag>=3.0.0 sphinxcontrib-jquery # - Version 0.11 is the first version that supports @@ -56,8 +55,3 @@ sphinxcontrib.plantuml>=0.11 # - Downgrade if successfully tested. # - Test with plantum set to "none" in conf.py. We don't want small # typo fixes to depend on UML diagrams. - -# Workaround for warning "'ImageDraw' object has no attribute 'textsize'" -# with pillow 10.0.0, see -# https://github.com/thesofproject/sof-docs/issues/472 -pillow<10 diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 17aff9e6..48d602f8 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,5 +1,12 @@ -# This file hardcodes validated versions with '==', -# see requirements-lax.txt for an alternative. +# Top-level dependencies for the *supported* documentation build. +# +# For a reproducible install, pair this with the lockfile that pins the +# full transitive tree to exact, validated versions: +# +# pip install -r scripts/requirements.txt -c scripts/constraints.txt +# +# For an unpinned "best effort" install (e.g. a drive-by typo fix), see +# requirements-lax.txt instead. sphinx>=7 breathe @@ -7,9 +14,3 @@ docutils sphinx_rtd_theme sphinxcontrib-plantuml sphinxcontrib-applehelp - - -# blockdiag is orphaned and not compatible with pillow>=10, -# see https://github.com/thesofproject/sof-docs/issues/472 -sphinxcontrib-blockdiag -pillow<10