Taskfile.yml

---
# https://taskfile.dev

version: '3'

dotenv: ['.env']

vars:
  PUBLIC_BRANCH: published
  # Throwaway branch used by public:preview. Never pushed, recreated on every
  # run and deleted again when the preview server stops.
  PREVIEW_BRANCH: publish-preview
  CURRENT_VERSION: 26.2
  # the markdown linter command, used by format:fix:path and check:rumdl
  RUMDL_CHECK: poetry run rumdl --config .markdownlint.jsonc check
  # the default lint targets: the docs tree plus the top-level markdown files
  DOCS_PATHS: ./docs ./*.md

tasks:

  default:
    summary: |
      List documented tasks
    silent: true
    cmds:
      - task --list

  versions:
    desc: List relevant versions for a zensical material bug report
    silent: true
    cmds:
      - |
        cat << EOF
        - Python: {{.PYTHON_VERSION}}
        - Zensical: {{.ZENSICAL_VERSION}}
        EOF
    vars:
      PYTHON_VERSION:
        sh: poetry run python --version
      ZENSICAL_VERSION:
        sh: poetry run zensical --version | cut -d " " -f -3

  clean:
    desc: Clean up working directory
    run: once
    cmds:
      - rm -rf site
      # Only ever the throwaway branch of public:preview, which normally deletes
      # itself; this catches the run that was killed before it could.
      - git branch -D {{.PREVIEW_BRANCH}} >/dev/null 2>&1 || true

  install:
    desc: Install needed dependencies
    # `run: once` because five tasks depend on this and `task check` resolves its
    # dependencies in parallel - without it, concurrent `poetry install`
    # invocations collide and one of them fails with a non-obvious exit 1.
    run: once
    cmds:
      - poetry install

  check:
    desc: Check documentation links, Markdown and YAML style, nav drift and build output
    # Sequential (`cmds`, not `deps`) on purpose. Task resolves `deps` in
    # parallel, which ran several `poetry install` invocations at once and, worse,
    # ran `zensical build` alongside the link and style checks - under that load
    # Zensical intermittently reports false "page does not exist" warnings and
    # `--strict` fails the build (zensical/zensical#641). Serialising costs a
    # little wall-clock and makes the stage deterministic.
    cmds:
      - task: check:links
      - task: check:rumdl
      - task: check:yamllint
      - task: check:navigation
      - task: check:output

  check:yamllint:
    desc: Fail if any YAML file deviates from the style in .yamllint
    deps:
      - install
    cmds:
      # `.` picks up the repo .yamllint, which is what makes a local run and a
      # CI run agree: GitHub runners have no user-level yamllint config.
      - poetry run yamllint .

  check:navigation:
    desc: Fail if nav.yml is out of sync with the docs/**/.pages files
    deps:
      - install
    cmds:
      - poetry run dec-tool build-navigation --check

  check:output:
    desc: Fail if a feature we reimplemented for Zensical regressed in the build
    deps:
      - build
    cmds:
      - poetry run dec-tool check-zensical-output

  check:links:
    desc: Check outgoing links
    deps:
      - install
    cmds:
      - poetry run linkcheckMarkdown -r docs 2>&1 | grep -v "ResourceWarning"

  test:
    desc: Run the Python unit test suite
    deps:
      - test:unit

  test:unit:
    desc: Run tests that need no external service
    deps:
      - install
    cmds:
      - poetry run pytest -m "not integration"

  test:integration:
    desc: Run tests that need a live CO instance
    deps:
      - install
    cmds:
      - poetry run pytest -m integration

  build:
    desc: Build the page
    deps:
      - task: install
    cmds:
      # `--clean` is not optional: with a warm .cache Zensical intermittently
      # reports false "page does not exist" warnings and --strict then fails the
      # build (zensical/zensical#641). Measured on this repo: 4 of 6 warm builds
      # failed, 6 of 6 passed with --clean. Costs ~5s; buys a deterministic build.
      - poetry run zensical build --strict --clean
      # Zensical bakes unpkg.com URLs into its JS bundle; rewrite them to the
      # vendored copies. Part of building, not of checking - without it the
      # published site issues third-party requests.
      - poetry run dec-tool localize-bundle-assets

  serve:
    desc: Serve the page on localhost with live reload (no post-build steps)
    # `zensical serve` rebuilds into site/ on every change, which overwrites what
    # the post-build steps in `build` produced - so this preview loads glightbox
    # and the ResizeObserver polyfill from unpkg rather than from our vendored
    # copies. Nothing downstream consumes a site/ left behind this way: `build`
    # and `check:output` rebuild, and `public:deploy` refuses a site/ whose
    # canonical URL is unversioned, which such a rebuild always is.
    #
    # Use `task public:preview` to see the site as it will actually be published.
    deps:
      - task: install
    cmds:
      - poetry run zensical serve

  pdf:
    desc: Build a single PDF of the whole site with pandoc and Typst
    summary: |
      Runs `task build`, then tools/build_pdf.py, which merges the per-page
      HTML in site/ along nav.yml into one document, converts it to Typst
      with pandoc and typesets it with Typst in the eccenca house style.
      Style, pandoc template, filter and the vendored fonts are in tools/pdf/;
      system fonts are ignored, so the PDF does not depend on the machine.

      Needs pandoc and typst on the PATH. The style is tested with typst
      0.15; another minor release prints a warning, since Typst layout
      changes between them.

      nav.yml is the page order: it is generated from the .pages files and
      gated by `task check`, so the PDF cannot drift from the sidebar. Pages
      outside the navigation - the homepage, tags.md, testing.md and the
      hidden Getting Started tutorial - are not included.

      Links the PDF cannot resolve within itself - downloadable resources,
      full-size screenshots, those excluded pages - point at the published
      site under the built version, or under `latest` for a dev build.

      The header and the title page print the build date and the commit,
      marked -dirty when tracked files have uncommitted changes.

      The merged HTML and the Typst source pandoc writes stay in dist/pdf/
      for debugging a rendering problem.

      On every push to main, .github/workflows/pdf.yml runs this task with
      the pinned pandoc and Typst and keeps the PDF as a workflow artifact.

      Tunables are options of `dec-tool build-pdf`, each with an environment
      variable fallback - see `poetry run dec-tool build-pdf --help`:
        --build-version / BUILD_VERSION - version stamped on the title page
                                          and folded into the default filename
                                          (default: {{.CURRENT_VERSION}})
        --output-file / PDF_OUT         - explicit output path
                                          (default:
                                          dist/documentation-eccenca-com-<ver>.pdf)
        --pandoc / PANDOC               - pandoc binary (default: pandoc)
        --typst / TYPST                 - typst binary (default: typst)
    deps:
      - task: build
    env:
      BUILD_VERSION: '{{.CURRENT_VERSION}}'
    cmds:
      - poetry run dec-tool build-pdf

  update:icons:
    desc: update the used eccenca icons from carbon
    cmds:
      - poetry run dec-tool update-icons \
          --version {{.GUI_VERSION}} \
          -o {{.ICON_DIR}} \
    vars:
      GUI_VERSION: main
      ICON_DIR: overrides/.icons/eccenca

  update:cmemc:
    desc: re-generates the cmemc command reference (needs local cmemc)
    cmds:
      - rm -rf {{.REFERENCE_DIR}}/*
      - cmemc manual --format markdown-multi-page --output-dir {{.REFERENCE_DIR}}
      - task: format:fix:path
        vars:
          PATHS: ./docs/automate/cmemc-command-line-interface
    vars:
      REFERENCE_DIR: docs/automate/cmemc-command-line-interface/command-reference

  update:cmem-client-api:
    desc: re-generates the cmem-client Python API reference (needs local cmem-client checkout)
    summary: |
      Regenerates docs/develop/cmem-client-api/api-reference from the
      docstrings of a local cmem-client checkout, via that repository's own
      `task docs:export-md` (see etc/export_api_md.py there).

      The landing page (docs/develop/cmem-client-api/index.md) and its
      .pages file are hand-written and untouched by this task - only the
      api-reference/ subtree is wiped and regenerated.

      This documents whatever branch/commit is currently checked out in
      CMEM_CLIENT_DIR, not necessarily the last release - check out the ref
      you want documented before running this.

        CMEM_CLIENT_DIR=/path/to/cmem-client task update:cmem-client-api
    preconditions:
      - sh: '[ -n "{{.CMEM_CLIENT_DIR}}" ]'
        msg: |
          CMEM_CLIENT_DIR is not set. Run this task with the path to a local
          cmem-client checkout, e.g.:

            CMEM_CLIENT_DIR=/path/to/cmem-client task update:cmem-client-api
    cmds:
      - rm -rf {{.REFERENCE_DIR}}/*
      - task -d {{.CMEM_CLIENT_DIR}} docs:export-md OUTPUT_DIR={{.ROOT_DIR}}/{{.REFERENCE_DIR}}
      - task: format:fix:path
        vars:
          PATHS: '{{.ROOT_DIR}}/{{.REFERENCE_DIR}}'
        ignore_error: true
    vars:
      REFERENCE_DIR: docs/develop/cmem-client-api/api-reference

  update:shape-reference:
    desc: re-generates the shape and datatype references (needs local CO)
    summary: >
      This task uses a running Corporate Memory to upload a specification for
      a custom markdown endpoint (see shapedocu.ttl in
      tools/update-shape-reference) for exporting.


      In order to use this task, a valid cmemc Corporate Memory connection needs
      to be available in the environment.
      This can be done by setting CMEMC_CONNECTION or by using the eval command.

        - CMEMC_CONNECTION=mycmem task update:shape-reference
        - eval $(cmemc -c mycmem config eval); task update:shape-reference
    cmds:
      - cmemc graph import --replace {{.SRC}}
      - cp {{.SRC}}/nodeshapedocu-head.md {{.NODESHAPES_MD}}
      - "{{.CURL}} {{.API}}/nodeshapedocu >>{{.NODESHAPES_MD}}"
      - cp {{.SRC}}/propertyshapedocu-head.md {{.PROPERTYSHAPES_MD}}
      - "{{.CURL}} {{.API}}/propertyshapedocu >>{{.PROPERTYSHAPES_MD}}"
      - cp {{.SRC}}/datatypedocu-head.md {{.DATATYPES_MD}}
      - "{{.CURL}} {{.API}}/datatypedocu >>{{.DATATYPES_MD}}"
    vars:
      DIR: docs/explore-and-author/graph-exploration/building-a-customized-user-interface
      TOKEN:
        sh: cmemc admin token
      DP:
        sh: cmemc config get DP_API_ENDPOINT
      CURL: "curl --silent -H 'Authorization: Bearer {{.TOKEN}}'"
      API: "{{.DP}}/api/custom"
      NODESHAPES_MD: "{{.DIR}}/node-shapes/index.md"
      PROPERTYSHAPES_MD: "{{.DIR}}/property-shapes/index.md"
      DATATYPES_MD: "{{.DIR}}/datatype-reference/index.md"
      SRC: tools/update-shape-reference

  update:di-reference:
    desc: update the task and operator reference pages (needs local CO)
    cmds:
      - poetry run dec-tool update-di-reference
      - task: format:fix:path
        vars:
          PATHS: ./docs/build/reference

  update:integrations:
    desc: update integrations page (needs data/integrations.yml)
    cmds:
      - poetry run dec-tool update-integrations
      - task: format:fix:path
        vars:
          PATHS: ./docs/build/integrations/index.md
        ignore_error: true

  update:navigation:
    desc: Regenerate nav.yml from the docs/**/.pages files
    deps:
      - install
    cmds:
      - poetry run dec-tool build-navigation

  public:versions:
    desc: List public documentation versions
    deps:
      - task: install
    cmds:
      - poetry run mike list -b {{.PUBLIC_BRANCH}}

  public:deploy:
    desc: Publish the working directory as version {{.CURRENT_VERSION}}
    summary: |
      Builds the site and commits it as version {{.CURRENT_VERSION}} on the
      {{.PUBLIC_BRANCH}} branch. Nothing is pushed - that stays a deliberate
      `git push` afterwards.

      Deliberately not `mike deploy`: that builds the site itself, without
      `--strict` and without the post-build steps below, and its rebuild would
      overwrite them anyway. So the build happens here and `dec-tool publish`
      hands the result to mike, which still owns versions.json, the aliases and
      the root redirect.

      MIKE_DOCS_VERSION is what makes the pages carry a versioned canonical URL;
      `dec-tool publish` refuses to commit a site/ built without it.
    deps:
      - task: install
    cmds:
      - MIKE_DOCS_VERSION={{.CURRENT_VERSION}} task build
      - >
        poetry run dec-tool publish
        --branch {{.PUBLIC_BRANCH}}
        --version {{.CURRENT_VERSION}}
        --alias latest

  public:serve:
    desc: Start a webserver to manually validate the public branch
    deps:
      - task: install
    cmds:
      - poetry run mike serve -b {{.PUBLIC_BRANCH}}

  public:preview:
    desc: Rehearse a deployment on a throwaway branch and serve it
    summary: |
      Deploys the working directory into a throwaway copy of the public branch,
      serves it on localhost, and deletes that branch again when the server
      stops. Nothing is pushed and {{.PUBLIC_BRANCH}} is never touched.

      The rehearsal is public:deploy itself, pointed at the throwaway branch, so
      what it serves is exactly what publishing would produce.

      Ctrl-C removes the branch. Should the process be killed harder than that,
      `task clean` removes it, as does the next run.

        task public:preview PORT=9000
    deps:
      - task: install
    vars:
      PORT: '{{.PORT | default "8002"}}'
    cmds:
      # Deleted when the server stops: `defer` runs on Ctrl-C too, since that
      # reaches mike and Task alike.
      - defer: git branch -D {{.PREVIEW_BRANCH}} >/dev/null 2>&1 || true
      # Start from the branch as it stands - locally if it exists, otherwise as
      # last fetched - so the rehearsal shows the new version among the existing
      # ones: version selector, outdated banner and root redirect included.
      - |
        base={{.PUBLIC_BRANCH}}
        git show-ref --verify --quiet refs/heads/{{.PUBLIC_BRANCH}} \
          || base=origin/{{.PUBLIC_BRANCH}}
        git branch -f --no-track {{.PREVIEW_BRANCH}} "$base"
      - task: public:deploy
        vars:
          PUBLIC_BRANCH: "{{.PREVIEW_BRANCH}}"
      - echo "Serving the deployment rehearsal on http://localhost:{{.PORT}} - Ctrl-C to stop and discard it"
      - poetry run mike serve -b {{.PREVIEW_BRANCH}} --dev-addr localhost:{{.PORT}}

  format:fix:path:
    internal: true
    summary: |
      Run rumdl (md-lint) with --fix on the markdown files below PATHS.

      PATHS is a space separated list of files and/or directories, which is
      used as the search roots of a find call, so directories are searched
      recursively:

        - task: format:fix:path
          vars:
            PATHS: ./docs/build/reference
    requires:
      vars: [PATHS]
    cmds:
      - >
        find {{.PATHS}} -type f -name '*.md' -print0
        | xargs -0 {{.RUMDL_CHECK}} --fix

  format:fix:
    desc: rumdl (md-lint) and apply possible style fixes
    cmds:
      - task: format:fix:path
        vars:
          PATHS: '{{.DOCS_PATHS}}'

  check:rumdl:
    desc: run rumdl (md-linter) for style issues without failing the check stage
    cmds:
      - mkdir -p ./dist
      - rm -f ./dist/md-lint-issues.xml
      - >
        find {{.DOCS_PATHS}} -type f -name '*.md' -print0
        | xargs -0 {{.RUMDL_CHECK}} --output-format junit > ./dist/md-lint-issues.xml || true
      - |
        find {{.DOCS_PATHS}} -type f -name '*.md' -print0 | xargs -0 {{.RUMDL_CHECK}} || {
          echo "rumdl reported markdown style issues; continuing without failing the check stage."
          true
        }