Rename tox target for pdf build

zuul jobs require target to be called pdf-docs instead of docs-pdf

otc_metadata/services.py

@@ -209,3 +209,22 @@ class Services(object):
                 doc_struct[srv["service_category"]].append(srv_res)
 
         return dict(categories=doc_struct)
+
+    def get_service_with_docs_by_service_type(self, service_type):
+        """Retrieve service and service docs by service_type
+
+        :param str service_type: Filter by service_type
+        """
+        res = dict()
+        res['service'] = {}
+        docs = []
+        services = self._service_data
+        for doc in services['documents']:
+            if doc['service_type'] == service_type:
+                docs.append(doc)
+        res['documents'] = docs
+        for service in services['services']:
+            if service['service_type'] == service_type:
+                res['service'] = service
+                break
+        return res

otc_metadata/templates/conf.py.j2

@@ -104,6 +104,9 @@ html_title = "{{ title }}"
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+# Do not include sources into the rendered results
+html_copy_source = False
+
 # -- Options for PDF output --------------------------------------------------
 latex_documents = [
 {%- if pdf_name is defined %}

otc_metadata/templates/tox.ini.j2

@@ -21,18 +21,28 @@ deps =
   -r{toxinidir}/requirements.txt
 commands = {posargs}
 
+# This env is invoked in the periodic pipeline and is therefore responsible to
+# build all relevant docs at once.
 [testenv:docs]
 deps = -r{toxinidir}/doc/requirements.txt
+allowlist_externals =
+  mkdir
+  cp
+  sh
 commands =
 {%- for doc in docs %}
   {[testenv:{{ doc.type }}]commands}
   {[testenv:json-{{ doc.type }}]commands}
 {%- endfor %}
 
-[testenv:docs-pdf]
+[testenv:pdf-docs]
 deps = -r{toxinidir}/doc/requirements.txt
 allowlist_externals =
+  rm
   mkdir
+  make
+  bash
+  cp
 commands =
   mkdir -p doc/build/pdf
 {%- for doc in docs %}
@@ -68,11 +78,12 @@ deps = -r{toxinidir}/doc/requirements.txt
 allowlist_externals =
   cp
   mkdir
-  find
+  sh
 commands =
   sphinx-build -W --keep-going -b json {{ loc }}/source doc/build/json/{{ doc.type }}
-  # Drop data useless for the search
+  # Drop data useless for the search - wrap it also with sh/xargs due to bugs
-  find doc/build/json -type d -and ( -name '_images' -or -name '_static' -or -name '_sources' ) -exec rm {:} ;
+  # in tox
+  sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf"
 {%- if doc.type == 'api-ref' %}
   mkdir -p api-ref/build/json
   cp -av doc/build/json/api-ref api-ref/build/json

tools-requirements.txt

@@ -1,2 +1,6 @@
 GitPython
-ruamel
+ruamel.yaml
+requests
+jinja2
+dirsync
+cookiecutter

tools/generate_doc_confpy.py

@@ -201,7 +201,7 @@ def process_repositories(args, service):
     repo_to.git.push(*push_args)
     if "github" in url_to:
         subprocess.run(
-            args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True
+            args=["gh", "pr", "create", "-f"], cwd=copy_to, check=False
         )
     elif "gitea" in url_to and args.token:
         open_pr(

tools/generate_doc_gitcontrol_repos.py

@@ -46,7 +46,7 @@ def process_services(args, services):
         has_wiki=False,
         default_delete_branch_after_merge=True,
         allow_merge_commit=False,
-        allow_squash_merge=False,
+        allow_squash_merge=True,
         allow_rebase_merge=False,
         default_merge_style="squash",
         branch_protections=[],

tools/sync_doc_repo.py

@@ -18,9 +18,9 @@ import argparse
 import logging
 import os
 import pathlib
-import shutil
 import subprocess
 import warnings
+from dirsync import sync
 
 from git import Repo
 from git import SymbolicReference
@@ -111,13 +111,21 @@ def process_repositories(args, service):
         new_branch.set_tracking_branch(remote_ref)
         new_branch.checkout()
 
-        shutil.copytree(
+        source_path = pathlib.Path(copy_from, doc["rst_location"])
-            pathlib.Path(copy_from, doc["rst_location"]),
+        target_path = pathlib.Path(copy_to, doc["rst_location"])
-            pathlib.Path(copy_to, doc["rst_location"]),
+        sync(
-            ignore=lambda a, b: ["conf.py"],
+            source_path,
-            dirs_exist_ok=True,
+            target_path,
+            'sync',
+            purge=True,
+            create=True,
+            content=True,
+            ignore=['conf.py']
         )
         repo_to.index.add([doc["rst_location"]])
+
+        for obj in repo_to.index.diff(None).iter_change_type('D'):
+            repo_to.index.remove([obj.b_path])
         if len(repo_to.index.diff("HEAD")) == 0:
             # Nothing to commit
             logging.debug("No changes.")

tools/update_zuul_project_configs.py

@@ -130,6 +130,7 @@ def process_repositories(args, service):
         zuul_templates = None
         zuul_jobs = dict()
         zuul_new_jobs = list()
+        zuul_vars = dict()
         zuul_config_updated = False
         for item in zuul_config:
             if "project" in item.keys():
@@ -137,10 +138,11 @@ def process_repositories(args, service):
                 zuul_templates = project.setdefault("templates", [])
                 if not zuul_templates:
                     zuul_templates = []
+                zuul_vars = project.setdefault("vars", {})
             elif "job" in item.keys():
                 job = item["job"]
                 zuul_jobs[job["name"]] = job
-        print(f"Existing jobs {zuul_jobs}")
+        logging.debug(f"Existing jobs {zuul_jobs}")
         if "helpcenter-base-jobs" not in zuul_templates:
             zuul_templates.append("helpcenter-base-jobs")
             zuul_config_updated = True
@@ -148,6 +150,7 @@ def process_repositories(args, service):
         job_suffix = (
             "-hc-int-jobs" if args.environment == "internal" else "-hc-jobs"
         )
+        sphinx_pdf_files = zuul_vars.setdefault('sphinx_pdf_files', [])
         for doc in data.docs_by_service_type(service["service_type"]):
             logging.debug(f"Analyzing document {doc}")
             if not doc.get("type"):
@@ -156,6 +159,12 @@ def process_repositories(args, service):
                 doc_type = "dev-guide"
             else:
                 doc_type = doc["type"]
+            # Collect all PDF files into sphinx_pdf_files var
+            pdf_name = doc.get('pdf_name')
+            if pdf_name and f"{pdf_name}.pdf" not in sphinx_pdf_files:
+                sphinx_pdf_files.append(f"{pdf_name}.pdf")
+                zuul_config_updated = True
+
             template_name = f"{doc_type}{job_suffix}"
             if doc_type in ["api-ref", "umn", "dev-guide"]:
                 if template_name not in zuul_templates:
@@ -190,6 +199,7 @@ def process_repositories(args, service):
                 if "project" in item.keys():
                     project = item["project"]
                     project["templates"] = zuul_templates
+                    project["vars"] = zuul_vars
                     # Ensure new jobs are in check
                     if len(zuul_new_jobs) > 0:
                         project.setdefault(