Index Tool for Search

Reviewed-by: Hasko, Vladimir <vladimir.hasko@t-systems.com>
Co-authored-by: Gode, Sebastian <sebastian.gode@t-systems.com>
Co-committed-by: Gode, Sebastian <sebastian.gode@t-systems.com>

README.rst

@@ -1,6 +1,9 @@
-===============================
-ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git
-===============================
+============
+otc-metadata
+============
+
+Link: ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git
+
 
 Metadata about OTC for Ecosystem
 
@@ -13,6 +16,39 @@ Note that this is a hard requirement.
 * Source: https://github.com/infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git
 
 Features
---------
+========
 
 * TODO
+
+Overview: service.yaml
+======================
+
+The :code:`service.yaml` file contains all data about services, service
+categories and the related documents of each service. The file is
+used as a base for several internal and external applications or
+websites like the Helpcenter 3.0 where the information about the document
+repositories and its properties are stored.
+
+File structure
+--------------
+
+The file is based on the yaml-file format and has three main sections
+which can be compared with database tables in a relational database.
+
+* documents: contains the information about every single document and its type
+  like umn, api-ref etc.
+
+* service category: contains the keyword and title of the service category
+
+* services: contains the repository information about the internal (Gitea) and
+  external location (GitHub) and all the necessary parameters of the service itself
+
+These sections, or better "tables" have
+their own keys and foreign keys so that the tables are linked together and
+the related information can be fetched.
+For the :code:`services` table
+the key is :code:`service_type` which has the foreign key in the
+:code:`documents` table. So a service can have multiple documents and each
+document can only be linked to one service.
+The key :code:`service_category` table is :code:`name` of the service category
+which is then used in the :code:`services` table as foreign key.

otc_metadata/services.py

@@ -113,6 +113,55 @@ class Services(object):
             res[cat]["docs"].append(res_doc)
         return res
 
+    def service_types_with_doc_types(self, environment=None):
+        """Retrieve type and title from services and corresponding docs.
+            As well as a list of all available doc types with title.
+
+        :param str environment: Optional service environment.
+        """
+        service_list = []
+        docs = []
+
+        for service in self.all_services:
+            if not service["service_title"]:
+                continue
+            if not service["service_type"]:
+                continue
+
+            doc_list = []
+            for doc in self.all_docs:
+                if "environment" in doc:
+                    if doc["environment"] != environment:
+                        continue
+                if doc["service_type"] == service["service_type"]:
+                    doc_list.append({
+                        "title": doc["title"],
+                        "type": doc["type"]
+                    })
+
+                new_doc = {
+                    "type": doc["type"],
+                    "title": doc["title"]
+                }
+                type_exists = any(
+                    doc_dict["type"] == new_doc["type"] for doc_dict in docs
+                )
+                if not type_exists:
+                    docs.append(new_doc)
+
+            service_list.append({
+                "service_title": service["service_title"],
+                "service_type": service["service_type"],
+                "docs": doc_list
+            })
+
+        res = {
+            "services": service_list,
+            "docs": docs
+        }
+
+        return res
+
     def docs_by_service_category(self, category, environment=None):
         """List services matching category
 

otc_metadata/templates/conf.py.j2

@@ -18,7 +18,10 @@ import os
 import sys
 
 extensions = [
-    'otcdocstheme'
+    'otcdocstheme',
+{%- if otc_sbv %}
+    'otc_sphinx_directives'
+{%- endif %}
 ]
 
 otcdocs_auto_name = False
@@ -34,6 +37,16 @@ otcdocs_git_fqdn = '{{ git_fqdn }}'
 otcdocs_git_type = '{{ git_type }}'
 {%- endif %}
 
+# Those variables are needed for indexing into OpenSearch
+otcdocs_doc_environment = '{{ doc_environment }}'
+otcdocs_doc_link = '{{ doc_link }}'
+otcdocs_doc_title = '{{ doc_title }}'
+otcdocs_doc_type = '{{ doc_type }}'
+otcdocs_service_category = '{{ service_category }}'
+otcdocs_service_title = '{{ service_title }}'
+otcdocs_service_type = '{{ service_type }}'
+otcdocs_search_environment = 'hc_de'
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

otc_metadata/templates/doc_requirements.txt.j2

@@ -2,3 +2,6 @@ sphinx>=2.0.0,!=2.1.0 # BSD
 otcdocstheme # Apache-2.0
 # releasenotes
 reno>=3.1.0 # Apache-2.0
+
+otc-sphinx-directives>=0.1.0
+git+https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata.git#egg=otc_metadata

otc_metadata/templates/index_sbv.rst.j2

@@ -0,0 +1,7 @@
+{{ sbv_title }}
+
+.. directive_wrapper::
+   :class: container-sbv
+
+   .. service_card::
+      :service_type: {{ service_type }}

otc_metadata/templates/tox.ini.j2

@@ -29,13 +29,17 @@ allowlist_externals =
   mkdir
   cp
   sh
+  rm
+  sphinx-build
 commands =
+  rm -rf doc/build/html doc/build/doctrees
+  sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html
 {%- for doc in docs %}
   {[testenv:{{ doc.type }}]commands}
   {[testenv:json-{{ doc.type }}]commands}
 {%- endfor %}
 
-[testenv:pdf-docs]
+[testenv:docs-pdf]
 deps = -r{toxinidir}/doc/requirements.txt
 allowlist_externals =
   rm
@@ -115,3 +119,7 @@ commands =
   cp {{ loc }}/build/pdf/{{ doc.pdf_name }}.pdf doc/build/pdf/
 {% endif %}
 {% endfor %}
+
+[doc8]
+ignore = D001
+extensions = .rst, .yaml

otc_metadata/tests/test_otc_metadata.py

@@ -21,8 +21,45 @@ Tests for `otc-metadata` module.
 
 from unittest import TestCase
 
+from otc_metadata import services
+
 
 class TestOtcMetadata(TestCase):
+    def setUp(self):
+        self.data = services.Services()
 
-    def test_something(self):
-        pass
+    def test_data_is_sorted(self):
+        curr = self.data
+        new = services.Services()
+        new._sort_data()
+        self.assertEqual(
+            curr._service_data, new._service_data, "Data is sorted properly"
+        )
+
+    def test_service_categories(self):
+        category = dict()
+        for cat in self.data._service_data["service_categories"]:
+            category[cat["name"]] = cat["title"]
+        for srv in self.data.all_services:
+            self.assertTrue(
+                srv["service_category"] in category,
+                f"Category {srv['service_category']} is present",
+            )
+
+    def test_doc_contains_required_data(self):
+        srv_types = dict()
+        for srv in self.data.all_services:
+            srv_types[srv["service_type"]] = srv
+        for doc in self.data.all_docs:
+            for attr in [
+                "rst_location",
+                "service_type",
+                "title",
+                "type",
+            ]:
+                self.assertIn(attr, doc, f"Document {doc} contains {attr}")
+            self.assertIn(
+                doc["service_type"],
+                srv_types,
+                f"Document {doc} contains valid service_type",
+            )

tools-requirements.txt

@@ -4,3 +4,4 @@ requests
 jinja2
 dirsync
 cookiecutter
+opensearch-py

tools/generate_doc_gitcontrol_repos.py

@@ -32,9 +32,11 @@ def process_services(args, services):
         block_on_rejected_reviews=True,
         dismiss_stale_approvals=True,
         enable_push=False,
+        enable_status_check=True,
         status_check_contexts=["gl/check"],
         enable_merge_whitelist=True,
         merge_whitelist_usernames=["zuul"],
+        required_approvals=1,
     )
     gitea_repo_template = dict(
         default_branch="main",

tools/update_zuul_project_configs.py

@@ -130,7 +130,6 @@ def process_repositories(args, service):
         zuul_templates = None
         zuul_jobs = dict()
         zuul_new_jobs = list()
-        zuul_vars = dict()
         zuul_config_updated = False
         for item in zuul_config:
             if "project" in item.keys():
@@ -138,11 +137,10 @@ def process_repositories(args, service):
                 zuul_templates = project.setdefault("templates", [])
                 if not zuul_templates:
                     zuul_templates = []
-                zuul_vars = project.setdefault("vars", {})
             elif "job" in item.keys():
                 job = item["job"]
                 zuul_jobs[job["name"]] = job
-        logging.debug(f"Existing jobs {zuul_jobs}")
+        print(f"Existing jobs {zuul_jobs}")
         if "helpcenter-base-jobs" not in zuul_templates:
             zuul_templates.append("helpcenter-base-jobs")
             zuul_config_updated = True
@@ -150,7 +148,6 @@ def process_repositories(args, service):
         job_suffix = (
             "-hc-int-jobs" if args.environment == "internal" else "-hc-jobs"
         )
-        sphinx_pdf_files = zuul_vars.setdefault('sphinx_pdf_files', [])
         for doc in data.docs_by_service_type(service["service_type"]):
             logging.debug(f"Analyzing document {doc}")
             if not doc.get("type"):
@@ -159,12 +156,6 @@ def process_repositories(args, service):
                 doc_type = "dev-guide"
             else:
                 doc_type = doc["type"]
-            # Collect all PDF files into sphinx_pdf_files var
-            pdf_name = doc.get('pdf_name')
-            if pdf_name and f"{pdf_name}.pdf" not in sphinx_pdf_files:
-                sphinx_pdf_files.append(f"{pdf_name}.pdf")
-                zuul_config_updated = True
-
             template_name = f"{doc_type}{job_suffix}"
             if doc_type in ["api-ref", "umn", "dev-guide"]:
                 if template_name not in zuul_templates:
@@ -199,7 +190,6 @@ def process_repositories(args, service):
                 if "project" in item.keys():
                     project = item["project"]
                     project["templates"] = zuul_templates
-                    project["vars"] = zuul_vars
                     # Ensure new jobs are in check
                     if len(zuul_new_jobs) > 0:
                         project.setdefault(

tox.ini

@@ -19,7 +19,6 @@ commands = {posargs}
 
 [flake8]
 # E123, E125 skipped as they are invalid PEP-8.
-
 show-source = True
 ignore = E123,E125,W503
 builtins = _

zuul.yaml

@@ -1,3 +1,4 @@
+---
 - project:
     merge-mode: squash-merge
     default-branch: main
@@ -8,4 +9,4 @@
     gate:
       jobs:
         - otc-tox-pep8
-        - otc-tox-py39
+        - otc-tox-py39