fixing the type for new document

.stestr.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-test_path=./otc_metadata/tests/
+test_path=./otc-metadata/tests
 top_dir=./

otc_metadata/data/__init__.py

@@ -28,7 +28,6 @@ def read_data(filename):
     with open(filepath, 'r') as fd:
         return yaml.safe_load(fd)
 
-
 def rewrite_data(filename, data):
     """Rewrites data formatting it
 

otc_metadata/docs.py

@@ -15,7 +15,7 @@ import copy
 
 import otc_metadata.data
 
-__all__ = ['Service']
+__all__ = ['Docs']
 
 BUILTIN_DATA = otc_metadata.data.read_data('docs.yaml')
 

otc_metadata/services.py

@@ -40,8 +40,7 @@ class Services(object):
 
         # sort docs list by <service_type>_<title>
         self._service_data["documents"] = sorted(
-            sorted_docs,
+            sorted_docs, key=lambda x: f"{x.get('service_type')}{x.get('title')}"
-            key=lambda x: f"{x.get('service_type')}{x.get('title')}"
         )
         # sort services by <service_type>_<service_title>
         self._service_data["services"] = sorted(
@@ -184,10 +183,7 @@ class Services(object):
                 ]
             if "repositories" in srv and environment:
                 for repo in srv["repositories"]:
-                    if (
+                    if "environment" in repo and repo["environment"] == environment:
-                        "environment" in repo
-                        and repo["environment"] == environment
-                    ):
                         srv_res["repository"] = repo["repo"]
             for doc in self.all_docs:
                 if (
@@ -209,22 +205,3 @@ class Services(object):
                 doc_struct[srv["service_category"]].append(srv_res)
 
         return dict(categories=doc_struct)
-
-    def get_service_with_docs_by_service_type(self, service_type):
-        """Retrieve service and service docs by service_type
-
-        :param str service_type: Filter by service_type
-        """
-        res = dict()
-        res['service'] = {}
-        docs = []
-        services = self._service_data
-        for doc in services['documents']:
-            if doc['service_type'] == service_type:
-                docs.append(doc)
-        res['documents'] = docs
-        for service in services['services']:
-            if service['service_type'] == service_type:
-                res['service'] = service
-                break
-        return res

otc_metadata/templates/conf.py.j2

@@ -104,9 +104,6 @@ html_title = "{{ title }}"
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
-# Do not include sources into the rendered results
-html_copy_source = False
-
 # -- Options for PDF output --------------------------------------------------
 latex_documents = [
 {%- if pdf_name is defined %}

otc_metadata/templates/tox.ini.j2

@@ -21,28 +21,18 @@ deps =
   -r{toxinidir}/requirements.txt
 commands = {posargs}
 
-# This env is invoked in the periodic pipeline and is therefore responsible to
-# build all relevant docs at once.
 [testenv:docs]
 deps = -r{toxinidir}/doc/requirements.txt
-allowlist_externals =
-  mkdir
-  cp
-  sh
 commands =
 {%- for doc in docs %}
   {[testenv:{{ doc.type }}]commands}
   {[testenv:json-{{ doc.type }}]commands}
 {%- endfor %}
 
-[testenv:pdf-docs]
+[testenv:docs-pdf]
 deps = -r{toxinidir}/doc/requirements.txt
-allowlist_externals =
+whitelist_externals =
-  rm
   mkdir
-  make
-  bash
-  cp
 commands =
   mkdir -p doc/build/pdf
 {%- for doc in docs %}
@@ -56,7 +46,7 @@ commands =
 # HTML version
 [testenv:{{ doc.type }}]
 deps = -r{toxinidir}/doc/requirements.txt
-allowlist_externals =
+whitelist_externals =
   cp
   mkdir
 commands =
@@ -75,15 +65,14 @@ commands =
 # Json version (for search)
 [testenv:json-{{ doc.type }}]
 deps = -r{toxinidir}/doc/requirements.txt
-allowlist_externals =
+whitelist_externals =
   cp
   mkdir
-  sh
+  find
 commands =
   sphinx-build -W --keep-going -b json {{ loc }}/source doc/build/json/{{ doc.type }}
-  # Drop data useless for the search - wrap it also with sh/xargs due to bugs
+  # Drop data useless for the search
-  # in tox
+  find doc/build/json -type d -and ( -name '_images' -or -name '_static' -or -name '_sources' ) -exec rm {:} ;
-  sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf"
 {%- if doc.type == 'api-ref' %}
   mkdir -p api-ref/build/json
   cp -av doc/build/json/api-ref api-ref/build/json
@@ -99,7 +88,7 @@ commands =
 # PDF version
 [testenv:{{ doc.type }}-pdf-docs]
 deps = -r{toxinidir}/doc/requirements.txt
-allowlist_externals =
+whitelist_externals =
   rm
   mkdir
   make

otc_metadata/tests/base.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2010-2011 OpenStack Foundation
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+
+
+class TestCase(TestCase):
+
+    """Test case base class for all unit tests."""

otc_metadata/tests/test_otc-metadata.py

@@ -19,10 +19,10 @@ test_otc-metadata
 Tests for `otc-metadata` module.
 """
 
-from unittest import TestCase
+from otc-metadata.tests import base
 
 
-class TestOtcMetadata(TestCase):
+class TestOtc-metadata(base.TestCase):
 
     def test_something(self):
         pass

setup.cfg

@@ -1,11 +1,11 @@
 [metadata]
 name = otc-metadata
 summary = Metadata about OTC for Ecosystem
-description_file =
+description-file =
     README.rst
 author = Open Telekom Cloud
-home_page = https://open.telekom.cloud/
+home-page = https://open.telekom.cloud/
-python_requires = >=3.6
+python-requires = >=3.6
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology

setup.py

@@ -11,6 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
 setuptools.setup(

test-requirements.txt

@@ -2,6 +2,9 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-stestr>=2.0.0 # Apache-2.0
+hacking>=3.0,<3.1 # Apache-2.0
-testtools>=2.2.0 # MIT
+
-flake8
+coverage>=4.0,!=4.4 # Apache-2.0
+python-subunit>=0.0.18 # Apache-2.0/BSD
+stestr>=1.0.0 # Apache-2.0
+testtools>=1.4.0 # MIT

tools-requirements.txt

@@ -1,6 +1,2 @@
 GitPython
-ruamel.yaml
+ruamel
-requests
-jinja2
-dirsync
-cookiecutter

tools/convert_data.py

@@ -1,3 +1,4 @@
+import copy
 import re
 
 import otc_metadata.services
@@ -14,9 +15,13 @@ for doc in new_data["documents"]:
     if link:
         print(f"Parsing {link}")
         #(p1, p2) = link.split("/")
-        doc["link"] = re.sub(r"/(.*)/(.*)/", r"/\2/\1/", link)
+        doc["link"] = re.sub(
+            r"/(.*)/(.*)/",
+            r"/\2/\1/",
+            link
+        )
 
 _yaml = YAML()
 _yaml.indent(mapping=2, sequence=4, offset=2)
-with open("new.yaml", "w") as fd:
+with open('new.yaml', 'w') as fd:
     _yaml.dump(new_data, fd)

tools/generate_docexports_data.py

@@ -13,9 +13,7 @@ def main():
 
     _yaml = YAML()
     _yaml.indent(mapping=2, sequence=4, offset=2)
-    sys.stdout.write(
+    sys.stdout.write("# Auto-generated by otc_metadata.generate_docexports.data\n")
-        "# Auto-generated by otc_metadata.generate_docexports.data\n"
-    )
     _yaml.dump(data.docs_html_by_category("internal"), sys.stdout)
 
 

tools/open_doc_issue.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 
 import argparse
-# import re
+import re
 import requests
-# import sys
+import sys
 
 import otc_metadata.services
 
@@ -12,7 +12,8 @@ api_session = requests.Session()
 
 def open_issue(args, repository, issue_data):
     req = dict(
-        title=issue_data["title"], body=issue_data["body"].replace("\\n", "\n")
+        title=issue_data["title"],
+        body=issue_data["body"].replace("\\n", "\n")
     )
     if "assignees" in issue_data:
         req["assignees"] = issue_data["assignees"]
@@ -20,54 +21,57 @@ def open_issue(args, repository, issue_data):
         req["labels"] = issue_data["labels"]
     print(req)
     rsp = api_session.post(
-        f"{args.api_url}/repos/{repository}/issues", json=req
+        f"{args.api_url}/repos/{repository}/issues",
+        json=req
     )
     if rsp.status_code != 201:
         print(rsp.text)
-    print(
+    print(f"Going to open issue with title {issue_data['title']} in {repository}")
-        f"Going to open issue with title {issue_data['title']} in {repository}"
-    )
 
 
 def main():
-    parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(description='Open Issue for every document.')
-        description="Open Issue for every document."
-    )
-    parser.add_argument("token", metavar="token", help="API token")
-    parser.add_argument("--api-url", help="API base url of the Git hoster")
-    parser.add_argument("--environment", help="Environment for the repository")
-    parser.add_argument("--title", required=True, help="Issue title")
-    parser.add_argument("--body", required=True, help="Issue body")
     parser.add_argument(
-        "--repo",
+        'token', metavar='token', help='API token')
-        help="Repository to report issue in (instead of doc repository).",
+    parser.add_argument(
+        '--api-url', help='API base url of the Git hoster'
     )
     parser.add_argument(
-        "--assignee",
+        '--environment', help='Environment for the repository'
-        help="Issue assignee to use instead of document service assignees.",
     )
     parser.add_argument(
-        "--labels",
+        '--title', required=True, help='Issue title'
-        help="Issue labels to use (comma separated list of label IDs).",
+    )
+    parser.add_argument(
+        '--body', required=True, help='Issue body'    
+    )
+    parser.add_argument(
+        '--repo', help='Repository to report issue in (instead of doc repository).'
+    )
+    parser.add_argument(
+        '--assignee', help='Issue assignee to use instead of document service assignees.'
+    )
+    parser.add_argument(
+        '--labels', help='Issue labels to use (comma separated list of label IDs).'
     )
     args = parser.parse_args()
     data = otc_metadata.services.Services()
-    api_session.headers.update({"Authorization": f"token {args.token}"})
+    api_session.headers.update({'Authorization': f"token {args.token}"})
 
     for doc in data.all_docs_full(environment=args.environment):
         issue_data=dict(
             title=args.title.format(**doc),
             body=args.body.format(**doc),
-            repository=doc["repository"],
+            repository=doc["repository"]
         )
         if "assignees" in doc:
             issue_data["assignees"] = doc["assignees"]
         if args.assignee:
             issue_data["assignees"] = [args.assignee]
         if args.labels:
-            issue_data["labels"] = [int(x) for x in args.labels.split(",")]
+            issue_data["labels"] = [int(x) for x in args.labels.split(',')]
         open_issue(args, args.repo or doc["repository"], issue_data)
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     main()

tools/sort_metadata.py

@@ -1,7 +1,7 @@
-# import copy
+import copy
 
 import otc_metadata.services
-# from ruamel.yaml import YAML
+from ruamel.yaml import YAML
 
 data = otc_metadata.services.Services()
 

tools/update_zuul_project_configs.py

@@ -15,15 +15,15 @@
 
 import argparse
 import logging
-# import os
+import os
 import pathlib
 import requests
 import subprocess
-# import warnings
+import warnings
 
 from git import exc
 from git import Repo
-# from git import SymbolicReference
+from git import SymbolicReference
 
 from ruamel.yaml import CommentedMap
 from ruamel.yaml import YAML
@@ -72,7 +72,7 @@ def process_repositories(args, service):
     workdir.mkdir(exist_ok=True)
 
     copy_to = None
-    # repo_to = None
+    repo_to = None
 
     for repo in service["repositories"]:
         logging.debug(f"Processing repository {repo}")
@@ -130,7 +130,6 @@ def process_repositories(args, service):
         zuul_templates = None
         zuul_jobs = dict()
         zuul_new_jobs = list()
-        zuul_vars = dict()
         zuul_config_updated = False
         for item in zuul_config:
             if "project" in item.keys():
@@ -138,11 +137,10 @@ def process_repositories(args, service):
                 zuul_templates = project.setdefault("templates", [])
                 if not zuul_templates:
                     zuul_templates = []
-                zuul_vars = project.setdefault("vars", {})
             elif "job" in item.keys():
                 job = item["job"]
                 zuul_jobs[job["name"]] = job
-        logging.debug(f"Existing jobs {zuul_jobs}")
+        print(f"Existing jobs {zuul_jobs}")
         if "helpcenter-base-jobs" not in zuul_templates:
             zuul_templates.append("helpcenter-base-jobs")
             zuul_config_updated = True
@@ -150,7 +148,6 @@ def process_repositories(args, service):
         job_suffix = (
             "-hc-int-jobs" if args.environment == "internal" else "-hc-jobs"
         )
-        sphinx_pdf_files = zuul_vars.setdefault('sphinx_pdf_files', [])
         for doc in data.docs_by_service_type(service["service_type"]):
             logging.debug(f"Analyzing document {doc}")
             if not doc.get("type"):
@@ -159,12 +156,6 @@ def process_repositories(args, service):
                 doc_type = "dev-guide"
             else:
                 doc_type = doc["type"]
-            # Collect all PDF files into sphinx_pdf_files var
-            pdf_name = doc.get('pdf_name')
-            if pdf_name and f"{pdf_name}.pdf" not in sphinx_pdf_files:
-                sphinx_pdf_files.append(f"{pdf_name}.pdf")
-                zuul_config_updated = True
-
             template_name = f"{doc_type}{job_suffix}"
             if doc_type in ["api-ref", "umn", "dev-guide"]:
                 if template_name not in zuul_templates:
@@ -199,7 +190,6 @@ def process_repositories(args, service):
                 if "project" in item.keys():
                     project = item["project"]
                     project["templates"] = zuul_templates
-                    project["vars"] = zuul_vars
                     # Ensure new jobs are in check
                     if len(zuul_new_jobs) > 0:
                         project.setdefault(
@@ -218,6 +208,7 @@ def process_repositories(args, service):
                         project["check"]["jobs"].extend(
                             [x["job"]["name"] for x in zuul_new_jobs])
 
+
             #yaml.indent(offset=2, sequence=2)
             with open(zuul_file_name, "w") as f:
                 yaml.dump(zuul_config, f)
@@ -232,8 +223,7 @@ def process_repositories(args, service):
             git_repo.index.commit(
                 (
                     "Update zuul.yaml file\n\n"
-                    "Performed-by: gitea/infra/otc-metadata"
+                    "Performed-by: gitea/infra/otc-metadata/tools/update_zuul_project_config.py"
-                    "/tools/update_zuul_project_config.py"
                 )
             )
             push_args = ["--set-upstream", "origin", branch_name]

tox.ini

@@ -1,26 +1,56 @@
 [tox]
 minversion = 3.2.0
-envlist = py3,pep8
+envlist = py39,pep8
+skipsdist = True
 ignore_basepython_conflict = true
 
+
 [testenv]
 basepython = python3
 usedevelop = True
 setenv =
    PYTHONWARNINGS=default::DeprecationWarning
+   OS_STDOUT_CAPTURE=1
+   OS_STDERR_CAPTURE=1
+   OS_TEST_TIMEOUT=60
 deps = -r{toxinidir}/test-requirements.txt
 commands = stestr run {posargs}
 
+[testenv:lower-constraints]
+deps = -r{toxinidir}/test-requirements.txt
+
 [testenv:pep8]
 commands = flake8 {posargs}
 
 [testenv:venv]
 commands = {posargs}
 
+[testenv:cover]
+setenv =
+    VIRTUAL_ENV={envdir}
+    PYTHON=coverage run --source otc_metadata --parallel-mode
+commands =
+    stestr run {posargs}
+    coverage combine
+    coverage html -d cover
+    coverage xml -o cover/coverage.xml
+
+[testenv:docs]
+deps = -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -W -b html doc/source doc/build/html
+
+[testenv:releasenotes]
+deps = {[testenv:docs]deps}
+commands =
+  sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+
+[testenv:debug]
+commands = oslo_debug_helper {posargs}
+
 [flake8]
 # E123, E125 skipped as they are invalid PEP-8.
 
 show-source = True
-ignore = E123,E125,W503
+ignore = E123,E125
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build

zuul.yaml

@@ -1,11 +1,19 @@
 - project:
     merge-mode: squash-merge
-    default-branch: main
+    default-branch: master
+    templates:
+      - publish-to-pypi
+      - publish-otc-docs-hc-pti
+      - release-notes-jobs
     check:
       jobs:
         - otc-tox-pep8
+        - otc-tox-py38
         - otc-tox-py39
+        - tox-functional
     gate:
       jobs:
         - otc-tox-pep8
+        - otc-tox-py38
         - otc-tox-py39
+        - tox-functional