fixing the type for new document

Reviewed-by: Goncharov, Artem <artem.goncharov@t-systems.com>
Co-authored-by: gtema <artem.goncharov@gmail.com>
Co-committed-by: gtema <artem.goncharov@gmail.com>

.stestr.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-test_path=./otc-metadata/tests
+test_path=./otc_metadata/tests/
 top_dir=./

otc_metadata/data/__init__.py

@@ -28,6 +28,7 @@ def read_data(filename):
     with open(filepath, 'r') as fd:
         return yaml.safe_load(fd)
 
+
 def rewrite_data(filename, data):
     """Rewrites data formatting it
 

otc_metadata/data/services.yaml

@@ -588,6 +588,14 @@ documents:
     service_type: mrs
     title: Component Operation Guide
     type: operation-guide
+  - hc_location: cmpntguide-lts/mrs
+    html_location: docs/mrs/component-operation-guide-lts
+    link: /mapreduce-service/operation-guide-lts/
+    pdf_name: mrs-component-operation-guide-lts
+    rst_location: doc/component-operation-guide-lts/source
+    service_type: mrs
+    title: Component Operation Guide (LTS)
+    type: operation-guide-lts
   - hc_location: usermanual/mrs
     html_location: docs/mrs/umn
     link: /mapreduce-service/umn/

otc_metadata/docs.py

@@ -15,7 +15,7 @@ import copy
 
 import otc_metadata.data
 
-__all__ = ['Docs']
+__all__ = ['Service']
 
 BUILTIN_DATA = otc_metadata.data.read_data('docs.yaml')
 

otc_metadata/services.py

@@ -40,7 +40,8 @@ class Services(object):
 
         # sort docs list by <service_type>_<title>
         self._service_data["documents"] = sorted(
-            sorted_docs, key=lambda x: f"{x.get('service_type')}{x.get('title')}"
+            sorted_docs,
+            key=lambda x: f"{x.get('service_type')}{x.get('title')}"
         )
         # sort services by <service_type>_<service_title>
         self._service_data["services"] = sorted(
@@ -183,7 +184,10 @@ class Services(object):
                 ]
             if "repositories" in srv and environment:
                 for repo in srv["repositories"]:
-                    if "environment" in repo and repo["environment"] == environment:
+                    if (
+                        "environment" in repo
+                        and repo["environment"] == environment
+                    ):
                         srv_res["repository"] = repo["repo"]
             for doc in self.all_docs:
                 if (

otc_metadata/templates/conf.py.j2

@@ -104,6 +104,9 @@ html_title = "{{ title }}"
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+# Do not include sources into the rendered results
+html_copy_source = False
+
 # -- Options for PDF output --------------------------------------------------
 latex_documents = [
 {%- if pdf_name is defined %}

otc_metadata/templates/tox.ini.j2

@@ -21,8 +21,14 @@ deps =
   -r{toxinidir}/requirements.txt
 commands = {posargs}
 
+# This env is invoked in the periodic pipeline and is therefore responsible to
+# build all relevant docs at once.
 [testenv:docs]
 deps = -r{toxinidir}/doc/requirements.txt
+allowlist_externals =
+  mkdir
+  cp
+  sh
 commands =
 {%- for doc in docs %}
   {[testenv:{{ doc.type }}]commands}
@@ -31,8 +37,12 @@ commands =
 
 [testenv:docs-pdf]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
+  rm
   mkdir
+  make
+  bash
+  cp
 commands =
   mkdir -p doc/build/pdf
 {%- for doc in docs %}
@@ -46,7 +56,7 @@ commands =
 # HTML version
 [testenv:{{ doc.type }}]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
   cp
   mkdir
 commands =
@@ -65,14 +75,15 @@ commands =
 # Json version (for search)
 [testenv:json-{{ doc.type }}]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
   cp
   mkdir
-  find
+  sh
 commands =
   sphinx-build -W --keep-going -b json {{ loc }}/source doc/build/json/{{ doc.type }}
-  # Drop data useless for the search
-  find doc/build/json -type d -and ( -name '_images' -or -name '_static' -or -name '_sources' ) -exec rm {:} ;
+  # Drop data useless for the search - wrap it also with sh/xargs due to bugs
+  # in tox
+  sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf"
 {%- if doc.type == 'api-ref' %}
   mkdir -p api-ref/build/json
   cp -av doc/build/json/api-ref api-ref/build/json
@@ -88,7 +99,7 @@ commands =
 # PDF version
 [testenv:{{ doc.type }}-pdf-docs]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
   rm
   mkdir
   make

otc_metadata/tests/base.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2010-2011 OpenStack Foundation
-# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from unittest import TestCase
-
-
-class TestCase(TestCase):
-
-    """Test case base class for all unit tests."""

otc_metadata/tests/test_otc_metadata.py

@@ -19,10 +19,10 @@ test_otc-metadata
 Tests for `otc-metadata` module.
 """
 
-from otc-metadata.tests import base
+from unittest import TestCase
 
 
-class TestOtc-metadata(base.TestCase):
+class TestOtcMetadata(TestCase):
 
     def test_something(self):
         pass

setup.cfg

@@ -1,11 +1,11 @@
 [metadata]
 name = otc-metadata
 summary = Metadata about OTC for Ecosystem
-description-file =
+description_file =
     README.rst
 author = Open Telekom Cloud
-home-page = https://open.telekom.cloud/
-python-requires = >=3.6
+home_page = https://open.telekom.cloud/
+python_requires = >=3.6
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology

setup.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
 setuptools.setup(

test-requirements.txt

@@ -2,9 +2,6 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking>=3.0,<3.1 # Apache-2.0
-
-coverage>=4.0,!=4.4 # Apache-2.0
-python-subunit>=0.0.18 # Apache-2.0/BSD
-stestr>=1.0.0 # Apache-2.0
-testtools>=1.4.0 # MIT
+stestr>=2.0.0 # Apache-2.0
+testtools>=2.2.0 # MIT
+flake8

tools-requirements.txt

@@ -1,2 +1,4 @@
 GitPython
-ruamel
+ruamel.yaml
+requests
+jinja2

tools/convert_data.py

@@ -1,4 +1,3 @@
-import copy
 import re
 
 import otc_metadata.services
@@ -15,13 +14,9 @@ for doc in new_data["documents"]:
     if link:
         print(f"Parsing {link}")
         # (p1, p2) = link.split("/")
-        doc["link"] = re.sub(
-            r"/(.*)/(.*)/",
-            r"/\2/\1/",
-            link
-        )
+        doc["link"] = re.sub(r"/(.*)/(.*)/", r"/\2/\1/", link)
 
 _yaml = YAML()
 _yaml.indent(mapping=2, sequence=4, offset=2)
-with open('new.yaml', 'w') as fd:
+with open("new.yaml", "w") as fd:
     _yaml.dump(new_data, fd)

tools/generate_docexports_data.py

@@ -13,7 +13,9 @@ def main():
 
     _yaml = YAML()
     _yaml.indent(mapping=2, sequence=4, offset=2)
-    sys.stdout.write("# Auto-generated by otc_metadata.generate_docexports.data\n")
+    sys.stdout.write(
+        "# Auto-generated by otc_metadata.generate_docexports.data\n"
+    )
     _yaml.dump(data.docs_html_by_category("internal"), sys.stdout)
 
 

tools/open_doc_issue.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 
 import argparse
-import re
+# import re
 import requests
-import sys
+# import sys
 
 import otc_metadata.services
 
@@ -12,8 +12,7 @@ api_session = requests.Session()
 
 def open_issue(args, repository, issue_data):
     req = dict(
-        title=issue_data["title"],
-        body=issue_data["body"].replace("\\n", "\n")
+        title=issue_data["title"], body=issue_data["body"].replace("\\n", "\n")
     )
     if "assignees" in issue_data:
         req["assignees"] = issue_data["assignees"]
@@ -21,57 +20,54 @@ def open_issue(args, repository, issue_data):
         req["labels"] = issue_data["labels"]
     print(req)
     rsp = api_session.post(
-        f"{args.api_url}/repos/{repository}/issues",
-        json=req
+        f"{args.api_url}/repos/{repository}/issues", json=req
     )
     if rsp.status_code != 201:
         print(rsp.text)
-    print(f"Going to open issue with title {issue_data['title']} in {repository}")
+    print(
+        f"Going to open issue with title {issue_data['title']} in {repository}"
+    )
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Open Issue for every document.')
+    parser = argparse.ArgumentParser(
+        description="Open Issue for every document."
+    )
+    parser.add_argument("token", metavar="token", help="API token")
+    parser.add_argument("--api-url", help="API base url of the Git hoster")
+    parser.add_argument("--environment", help="Environment for the repository")
+    parser.add_argument("--title", required=True, help="Issue title")
+    parser.add_argument("--body", required=True, help="Issue body")
     parser.add_argument(
-        'token', metavar='token', help='API token')
-    parser.add_argument(
-        '--api-url', help='API base url of the Git hoster'
+        "--repo",
+        help="Repository to report issue in (instead of doc repository).",
     )
     parser.add_argument(
-        '--environment', help='Environment for the repository'
+        "--assignee",
+        help="Issue assignee to use instead of document service assignees.",
     )
     parser.add_argument(
-        '--title', required=True, help='Issue title'
-    )
-    parser.add_argument(
-        '--body', required=True, help='Issue body'    
-    )
-    parser.add_argument(
-        '--repo', help='Repository to report issue in (instead of doc repository).'
-    )
-    parser.add_argument(
-        '--assignee', help='Issue assignee to use instead of document service assignees.'
-    )
-    parser.add_argument(
-        '--labels', help='Issue labels to use (comma separated list of label IDs).'
+        "--labels",
+        help="Issue labels to use (comma separated list of label IDs).",
     )
     args = parser.parse_args()
     data = otc_metadata.services.Services()
-    api_session.headers.update({'Authorization': f"token {args.token}"})
+    api_session.headers.update({"Authorization": f"token {args.token}"})
 
     for doc in data.all_docs_full(environment=args.environment):
         issue_data = dict(
             title=args.title.format(**doc),
             body=args.body.format(**doc),
-            repository=doc["repository"]
+            repository=doc["repository"],
         )
         if "assignees" in doc:
             issue_data["assignees"] = doc["assignees"]
         if args.assignee:
             issue_data["assignees"] = [args.assignee]
         if args.labels:
-            issue_data["labels"] = [int(x) for x in args.labels.split(',')]
+            issue_data["labels"] = [int(x) for x in args.labels.split(",")]
         open_issue(args, args.repo or doc["repository"], issue_data)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

tools/sort_metadata.py

@@ -1,7 +1,7 @@
-import copy
+# import copy
 
 import otc_metadata.services
-from ruamel.yaml import YAML
+# from ruamel.yaml import YAML
 
 data = otc_metadata.services.Services()
 

tools/update_zuul_project_configs.py

@@ -15,15 +15,15 @@
 
 import argparse
 import logging
-import os
+# import os
 import pathlib
 import requests
 import subprocess
-import warnings
+# import warnings
 
 from git import exc
 from git import Repo
-from git import SymbolicReference
+# from git import SymbolicReference
 
 from ruamel.yaml import CommentedMap
 from ruamel.yaml import YAML
@@ -72,7 +72,7 @@ def process_repositories(args, service):
     workdir.mkdir(exist_ok=True)
 
     copy_to = None
-    repo_to = None
+    # repo_to = None
 
     for repo in service["repositories"]:
         logging.debug(f"Processing repository {repo}")
@@ -208,7 +208,6 @@ def process_repositories(args, service):
                         project["check"]["jobs"].extend(
                             [x["job"]["name"] for x in zuul_new_jobs])
 
-
             # yaml.indent(offset=2, sequence=2)
             with open(zuul_file_name, "w") as f:
                 yaml.dump(zuul_config, f)
@@ -223,7 +222,8 @@ def process_repositories(args, service):
             git_repo.index.commit(
                 (
                     "Update zuul.yaml file\n\n"
-                    "Performed-by: gitea/infra/otc-metadata/tools/update_zuul_project_config.py"
+                    "Performed-by: gitea/infra/otc-metadata"
+                    "/tools/update_zuul_project_config.py"
                 )
             )
             push_args = ["--set-upstream", "origin", branch_name]

tox.ini

@@ -1,56 +1,26 @@
 [tox]
 minversion = 3.2.0
-envlist = py39,pep8
-skipsdist = True
+envlist = py3,pep8
 ignore_basepython_conflict = true
 
-
 [testenv]
 basepython = python3
 usedevelop = True
 setenv =
    PYTHONWARNINGS=default::DeprecationWarning
-   OS_STDOUT_CAPTURE=1
-   OS_STDERR_CAPTURE=1
-   OS_TEST_TIMEOUT=60
 deps = -r{toxinidir}/test-requirements.txt
 commands = stestr run {posargs}
 
-[testenv:lower-constraints]
-deps = -r{toxinidir}/test-requirements.txt
-
 [testenv:pep8]
 commands = flake8 {posargs}
 
 [testenv:venv]
 commands = {posargs}
 
-[testenv:cover]
-setenv =
-    VIRTUAL_ENV={envdir}
-    PYTHON=coverage run --source otc_metadata --parallel-mode
-commands =
-    stestr run {posargs}
-    coverage combine
-    coverage html -d cover
-    coverage xml -o cover/coverage.xml
-
-[testenv:docs]
-deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -W -b html doc/source doc/build/html
-
-[testenv:releasenotes]
-deps = {[testenv:docs]deps}
-commands =
-  sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
-
-[testenv:debug]
-commands = oslo_debug_helper {posargs}
-
 [flake8]
 # E123, E125 skipped as they are invalid PEP-8.
 
 show-source = True
-ignore = E123,E125
+ignore = E123,E125,W503
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build

zuul.yaml

@@ -1,19 +1,11 @@
 - project:
     merge-mode: squash-merge
-    default-branch: master
-    templates:
-      - publish-to-pypi
-      - publish-otc-docs-hc-pti
-      - release-notes-jobs
+    default-branch: main
     check:
       jobs:
         - otc-tox-pep8
-        - otc-tox-py38
         - otc-tox-py39
-        - tox-functional
     gate:
       jobs:
         - otc-tox-pep8
-        - otc-tox-py38
         - otc-tox-py39
-        - tox-functional