167 lines
4.3 KiB
Python
167 lines
4.3 KiB
Python
import otc_metadata
|
|
import argparse
|
|
import logging
|
|
from opensearchpy import OpenSearch
|
|
|
|
|
|
metadata = otc_metadata.Services()
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(
|
|
description="Create Index data for search inside OpenSearch"
|
|
)
|
|
parser.add_argument(
|
|
"--target-environment",
|
|
required=True,
|
|
help="Environment to be used as a source",
|
|
)
|
|
parser.add_argument(
|
|
'--delete-index',
|
|
action='store_true',
|
|
help='Option deletes old index with the same name and creates new '
|
|
'one.'
|
|
)
|
|
parser.add_argument(
|
|
'--all-doc-types',
|
|
action='store_true',
|
|
help='Upload all doc-types instead of only umn, api-ref and dev'
|
|
)
|
|
parser.add_argument(
|
|
'--debug',
|
|
action='store_true',
|
|
help='Enable Debug mode'
|
|
)
|
|
parser.add_argument(
|
|
'--hosts',
|
|
metavar='<host:port>',
|
|
nargs='+',
|
|
default=['localhost:9200'],
|
|
help='Provide one or multiple host:port values '
|
|
'separated by space for multiple hosts.\n'
|
|
'Default: localhost:9200'
|
|
)
|
|
parser.add_argument(
|
|
'--index',
|
|
metavar='<index>',
|
|
default='test-index',
|
|
help="OpenSearch / ElasticSearch index name.\n"
|
|
'Default: test-index'
|
|
)
|
|
parser.add_argument(
|
|
'--username',
|
|
metavar='<username>',
|
|
required=True,
|
|
help='Username for the connection.'
|
|
)
|
|
parser.add_argument(
|
|
'--password',
|
|
metavar='<password>',
|
|
required=True,
|
|
help='Password for the connection.'
|
|
)
|
|
parser.add_argument(
|
|
"--cloud-environment",
|
|
required=True,
|
|
default="eu_de",
|
|
help="Cloud Environment. Default: eu_de",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
return args
|
|
|
|
|
|
def main():
|
|
|
|
args = parse_args()
|
|
|
|
if args.debug:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
logging.debug("Obtaining data from otc_metadata")
|
|
data = getData(
|
|
cloud_environment=args.cloud_environment,
|
|
environment=args.target_environment,
|
|
all_doc_types=args.all_doc_types
|
|
)
|
|
|
|
sorted_services = sortData(data['services'], sort_key='service_title')
|
|
sorted_data = {'services': sorted_services, 'docs': data['docs']}
|
|
|
|
logging.debug("Indexing data into OpenSearch")
|
|
indexData(
|
|
deleteIndex=args.delete_index,
|
|
hosts=args.hosts,
|
|
index=args.index,
|
|
username=args.username,
|
|
password=args.password,
|
|
data=sorted_data
|
|
)
|
|
|
|
|
|
def filter_docs(metadata):
|
|
allowed_types = ["umn", "api-ref", "dev"]
|
|
metadata['docs'] = [doc for doc in metadata['docs']
|
|
if doc['type'] in allowed_types]
|
|
return metadata
|
|
|
|
|
|
def getData(cloud_environment, environment, all_doc_types):
|
|
metadatadata = metadata.service_types_with_doc_types(
|
|
cloud_environment=cloud_environment,
|
|
environment=environment
|
|
)
|
|
final_data = metadatadata
|
|
if not all_doc_types:
|
|
final_data = filter_docs(metadatadata)
|
|
return final_data
|
|
|
|
|
|
def sortData(data, sort_key):
|
|
return sorted(data, key=lambda x: x[sort_key])
|
|
|
|
|
|
def indexData(deleteIndex, hosts, index, username, password, data):
|
|
hosts = generate_os_host_list(hosts)
|
|
client = OpenSearch(
|
|
hosts=hosts,
|
|
http_compress=True,
|
|
http_auth=(username, password),
|
|
use_ssl=True,
|
|
verify_certs=True,
|
|
ssl_assert_hostname=False,
|
|
ssl_show_warn=False
|
|
)
|
|
|
|
if deleteIndex is True:
|
|
logging.debug("Deleting Index")
|
|
delete_index(client, index)
|
|
|
|
logging.debug("Started creating Index")
|
|
create_index(client, index, data)
|
|
logging.debug("Finished creating Index")
|
|
|
|
|
|
def generate_os_host_list(hosts):
|
|
host_list = []
|
|
for host in hosts:
|
|
raw_host = host.split(':')
|
|
if len(raw_host) != 2:
|
|
raise Exception('--hosts parameter does not match the following '
|
|
'format: hostname:port')
|
|
json_host = {'host': raw_host[0], 'port': int(raw_host[1])}
|
|
host_list.append(json_host)
|
|
return host_list
|
|
|
|
|
|
def create_index(client, index, data):
|
|
client.indices.create(index=index)
|
|
return client.index(index=index, body=data)
|
|
|
|
|
|
def delete_index(client, index):
|
|
return client.indices.delete(index=index, ignore=[400, 404])
|
|
|
|
|
|
main()
|