fix: adjust requires and templates

README.md

@@ -1,8 +1,17 @@
 # namekoplus
 
+[![Upload Python Package](https://github.com/Bryanthelol/namekoplus/actions/workflows/python-publish.yml/badge.svg?event=release)](https://github.com/Bryanthelol/namekoplus/actions/workflows/python-publish.yml)
+
 A lightweight Python distributed microservice solution
 
-## Command Line Tool Usage
+## Installation
+
+```shell
+python3 -m pip install namekoplus
+```
+
+
+## CLI Usage
 
 ### Checkout Command
 
@@ -27,5 +36,5 @@ namekoplus init --directory <dir_name> --type <template_type>
 
 See Documents: 
 
-- [中文](https://doc.bearcatlog.com/)
-- [English](https://legendary-sopapillas-e2626d.netlify.app/)
+- [English](https://legendary-sopapillas-e2626d.netlify.app/)
+- [中文](https://doc.bearcatlog.com/)

namekoplus/chassis-agent/metric-configs/grafana.json.mako

@@ -28,15 +28,14 @@
   "links": [],
   "liveNow": false,
   "panels": [
-    % for grafana_dict in grafana_list:
+% for grafana_dict in grafana_list:
     {
       "aliasColors": {},
       "bars": false,
       "dashLength": 10,
       "dashes": false,
       "datasource": {
-        "type": "statsd",
-        "uid": "5qrwjFCnk"
+        "type": "statsd"
       },
       "fill": 1,
       "fillGradient": 0,
@@ -47,7 +46,7 @@
         "y": 0
       },
       "hiddenSeries": false,
-      "id": 5,
+      "id": ${loop.index},
       "legend": {
         "alignAsTable": true,
         "avg": true,
@@ -92,7 +91,7 @@
       "title": "${service_name} | ${grafana_dict['stat_name']}",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": ${loop.index},
         "value_type": "individual"
       },
       "type": "graph",
@@ -116,8 +115,13 @@
       "yaxis": {
         "align": false
       }
+
+  % if grafana_dict['is_last'] == 1:
     }
-    % endfor
+  % else:
+    },
+  % endif
+% endfor
   ],
   "refresh": "3s",
   "schemaVersion": 36,

namekoplus/command.py

@@ -9,6 +9,10 @@ import shortuuid
 from python_on_whales import DockerException, ClientNotFoundError, DockerClient, docker
 from mako.template import Template
 
+INIT_TYPE_CHOICES = ['all', 'rpc', 'event', 'http', 'timer', 'demo']
+MIDDLEWARE_CHOICES = ['rabbitmq', 'metrics']
+TEST_TYPE_CHOICES = ['unit']
+
 
 def check_docker():
     """
@@ -23,9 +27,10 @@ def check_docker():
         click.echo('Please start docker correctly', err=True)
         raise
 
-    if not docker.compose.is_installed():
-        click.echo('Please install docker-compose first', err=True)
-        raise
+    # TODO 目前在某些系统环境的检查不准确，暂时去除这个检查逻辑
+    # if not docker.compose.is_installed():
+    #     click.echo('Please install docker-compose first', err=True)
+    #     raise
 
 
 @contextmanager
@@ -104,7 +109,7 @@ def start_statsd_agent():
 
 def start_statsd_exporter():
     with status(f'Starting statsd exporter'):
-        statsd_mapping_file_path = os.path.join('.', 'statsd_mapping.yml')
+        statsd_mapping_file_path = os.getcwd() + '/statsd_mapping.yml'
         returned_string = docker.run(image='prom/statsd-exporter:latest', name='statsd-exporter', pull='missing',
                                      detach=True, restart='always', tty=True, hostname='statsd-exporter',
                                      publish=[(9125, 9125, 'udp'), (9102, 9102)], interactive=True,
@@ -131,7 +136,7 @@ def start_grafana():
         grafana_conf_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
         grafana_provisioning_path = os.path.join(grafana_conf_dir, 'grafana_conf/provisioning')
         grafana_config_path = os.path.join(grafana_conf_dir, 'grafana_conf/config/grafana.ini')
-        grafana_dashboard_path = os.path.join(grafana_conf_dir, 'grafana_conf/dashboards')
+        grafana_dashboard_path = os.path.join(os.getcwd(), 'grafana_dashboards')
         returned_string = docker.run(image='grafana/grafana:latest', name='grafana', hostname='grafana',
                                      detach=True, restart='always', tty=True, interactive=True,
                                      publish=[(3100, 3000)], pull='missing',
@@ -153,6 +158,7 @@ def stop_network(network_name):
 
 
 def start_metric_servers():
+    # TODO 检查相应容器是否已启动，如果启动，则先删除
     start_network('metric_servers')
     sleep(0.5)
     start_prometheus()
@@ -232,7 +238,7 @@ def cli():
 @click.option('-t', '--type', '_type',
               default='all',
               show_default=True,
-              type=click.Choice(['all', 'rpc', 'event', 'http', 'timer', 'demo'], case_sensitive=False),
+              type=click.Choice(INIT_TYPE_CHOICES, case_sensitive=False),
               help='The template type of nameko service')
 def init(directory, _type):
     """
@@ -257,7 +263,7 @@ def init(directory, _type):
 @cli.command()
 @click.option('-m', '--middleware',
               required=True,
-              type=click.Choice(['rabbitmq', 'metrics'], case_sensitive=False),
+              type=click.Choice(MIDDLEWARE_CHOICES, case_sensitive=False),
               help='The middleware name')
 def start(middleware):
     """
@@ -270,7 +276,7 @@ def start(middleware):
 @cli.command()
 @click.option('-m', '--middleware',
               required=True,
-              type=click.Choice(['rabbitmq', 'metrics'], case_sensitive=False),
+              type=click.Choice(MIDDLEWARE_CHOICES, case_sensitive=False),
               help='The middleware name')
 def stop(middleware):
     """
@@ -287,7 +293,7 @@ def stop(middleware):
 @click.option('-t', '--type', '_type',
               default='unit',
               show_default=True,
-              type=click.Choice(['unit'], case_sensitive=False),
+              type=click.Choice(TEST_TYPE_CHOICES, case_sensitive=False),
               help='The test type of the nameko service')
 def test_gen(directory, _type):
     """
@@ -316,12 +322,17 @@ def metric_config_gen(module, class_name_str):
     """
     Generate metric config for nameko services.
     """
+    import sys
     from statsd.client.timer import Timer
+    sys.path.append(os.getcwd())
+    for root, dirs, files in os.walk(os.getcwd()):
+        for _dir in dirs:
+            sys.path.append(os.path.join(root, _dir))
 
     # Extract information of statsd config from the class of nameko service
-    dest_dir = module.split('.')[0]
     file_name = module.split('.')[-1]
     _module = __import__(module)
+
     config_list = []
     for class_name in class_name_str.split(','):
         members = inspect.getmembers(getattr(getattr(_module, file_name), class_name), predicate=inspect.isfunction)
@@ -338,24 +349,35 @@ def metric_config_gen(module, class_name_str):
                 })
 
     # Generate one file of statsd config yaml for statsd exporter
-    metric_configs_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
-    template_file_path = os.path.join(metric_configs_dir, 'statsd_mapping.yml.mako')
-    output_file = os.path.join('.', 'statsd_mapping.yml')
-    template_to_file(template_file=template_file_path, dest=output_file, output_encoding='utf-8',
-                     **{'config_list': config_list})
+    with status(f'Creating statsd_mapping.yml'):
+        metric_configs_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
+        template_file_path = os.path.join(metric_configs_dir, 'statsd_mapping.yml.mako')
+        output_file = os.path.join('.', 'statsd_mapping.yml')
+        template_to_file(template_file=template_file_path, dest=output_file, output_encoding='utf-8',
+                         **{'config_list': config_list})
 
     # Generate files of json for grafana dashboard
-    for class_name in class_name_str.split(','):
-        grafana_list = []
-        for config in config_list:
-            if config['class_name'] == class_name:
-                grafana_list.append(config)
-        grafana_configs_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
-        grafana_file_path = os.path.join(grafana_configs_dir, 'grafana.json.mako')
-        output_file = os.path.join(dest_dir, f'{class_name}_Grafana.json')
-        template_to_file(template_file=grafana_file_path, dest=output_file, output_encoding='utf-8',
-                         **{'service_name': class_name, 'uid': shortuuid.uuid(),
-                            'grafana_list': grafana_list})
+    if not os.access('grafana_dashboards', os.F_OK):
+        with status(f'Creating directory {os.path.abspath("grafana_dashboards")!r}'):
+            os.makedirs('grafana_dashboards')
+
+    with status(f'Creating files of Grafana.json into the directory of grafana_dashboards'):
+        for class_name in class_name_str.split(','):
+            grafana_list = []
+            for config in config_list:
+                if config['class_name'] == class_name:
+                    grafana_list.append(config)
+            for idx, grafana_dict in enumerate(grafana_list):
+                if idx + 1 == len(grafana_list):
+                    grafana_dict['is_last'] = 1
+                else:
+                    grafana_dict['is_last'] = 0
+            grafana_configs_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
+            grafana_file_path = os.path.join(grafana_configs_dir, 'grafana.json.mako')
+            output_file = os.path.join('grafana_dashboards', f'{class_name}_Grafana.json')
+            template_to_file(template_file=grafana_file_path, dest=output_file, output_encoding='utf-8',
+                             **{'service_name': class_name, 'uid': shortuuid.uuid(),
+                                'grafana_list': grafana_list})
 
 
 if __name__ == '__main__':

namekoplus/templates/all/config.yml

@@ -2,7 +2,8 @@ AMQP_URI: pyamqp://${RABBIT_USER:admin}:${RABBIT_PASSWORD:admin}@${RABBIT_HOST:l
 WEB_SERVER_ADDRESS: '0.0.0.0:8000'
 RPC_EXCHANGE: 'nameko-rpc'
 
-max_workers: 10
+max_workers: 20
+PREFETCH_COUNT: 20
 parent_calls_tracked: 20
 
 LOGGING:

namekoplus/templates/demo/config.yml

@@ -1,5 +1,6 @@
 AMQP_URI: pyamqp://${RABBIT_USER:admin}:${RABBIT_PASSWORD:admin}@${RABBIT_HOST:localhost}:${RABBIT_PORT:5672}/
 RPC_EXCHANGE: 'nameko-rpc'
 
-max_workers: 10
+max_workers: 20
+PREFETCH_COUNT: 20
 parent_calls_tracked: 20

namekoplus/templates/event/config.yml

@@ -1,7 +1,8 @@
 AMQP_URI: pyamqp://${RABBIT_USER:admin}:${RABBIT_PASSWORD:admin}@${RABBIT_HOST:localhost}:${RABBIT_PORT:5672}/
 RPC_EXCHANGE: 'nameko-rpc'
 
-max_workers: 10
+max_workers: 20
+PREFETCH_COUNT: 20
 parent_calls_tracked: 20
 
 LOGGING:

namekoplus/templates/http/config.yml

@@ -2,7 +2,8 @@ AMQP_URI: pyamqp://${RABBIT_USER:admin}:${RABBIT_PASSWORD:admin}@${RABBIT_HOST:l
 WEB_SERVER_ADDRESS: '0.0.0.0:8000'
 RPC_EXCHANGE: 'nameko-rpc'
 
-max_workers: 10
+max_workers: 20
+PREFETCH_COUNT: 20
 parent_calls_tracked: 20
 
 LOGGING:

namekoplus/templates/rpc/config.yml

@@ -1,7 +1,8 @@
 AMQP_URI: pyamqp://${RABBIT_USER:admin}:${RABBIT_PASSWORD:admin}@${RABBIT_HOST:localhost}:${RABBIT_PORT:5672}/
 RPC_EXCHANGE: 'nameko-rpc'
 
-max_workers: 10
+max_workers: 20
+PREFETCH_COUNT: 20
 parent_calls_tracked: 20
 
 LOGGING:

namekoplus/templates/timer/config.yml

@@ -1,7 +1,8 @@
 AMQP_URI: pyamqp://${RABBIT_USER:admin}:${RABBIT_PASSWORD:admin}@${RABBIT_HOST:localhost}:${RABBIT_PORT:5672}/
 RPC_EXCHANGE: 'nameko-rpc'
 
-max_workers: 10
+max_workers: 20
+PREFETCH_COUNT: 20
 parent_calls_tracked: 20
 
 LOGGING:

setup.py

@@ -9,13 +9,12 @@ with open(path.join(here, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='namekoplus',
-    version='0.4.0',
+    version='0.4.27',
     description='A lightweight Python distributed microservice solution',
     long_description=long_description,
     long_description_content_type='text/markdown',
     url='',
     project_urls={
-        'Documentation': 'https://doc.bearcatlog.com/',
         'Source Code': 'https://github.com/Bryanthelol/namekoplus',
         'Bug Tracker': 'https://github.com/Bryanthelol/namekoplus/issues',
     },
@@ -50,7 +49,7 @@ setup(
     install_requires=[
         'nameko==3.0.0rc11',
         'click==8.1.5',
-        'python-on-whales==0.63.0',
+        'python-on-whales==0.65.0',
         'pytest==7.4.0',
         'mako==1.2.4',
         'shortuuid==1.0.11'
@@ -58,21 +57,15 @@ setup(
     extras_require={
         'ha': ['tenacity==8.2.2',
                'cachetools==5.3.0',
-               'circuitbreaker==2.0.0',
-               'statsd==4.0.1',
-               'logstash_formatter==0.5.17',
-               'nameko-sentry==1.0.0',
+               'circuitbreaker==2.0.0'],
+        'ob': ['nameko-sentry==1.0.0',
                'nameko-tracer==1.4.0'],
-        'apiflask': ['apiflask>=1.3.1',
-                     'gevent>=22.10.2',
-                     'gunicorn==20.1.0'],
-        'rocketry': ['rocketry==2.4.0'],
-        'gutter': ['gutter==0.5.0'],
-        'mysql': ['pymysql==1.0.3',
-                  'sqlalchemy==2.0.15',
-                  'sqlacodegen==2.3.0',
-                  'alembic==1.11.1'],
+        'log': ['loguru==0.7.2',
+                'logstash_formatter==0.5.17'],
+        'metric': ['statsd==4.0.1'],
+        'schema': ['marshmallow==3.20.1'],
         'security': ['cryptography'],
-        'dev': ['environs==9.5.0']
+        'dev': ['python-dotenv==1.0.0',
+                'environs==9.5.0']
     },
 )