diff --git a/README.md b/README.md index 75ad1a5..6f658d2 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,29 @@ chassis <ˈʃæsi> n. 底盘，底架 -寓意服务基座 \ No newline at end of file +仓库名寓意服务基座，基础库。 + + +微服务底座的目的是简化服务的创建过程，同时确保开发者拥有一套所有服务都要遵循的标准。 + +包含的特性 +1. 错误报告 +2. 日志收集 +3. Metrics 数据收集 +4. 配置获取 +5. 数据存储设置（ORM） +6. 健康检查 +7. 服务注册和发现 +8. 模板代码 + + +实现的功能 +- 服务发现 +- 可观测 +- 传输 +- 负载均衡 +- 限流降级（消息确认+熔断） + + +安装方法： +`python3 -m pip install -U https://gitea.bearcatlog.com/BryantStudio/chassis.git` diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chassis.py b/chassis.py new file mode 100644 index 0000000..5fd51d8 --- /dev/null +++ b/chassis.py @@ -0,0 +1,41 @@ +import logging + +from jaeger_client import Config +from logstash_formatter import LogstashFormatterV1 +from statsd import StatsClient + + +def init_statsd(prefix=None, host=None, port=8125): + statsd = StatsClient(host, port, prefix=prefix) + return statsd + + +def init_logger(): + logger = logging.getLogger() + handler = logging.StreamHandler() + formatter = LogstashFormatterV1() + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + +def init_tracer(service): + config = Config( + config={ + 'sampler': { + 'type': 'const', + 'param': 1, + }, + 'local_agent': { + 'reporting_host': "jaeger", + 'reporting_port': 5775, + }, + 'logging': True, + 'reporter_batch_size': 1, + }, + + service_name=service, + ) + + # this call also sets opentracing.tracer + return config.initialize_tracer() diff --git a/chassis_agent/.dockerignore b/chassis_agent/.dockerignore new file mode 100644 index 0000000..36880ba --- /dev/null +++ b/chassis_agent/.dockerignore @@ -0,0 +1,7 @@ +.venv +.git +.gitignore +.dockerignore +/log/* +/tmp/* +/test/reports/* \ No newline at end of file diff --git a/chassis_agent/Admin Service.json b/chassis_agent/Admin Service.json new file mode 100644 index 0000000..3a99fcb --- /dev/null +++ b/chassis_agent/Admin Service.json @@ -0,0 +1,261 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.6.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "refresh": false, + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${Prometheus}", + "fill": 1, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "admin_service_hello", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "admin_service_hello {{quantile}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Admin Service | Hello", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${Prometheus}", + "fill": 1, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "admin_service_error", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "admin_service_error {{quantile}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Admin Service | Error", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Admin Service", + "version": 5 +} \ No newline at end of file diff --git a/chassis_agent/docker-compose.yml b/chassis_agent/docker-compose.yml new file mode 100644 index 0000000..21f752e --- /dev/null +++ b/chassis_agent/docker-compose.yml @@ -0,0 +1,32 @@ +version: "3" + +services: + statsd-agent: + image: statsd/statsd:latest + restart: always + ports: + - "8125:8125/udp" + - "8126:8126" + volumes: + - "./statsd_config.js:/usr/src/app/config.js" + networks: + chassis_agent: + statsd-exporter: + image: prom/statsd-exporter:latest + hostname: "statsd-exporter" + command: "--statsd.mapping-config=/tmp/statsd_mapping.yml" + ports: + - "9102:9102" + - "9125:9125/udp" + volumes: + - "./statsd_mapping.yml:/tmp/statsd_mapping.yml" + networks: + chassis_agent: + +networks: + chassis_agent: + driver: bridge + +#volumes: +# chassis_agent_data: +# driver: local diff --git a/chassis_agent/statsd_config.js b/chassis_agent/statsd_config.js new file mode 100644 index 0000000..7a974e0 --- /dev/null +++ b/chassis_agent/statsd_config.js @@ -0,0 +1,8 @@ +(function () { + return { + "port": 8125, + "backends": ["./backends/repeater"], + "repeater": [{host: 'statsd-exporter', port: 9125}], + }; +})(); + diff --git a/chassis_agent/statsd_mapping.yml b/chassis_agent/statsd_mapping.yml new file mode 100644 index 0000000..9b4947d --- /dev/null +++ b/chassis_agent/statsd_mapping.yml @@ -0,0 +1,41 @@ +mappings: +- match: "admin-service.admin-service.hello" + observer_type: summary + name: "hello" + labels: + provider: "$2" + outcome: "$3" + job: "admin-service" + summary_options: + quantiles: + - quantile: 0.99 + error: 0.001 + - quantile: 0.95 + error: 0.01 + - quantile: 0.9 + error: 0.05 + - quantile: 0.5 + error: 0.005 + max_summary_age: 30s + summary_age_buckets: 3 + stream_buffer_size: 1000 +- match: "admin-service.admin-service.error" + observer_type: summary + name: "error" + labels: + provider: "$2" + outcome: "$3" + job: "admin-service" + summary_options: + quantiles: + - quantile: 0.99 + error: 0.001 + - quantile: 0.95 + error: 0.01 + - quantile: 0.9 + error: 0.05 + - quantile: 0.5 + error: 0.005 + max_summary_age: 30s + summary_age_buckets: 3 + stream_buffer_size: 1000 \ No newline at end of file diff --git a/nameko_demo/config.yml b/nameko_demo/config.yml new file mode 100644 index 0000000..acf0bf2 --- /dev/null +++ b/nameko_demo/config.yml @@ -0,0 +1,24 @@ +AMQP_URI: pyamqp://${RABBIT_USER:guest}:${RABBIT_PASSWORD:guest}@${RABBIT_HOST:localhost}:${RABBIT_PORT:5672}/ +RPC_EXCHANGE: 'nameko-rpc' + +max_workers: 10 +parent_calls_tracked: 20 + +LOGGING: + version: 1 + formatters: + tracer: + (): nameko_tracer.formatters.PrettyJSONFormatter + handlers: + tracer: + class: logging.StreamHandler + formatter: tracer + loggers: + nameko_tracer: + level: INFO + handlers: [tracer] + +SENTRY: + DSN: "xxxx" + CLIENT_CONFIG: + site: "my site name" \ No newline at end of file diff --git a/nameko_demo/demo.py b/nameko_demo/demo.py new file mode 100644 index 0000000..7510951 --- /dev/null +++ b/nameko_demo/demo.py @@ -0,0 +1,44 @@ +import json +import datetime +import requests +from nameko.web.handlers import http +from nameko.timer import timer +from statsd import StatsClient +from circuitbreaker import circuit + + +class DemoChassisService: + name = "demo_chassis_service" + + statsd = StatsClient('localhost', 8125, prefix='simplebank-nameko_demo') + + @http('GET', '/health') + @statsd.timer('health') + def health(self, _request): + return json.dumps({'ok': datetime.datetime.utcnow().__str__()}) + + @http('GET', '/external') + @circuit(failure_threshold=5, expected_exception=ConnectionError) + @statsd.timer('external') + def external_request(self, _request): + response = requests.get('https://jsonplaceholder.typicode.com/posts/1') + return json.dumps({'code': response.status_code, 'body': response.text}) + + @http('GET', '/error') + @circuit(failure_threshold=5, expected_exception=ZeroDivisionError) + @statsd.timer('http_error') + def error_http_request(self): + return json.dumps({1 / 0}) + + +class HealthCheckService: + name = "health_check_service" + + statsd = StatsClient('localhost', 8125, prefix='simplebank-nameko_demo') + + @timer(interval=10) + @statsd.timer('check_demo_service') + def check_demo_service(self): + response = requests.get('http://0.0.0.0:8000/health') + print("DemoChassisService HEALTH CHECK: status_code {}, response: {}".format( + response.status_code, response.text)) diff --git a/nameko_demo/events_demo.py b/nameko_demo/events_demo.py new file mode 100644 index 0000000..be5195e --- /dev/null +++ b/nameko_demo/events_demo.py @@ -0,0 +1,41 @@ +from nameko.events import BROADCAST, EventDispatcher, event_handler +from nameko.rpc import rpc +from nameko.timer import timer + + +class EventPublisherService: + name = "publisher_service" + + dispatch = EventDispatcher() + + @rpc + def publish(self, event_type, payload): + self.dispatch(event_type, payload) + + +class AnEventListenerService: + name = "an_event_listener_service" + + @event_handler("publisher_service", "an_event") + def consume_an_event(self, payload): + print("service {} received:".format(self.name), payload) + + +class AnotherEventListenerService: + name = "another_event_listener_service" + + @event_handler("publisher_service", "another_event") + def consume_another_event(self, payload): + print("service {} received:".format(self.name), payload) + + +class ListenBothEventsService: + name = "listen_both_events_service" + + @event_handler("publisher_service", "an_event") + def consume_an_event(self, payload): + print("service {} received:".format(self.name), payload) + + @event_handler("publisher_service", "another_event") + def consume_another_event(self, payload): + print("service {} received:".format(self.name), payload) diff --git a/nameko_demo/http_demo.py b/nameko_demo/http_demo.py new file mode 100644 index 0000000..25e4b3a --- /dev/null +++ b/nameko_demo/http_demo.py @@ -0,0 +1,25 @@ +import json +from nameko.web.handlers import http +from werkzeug.wrappers import Response +from nameko_sentry import SentryReporter + + +class HttpDemoService: + name = "http_demo_service" + sentry = SentryReporter() + + @http("GET", "/broken") + def broken(self, request): + raise ConnectionRefusedError() + + @http('GET', '/books/') + def demo_get(self, request, uuid): + data = {'id': uuid, 'title': 'The unbearable lightness of being', + 'author': 'Milan Kundera'} + return Response(json.dumps({'book': data}), + mimetype='application/json') + + @http('POST', '/books') + def demo_post(self, request): + return Response(json.dumps({'book': request.data.decode()}), + mimetype='application/json') diff --git a/nameko_demo/requirements.txt b/nameko_demo/requirements.txt new file mode 100644 index 0000000..862eaf1 --- /dev/null +++ b/nameko_demo/requirements.txt @@ -0,0 +1,9 @@ +nameko==2.14.1 +nameko-tracer==1.4.0 +logstash_formatter==0.5.17 +circuitbreaker==2.0.0 +statsd==4.0.1 +gutter==0.5.0 +request-id==1.0.1 +nameko-sentry==1.0.0 +pyopenssl==23.1.1 diff --git a/nameko_demo/rpc_demo.py b/nameko_demo/rpc_demo.py new file mode 100644 index 0000000..958e0ac --- /dev/null +++ b/nameko_demo/rpc_demo.py @@ -0,0 +1,20 @@ +from nameko.rpc import rpc, RpcProxy + + +class RpcResponderDemoService: + name = "rpc_responder_demo_service" + + @rpc + def hello(self, name): + return "Hello, {}!".format(name) + + +class RpcCallerDemoService: + name = "rpc_caller_demo_service" + + remote = RpcProxy("rpc_responder_demo_service") + + @rpc + def remote_hello(self, value="John Doe"): + res = u"{}".format(value) + return self.remote.hello(res) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..eaf343a --- /dev/null +++ b/setup.py @@ -0,0 +1,54 @@ +from setuptools import setup, find_packages +from codecs import open +from os import path + +here = path.abspath(path.dirname(__file__)) + +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name='chassis', + version='0.0.1', + description='basic sdk', + long_description=long_description, + url='https://gitea.bearcatlog.com/BryantStudio/chassis', + + author='BryantStudio Engineering', + author_email='bryantsisu@qq.book', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3', + 'Topic :: Software Development :: Libraries :: Python Modules', + + "Programming Language :: Python", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Programming Language :: Python :: 3", + "Topic :: Internet", + "Topic :: Software Development :: Libraries :: Python Modules", + "Intended Audience :: Developers", + ], + + keywords='microservices basic sdk', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + install_requires=[ + 'nameko==2.14.1', + 'nameko-tracer==1.3.0', + 'logstash_formatter==0.5.17', + 'circuitbreaker==2.0.0', + 'statsd==4.0.1', + 'gutter==0.5.0', + 'request-id==1.0.1', + 'nameko-sentry==1.0.0', + 'pyopenssl==23.1.1', + ], +)