Bryant
/
namekoplus
mirror of https://github.com/Bryanthelol/namekoplus
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add metrics for nameko services

This commit is contained in:
BryantHe 2023-07-27 14:02:01 +08:00
parent 58a99d9d2c
commit 857d26d092
10 changed files with 1275 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
MANIFEST.in
View File

@ -1,5 +1,5 @@
include *.py *.md *.mako include *.py *.md *.mako *.js *.yml *.yaml *.ini
recursive-include namekoplus/chassis *.py *.mako recursive-include namekoplus/chassis *.py *.mako *.js *.yaml *.ini
recursive-include namekoplus/templates *.py *.yml *.mako recursive-include namekoplus/templates *.py *.yml *.yaml *.mako *.js *.ini
recursive-include namekoplus/chassis-agent *.py *.yml *.mako recursive-include namekoplus/chassis-agent *.py *.yml *.yaml *.mako *.js *.ini
recursive-include namekoplus/tests *.py *.mako recursive-include namekoplus/tests *.py *.mako *.js *.yaml *.ini

164
namekoplus/chassis-agent/metric-configs/grafana.json.mako Normal file
View File

@ -0,0 +1,164 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 20,
"links": [],
"liveNow": false,
"panels": [
% for grafana_dict in grafana_list:
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "statsd",
"uid": "5qrwjFCnk"
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 12,
"w": 12,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 5,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": false,
"hideZero": false,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.5.5",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "${grafana_dict['stat_name']}",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
"legendFormat": "${grafana_dict['stat_name']} {{quantile}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "${service_name} | ${grafana_dict['stat_name']}",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
% endfor
],
"refresh": "3s",
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-12h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"3s",
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "${service_name}",
"uid": "${uid}",
"version": 7,
"weekStart": ""
}

756
namekoplus/chassis-agent/metric-configs/grafana_conf/config/grafana.ini Normal file
View File

@ -0,0 +1,756 @@
##################### Grafana Configuration Example #####################
#
# Everything has defaults so you only need to uncomment things you want to
# change
# possible values : production, development
;app_mode = production
# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
;instance_name = ${HOSTNAME}
#################################### Paths ####################################
[paths]
# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
;data = /var/lib/grafana
# Temporary files in `data` directory older than given duration will be removed
;temp_data_lifetime = 24h
# Directory where grafana can store logs
;logs = /var/log/grafana
# Directory where grafana will automatically scan and look for plugins
;plugins = /var/lib/grafana/plugins
# folder that contains provisioning config files that grafana will apply on startup and while running.
;provisioning = conf/provisioning
#################################### Server ####################################
[server]
# Protocol (http, https, h2, socket)
;protocol = http
# The ip address to bind to, empty will bind to all interfaces
;http_addr =
# The http port to use
;http_port = 3000
# The public facing domain name used to access grafana from a browser
;domain = localhost
# Redirect to correct domain if host header does not match domain
# Prevents DNS rebinding attacks
;enforce_domain = false
# The full public facing url you use in browser, used for redirects and emails
# If you use reverse proxy and sub path specify full url (with sub path)
;root_url = %(protocol)s://%(domain)s:%(http_port)s/
# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
;serve_from_sub_path = false
# Log web requests
;router_logging = false
# the path relative working path
;static_root_path = public
# enable gzip
;enable_gzip = false
# https certs & key file
;cert_file =
;cert_key =
# Unix socket path
;socket =
#################################### Database ####################################
[database]
# You can configure the database connection by specifying type, host, name, user and password
# as separate properties or as on string using the url properties.
# Either "mysql", "postgres" or "sqlite3", it's your choice
;type = sqlite3
;host = 127.0.0.1:3306
;name = grafana
;user = root
# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
;password =
# Use either URL or the previous fields to configure the database
# Example: mysql://user:secret@host:port/database
;url =
# For "postgres" only, either "disable", "require" or "verify-full"
;ssl_mode = disable
;ca_cert_path =
;client_key_path =
;client_cert_path =
;server_cert_name =
# For "sqlite3" only, path relative to data_path setting
;path = grafana.db
# Max idle conn setting default is 2
;max_idle_conn = 2
# Max conn setting default is 0 (mean not set)
;max_open_conn =
# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
;conn_max_lifetime = 14400
# Set to true to log the sql calls and execution times.
;log_queries =
# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared)
;cache_mode = private
#################################### Cache server #############################
[remote_cache]
# Either "redis", "memcached" or "database" default is "database"
;type = database
# cache connectionstring options
# database: will use Grafana primary database.
# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'.
# memcache: 127.0.0.1:11211
;connstr =
#################################### Data proxy ###########################
[dataproxy]
# This enables data proxy logging, default is false
;logging = false
# How long the data proxy should wait before timing out default is 30 (seconds)
;timeout = 30
# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
;send_user_header = false
#################################### Analytics ####################################
[analytics]
# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
# No ip addresses are being tracked, only simple counters to track
# running instances, dashboard and error counts. It is very helpful to us.
# Change this option to false to disable reporting.
;reporting_enabled = true
# Set to false to disable all checks to https://grafana.net
# for new vesions (grafana itself and plugins), check is used
# in some UI views to notify that grafana or plugin update exists
# This option does not cause any auto updates, nor send any information
# only a GET request to http://grafana.com to get latest versions
;check_for_updates = true
# Google Analytics universal tracking code, only enabled if you specify an id here
;google_analytics_ua_id =
# Google Tag Manager ID, only enabled if you specify an id here
;google_tag_manager_id =
#################################### Security ####################################
[security]
# disable creation of admin user on first start of grafana
;disable_initial_admin_creation = false
# default admin user, created on startup
;admin_user = admin
# default admin password, can be changed before first start of grafana, or in profile settings
;admin_password = admin
# used for signing
;secret_key = SW2YcwTIb9zpOOhoPsMm
# disable gravatar profile images
;disable_gravatar = false
# data source proxy whitelist (ip_or_domain:port separated by spaces)
;data_source_proxy_whitelist =
# disable protection against brute force login attempts
;disable_brute_force_login_protection = false
# set to true if you host Grafana behind HTTPS. default is false.
;cookie_secure = false
# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled"
;cookie_samesite = none
# set to true if you want to allow browsers to render Grafana in a <frame>, <iframe>, <embed> or <object>. default is false.
allow_embedding = true
# Set to true if you want to enable http strict transport security (HSTS) response header.
# This is only sent when HTTPS is enabled in this configuration.
# HSTS tells browsers that the site should only be accessed using HTTPS.
# The default version will change to true in the next minor release, 6.3.
;strict_transport_security = false
# Sets how long a browser should cache HSTS. Only applied if strict_transport_security is enabled.
;strict_transport_security_max_age_seconds = 86400
# Set to true if to enable HSTS preloading option. Only applied if strict_transport_security is enabled.
;strict_transport_security_preload = false
# Set to true if to enable the HSTS includeSubDomains option. Only applied if strict_transport_security is enabled.
;strict_transport_security_subdomains = false
# Set to true to enable the X-Content-Type-Options response header.
# The X-Content-Type-Options response HTTP header is a marker used by the server to indicate that the MIME types advertised
# in the Content-Type headers should not be changed and be followed. The default will change to true in the next minor release, 6.3.
;x_content_type_options = false
# Set to true to enable the X-XSS-Protection header, which tells browsers to stop pages from loading
# when they detect reflected cross-site scripting (XSS) attacks. The default will change to true in the next minor release, 6.3.
;x_xss_protection = false
#################################### Snapshots ###########################
[snapshots]
# snapshot sharing options
;external_enabled = true
;external_snapshot_url = https://snapshots-origin.raintank.io
;external_snapshot_name = Publish to snapshot.raintank.io
# Set to true to enable this Grafana instance act as an external snapshot server and allow unauthenticated requests for
# creating and deleting snapshots.
;public_mode = false
# remove expired snapshot
;snapshot_remove_expired = true
#################################### Dashboards History ##################
[dashboards]
# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
;versions_to_keep = 20
# Minimum dashboard refresh interval. When set, this will restrict users to set the refresh interval of a dashboard lower than given interval. Per default this is 5 seconds.
# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
;min_refresh_interval = 5s
#################################### Users ###############################
[users]
# disable user signup / registration
;allow_sign_up = true
# Allow non admin users to create organizations
;allow_org_create = true
# Set to true to automatically assign new users to the default organization (id 1)
;auto_assign_org = true
# Set this value to automatically add new users to the provided organization (if auto_assign_org above is set to true)
;auto_assign_org_id = 1
# Default role new users will be automatically assigned (if disabled above is set to true)
;auto_assign_org_role = Viewer
# Require email validation before sign up completes
;verify_email_enabled = false
# Background text for the user field on the login page
;login_hint = email or username
;password_hint = password
# Default UI theme ("dark" or "light")
;default_theme = dark
# External user management, these options affect the organization users view
;external_manage_link_url =
;external_manage_link_name =
;external_manage_info =
# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
;viewers_can_edit = false
# Editors can administrate dashboard, folders and teams they create
;editors_can_admin = false
[auth]
# Login cookie name
;login_cookie_name = grafana_session
# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days,
;login_maximum_inactive_lifetime_days = 7
# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days.
;login_maximum_lifetime_days = 30
# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes.
;token_rotation_interval_minutes = 10
# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
;disable_login_form = false
# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
;disable_signout_menu = false
# URL to redirect the user to after sign out
;signout_redirect_url =
# Set to true to attempt login with OAuth automatically, skipping the login screen.
# This setting is ignored if multiple OAuth providers are configured.
;oauth_auto_login = false
# OAuth state max age cookie duration. Defaults to 60 seconds.
;oauth_state_cookie_max_age = 60
# limit of api_key seconds to live before expiration
;api_key_max_seconds_to_live = -1
#################################### Anonymous Auth ######################
[auth.anonymous]
# enable anonymous access
enabled = true
# specify organization name that should be used for unauthenticated users
;org_name = Main Org.
# specify role for unauthenticated users
;org_role = Viewer
#################################### Github Auth ##########################
[auth.github]
;enabled = false
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = user:email,read:org
;auth_url = https://github.com/login/oauth/authorize
;token_url = https://github.com/login/oauth/access_token
;api_url = https://api.github.com/user
;allowed_domains =
;team_ids =
;allowed_organizations =
#################################### GitLab Auth #########################
[auth.gitlab]
;enabled = false
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = api
;auth_url = https://gitlab.com/oauth/authorize
;token_url = https://gitlab.com/oauth/token
;api_url = https://gitlab.com/api/v4
;allowed_domains =
;allowed_groups =
#################################### Google Auth ##########################
[auth.google]
;enabled = false
;allow_sign_up = true
;client_id = some_client_id
;client_secret = some_client_secret
;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
;auth_url = https://accounts.google.com/o/oauth2/auth
;token_url = https://accounts.google.com/o/oauth2/token
;api_url = https://www.googleapis.com/oauth2/v1/userinfo
;allowed_domains =
;hosted_domain =
#################################### Grafana.com Auth ####################
[auth.grafana_com]
;enabled = false
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = user:email
;allowed_organizations =
#################################### Azure AD OAuth #######################
[auth.azuread]
;name = Azure AD
;enabled = false
;allow_sign_up = true
;client_id = some_client_id
;client_secret = some_client_secret
;scopes = openid email profile
;auth_url = https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize
;token_url = https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/token
;allowed_domains =
;allowed_groups =
#################################### Okta OAuth #######################
[auth.okta]
;name = Okta
;enabled = false
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = openid profile email groups
;auth_url = https://<tenant-id>.okta.com/oauth2/v1/authorize
;token_url = https://<tenant-id>.okta.com/oauth2/v1/token
;api_url = https://<tenant-id>.okta.com/oauth2/v1/userinfo
;allowed_domains =
;allowed_groups =
;role_attribute_path =
#################################### Generic OAuth ##########################
[auth.generic_oauth]
;enabled = false
;name = OAuth
;allow_sign_up = true
;client_id = some_id
;client_secret = some_secret
;scopes = user:email,read:org
;email_attribute_name = email:primary
;email_attribute_path =
;auth_url = https://foo.bar/login/oauth/authorize
;token_url = https://foo.bar/login/oauth/access_token
;api_url = https://foo.bar/user
;allowed_domains =
;team_ids =
;allowed_organizations =
;role_attribute_path =
;tls_skip_verify_insecure = false
;tls_client_cert =
;tls_client_key =
;tls_client_ca =
#################################### Basic Auth ##########################
[auth.basic]
;enabled = true
#################################### Auth Proxy ##########################
[auth.proxy]
;enabled = false
;header_name = X-WEBAUTH-USER
;header_property = username
;auto_sign_up = true
;sync_ttl = 60
;whitelist = 192.168.1.1, 192.168.2.1
;headers = Email:X-User-Email, Name:X-User-Name
# Read the auth proxy docs for details on what the setting below enables
;enable_login_token = false
#################################### Auth LDAP ##########################
[auth.ldap]
;enabled = false
;config_file = /etc/grafana/ldap.toml
;allow_sign_up = true
# LDAP backround sync (Enterprise only)
# At 1 am every day
;sync_cron = "0 0 1 * * *"
;active_sync_enabled = true
#################################### SMTP / Emailing ##########################
[smtp]
;enabled = false
;host = localhost:25
;user =
# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
;password =
;cert_file =
;key_file =
;skip_verify = false
;from_address = admin@grafana.localhost
;from_name = Grafana
# EHLO identity in SMTP dialog (defaults to instance_name)
;ehlo_identity = dashboard.example.com
[emails]
;welcome_email_on_sign_up = false
;templates_pattern = emails/*.html
#################################### Logging ##########################
[log]
# Either "console", "file", "syslog". Default is console and file
# Use space to separate multiple modes, e.g. "console file"
;mode = console file
# Either "debug", "info", "warn", "error", "critical", default is "info"
;level = info
# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
;filters =
# For "console" mode only
[log.console]
;level =
# log line format, valid options are text, console and json
;format = console
# For "file" mode only
[log.file]
;level =
# log line format, valid options are text, console and json
;format = text
# This enables automated log rotate(switch of following options), default is true
;log_rotate = true
# Max line number of single file, default is 1000000
;max_lines = 1000000
# Max size shift of single file, default is 28 means 1 << 28, 256MB
;max_size_shift = 28
# Segment log daily, default is true
;daily_rotate = true
# Expired days of log file(delete after max days), default is 7
;max_days = 7
[log.syslog]
;level =
# log line format, valid options are text, console and json
;format = text
# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
;network =
;address =
# Syslog facility. user, daemon and local0 through local7 are valid.
;facility =
# Syslog tag. By default, the process' argv[0] is used.
;tag =
#################################### Usage Quotas ########################
[quota]
; enabled = false
#### set quotas to -1 to make unlimited. ####
# limit number of users per Org.
; org_user = 10
# limit number of dashboards per Org.
; org_dashboard = 100
# limit number of data_sources per Org.
; org_data_source = 10
# limit number of api_keys per Org.
; org_api_key = 10
# limit number of orgs a user can create.
; user_org = 10
# Global limit of users.
; global_user = -1
# global limit of orgs.
; global_org = -1
# global limit of dashboards
; global_dashboard = -1
# global limit of api_keys
; global_api_key = -1
# global limit on number of logged in users.
; global_session = -1
#################################### Alerting ############################
[alerting]
# Disable alerting engine & UI features
;enabled = true
# Makes it possible to turn off alert rule execution but alerting UI is visible
;execute_alerts = true
# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
;error_or_timeout = alerting
# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
;nodata_or_nullvalues = no_data
# Alert notifications can include images, but rendering many images at the same time can overload the server
# This limit will protect the server from render overloading and make sure notifications are sent out quickly
;concurrent_render_limit = 5
# Default setting for alert calculation timeout. Default value is 30
;evaluation_timeout_seconds = 30
# Default setting for alert notification timeout. Default value is 30
;notification_timeout_seconds = 30
# Default setting for max attempts to sending alert notifications. Default value is 3
;max_attempts = 3
# Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
;min_interval_seconds = 1
#################################### Explore #############################
[explore]
# Enable the Explore section
;enabled = true
#################################### Internal Grafana Metrics ##########################
# Metrics available at HTTP API Url /metrics
[metrics]
# Disable / Enable internal metrics
;enabled = true
# Graphite Publish interval
;interval_seconds = 10
# Disable total stats (stat_totals_*) metrics to be generated
;disable_total_stats = false
#If both are set, basic auth will be required for the metrics endpoint.
; basic_auth_username =
; basic_auth_password =
# Send internal metrics to Graphite
[metrics.graphite]
# Enable by setting the address setting (ex localhost:2003)
;address =
;prefix = prod.grafana.%(instance_name)s.
#################################### Grafana.com integration ##########################
# Url used to import dashboards directly from Grafana.com
[grafana_com]
;url = https://grafana.com
#################################### Distributed tracing ############
[tracing.jaeger]
# Enable by setting the address sending traces to jaeger (ex localhost:6831)
;address = localhost:6831
# Tag that will always be included in when creating new spans. ex (tag1:value1,tag2:value2)
;always_included_tag = tag1:value1
# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote
;sampler_type = const
# jaeger samplerconfig param
# for "const" sampler, 0 or 1 for always false/true respectively
# for "probabilistic" sampler, a probability between 0 and 1
# for "rateLimiting" sampler, the number of spans per second
# for "remote" sampler, param is the same as for "probabilistic"
# and indicates the initial sampling rate before the actual one
# is received from the mothership
;sampler_param = 1
# Whether or not to use Zipkin propagation (x-b3- HTTP headers).
;zipkin_propagation = false
# Setting this to true disables shared RPC spans.
# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
;disable_shared_zipkin_spans = false
#################################### External image storage ##########################
[external_image_storage]
# Used for uploading images to public servers so they can be included in slack/email messages.
# you can choose between (s3, webdav, gcs, azure_blob, local)
;provider =
[external_image_storage.s3]
;endpoint =
;path_style_access =
;bucket =
;region =
;path =
;access_key =
;secret_key =
[external_image_storage.webdav]
;url =
;public_url =
;username =
;password =
[external_image_storage.gcs]
;key_file =
;bucket =
;path =
[external_image_storage.azure_blob]
;account_name =
;account_key =
;container_name =
[external_image_storage.local]
# does not require any configuration
[rendering]
# Options to configure a remote HTTP image rendering service, e.g. using https://github.com/grafana/grafana-image-renderer.
# URL to a remote HTTP image renderer service, e.g. http://localhost:8081/render, will enable Grafana to render panels and dashboards to PNG-images using HTTP requests to an external service.
;server_url =
# If the remote HTTP image renderer service runs on a different server than the Grafana server you may have to configure this to a URL where Grafana is reachable, e.g. http://grafana.domain/.
;callback_url =
# Concurrent render request limit affects when the /render HTTP endpoint is used. Rendering many images at the same time can overload the server,
# which this setting can help protect against by only allowing a certain amount of concurrent requests.
;concurrent_render_request_limit = 30
[panels]
# If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.
;disable_sanitize_html = false
[plugins]
;enable_alpha = false
;app_tls_skip_verify_insecure = false
# Enter a comma-separated list of plugin identifiers to identify plugins that are allowed to be loaded even if they lack a valid signature.
;allow_loading_unsigned_plugins =
#################################### Grafana Image Renderer Plugin ##########################
[plugin.grafana-image-renderer]
# Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert.
# See ICU’s metaZones.txt (https://cs.chromium.org/chromium/src/third_party/icu/source/data/misc/metaZones.txt) for a list of supported
# timezone IDs. Fallbacks to TZ environment variable if not set.
;rendering_timezone =
# Instruct headless browser instance to use a default language when not provided by Grafana, e.g. when rendering panel image of alert.
# Please refer to the HTTP header Accept-Language to understand how to format this value, e.g. 'fr-CH, fr;q=0.9, en;q=0.8, de;q=0.7, *;q=0.5'.
;rendering_language =
# Instruct headless browser instance to use a default device scale factor when not provided by Grafana, e.g. when rendering panel image of alert.
# Default is 1. Using a higher value will produce more detailed images (higher DPI), but will require more disk space to store an image.
;rendering_viewport_device_scale_factor =
# Instruct headless browser instance whether to ignore HTTPS errors during navigation. Per default HTTPS errors are not ignored. Due to
# the security risk it's not recommended to ignore HTTPS errors.
;rendering_ignore_https_errors =
# Instruct headless browser instance whether to capture and log verbose information when rendering an image. Default is false and will
# only capture and log error messages. When enabled, debug messages are captured and logged as well.
# For the verbose information to be included in the Grafana server log you have to adjust the rendering log level to debug, configure
# [log].filter = rendering:debug.
;rendering_verbose_logging =
# Instruct headless browser instance whether to output its debug and error messages into running process of remote rendering service.
# Default is false. This can be useful to enable (true) when troubleshooting.
;rendering_dumpio =
# Additional arguments to pass to the headless browser instance. Default is --no-sandbox. The list of Chromium flags can be found
# here (https://peter.sh/experiments/chromium-command-line-switches/). Multiple arguments is separated with comma-character.
;rendering_args =
# You can configure the plugin to use a different browser binary instead of the pre-packaged version of Chromium.
# Please note that this is not recommended, since you may encounter problems if the installed version of Chrome/Chromium is not
# compatible with the plugin.
;rendering_chrome_bin =
# Instruct how headless browser instances are created. Default is 'default' and will create a new browser instance on each request.
# Mode 'clustered' will make sure that only a maximum of browsers/incognito pages can execute concurrently.
# Mode 'reusable' will have one browser instance and will create a new incognito page on each request.
;rendering_mode =
# When rendering_mode = clustered you can instruct how many browsers or incognito pages can execute concurrently. Default is 'browser'
# and will cluster using browser instances.
# Mode 'context' will cluster using incognito pages.
;rendering_clustering_mode =
# When rendering_mode = clustered you can define maximum number of browser instances/incognito pages that can execute concurrently..
;rendering_clustering_max_concurrency =
# Limit the maxiumum viewport width, height and device scale factor that can be requested.
;rendering_viewport_max_width =
;rendering_viewport_max_height =
;rendering_viewport_max_device_scale_factor =
# Change the listening host and port of the gRPC server. Default host is 127.0.0.1 and default port is 0 and will automatically assign
# a port not in use.
;grpc_host =
;grpc_port =
[enterprise]
# Path to a valid Grafana Enterprise license.jwt file
;license_path =
[feature_toggles]
# enable features, separated by spaces
;enable =

27
namekoplus/chassis-agent/metric-configs/grafana_conf/provisioning/dashboards/all.yaml Normal file
View File

@ -0,0 +1,27 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: false
editable: false
options:
path: /var/lib/grafana/dashboards

25
namekoplus/chassis-agent/metric-configs/grafana_conf/provisioning/datasources/all.yaml Normal file
View File

@ -0,0 +1,25 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
datasources:
- access: 'proxy'
editable: true
is_default: true
name: 'statsd'
org_id: 1
type: 'prometheus'
url: 'http://prometheus:9090'
version: 1

42
namekoplus/chassis-agent/metric-configs/prometheus_conf/prometheus.yml Normal file
View File

@ -0,0 +1,42 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
global:
scrape_interval: 3s # By default, scrape targets every 15 seconds.
# Attach these labels to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
external_labels:
stack: "statsd"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: "prometheus"
# Override the global default and scrape targets from this job every 5 seconds.
scrape_interval: 3s
static_configs:
- targets: ["localhost:9090"]
- job_name: 'statsd_exporter'
scrape_interval: 3s
static_configs:
- targets: ['statsd-exporter:9102']
labels:
exporter: 'statsd'

8
namekoplus/chassis-agent/metric-configs/statsd_config.js Normal file
View File

@ -0,0 +1,8 @@
(function () {
return {
"port": 8125,
"backends": ["./backends/repeater"],
"repeater": [{host: 'statsd-exporter', port: 9125}],
};
})();

23
namekoplus/chassis-agent/metric-configs/statsd_mapping.yml.mako Normal file
View File

@ -0,0 +1,23 @@
mappings:
% for config_dict in config_list:
- match: "${config_dict['statsd_prefix']}.${config_dict['stat_name']}"
observer_type: summary
name: "${config_dict['stat_name']}"
labels:
provider: "$2"
outcome: "$3"
job: "${config_dict['statsd_prefix']}"
summary_options:
quantiles:
- quantile: 0.99
error: 0.001
- quantile: 0.95
error: 0.01
- quantile: 0.9
error: 0.05
- quantile: 0.5
error: 0.005
max_summary_age: 30s
summary_age_buckets: 3
stream_buffer_size: 1000
% endfor

250
namekoplus/command.py
View File

@ -1,9 +1,13 @@
import inspect
import os import os
import shutil import shutil
from contextlib import contextmanager from contextlib import contextmanager
from time import sleep
import click import click
from python_on_whales import DockerException, ClientNotFoundError, DockerClient, docker as docker_testing import shortuuid
from python_on_whales import DockerException, ClientNotFoundError, DockerClient, docker
from mako.template import Template
def check_docker(): def check_docker():
@ -11,7 +15,7 @@ def check_docker():
Check if docker and docker compose are installed and running. Check if docker and docker compose are installed and running.
""" """
try: try:
docker_testing.ps() docker.ps()
except ClientNotFoundError: except ClientNotFoundError:
click.echo('Please install docker first', err=True) click.echo('Please install docker first', err=True)
raise raise
@ -19,7 +23,7 @@ def check_docker():
click.echo('Please start docker correctly', err=True) click.echo('Please start docker correctly', err=True)
raise raise
if not docker_testing.compose.is_installed(): if not docker.compose.is_installed():
click.echo('Please install docker-compose first', err=True) click.echo('Please install docker-compose first', err=True)
raise raise
@ -63,6 +67,159 @@ def copy_files(src_dir, dest_dir):
shutil.copy(src_file_path, output_file) shutil.copy(src_file_path, output_file)
def template_to_file(
template_file: str, dest: str, output_encoding: str, **kw
) -> None:
template = Template(filename=template_file)
try:
output = template.render_unicode(**kw).encode(output_encoding)
except Exception as e:
click.echo('Template rendering failed.', err=True)
raise
else:
with open(dest, "wb") as f:
f.write(output)
def start_rabbitmq():
docker_compose_file_dir = os.path.join(get_directory('chassis-agent'), 'rabbitmq')
for file_ in os.listdir(docker_compose_file_dir):
compose_file_path = os.path.join(docker_compose_file_dir, file_)
with status(f'Starting rabbitmq'):
temp_docker = DockerClient(compose_files=[compose_file_path])
temp_docker.compose.up(detach=True)
def start_statsd_agent():
with status(f'Starting statsd agent'):
metric_configs_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
statsd_config_file_path = os.path.join(metric_configs_dir, 'statsd_config.js')
returned_string = docker.run(image='statsd/statsd:latest', name='statsd-agent', hostname='statsd-agent',
detach=True, restart='always', interactive=True, tty=True,
publish=[(8125, 8125, 'udp'), (8126, 8126)], pull='missing',
volumes=[(statsd_config_file_path, '/usr/src/app/config.js', 'rw')],
networks=['metric_servers'])
click.echo('\nContainer ID: ' + str(returned_string) + '\n')
def start_statsd_exporter():
with status(f'Starting statsd exporter'):
statsd_mapping_file_path = os.path.join('.', 'statsd_mapping.yml')
returned_string = docker.run(image='prom/statsd-exporter:latest', name='statsd-exporter', pull='missing',
detach=True, restart='always', tty=True, hostname='statsd-exporter',
publish=[(9125, 9125, 'udp'), (9102, 9102)], interactive=True,
command=['--statsd.mapping-config=/tmp/statsd_mapping.yml'],
volumes=[(statsd_mapping_file_path, '/tmp/statsd_mapping.yml', 'rw')],
networks=['metric_servers'])
click.echo('\nContainer ID: ' + str(returned_string) + '\n')
def start_prometheus():
with status(f'Starting prometheus'):
prometheus_conf_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
prometheus_conf_file_path = os.path.join(prometheus_conf_dir, 'prometheus_conf/prometheus.yml')
returned_string = docker.run(image='prom/prometheus:latest', name='prometheus', hostname='prometheus',
detach=True, restart='always', tty=True, interactive=True,
publish=[(9193, 9090)], pull='missing',
volumes=[(prometheus_conf_file_path, '/etc/prometheus/prometheus.yml', 'rw')],
networks=['metric_servers'])
click.echo('\nContainer ID: ' + str(returned_string) + '\n')
def start_grafana():
with status(f'Starting grafana'):
grafana_conf_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
grafana_provisioning_path = os.path.join(grafana_conf_dir, 'grafana_conf/provisioning')
grafana_config_path = os.path.join(grafana_conf_dir, 'grafana_conf/config/grafana.ini')
grafana_dashboard_path = os.path.join(grafana_conf_dir, 'grafana_conf/dashboards')
returned_string = docker.run(image='grafana/grafana:latest', name='grafana', hostname='grafana',
detach=True, restart='always', tty=True, interactive=True,
publish=[(3100, 3000)], pull='missing',
volumes=[(grafana_provisioning_path, '/etc/grafana/provisioning', 'rw'),
(grafana_config_path, '/etc/grafana/grafana.ini', 'rw'),
(grafana_dashboard_path, '/var/lib/grafana/dashboards', 'rw')],
networks=['metric_servers'])
click.echo('\nContainer ID: ' + str(returned_string) + '\n')
def start_network(network_name):
with status(f'Starting network {network_name}'):
docker.network.create(network_name, driver='bridge')
def stop_network(network_name):
with status(f'Stopping network {network_name}'):
docker.network.remove(network_name)
def start_metric_servers():
start_network('metric_servers')
sleep(0.5)
start_prometheus()
sleep(0.5)
start_statsd_exporter()
sleep(0.5)
start_statsd_agent()
sleep(0.5)
start_grafana()
def stop_rabbitmq():
docker_compose_file_dir = os.path.join(get_directory('chassis-agent'), 'rabbitmq')
for file_ in os.listdir(docker_compose_file_dir):
compose_file_path = os.path.join(docker_compose_file_dir, file_)
with status(f'Stopping rabbitmq'):
temp_docker = DockerClient(compose_files=[compose_file_path])
temp_docker.compose.down()
def stop_statsd_agent():
with status(f'Stopping statsd agent'):
docker.remove('statsd-agent', force=True)
click.echo('\nContainer is removed.' + '\n')
def stop_statsd_exporter():
with status(f'Stopping statsd exporter'):
docker.remove('statsd-exporter', force=True)
click.echo('\nContainer is removed.' + '\n')
def stop_prometheus():
with status(f'Stopping prometheus'):
docker.remove('prometheus', force=True)
click.echo('\nContainer is removed.' + '\n')
def stop_grafana():
with status(f'Stopping grafana'):
docker.remove('grafana', force=True)
click.echo('\nContainer is removed.' + '\n')
def stop_metric_servers():
stop_statsd_agent()
sleep(0.5)
stop_statsd_exporter()
sleep(0.5)
stop_prometheus()
sleep(0.5)
stop_grafana()
sleep(0.5)
stop_network('metric_servers')
middleware_starting_dict = {
'rabbitmq': start_rabbitmq,
'metrics': start_metric_servers,
}
middleware_stopping_dict = {
'rabbitmq': stop_rabbitmq,
'metrics': stop_metric_servers,
}
@click.group() @click.group()
def cli(): def cli():
pass pass
@ -100,49 +257,27 @@ def init(directory, _type):
@cli.command() @cli.command()
@click.option('-m', '--middleware', @click.option('-m', '--middleware',
required=True, required=True,
type=click.Choice(['rabbitmq'], case_sensitive=False), type=click.Choice(['rabbitmq', 'metrics'], case_sensitive=False),
help='The middleware name') help='The middleware name')
@click.option('-u', '--user', def start(middleware):
required=False,
help='The user name of the middleware')
@click.option('-p', '--password',
required=False,
help='The password of the middleware')
def start(middleware, user, password):
""" """
Start a middleware that the nameko service depends on. Start a middleware that the nameko service depends on.
""" """
check_docker() check_docker()
middleware_starting_dict.get(middleware)()
if user and password:
os.environ['RABBITMQ_DEFAULT_USER'] = user
os.environ['RABBITMQ_DEFAULT_PASS'] = password
docker_compose_file_dir = os.path.join(get_directory('chassis-agent'), middleware)
for file_ in os.listdir(docker_compose_file_dir):
compose_file_path = os.path.join(docker_compose_file_dir, file_)
with status(f'Starting {middleware}'):
docker = DockerClient(compose_files=[compose_file_path])
docker.compose.up(detach=True)
@cli.command() @cli.command()
@click.option('-m', '--middleware', @click.option('-m', '--middleware',
required=True, required=True,
type=click.Choice(['rabbitmq'], case_sensitive=False), type=click.Choice(['rabbitmq', 'metrics'], case_sensitive=False),
help='The middleware name') help='The middleware name')
def stop(middleware): def stop(middleware):
""" """
Stop a middleware that the nameko service depends on. Stop a middleware that the nameko service depends on.
""" """
check_docker() check_docker()
middleware_stopping_dict.get(middleware)()
docker_compose_file_dir = os.path.join(get_directory('chassis-agent'), middleware)
for file_ in os.listdir(docker_compose_file_dir):
compose_file_path = os.path.join(docker_compose_file_dir, file_)
with status(f'Stopping {middleware}'):
docker = DockerClient(compose_files=[compose_file_path])
docker.compose.down()
@cli.command() @cli.command()
@ -170,5 +305,58 @@ def test_gen(directory, _type):
copy_files(tests_dir, directory) copy_files(tests_dir, directory)
@cli.command()
@click.option('-m', '--module',
required=True,
help='The module name where the nameko service exists')
@click.option('-c', '--class', 'class_name_str',
required=True,
help='The class name of the nameko service')
def metric_config_gen(module, class_name_str):
"""
Generate metric config for nameko services.
"""
from statsd.client.timer import Timer
# Extract information of statsd config from the class of nameko service
dest_dir = module.split('.')[0]
file_name = module.split('.')[-1]
_module = __import__(module)
config_list = []
for class_name in class_name_str.split(','):
members = inspect.getmembers(getattr(getattr(_module, file_name), class_name), predicate=inspect.isfunction)
for member_tuple in members:
name, _obj = member_tuple
unwrap = inspect.getclosurevars(_obj)
if unwrap.nonlocals.get('self') and isinstance(unwrap.nonlocals['self'], Timer):
statsd_prefix = unwrap.nonlocals['self'].client._prefix
stat_name = unwrap.nonlocals['self'].stat
config_list.append({
'statsd_prefix': statsd_prefix,
'stat_name': stat_name,
'class_name': class_name
})
# Generate one file of statsd config yaml for statsd exporter
metric_configs_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
template_file_path = os.path.join(metric_configs_dir, 'statsd_mapping.yml.mako')
output_file = os.path.join('.', 'statsd_mapping.yml')
template_to_file(template_file=template_file_path, dest=output_file, output_encoding='utf-8',
**{'config_list': config_list})
# Generate files of json for grafana dashboard
for class_name in class_name_str.split(','):
grafana_list = []
for config in config_list:
if config['class_name'] == class_name:
grafana_list.append(config)
grafana_configs_dir = os.path.join(get_directory('chassis-agent'), 'metric-configs')
grafana_file_path = os.path.join(grafana_configs_dir, 'grafana.json.mako')
output_file = os.path.join(dest_dir, f'{class_name}_Grafana.json')
template_to_file(template_file=grafana_file_path, dest=output_file, output_encoding='utf-8',
**{'service_name': class_name, 'uid': shortuuid.uuid(),
'grafana_list': grafana_list})
if __name__ == '__main__': if __name__ == '__main__':
cli() cli()

12
setup.py
View File

@ -9,7 +9,7 @@ with open(path.join(here, 'README.md'), encoding='utf-8') as f:
setup( setup(
name='namekoplus', name='namekoplus',
version='0.3.2', version='0.4.0',
description='A lightweight Python distributed microservice solution', description='A lightweight Python distributed microservice solution',
long_description=long_description, long_description=long_description,
long_description_content_type='text/markdown', long_description_content_type='text/markdown',
@ -51,7 +51,9 @@ setup(
'nameko==3.0.0rc11', 'nameko==3.0.0rc11',
'click==8.1.5', 'click==8.1.5',
'python-on-whales==0.63.0', 'python-on-whales==0.63.0',
'pytest==7.4.0' 'pytest==7.4.0',
'mako==1.2.4',
'shortuuid==1.0.11'
], ],
extras_require={ extras_require={
'ha': ['tenacity==8.2.2', 'ha': ['tenacity==8.2.2',
@ -60,8 +62,7 @@ setup(
'statsd==4.0.1', 'statsd==4.0.1',
'logstash_formatter==0.5.17', 'logstash_formatter==0.5.17',
'nameko-sentry==1.0.0', 'nameko-sentry==1.0.0',
'nameko-tracer==1.4.0', 'nameko-tracer==1.4.0'],
'shortuuid==1.0.11'],
'apiflask': ['apiflask>=1.3.1', 'apiflask': ['apiflask>=1.3.1',
'gevent>=22.10.2', 'gevent>=22.10.2',
'gunicorn==20.1.0'], 'gunicorn==20.1.0'],
@ -72,7 +73,6 @@ setup(
'sqlacodegen==2.3.0', 'sqlacodegen==2.3.0',
'alembic==1.11.1'], 'alembic==1.11.1'],
'security': ['cryptography'], 'security': ['cryptography'],
'dev': ['mako==1.2.4', 'dev': ['environs==9.5.0']
'environs==9.5.0']
}, },
) )