边侧监控选型

1
2
3
4
5

mv prometheus-*.tar.gz prometheus.tar.gz
tar xvfz prometheus.tar.gz
cd prometheus
mkdir -p data
./prometheus

1
2

mkdir -p /etc/prometheus
docker run -p 9090:9090 -v /etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

groups:
- name: example
  rules:
  - alert: HighErrorRate
    expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5
    for: 10m
    labels:
      severity: page
    annotations:
      summary: High request latency
      description: description info

1
2

{{$labels.<labelname>}}
{{$value}}

1
2
3
4
5

groups:
  - name: example
    rules:
    - record: job:http_inprogress_requests:sum
      expr: sum(http_inprogress_requests) by (job)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

global:
  resolve_timeout: 5m

route:
  group_by: ['alertname']
  group_wait: 5s
  group_interval: 60s
  repeat_interval: 1h
  receiver: 'web.hook'
receivers:
- name: 'web.hook'
  webhook_configs:
  - url: 'http://127.0.0.1:5001/'
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

route:
  receiver: 'default-receiver'
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  group_by: [cluster, alertname]
  routes:
  - receiver: 'database-pager'
    group_wait: 10s
    match_re:
      service: mysql|cassandra
  - receiver: 'frontend-pager'
    group_by: [product, environment]
    match:
      team: frontend

1
2
3
4
5
6
7

- source_match:
    alertname: NodeDown
    severity: critical
  target_match:
    severity: critical
  equal:
    - node

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

scrape_configs:
  - job_name: 'federate'
    scrape_interval: 15s
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{job="prometheus"}'
        - '{__name__=~"job:.*"}'
        - '{__name__=~"node.*"}'
    static_configs:
      - targets:
        - '192.168.77.11:9090'
        - '192.168.77.12:9090'

1

alertmanager  --web.listen-address=":9093" --cluster.listen-address="127.0.0.1:8001" --config.file=/etc/prometheus/alertmanager.yml  --storage.path=/data/alertmanager/

1

alertmanager  --web.listen-address=":9094" --cluster.listen-address="127.0.0.1:8002" --cluster.peer=127.0.0.1:8001 --config.file=/etc/prometheus/alertmanager.yml  --storage.path=/data/alertmanager2/

1

helm install kube-prometheus bitnami/kube-prometheus --namespace monitor --create-namespace --version 8.15.3

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

NAME                                                     READY   STATUS    RESTARTS   AGE
pod/kube-prometheus-node-exporter-k5gd6                  1/1     Running   0          43m
pod/kube-prometheus-blackbox-exporter-8567bbfd99-7k8ms   1/1     Running   0          43m
pod/kube-prometheus-kube-state-metrics-5c9f68d87-5vdn6   1/1     Running   0          43m
pod/kube-prometheus-operator-c9c58b595-mv6kk             1/1     Running   0          43m
pod/alertmanager-kube-prometheus-alertmanager-0          2/2     Running   0          40m
pod/prometheus-kube-prometheus-prometheus-0              2/2     Running   0          40m

NAME                                         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                      AGE
service/kube-prometheus-alertmanager         ClusterIP   10.43.34.93    <none>        9093/TCP                     43m
service/kube-prometheus-prometheus           ClusterIP   10.43.96.98    <none>        9090/TCP                     43m
service/kube-prometheus-kube-state-metrics   ClusterIP   10.43.31.57    <none>        8080/TCP                     43m
service/kube-prometheus-operator             ClusterIP   10.43.33.45    <none>        8080/TCP                     43m
service/kube-prometheus-blackbox-exporter    ClusterIP   10.43.54.206   <none>        19115/TCP                    43m
service/kube-prometheus-node-exporter        ClusterIP   10.43.135.99   <none>        9100/TCP                     43m
service/alertmanager-operated                ClusterIP   None           <none>        9093/TCP,9094/TCP,9094/UDP   40m
service/prometheus-operated                  ClusterIP   None           <none>        9090/TCP                     40m

NAME                                           DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/kube-prometheus-node-exporter   1         1         1       1            1           <none>          43m

NAME                                                 READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/kube-prometheus-blackbox-exporter    1/1     1            1           43m
deployment.apps/kube-prometheus-kube-state-metrics   1/1     1            1           43m
deployment.apps/kube-prometheus-operator             1/1     1            1           43m

NAME                                                           DESIRED   CURRENT   READY   AGE
replicaset.apps/kube-prometheus-blackbox-exporter-8567bbfd99   1         1         1       43m
replicaset.apps/kube-prometheus-kube-state-metrics-5c9f68d87   1         1         1       43m
replicaset.apps/kube-prometheus-operator-c9c58b595             1         1         1       43m

NAME                                                         READY   AGE
statefulset.apps/alertmanager-kube-prometheus-alertmanager   1/1     40m
statefulset.apps/prometheus-kube-prometheus-prometheus       1/1     40m

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# ref: https://artifacthub.io/packages/helm/bitnami/kube-prometheus
kubectl create ns monitor
helm install prometheus-stack bitnami/kube-prometheus -n monitor --version 8.15.4 \
  --set global.storageClass=local-path \
  --set prometheus.persistence.enabled=true \
  --set alertmanager.persistence.enabled=true \
  --set prometheus.retention="72h" \
  --set alertmanager.retention="72h" \
  --set kubelet.serviceMonitor.resourcePath=/metrics/resource \
  --set prometheus.service.type=LoadBalancer \
  --set alertmanager.service.type=LoadBalancer \
  --set prometheus.ruleSelector.matchLabels.monitor=default \
  --set alertmanager.configSelector.matchLabels.monitor=default \
  --set alertmanager.externalConfig=true

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: example-app
  namespace: monitor
  labels:
    team: frontend
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app: example-app
  endpoints:
    - port: 80

1
2
3

spec:
  namespaceSelector:
    any: true

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    monitor: default
  name: prometheus-example-rules
spec:
  groups:
  - name: example-rules
    rules:
    - alert: ExampleAlert
      expr: vector(1)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: config-example
  labels:
    monitor: default
spec:
  route:
    groupBy: ['job']
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 12h
    receiver: 'webhook'
  receivers:
  - name: 'webhook'
    webhookConfigs:
    - url: 'http://example.com/'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

global:
  resolve_timeout: 5m
route:
  group_by: ['job']
  group_wait: 5s
  group_interval: 1m
  repeat_interval: 15m
  receiver: 'null'
  routes:
  - matchers:
    - alertname = Watchdog
    receiver: 'null'
  - matchers:
    - severity =~ "critical|warning"
    receiver: webhook

receivers:
- name: 'null'
- name: 'webhook'
  webhook_configs:
  - url: 'http://edge-webhook:9095/api/v1/alert'
    send_resolved: true

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

apiVersion: monitoring.coreos.com/v1
kind: Probe
metadata:
  name: domain-probe
  namespace: monitoring
  labels:
    monitor: default
spec:
  jobName: domain-probe
  prober:
    url: prometheus-stack-kube-prom-blackbox-exporter:19115
  module: http_2xx
  targets:
    # ingress:{}
    staticConfig:
      static:
        - baidu.com

1

mtail --progs /etc/mtail --logs /var/log/syslog --logs /var/log/ntp/peerstats --novm_logs_runtime_errors

1
2
3
4
5
6

counter total
/^[a-z]+((?P<response_size>\d+)|-)$/ {
  $1 != "-" {
    total = $response_size
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# The `syslog' decorator defines a procedure.  When a block of mtail code is
# "decorated", it is called before entering the block.  The block is entered
# when the keyword `next' is reached.
def syslog {
    /(?P<date>(?P<legacy_date>\w+\s+\d+\s+\d+:\d+:\d+)|(?P<rfc3339_date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+[+-]\d{2}:\d{2}))/ +
        /\s+(?:\w+@)?(?P<hostname>[\w\.-]+)\s+(?P<application>[\w\.-]+)(?:\[(?P<pid>\d+)\])?:\s+(?P<message>.*)/ {
        # If the legacy_date regexp matched, try this format.
        len($legacy_date) > 0 {
            strptime($legacy_date, "Jan _2 15:04:05")
        }
        # If the RFC3339 style matched, parse it this way.
        len($rfc3339_date) > 0 {
            strptime($rfc3339_date, "2006-01-02T15:04:05-07:00")
        }
        # Call into the decorated block
        next
    }
}

1
2
3
4

@syslog
/foo/ {
  ...
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

/pattern/ {
  ...
} else {
  ...
}

{
  /pattern1/ {}
  /pattern2/ {}
  otherwise {}
}

1
2
3
4
5

counter error_line_total by filename

/ERROR/ {
    error_line_total[getfilename()]++
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

groups:
  - name: LogStatExporter
    rules:
      - alert: FileErrorAlert
        expr: sum(increase(error_line_total[1m])) by (filename) > 0
        labels:
          severity: critical
        annotations:
          summary: "Log file {{ $labels.filename }} has excessive errors"
          description: "Log file {{ $labels.filename }} has experienced {{ $value }} errors in the last 1m"

1
2
3
4
5
6
7

helm install loggie https://ghproxy.com/https://github.com/loggie-io/installation/releases/download/v1.4.0/loggie-v1.4.0.tgz -n monitor \
--set image=loggieio/loggie:v1.4.1 \
--set config.loggie.discovery.enabled=true \
--set config.loggie.discovery.kubernetes.containerRuntime=docker \
--set config.loggie.discovery.kubernetes.rootFsCollectionEnabled=true \
--set config.loggie.discovery.kubernetes.parseStdout=true \
--set serviceMonitor.enabled=true

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

apiVersion: loggie.io/v1beta1
kind: ClusterLogConfig
metadata:
  name: error-log-grep
spec:
  selector:
    type: pod
    labelSelector:
      alertError: open
  pipeline:
    sources: |
      - type: file
        name: stdout
        addonMeta: true
        paths:
          - stdout
    sink: |
      type: alertWebhook
      addr: http://edge-webhook:9095/api/v1/loggie
    interceptors: |
      - type: rateLimit
        qps: 4096
      - type: logAlert
        matcher:
          contains: ["ERROR"]
        sendOnlyMatched: true

1

loki -config.file=loki.local-config.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

auth_enabled: false

server:
  http_listen_port: 3100

common:
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory
  replication_factor: 1
  path_prefix: /tmp/loki

schema_config:
  configs:
  - from: 2020-05-15
    store: boltdb-shipper
    object_store: filesystem
    schema: v11
    index:
      prefix: index_
      period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /data/loki/boltdb-shipper-active
    cache_location: /data/loki/boltdb-shipper-cache
    cache_ttl: 24h
    shared_store: filesystem
  filesystem:
    directory: /data/loki/chunks

table_manager:
  retention_deletes_enabled: true
  retention_period: 72h

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /var/log/positions.yaml # This location needs to be writeable by Promtail.

client:
  backoff_config:
    max_period: 5s
    max_retries: 20
    min_period: 100ms
  batchsize: 102400
  batchwait: 1s
  timeout: 10s

clients:
  - url: http://ip_or_hostname_where_Loki_run:3100/loki/api/v1/push

scrape_configs:
 - job_name: system
   pipeline_stages:
   static_configs:
   - targets:
      - localhost
     labels:
      job: varlogs  # A `job` label is fairly standard in prometheus and useful for linking metrics and logs.
      host: yourhost # A `host` label will help identify logs from this machine vs others
      __path__: /var/log/*.log  # The path matching uses a third party library: https://github.com/bmatcuk/doublestar

1
2
3
4
5
6

helm install loki-stack bitnami/grafana-loki -n monitor --version 2.10.1 \
--set global.storageClass=longhorn \
--set metrics.enabled=true \
--set metrics.serviceMonitor.enabled=true \
--set gateway.service.type=LoadBalancer \
--set gateway.service.ports.http=9092