分布式系统可观测性之应用业务指标监控

# HELP http_requests_total The total number of HTTP requests.
# TYPE http_requests_total counter
http_requests_total{method="post",code="200"} 1027
http_requests_total{method="post",code="400"}    3

# Escaping in label values:
msdos_file_access_time_seconds{path="C:\\DIR\\FILE.TXT",error="Cannot find file:\n\"FILE.TXT\""} 1.458255915e9

# Minimalistic line:
metric_without_timestamp_and_labels 12.47

# A weird metric from before the epoch:
something_weird{problem="division by zero"} +Inf -3982045

# A histogram, which has a pretty complex representation in the text format:
# HELP http_request_duration_seconds A histogram of the request duration.
# TYPE http_request_duration_seconds histogram
http_request_duration_seconds_bucket{le="0.05"} 24054
http_request_duration_seconds_bucket{le="0.1"} 33444
http_request_duration_seconds_bucket{le="0.2"} 100392
http_request_duration_seconds_bucket{le="0.5"} 129389
http_request_duration_seconds_bucket{le="1"} 133988
http_request_duration_seconds_bucket{le="+Inf"} 144320
http_request_duration_seconds_sum 53423
http_request_duration_seconds_count 144320

# Finally a summary, which has a complex representation, too:
# HELP rpc_duration_seconds A summary of the RPC duration in seconds.
# TYPE rpc_duration_seconds summary
rpc_duration_seconds{quantile="0.01"} 3102
rpc_duration_seconds{quantile="0.05"} 3272
rpc_duration_seconds{quantile="0.5"} 4773
rpc_duration_seconds{quantile="0.9"} 9001
rpc_duration_seconds{quantile="0.99"} 76656
rpc_duration_seconds_sum 1.7560473e+07
rpc_duration_seconds_count 2693

metric_name [
  "{" label_name "=" `"` label_value `"` { "," label_name "=" `"` label_value `"` } [ "," ] "}"
] value [ timestamp ]

<dependency>
  <groupId>org.springframework.boot</groupId>
  <artifactId>spring-boot-starter-actuator</artifactId>
</dependency>

<dependency>
  <groupId>io.micrometer</groupId>
  <artifactId>micrometer-registry-prometheus</artifactId>
  <version>${micrometer.version}</version>
</dependency>

management.metrics.enable.jvm=false
management.metrics.enable.process=false
management.metrics.enable.system=false

1management.server.port=xxxx

@PropertySource(value = {"classpath:/micrometer.properties"})
@Configuration
public class MetricsConfig {

    @Bean
    public TimedAspect timedAspect(MeterRegistry registry){
        return new TimedAspect(registry, (pjp) -> Tags.empty());
    }

    @Bean
    public CountedAspect countedAspect(MeterRegistry registry){
        return new CountedAspect(registry);
    }

    @Bean
    public PrometheusMetricScrapeEndpoint prometheusMetricScrapeEndpoint(CollectorRegistry collectorRegistry){
        return new PrometheusMetricScrapeEndpoint(collectorRegistry);
    }

    @Bean
    public PrometheusMetricScrapeMvcEndpoint prometheusMvcEndpoint(PrometheusMetricScrapeEndpoint delegate){
        return new PrometheusMetricScrapeMvcEndpoint(delegate);
    }

}

<dependency>
  <groupId>xxx</groupId>
  <artifactId>micrometer-spring-boot-starter</artifactId>
</dependency>

@Timed(value = "biz.print", percentiles = {0.95, 0.99}, description = "metrics of print")
public String print(PrintData printData) {

}

biz_print_seconds_count{exception="none"} 4.0
biz_print_seconds_sum{exception="none"} 7.783213927
biz_print_seconds_max{exception="none"} 6.14639717
biz_print_seconds{exception="NullPointerException"} 0.318767104
biz_print_seconds{exception="none",quantile="0.95",} 0.58720256
biz_print_seconds{exception="none",quantile="0.99",} 6.157238272

try {
    return pjp.proceed();
} catch (Exception ex) {
    exceptionClass = ex.getClass().getSimpleName();
    throw ex;
} finally {
    try {
        sample.stop(Timer.builder(metricName)
                    .description(timed.description().isEmpty() ? null : timed.description())
                    .tags(timed.extraTags())
                    .tags(EXCEPTION_TAG, exceptionClass)
                    .tags(tagsBasedOnJoinPoint.apply(pjp))
                    .publishPercentileHistogram(timed.histogram())
                    .publishPercentiles(timed.percentiles().length == 0 ? null : timed.percentiles())
                    .register(registry));
    } catch (Exception e) {
        // ignoring on purpose
    }
}

@Timed(value = "biz.print", recordFailuresOnly = true, description = "metrics of print")
public String print(PrintData printData) {

}

biz_print_failure_total{class="com.xxx.print.service.impl.PrintServiceImpl",exception="NullPointerException",method="print",result="failure",} 4.0

private void record(ProceedingJoinPoint pjp, Counted counted, String exception, String result){
    counter(pjp, counted)
            .tag(EXCEPTION_TAG, exception)
            .tag(RESULT_TAG, result)
            .register(meterRegistry)
            .increment();
}

private Counter.Builder counter(ProceedingJoinPoint pjp, Counted counted){
    Counter.Builder builder = Counter.builder(counted.value()).tags(tagsBasedOnJoinPoint.apply(pjp));
    String description = counted.description();
    if (!description.isEmpty()) {
        builder.description(description);
    }
    return builder;
}

@Autowired
private PrometheusMeterRegistry meterRegistry;

public void buildGauge(Long workId) {
    Gauge.builder("biz.alphard.snowFlakeIdGenerator.workId", workId, Long::longValue)
            .description("alphard snowFlakeIdGenerator workId")
            .tag("workId", workId.toString())
            .register(meterRegistry).measure();
}

biz_alphard_snowFlakeIdGenerator_workId{workId="2"} 2

management.metrics.distribution.sla[biz.print]=300ms,400ms,500ms,1s,10s

biz_print_seconds_bucket{exception="none",le="0.3",} 1.0
biz_print_seconds_bucket{exception="none",le="0.4",} 3.0
biz_print_seconds_bucket{exception="none",le="0.5",} 10.0
biz_print_seconds_bucket{exception="none",le="0.6",} 11.0
biz_print_seconds_bucket{exception="none",le="1.0",} 11.0
biz_print_seconds_bucket{exception="none",le="10.0",} 12.0
biz_print_seconds_bucket{exception="none",le="+Inf",} 12.0

apiVersion: v1
kind: Service
metadata:
  name: print-svc
  labels:
    monitor/metrics: ""
spec:
  ports:
  - name: custom-metrics
    port: xxxx
    targetPort: xxxx
    protocol: TCP
  type: ClusterIP
  selector:
    app: print-test

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: metrics
  labels:
    app: metric-monitor
spec:
  namespaceSelector:
    any: true
  endpoints:
  - interval: 15s
    port: custom-metrics
    path: "/manage/prometheusMetric"
  selector:
    matchLabels:
      monitor/metrics: ""