OpenTelemetry 实战：从零实现应用指标监控

docker run  -d -p 9292:9090 --name prometheus \
-v /prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
quay.io/prometheus/prometheus:v2.49.1 \
--config.file=/etc/prometheus/prometheus.yml \
--storage.tsdb.path=/prometheus \
--web.console.libraries=/etc/prometheus/console_libraries \
--web.console.templates=/etc/prometheus/consoles \
--enable-feature=exemplar-storage \
--enable-feature=otlp-write-receiver

exporters:
  debug:
  otlp:
    endpoint: "jaeger:4317"
    tls:
      insecure: true
  otlphttp/prometheus:
    endpoint: http://prometheus:9292/api/v1/otlp
    tls:
      insecure: true      

processors:
  batch:

service:
  pipelines:
    traces:
      receivers:
      - otlp
      processors: [batch]
      exporters:
      - otlp
      - debug        
    metrics:
      exporters:
      - otlphttp/prometheus
      - debug
      processors:
      - batch
      receivers:
      - otlp

java -javaagent:opentelemetry-javaagent-2.4.0-SNAPSHOT.jar \
-Dotel.traces.exporter=otlp \
-Dotel.metrics.exporter=otlp \
-Dotel.logs.exporter=none \
-Dotel.service.name=java-demo \
-Dotel.exporter.otlp.protocol=grpc \
-Dotel.propagators=tracecontext,baggage \
-Dotel.exporter.otlp.endpoint=http://127.0.0.1:5317 -jar target/demo-0.0.1-SNAPSHOT.jar

# Run go app
export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:5317 OTEL_RESOURCE_ATTRIBUTES=service.name=k8s-combat
./k8s-combat

2024-07-22T06:34:08.060Z info MetricsExporter {"kind": "exporter", "data_type": "metrics", "name": "debug", "resource metrics": 1, "metrics": 18, "data points": 44}

func initMeterProvider() *sdkmetric.MeterProvider {  
    ctx := context.Background()  
  
    exporter, err := otlpmetricgrpc.New(ctx)  
    if err != nil {  
       log.Printf("new otlp metric grpc exporter failed: %v", err)  
    }  
    mp := sdkmetric.NewMeterProvider(  
       sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter)),  
       sdkmetric.WithResource(initResource()),  
    )    otel.SetMeterProvider(mp)  
    return mp  
}

mp := initMeterProvider()
defer func() {
 if err := mp.Shutdown(context.Background()); err != nil {
  log.Printf("Error shutting down meter provider: %v", err)
 }
}()

err := runtime.Start(runtime.WithMinimumReadMemStatsInterval(time.Second))
if err != nil {
    log.Fatal(err)
}

var meter =  otel.Meter("test.io/k8s/combat")  
apiCounter, err = meter.Int64Counter(  
    "api.counter",  
    metric.WithDescription("Number of API calls."),  
    metric.WithUnit("{call}"),  
)  
if err != nil {  
    log.Err(err)  
}

func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {  
    defer apiCounter.Add(ctx, 1)  
    return &pb.HelloReply{Message: fmt.Sprintf("hostname:%s, in:%s, md:%v", name, in.Name, md)}, nil  
}

messageInCounter = meter    
        .counterBuilder(MESSAGE_IN_COUNTER)    
        .setUnit("{message}")    
        .setDescription("The total number of messages received for this topic.")    
        .buildObserver();

public static void registerObservers() {      
    Meter meter = MetricsRegistration.getMeter();      
      
    meter.gaugeBuilder("pulsar_producer_num_msg_send")      
            .setDescription("The number of messages published in the last interval")      
            .ofLongs()      
            .buildWithCallback(      
                    r -> recordProducerMetrics(r, ProducerStats::getNumMsgsSent));  
  
private static void recordProducerMetrics(ObservableLongMeasurement observableLongMeasurement, Function<ProducerStats, Long> getter) {      
    for (Producer producer : CollectionHelper.PRODUCER_COLLECTION.list()) {      
        ProducerStats stats = producer.getStats();      
        String topic = producer.getTopic();      
        if (topic.endsWith(RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX)) {      
            continue;      
        }        observableLongMeasurement.record(getter.apply(stats),      
                Attributes.of(PRODUCER_NAME, producer.getProducerName(), TOPIC, topic));      
    }}

export OTEL_METRICS_EXPORTER=prometheus

scrape_configs:
  - job_name: "k8s-combat"
    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.
    static_configs:
      - targets: ["k8s-combat:9464"]

apiVersion: operator.victoriametrics.com/v1beta1  
kind: VMPodScrape  
metadata:  
  name: isito-pod-scrape  
spec:  
  podMetricsEndpoints:  
    - scheme: http  
      scrape_interval: "30s"  
      scrapeTimeout: "30s"  
      path: /stats/prometheus  
      metricRelabelConfigs:  
        - regex: ^envoy_.*|^url\_\_\_\_.*|istio_request_bytes_sum|istio_request_bytes_count|istio_response_bytes_sum|istio_request_bytes_sum|istio_request_duration_milliseconds_sum|istio_response_bytes_count|istio_request_duration_milliseconds_count|^ostrich_apigateway.*|istio_request_messages_total|istio_response_messages_total  
          action: drop_metrics  
  namespaceSelector:  
    any: true