深入剖析Alertmanager：解锁告警管理的核心逻辑

func hashAlert(a *types.Alert) uint64 {  
    const sep = '\xff'  
  
    hb := hashBuffers.Get().(*hashBuffer)  
    defer hashBuffers.Put(hb)  
    b := hb.buf[:0]  
  
    names := make(model.LabelNames, 0, len(a.Labels))  
  
    for ln := range a.Labels {  
       names = append(names, ln)  
    }  
    sort.Sort(names)  
  
    for _, ln := range names {  
       b = append(b, string(ln)...)  
       b = append(b, sep)  
       b = append(b, string(a.Labels[ln])...)  
       b = append(b, sep)  
    }  
  
    hash := xxhash.Sum64(b)  
  
    return hash  
}

func (n *DedupStage) Exec(ctx context.Context, _ *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {  
    gkey, ok := GroupKey(ctx)  
    if !ok {  
       return ctx, nil, errors.New("group key missing")  
    }  
  
    repeatInterval, ok := RepeatInterval(ctx)  
    if !ok {  
       return ctx, nil, errors.New("repeat interval missing")  
    }  
  
    firingSet := map[uint64]struct{}{}  
    resolvedSet := map[uint64]struct{}{}  
    firing := []uint64{}  
    resolved := []uint64{}  
  
    var hash uint64  
    for _, a := range alerts {  
       hash = n.hash(a)  
       if a.Resolved() {  
          resolved = append(resolved, hash)  
          resolvedSet[hash] = struct{}{}  
       } else {  
          firing = append(firing, hash)  
          firingSet[hash] = struct{}{}  
       }  
    }  
  
    ctx = WithFiringAlerts(ctx, firing)  
    ctx = WithResolvedAlerts(ctx, resolved)  
  
    entries, err := n.nflog.Query(nflog.QGroupKey(gkey), nflog.QReceiver(n.recv))  
    if err != nil && !errors.Is(err, nflog.ErrNotFound) {  
       return ctx, nil, err  
    }  
  
    var entry *nflogpb.Entry  
    switch len(entries) {  
    case 0:  
    case 1:  
       entry = entries[0]  
    default:  
       return ctx, nil, fmt.Errorf("unexpected entry result size %d", len(entries))  
    }  
  
    if n.needsUpdate(entry, firingSet, resolvedSet, repeatInterval) {  
       return ctx, alerts, nil  
    }  
    return ctx, nil, nil  
}

route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default_receiver'
  routes:
  - match:
      severity: 'critical'
      service: 'backend'
    receiver: 'backend_critical_receiver'
    continue: false
  - match:
      severity: 'warning'
      service: 'backend'
    receiver: 'backend_warning_receiver'
    continue: true
  - match:
      service: 'frontend'
    receiver: 'frontend_receiver'

// inhibit/inhibitor.go
func NewInhibitor(ap provider.Alerts, rs []config.InhibitRule, mk types.AlertMarker, logger *slog.Logger) *Inhibitor {  
    ih := &Inhibitor{  
       alerts: ap,  
       marker: mk,  
       logger: logger,  
    }  
    for _, cr := range rs {  
       r := NewInhibitRule(cr)  
       ih.rules = append(ih.rules, r)  
    }  
    return ih  
}

// inhibit/inhibit.go
func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {  
    fp := lset.Fingerprint()  
  
    for _, r := range ih.rules {  
       if !r.TargetMatchers.Matches(lset) {  
          // If target side of rule doesn't match, we don't need to look any further.  
          continue  
       }  
       // If we are here, the target side matches. If the source side matches, too, we  
       // need to exclude inhibiting alerts for which the same is true.       
       if inhibitedByFP, eq := r.hasEqual(lset, r.SourceMatchers.Matches(lset)); eq {  
          ih.marker.SetInhibited(fp, inhibitedByFP.String())  
          return true  
       }  
    }  
    ih.marker.SetInhibited(fp)  
  
    return false  
}

// MuteStage filters alerts through a Muter.type MuteStage struct {  
    muter   types.Muter  
    metrics *Metrics  
}  
  
// NewMuteStage return a new MuteStage.  
func NewMuteStage(m types.Muter, metrics *Metrics) *MuteStage {  
    return &MuteStage{muter: m, metrics: metrics}  
}  
  
// Exec implements the Stage interface.func (n *MuteStage) Exec(ctx context.Context, logger *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {  
    var (  
       filtered []*types.Alert  
       muted    []*types.Alert  
    )  
    for _, a := range alerts {  
       // TODO(fabxc): increment total alerts counter.  
       // Do not send the alert if muted.  
       if n.muter.Mutes(a.Labels) {  
          muted = append(muted, a)  
       } else {  
          filtered = append(filtered, a)  
       }  
       // TODO(fabxc): increment muted alerts counter if muted.  
    }  
    if len(muted) > 0 {  
  
       var reason string  
       switch n.muter.(type) {  
       case *silence.Silencer:  
          reason = SuppressedReasonSilence  
       case *inhibit.Inhibitor:  
          reason = SuppressedReasonInhibition  
       default:  
       }  
       n.metrics.numNotificationSuppressedTotal.WithLabelValues(reason).Add(float64(len(muted)))  
       logger.Debug("Notifications will not be sent for muted alerts", "alerts", fmt.Sprintf("%v", muted), "reason", reason)  
    }  
  
    return ctx, filtered, nil  
}

type Alert struct {
 GeneratorURL strfmt.URI `json:"generatorURL,omitempty"`
 Labels LabelSet `json:"labels"`
}

type AlertGroup struct {  
    Alerts []*GettableAlert `json:"alerts"`
    Labels LabelSet `json:"labels"`
    Receiver *Receiver `json:"receiver"`
}

type Route struct {  
    parent *Route  
  
    // The configuration parameters for matches of this route.  
    RouteOpts RouteOpts  
  
    // Matchers an alert has to fulfill to match    // this route.    
    Matchers labels.Matchers  
  
    // If true, an alert matches further routes on the same level.  
    Continue bool  
  
    // Children routes of this route.  
    Routes []*Route  
}

func (api *API) postAlertsHandler(params alert_ops.PostAlertsParams) middleware.Responder {  
    logger := api.requestLogger(params.HTTPRequest)  
  
    alerts := OpenAPIAlertsToAlerts(params.Alerts)  
    now := time.Now()  
  
    api.mtx.RLock()  
    resolveTimeout := time.Duration(api.alertmanagerConfig.Global.ResolveTimeout)  
    api.mtx.RUnlock()  
  
    for _, alert := range alerts {  
       alert.UpdatedAt = now  
  
       // Ensure StartsAt is set.  
       if alert.StartsAt.IsZero() {  
          if alert.EndsAt.IsZero() {  
             alert.StartsAt = now  
          } else {  
             alert.StartsAt = alert.EndsAt  
          }  
       }  
       // If no end time is defined, set a timeout after which an alert  
       // is marked resolved if it is not updated.       if alert.EndsAt.IsZero() {  
          alert.Timeout = true  
          alert.EndsAt = now.Add(resolveTimeout)  
       }  
       if alert.EndsAt.After(time.Now()) {  
          api.m.Firing().Inc()  
       } else {  
          api.m.Resolved().Inc()  
       }  
    }  
  
    // Make a best effort to insert all alerts that are valid.  
    var (  
       validAlerts    = make([]*types.Alert, 0, len(alerts))  
       validationErrs = &types.MultiError{}  
    )  
    for _, a := range alerts {  
       removeEmptyLabels(a.Labels)  
  
       if err := a.Validate(); err != nil {  
          validationErrs.Add(err)  
          api.m.Invalid().Inc()  
          continue  
       }  
       validAlerts = append(validAlerts, a)  
    }  
    if err := api.alerts.Put(validAlerts...); err != nil {  
       logger.Error("Failed to create alerts", "err", err)  
       return alert_ops.NewPostAlertsInternalServerError().WithPayload(err.Error())  
    }  
  
    if validationErrs.Len() > 0 {  
       logger.Error("Failed to validate alerts", "err", validationErrs.Error())  
       return alert_ops.NewPostAlertsBadRequest().WithPayload(validationErrs.Error())  
    }  
  
    return alert_ops.NewPostAlertsOK()  
}

func (d *Dispatcher) Run() {  
    d.done = make(chan struct{})  
  
    d.mtx.Lock()  
    d.aggrGroupsPerRoute = map[*Route]map[model.Fingerprint]*aggrGroup{}  
    d.aggrGroupsNum = 0  
    d.metrics.aggrGroups.Set(0)  
    d.ctx, d.cancel = context.WithCancel(context.Background())  
    d.mtx.Unlock()  
  
    d.run(d.alerts.Subscribe())  
    close(d.done)  
}

func (d *Dispatcher) run(it provider.AlertIterator) {  
    maintenance := time.NewTicker(30 * time.Second)  
    defer maintenance.Stop()  
  
    defer it.Close()  
  
    for {  
       select {  
       case alert, ok := <-it.Next():  
          if !ok {  
             // Iterator exhausted for some reason.  
             if err := it.Err(); err != nil {  
                d.logger.Error("Error on alert update", "err", err)  
             }  
             return  
          }  
  
          d.logger.Debug("Received alert", "alert", alert)  
  
          // Log errors but keep trying.  
          if err := it.Err(); err != nil {  
             d.logger.Error("Error on alert update", "err", err)  
             continue  
          }  
  
          now := time.Now()  
          for _, r := range d.route.Match(alert.Labels) {  
             d.processAlert(alert, r)  
          }  
          d.metrics.processingDuration.Observe(time.Since(now).Seconds())  
  
       case <-maintenance.C:  
          d.doMaintenance()  
       case <-d.ctx.Done():  
          return  
       }  
    }  
}

func (d *Dispatcher) processAlert(alert *types.Alert, route *Route) {  
    groupLabels := getGroupLabels(alert, route)  
  
    fp := groupLabels.Fingerprint()  
  
    d.mtx.Lock()  
    defer d.mtx.Unlock()  
  
    routeGroups, ok := d.aggrGroupsPerRoute[route]  
    if !ok {  
       routeGroups = map[model.Fingerprint]*aggrGroup{}  
       d.aggrGroupsPerRoute[route] = routeGroups  
    }  
  
    ag, ok := routeGroups[fp]  
    if ok {  
       ag.insert(alert)  
       return  
    }  
  
    // If the group does not exist, create it. But check the limit first.  
    if limit := d.limits.MaxNumberOfAggregationGroups(); limit > 0 && d.aggrGroupsNum >= limit {  
       d.metrics.aggrGroupLimitReached.Inc()  
       d.logger.Error("Too many aggregation groups, cannot create new group for alert", "groups", d.aggrGroupsNum, "limit", limit, "alert", alert.Name())  
       return  
    }  
  
    ag = newAggrGroup(d.ctx, groupLabels, route, d.timeout, d.logger)  
    routeGroups[fp] = ag  
    d.aggrGroupsNum++  
    d.metrics.aggrGroups.Inc()  
  
    // Insert the 1st alert in the group before starting the group's run()  
    // function, to make sure that when the run() will be executed the 1st    
    // alert is already there.    
    ag.insert(alert)  
  
    go ag.run(func(ctx context.Context, alerts ...*types.Alert) bool {  
       _, _, err := d.stage.Exec(ctx, d.logger, alerts...)  
       if err != nil {  
          logger := d.logger.With("num_alerts", len(alerts), "err", err)  
          if errors.Is(ctx.Err(), context.Canceled) {  
             // It is expected for the context to be canceled on  
             // configuration reload or shutdown. In this case, the             
             // message should only be logged at the debug level.             
             logger.Debug("Notify for alerts failed")  
          } else {  
             logger.Error("Notify for alerts failed")  
          }  
       }  
       return err == nil  
    })  
}

func (pb *PipelineBuilder) New(  
    receivers map[string][]Integration,  
    wait func() time.Duration,  
    inhibitor *inhibit.Inhibitor,  
    silencer *silence.Silencer,  
    intervener *timeinterval.Intervener,  
    marker types.GroupMarker,  
    notificationLog NotificationLog,  
    peer Peer,  
) RoutingStage {  
    rs := make(RoutingStage, len(receivers))  
  
    ms := NewGossipSettleStage(peer)  
    is := NewMuteStage(inhibitor, pb.metrics)  
    tas := NewTimeActiveStage(intervener, marker, pb.metrics)  
    tms := NewTimeMuteStage(intervener, marker, pb.metrics)  
    ss := NewMuteStage(silencer, pb.metrics)  
  
    for name := range receivers {  
       st := createReceiverStage(name, receivers[name], wait, notificationLog, pb.metrics)  
       rs[name] = MultiStage{ms, is, tas, tms, ss, st}  
    }  
  
    pb.metrics.InitializeFor(receivers)  
  
    return rs  
}

func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {  
    var sent []*types.Alert  
  
    // If we shouldn't send notifications for resolved alerts, but there are only  
    // resolved alerts, report them all as successfully notified (we still want the    // notification log to log them for the next run of DedupStage).    
    if !r.integration.SendResolved() {  
       firing, ok := FiringAlerts(ctx)  
       if !ok {  
          return ctx, nil, errors.New("firing alerts missing")  
       }  
       if len(firing) == 0 {  
          return ctx, alerts, nil  
       }  
       for _, a := range alerts {  
          if a.Status() != model.AlertResolved {  
             sent = append(sent, a)  
          }  
       }  
    } else {  
       sent = alerts  
    }  
  
    b := backoff.NewExponentialBackOff()  
    b.MaxElapsedTime = 0 // Always retry.  
  
    tick := backoff.NewTicker(b)  
    defer tick.Stop()  
  
    var (  
       i    = 0  
       iErr error  
    )  
  
    l = l.With("receiver", r.groupName, "integration", r.integration.String())  
    if groupKey, ok := GroupKey(ctx); ok {  
       l = l.With("aggrGroup", groupKey)  
    }  
  
    for {  
       i++  
       // Always check the context first to not notify again.  
       select {  
       case <-ctx.Done():  
          if iErr == nil {  
             iErr = ctx.Err()  
             if errors.Is(iErr, context.Canceled) {  
                iErr = NewErrorWithReason(ContextCanceledReason, iErr)  
             } else if errors.Is(iErr, context.DeadlineExceeded) {  
                iErr = NewErrorWithReason(ContextDeadlineExceededReason, iErr)  
             }  
          }  
  
          if iErr != nil {  
             return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr)  
          }  
          return ctx, nil, nil  
       default:  
       }  
  
       select {  
       case <-tick.C:  
          now := time.Now()  
          retry, err := r.integration.Notify(ctx, sent...)  
          dur := time.Since(now)  
          r.metrics.notificationLatencySeconds.WithLabelValues(r.labelValues...).Observe(dur.Seconds())  
          r.metrics.numNotificationRequestsTotal.WithLabelValues(r.labelValues...).Inc()  
          if err != nil {  
             r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc()  
             if !retry {  
                return ctx, alerts, fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err)  
             }  
             if ctx.Err() == nil {  
                if iErr == nil || err.Error() != iErr.Error() {  
                   // Log the error if the context isn't done and the error isn't the same as before.  
                   l.Warn("Notify attempt failed, will retry later", "attempts", i, "err", err)  
                }  
                // Save this error to be able to return the last seen error by an  
                // integration upon context timeout.                
                iErr = err  
             }  
          } else {  
             l := l.With("attempts", i, "duration", dur)  
             if i <= 1 {  
                l = l.With("alerts", fmt.Sprintf("%v", alerts))  
                l.Debug("Notify success")  
             } else {  
                l.Info("Notify success")  
             }  
  
             return ctx, alerts, nil  
          }  
       case <-ctx.Done():  
          if iErr == nil {  
             iErr = ctx.Err()  
             if errors.Is(iErr, context.Canceled) {  
                iErr = NewErrorWithReason(ContextCanceledReason, iErr)  
             } else if errors.Is(iErr, context.DeadlineExceeded) {  
                iErr = NewErrorWithReason(ContextDeadlineExceededReason, iErr)  
             }  
          }  
          if iErr != nil {  
             return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr)  
          }  
          return ctx, nil, nil  
       }  
    }  
}