来聊聊 Redis 哨兵如何主观认定下线

int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
//......    
    //100ms一次，如果是哨兵模式则运行哨兵的时间事件
    run_with_period(100) {
        if (server.sentinel_mode) sentinelTimer();
    }
//......
}

void sentinelTimer(void) {
    // 前置检查事件定期任务是否因为系统负载过大或者各种原因导致时钟回拨，或者处理过长，进入tilt模式，该模式哨兵只会定期发送和接收命令
    sentinelCheckTiltCondition();
    //监听的master节点作为参数传入，进行逐个通信处理
    sentinelHandleDictOfRedisInstances(sentinel.masters);
    //......
    //随机调整执行频率避免同时执行，确保提高选举一次性成功的概率
    server.hz = REDIS_DEFAULT_HZ + rand() % REDIS_DEFAULT_HZ;
}

void sentinelHandleDictOfRedisInstances(dict *instances) {
     //.......
    //迭代出每一个master实例再对主节点进行处理
    di = dictGetIterator(instances);
    while((de = dictNext(di)) != NULL) {
        sentinelRedisInstance *ri = dictGetVal(de);
        //迭代并处理每一个master实例
        sentinelHandleRedisInstance(ri);
        //.......
    }
    //.......
}

//这个入参包含恰哨兵实例和当前主节点的从节点信息
void sentinelHandleRedisInstance(sentinelRedisInstance *ri) {
    /* ========== MONITORING HALF ============ */
    /* Every kind of instance */
    //1. 尝试和断连的实例重新建立连接
    sentinelReconnectInstance(ri);
    //2. 向实例发送ping和info等命令
    sentinelSendPeriodicCommands(ri);

  //......

    /* Every kind of instance */
    //3. 主观判断是否下线
    sentinelCheckSubjectivelyDown(ri);

    /* Masters and slaves */
    if (ri->flags & (SRI_MASTER|SRI_SLAVE)) {
        /* Nothing so far. */
    }

    /* Only masters */
    if (ri->flags & SRI_MASTER) {
        //4. 检查其当前是否客观下线
        sentinelCheckObjectivelyDown(ri);
        //5.  判断是否要进行故障切换，如果要启动故障切换，则获取其他哨兵对于该节点的判断
        if (sentinelStartFailoverIfNeeded(ri))
            sentinelAskMasterStateToOtherSentinels(ri,SENTINEL_ASK_FORCED);
        //6. 执行故障切换
        sentinelFailoverStateMachine(ri);
        //7. 再次获取哨兵实例对主节点状态的判断
        sentinelAskMasterStateToOtherSentinels(ri,SENTINEL_NO_FLAGS);
    }
}

typedef struct sentinelRedisInstance {
    //......
    //异步发送命令的连接
    redisAsyncContext *cc; /* Hiredis context for commands. */
    //pub/sub发送通道，用于处理频道消息的收发
    redisAsyncContext *pc; 
 //......
}

void sentinelReconnectInstance(sentinelRedisInstance *ri) {
    if (!(ri->flags & SRI_DISCONNECTED)) return;

    /* Commands connection. */
    //如果命令指针cc为空，则进行一次异步重连
    if (ri->cc == NULL) {
     //基于外部遍历传入的master指针进行异步重连
        ri->cc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,REDIS_BIND_ADDR);
        //如果连接失败则调用sentinelKillLink销毁该连接
        if (ri->cc->err) {
            sentinelEvent(REDIS_DEBUG,"-cmd-link-reconnection",ri,"%@ #%s",
                ri->cc->errstr);
            sentinelKillLink(ri,ri->cc);
        } else {
          //......
        }
    }
   
    //检查发布订阅pc,如果为空则将外部传入的master信息通过异步的方式重新和频道建立连接
    if ((ri->flags & (SRI_MASTER|SRI_SLAVE)) && ri->pc == NULL) {
        ri->pc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,REDIS_BIND_ADDR);
        if (ri->pc->err) {
            sentinelEvent(REDIS_DEBUG,"-pubsub-link-reconnection",ri,"%@ #%s",
                ri->pc->errstr);
            sentinelKillLink(ri,ri->pc);
        } else {
            //......
          
        }
    }
  //......
}

void sentinelSendPeriodicCommands(sentinelRedisInstance *ri) {
    //......
 //和其他哨兵处理的逻辑
    if ((ri->flags & SRI_SENTINEL) == 0 &&
        (ri->info_refresh == 0 ||
        (now - ri->info_refresh) > info_period))
    {
       //......
    } else if ((now - ri->last_pong_time) > ping_period) {//超过ping间隔发ping
       
        sentinelSendPing(ri);
    } else if ((now - ri->last_pub_time) > SENTINEL_PUBLISH_PERIOD) {//超过pub间隔通过cc发送当前哨兵个人信息
        sentinelSendHello(ri);
    }
}

int sentinelSendPing(sentinelRedisInstance *ri) {
 //通过cc异步命令接口发送ping
    int retval = redisAsyncCommand(ri->cc,
        sentinelPingReplyCallback, NULL, "PING");
      //如果得到正常响应则更新last_ping_time 
    if (retval == REDIS_OK) {
        ri->pending_commands++;
       
        if (ri->last_ping_time == 0) ri->last_ping_time = mstime();
        return 1;
    } else {
        return 0;
    }
}

int sentinelSendHello(sentinelRedisInstance *ri) {
   //......

    /* Format and send the Hello message. */
    //将哨兵ip 端口以及master地址信息数据拼接到payload中
    snprintf(payload,sizeof(payload),
        "%s,%d,%s,%llu," /* Info about this sentinel. */
        "%s,%s,%d,%llu", /* Info about current master. */
        announce_ip, announce_port, server.runid,
        (unsigned long long) sentinel.current_epoch,
        /* --- */
        master->name,master_addr->ip,master_addr->port,
        (unsigned long long) master->config_epoch);
        //通过cc异步发送到__sentinel__:hello频道中
    retval = redisAsyncCommand(ri->cc,
        sentinelPublishReplyCallback, NULL, "PUBLISH %s %s",
            SENTINEL_HELLO_CHANNEL,payload);
    if (retval != REDIS_OK) return REDIS_ERR;
    ri->pending_commands++;
    return REDIS_OK;
}

void sentinelCheckSubjectivelyDown(sentinelRedisInstance *ri) {
   //......
   
    //如果上一次到现在的间隔elapsed 大于down_after_period ，则当前哨兵会主观认定其下线
    if (elapsed > ri->down_after_period ||
    //或者当前哨兵认定它是master而其他报告长时间的反馈都是从节点，则当前哨兵会主观认定其下线
        (ri->flags & SRI_MASTER &&
         ri->role_reported == SRI_SLAVE &&
         mstime() - ri->role_reported_time >
          (ri->down_after_period+SENTINEL_INFO_PERIOD*2)))
    {
       
        //通过 +sdown这个channel 发送主观下线的消息
        if ((ri->flags & SRI_S_DOWN) == 0) {
            sentinelEvent(REDIS_WARNING,"+sdown",ri,"%@");
            //设置当前监控的master实例为主观下线
            ri->s_down_since_time = mstime();
            ri->flags |= SRI_S_DOWN;
        }
    } else {
       //......
    }
}