snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/balancer/outlierdetection/balancer.go

@@ -0,0 +1,927 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package outlierdetection provides an implementation of the outlier detection
+// LB policy, as defined in
+// https://github.com/grpc/proposal/blob/master/A50-xds-outlier-detection.md.
+package outlierdetection
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	rand "math/rand/v2"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/internal/balancer/gracefulswitch"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/channelz"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Globals to stub out in tests.
+var (
+	afterFunc = time.AfterFunc
+	now       = time.Now
+)
+
+// Name is the name of the outlier detection balancer.
+const Name = "outlier_detection_experimental"
+
+var (
+	ejectionsEnforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:        "grpc.lb.outlier_detection.ejections_enforced",
+		Description: "EXPERIMENTAL. Number of outlier ejections enforced by detection method",
+		Unit:        "{ejection}",
+		Labels:      []string{"grpc.target", "grpc.lb.outlier_detection.detection_method"},
+		Default:     false,
+	})
+
+	ejectionsUnenforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:        "grpc.lb.outlier_detection.ejections_unenforced",
+		Description: "EXPERIMENTAL. Number of unenforced outlier ejections due to either `max_ejection_percentage` or `enforcement_percentage`",
+		Unit:        "{ejection}",
+		Labels:      []string{"grpc.target", "grpc.lb.outlier_detection.detection_method", "grpc.lb.outlier_detection.unenforced_reason"},
+		Default:     false,
+	})
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &outlierDetectionBalancer{
+		ClientConn:      cc,
+		closed:          grpcsync.NewEvent(),
+		done:            grpcsync.NewEvent(),
+		addrs:           make(map[string]*endpointInfo),
+		scUpdateCh:      buffer.NewUnbounded[any](),
+		pickerUpdateCh:  buffer.NewUnbounded[any](),
+		channelzParent:  bOpts.ChannelzParent,
+		endpoints:       resolver.NewEndpointMap[*endpointInfo](),
+		metricsRecorder: cc.MetricsRecorder(), // we use an explicit field instead of using cc.MetricsRecorder() so we can override the metric recorder in tests.
+		target:          bOpts.Target.String(),
+	}
+	b.logger = prefixLogger(b)
+	b.logger.Infof("Created")
+	b.child = synchronizingBalancerWrapper{lb: gracefulswitch.NewBalancer(b, bOpts)}
+	go b.run()
+	return b
+}
+
+func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	lbCfg := &LBConfig{
+		// Default top layer values as documented in A50.
+		Interval:           iserviceconfig.Duration(10 * time.Second),
+		BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+		MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+		MaxEjectionPercent: 10,
+	}
+
+	// This unmarshalling handles underlying layers sre and fpe which have their
+	// own defaults for their fields if either sre or fpe are present.
+	if err := json.Unmarshal(s, lbCfg); err != nil { // Validates child config if present as well.
+		return nil, fmt.Errorf("xds: unable to unmarshal LBconfig: %s, error: %v", string(s), err)
+	}
+
+	// Note: in the xds flow, these validations will never fail. The xdsclient
+	// performs the same validations as here on the xds Outlier Detection
+	// resource before parsing resource into JSON which this function gets
+	// called with. A50 defines two separate places for these validations to
+	// take place, the xdsclient and this ParseConfig method. "When parsing a
+	// config from JSON, if any of these requirements is violated, that should
+	// be treated as a parsing error." - A50
+	switch {
+	// "The google.protobuf.Duration fields interval, base_ejection_time, and
+	// max_ejection_time must obey the restrictions in the
+	// google.protobuf.Duration documentation and they must have non-negative
+	// values." - A50
+	// Approximately 290 years is the maximum time that time.Duration (int64)
+	// can represent. The restrictions on the protobuf.Duration field are to be
+	// within +-10000 years. Thus, just check for negative values.
+	case lbCfg.Interval < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.interval = %s; must be >= 0", lbCfg.Interval)
+	case lbCfg.BaseEjectionTime < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.base_ejection_time = %s; must be >= 0", lbCfg.BaseEjectionTime)
+	case lbCfg.MaxEjectionTime < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_time = %s; must be >= 0", lbCfg.MaxEjectionTime)
+
+	// "The fields max_ejection_percent,
+	// success_rate_ejection.enforcement_percentage,
+	// failure_percentage_ejection.threshold, and
+	// failure_percentage.enforcement_percentage must have values less than or
+	// equal to 100." - A50
+	case lbCfg.MaxEjectionPercent > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_percent = %v; must be <= 100", lbCfg.MaxEjectionPercent)
+	case lbCfg.SuccessRateEjection != nil && lbCfg.SuccessRateEjection.EnforcementPercentage > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = %v; must be <= 100", lbCfg.SuccessRateEjection.EnforcementPercentage)
+	case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.Threshold > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = %v; must be <= 100", lbCfg.FailurePercentageEjection.Threshold)
+	case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.EnforcementPercentage > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = %v; must be <= 100", lbCfg.FailurePercentageEjection.EnforcementPercentage)
+	}
+	return lbCfg, nil
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+// scUpdate wraps a subConn update to be sent to the child balancer.
+type scUpdate struct {
+	scw   *subConnWrapper
+	state balancer.SubConnState
+}
+
+type ejectionUpdate struct {
+	scw       *subConnWrapper
+	isEjected bool // true for ejected, false for unejected
+}
+
+type lbCfgUpdate struct {
+	lbCfg *LBConfig
+	// to make sure picker is updated synchronously.
+	done chan struct{}
+}
+
+type scHealthUpdate struct {
+	scw   *subConnWrapper
+	state balancer.SubConnState
+}
+
+type outlierDetectionBalancer struct {
+	balancer.ClientConn
+	// These fields are safe to be accessed without holding any mutex because
+	// they are synchronized in run(), which makes these field accesses happen
+	// serially.
+	//
+	// childState is the latest balancer state received from the child.
+	childState balancer.State
+	// recentPickerNoop represents whether the most recent picker sent upward to
+	// the balancer.ClientConn is a noop picker, which doesn't count RPC's. Used
+	// to suppress redundant picker updates.
+	recentPickerNoop bool
+
+	closed          *grpcsync.Event
+	done            *grpcsync.Event
+	logger          *grpclog.PrefixLogger
+	channelzParent  channelz.Identifier
+	metricsRecorder estats.MetricsRecorder
+	target          string
+
+	child synchronizingBalancerWrapper
+
+	// mu guards access to the following fields. It also helps to synchronize
+	// behaviors of the following events: config updates, firing of the interval
+	// timer, SubConn State updates, SubConn address updates, and child state
+	// updates.
+	//
+	// For example, when we receive a config update in the middle of the
+	// interval timer algorithm, which uses knobs present in the config, the
+	// balancer will wait for the interval timer algorithm to finish before
+	// persisting the new configuration.
+	//
+	// Another example would be the updating of the endpoints or addrs map, such
+	// as from a SubConn address update in the middle of the interval timer
+	// algorithm which uses endpoints. This balancer waits for the interval
+	// timer algorithm to finish before making the update to the endpoints map.
+	//
+	// This mutex is never held when calling methods on the child policy
+	// (within the context of a single goroutine).
+	mu sync.Mutex
+	// endpoints stores pointers to endpointInfo objects for each endpoint.
+	endpoints *resolver.EndpointMap[*endpointInfo]
+	// addrs stores pointers to endpointInfo objects for each address. Addresses
+	// belonging to the same endpoint point to the same object.
+	addrs                 map[string]*endpointInfo
+	cfg                   *LBConfig
+	timerStartTime        time.Time
+	intervalTimer         *time.Timer
+	inhibitPickerUpdates  bool
+	updateUnconditionally bool
+	numEndpointsEjected   int // For fast calculations of percentage of endpoints ejected
+
+	scUpdateCh     *buffer.Unbounded[any]
+	pickerUpdateCh *buffer.Unbounded[any]
+}
+
+// noopConfig returns whether this balancer is configured with a logical no-op
+// configuration or not.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) noopConfig() bool {
+	return b.cfg.SuccessRateEjection == nil && b.cfg.FailurePercentageEjection == nil
+}
+
+// onIntervalConfig handles logic required specifically on the receipt of a
+// configuration which specifies to count RPC's and periodically perform passive
+// health checking based on heuristics defined in configuration every configured
+// interval.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) onIntervalConfig() {
+	var interval time.Duration
+	if b.timerStartTime.IsZero() {
+		b.timerStartTime = time.Now()
+		for _, epInfo := range b.endpoints.All() {
+			epInfo.callCounter.clear()
+		}
+		interval = time.Duration(b.cfg.Interval)
+	} else {
+		interval = time.Duration(b.cfg.Interval) - now().Sub(b.timerStartTime)
+		if interval < 0 {
+			interval = 0
+		}
+	}
+	b.intervalTimer = afterFunc(interval, b.intervalTimerAlgorithm)
+}
+
+// onNoopConfig handles logic required specifically on the receipt of a
+// configuration which specifies the balancer to be a noop.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) onNoopConfig() {
+	// "If a config is provided with both the `success_rate_ejection` and
+	// `failure_percentage_ejection` fields unset, skip starting the timer and
+	// do the following:"
+	// "Unset the timer start timestamp."
+	b.timerStartTime = time.Time{}
+	for _, epInfo := range b.endpoints.All() {
+		// "Uneject all currently ejected endpoints."
+		if !epInfo.latestEjectionTimestamp.IsZero() {
+			b.unejectEndpoint(epInfo)
+		}
+		// "Reset each endpoint's ejection time multiplier to 0."
+		epInfo.ejectionTimeMultiplier = 0
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	lbCfg, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		b.logger.Errorf("received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig)
+		return balancer.ErrBadResolverState
+	}
+
+	// Reject whole config if child policy doesn't exist, don't persist it for
+	// later.
+	bb := balancer.Get(lbCfg.ChildPolicy.Name)
+	if bb == nil {
+		return fmt.Errorf("outlier detection: child balancer %q not registered", lbCfg.ChildPolicy.Name)
+	}
+
+	// It is safe to read b.cfg here without holding the mutex, as the only
+	// write to b.cfg happens later in this function. This function is part of
+	// the balancer.Balancer API, so it is guaranteed to be called in a
+	// synchronous manner, so it cannot race with this read.
+	if b.cfg == nil || b.cfg.ChildPolicy.Name != lbCfg.ChildPolicy.Name {
+		if err := b.child.switchTo(bb); err != nil {
+			return fmt.Errorf("outlier detection: error switching to child of type %q: %v", lbCfg.ChildPolicy.Name, err)
+		}
+	}
+
+	b.mu.Lock()
+	// Inhibit child picker updates until this UpdateClientConnState() call
+	// completes. If needed, a picker update containing the no-op config bit
+	// determined from this config and most recent state from the child will be
+	// sent synchronously upward at the end of this UpdateClientConnState()
+	// call.
+	b.inhibitPickerUpdates = true
+	b.updateUnconditionally = false
+	b.cfg = lbCfg
+
+	newEndpoints := resolver.NewEndpointMap[bool]()
+	for _, ep := range s.ResolverState.Endpoints {
+		newEndpoints.Set(ep, true)
+		if _, ok := b.endpoints.Get(ep); !ok {
+			b.endpoints.Set(ep, newEndpointInfo())
+		}
+	}
+
+	for ep := range b.endpoints.All() {
+		if _, ok := newEndpoints.Get(ep); !ok {
+			b.endpoints.Delete(ep)
+		}
+	}
+
+	// populate the addrs map.
+	b.addrs = map[string]*endpointInfo{}
+	for _, ep := range s.ResolverState.Endpoints {
+		epInfo, _ := b.endpoints.Get(ep)
+		for _, addr := range ep.Addresses {
+			if _, ok := b.addrs[addr.Addr]; ok {
+				b.logger.Errorf("Endpoints contain duplicate address %q", addr.Addr)
+				continue
+			}
+			b.addrs[addr.Addr] = epInfo
+		}
+	}
+
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+
+	if b.noopConfig() {
+		b.onNoopConfig()
+	} else {
+		b.onIntervalConfig()
+	}
+	b.mu.Unlock()
+
+	err := b.child.updateClientConnState(balancer.ClientConnState{
+		ResolverState:  s.ResolverState,
+		BalancerConfig: b.cfg.ChildPolicy.Config,
+	})
+
+	done := make(chan struct{})
+	b.pickerUpdateCh.Put(lbCfgUpdate{
+		lbCfg: lbCfg,
+		done:  done,
+	})
+	<-done
+
+	return err
+}
+
+func (b *outlierDetectionBalancer) ResolverError(err error) {
+	b.child.resolverError(err)
+}
+
+func (b *outlierDetectionBalancer) updateSubConnState(scw *subConnWrapper, state balancer.SubConnState) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	scw.setLatestConnectivityState(state.ConnectivityState)
+	b.scUpdateCh.Put(&scUpdate{
+		scw:   scw,
+		state: state,
+	})
+}
+
+func (b *outlierDetectionBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *outlierDetectionBalancer) Close() {
+	b.closed.Fire()
+	<-b.done.Done()
+	b.child.closeLB()
+
+	b.scUpdateCh.Close()
+	b.pickerUpdateCh.Close()
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+}
+
+func (b *outlierDetectionBalancer) ExitIdle() {
+	b.child.exitIdle()
+}
+
+// wrappedPicker delegates to the child policy's picker, and when the request
+// finishes, it increments the corresponding counter in the map entry referenced
+// by the subConnWrapper that was picked. If both the `success_rate_ejection`
+// and `failure_percentage_ejection` fields are unset in the configuration, this
+// picker will not count.
+type wrappedPicker struct {
+	childPicker balancer.Picker
+	noopPicker  bool
+}
+
+func (wp *wrappedPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	pr, err := wp.childPicker.Pick(info)
+	if err != nil {
+		return balancer.PickResult{}, err
+	}
+
+	done := func(di balancer.DoneInfo) {
+		if !wp.noopPicker {
+			incrementCounter(pr.SubConn, di)
+		}
+		if pr.Done != nil {
+			pr.Done(di)
+		}
+	}
+	scw, ok := pr.SubConn.(*subConnWrapper)
+	if !ok {
+		// This can never happen, but check is present for defensive
+		// programming.
+		logger.Errorf("Picked SubConn from child picker is not a SubConnWrapper")
+		return balancer.PickResult{
+			SubConn:  pr.SubConn,
+			Done:     done,
+			Metadata: pr.Metadata,
+		}, nil
+	}
+	return balancer.PickResult{
+		SubConn:  scw.SubConn,
+		Done:     done,
+		Metadata: pr.Metadata,
+	}, nil
+}
+
+func incrementCounter(sc balancer.SubConn, info balancer.DoneInfo) {
+	scw, ok := sc.(*subConnWrapper)
+	if !ok {
+		// Shouldn't happen, as comes from child
+		return
+	}
+
+	// After reading callCounter.activeBucket in this picker a swap call can
+	// concurrently change what activeBucket points to. A50 says to swap the
+	// pointer, which will cause this race to write to deprecated memory the
+	// interval timer algorithm will never read, which makes this race alright.
+	epInfo := scw.endpointInfo
+	if epInfo == nil {
+		return
+	}
+	ab := epInfo.callCounter.activeBucket.Load()
+
+	if info.Err == nil {
+		atomic.AddUint32(&ab.numSuccesses, 1)
+	} else {
+		atomic.AddUint32(&ab.numFailures, 1)
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateState(s balancer.State) {
+	b.pickerUpdateCh.Put(s)
+}
+
+func (b *outlierDetectionBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	oldListener := opts.StateListener
+	scw := &subConnWrapper{
+		addresses:         addrs,
+		scUpdateCh:        b.scUpdateCh,
+		listener:          oldListener,
+		latestHealthState: balancer.SubConnState{ConnectivityState: connectivity.Connecting},
+	}
+	opts.StateListener = func(state balancer.SubConnState) { b.updateSubConnState(scw, state) }
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	sc, err := b.ClientConn.NewSubConn(addrs, opts)
+	if err != nil {
+		return nil, err
+	}
+	scw.SubConn = sc
+	if len(addrs) != 1 {
+		return scw, nil
+	}
+	epInfo, ok := b.addrs[addrs[0].Addr]
+	if !ok {
+		return scw, nil
+	}
+	epInfo.sws = append(epInfo.sws, scw)
+	scw.endpointInfo = epInfo
+	if !epInfo.latestEjectionTimestamp.IsZero() {
+		scw.eject()
+	}
+	return scw, nil
+}
+
+func (b *outlierDetectionBalancer) RemoveSubConn(sc balancer.SubConn) {
+	b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc)
+}
+
+// removeSubConnFromEndpointMapEntry removes the scw from its map entry if
+// present.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) removeSubConnFromEndpointMapEntry(scw *subConnWrapper) {
+	epInfo := scw.endpointInfo
+	if epInfo == nil {
+		return
+	}
+	for i, sw := range epInfo.sws {
+		if scw == sw {
+			epInfo.sws = append(epInfo.sws[:i], epInfo.sws[i+1:]...)
+			return
+		}
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) {
+	b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc)
+}
+
+// handleSubConnUpdate stores the recent state and forward the update.
+func (b *outlierDetectionBalancer) handleSubConnUpdate(u *scUpdate) {
+	scw := u.scw
+	scw.clearHealthListener()
+	b.child.updateSubConnState(scw, u.state)
+	if u.state.ConnectivityState == connectivity.Shutdown {
+		b.mu.Lock()
+		b.removeSubConnFromEndpointMapEntry(scw)
+		b.mu.Unlock()
+	}
+}
+
+func (b *outlierDetectionBalancer) handleSubConnHealthUpdate(u *scHealthUpdate) {
+	b.child.updateSubConnHealthState(u.scw, u.state)
+}
+
+// handleEjectedUpdate handles any SubConns that get ejected/unejected, and
+// forwards the appropriate corresponding subConnState to the child policy.
+func (b *outlierDetectionBalancer) handleEjectedUpdate(u *ejectionUpdate) {
+	b.child.handleEjectionUpdate(u)
+}
+
+// handleChildStateUpdate forwards the picker update wrapped in a wrapped picker
+// with the noop picker bit present.
+func (b *outlierDetectionBalancer) handleChildStateUpdate(u balancer.State) {
+	b.childState = u
+	b.mu.Lock()
+	if b.inhibitPickerUpdates {
+		// If a child's state is updated during the suppression of child
+		// updates, the synchronous handleLBConfigUpdate function with respect
+		// to UpdateClientConnState should return a picker unconditionally.
+		b.updateUnconditionally = true
+		b.mu.Unlock()
+		return
+	}
+	noopCfg := b.noopConfig()
+	b.mu.Unlock()
+	b.recentPickerNoop = noopCfg
+	b.ClientConn.UpdateState(balancer.State{
+		ConnectivityState: b.childState.ConnectivityState,
+		Picker: &wrappedPicker{
+			childPicker: b.childState.Picker,
+			noopPicker:  noopCfg,
+		},
+	})
+}
+
+// handleLBConfigUpdate compares whether the new config is a noop config or not,
+// to the noop bit in the picker if present. It updates the picker if this bit
+// changed compared to the picker currently in use.
+func (b *outlierDetectionBalancer) handleLBConfigUpdate(u lbCfgUpdate) {
+	lbCfg := u.lbCfg
+	noopCfg := lbCfg.SuccessRateEjection == nil && lbCfg.FailurePercentageEjection == nil
+	// If the child has sent its first update and this config flips the noop
+	// bit compared to the most recent picker update sent upward, then a new
+	// picker with this updated bit needs to be forwarded upward. If a child
+	// update was received during the suppression of child updates within
+	// UpdateClientConnState(), then a new picker needs to be forwarded with
+	// this updated state, irregardless of whether this new configuration flips
+	// the bit.
+	if b.childState.Picker != nil && noopCfg != b.recentPickerNoop || b.updateUnconditionally {
+		b.recentPickerNoop = noopCfg
+		b.ClientConn.UpdateState(balancer.State{
+			ConnectivityState: b.childState.ConnectivityState,
+			Picker: &wrappedPicker{
+				childPicker: b.childState.Picker,
+				noopPicker:  noopCfg,
+			},
+		})
+	}
+	b.inhibitPickerUpdates = false
+	b.updateUnconditionally = false
+	close(u.done)
+}
+
+func (b *outlierDetectionBalancer) run() {
+	defer b.done.Fire()
+	for {
+		select {
+		case update, ok := <-b.scUpdateCh.Get():
+			if !ok {
+				return
+			}
+			b.scUpdateCh.Load()
+			if b.closed.HasFired() { // don't send SubConn updates to child after the balancer has been closed
+				return
+			}
+			switch u := update.(type) {
+			case *scUpdate:
+				b.handleSubConnUpdate(u)
+			case *ejectionUpdate:
+				b.handleEjectedUpdate(u)
+			case *scHealthUpdate:
+				b.handleSubConnHealthUpdate(u)
+			}
+		case update, ok := <-b.pickerUpdateCh.Get():
+			if !ok {
+				return
+			}
+			b.pickerUpdateCh.Load()
+			if b.closed.HasFired() { // don't send picker updates to grpc after the balancer has been closed
+				return
+			}
+			switch u := update.(type) {
+			case balancer.State:
+				b.handleChildStateUpdate(u)
+			case lbCfgUpdate:
+				b.handleLBConfigUpdate(u)
+			}
+		case <-b.closed.Done():
+			return
+		}
+	}
+}
+
+// intervalTimerAlgorithm ejects and unejects endpoints based on the Outlier
+// Detection configuration and data about each endpoint from the previous
+// interval.
+func (b *outlierDetectionBalancer) intervalTimerAlgorithm() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.timerStartTime = time.Now()
+
+	for _, epInfo := range b.endpoints.All() {
+		epInfo.callCounter.swap()
+	}
+
+	if b.cfg.SuccessRateEjection != nil {
+		b.successRateAlgorithm()
+	}
+
+	if b.cfg.FailurePercentageEjection != nil {
+		b.failurePercentageAlgorithm()
+	}
+
+	for _, epInfo := range b.endpoints.All() {
+		if epInfo.latestEjectionTimestamp.IsZero() && epInfo.ejectionTimeMultiplier > 0 {
+			epInfo.ejectionTimeMultiplier--
+			continue
+		}
+		if epInfo.latestEjectionTimestamp.IsZero() {
+			// Endpoint is already not ejected, so no need to check for whether
+			// to uneject the endpoint below.
+			continue
+		}
+		et := time.Duration(b.cfg.BaseEjectionTime) * time.Duration(epInfo.ejectionTimeMultiplier)
+		met := max(time.Duration(b.cfg.BaseEjectionTime), time.Duration(b.cfg.MaxEjectionTime))
+		uet := epInfo.latestEjectionTimestamp.Add(min(et, met))
+		if now().After(uet) {
+			b.unejectEndpoint(epInfo)
+		}
+	}
+
+	// This conditional only for testing (since the interval timer algorithm is
+	// called manually), will never hit in production.
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+	b.intervalTimer = afterFunc(time.Duration(b.cfg.Interval), b.intervalTimerAlgorithm)
+}
+
+// endpointsWithAtLeastRequestVolume returns a slice of endpoint information of
+// all endpoints with at least request volume passed in.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) endpointsWithAtLeastRequestVolume(requestVolume uint32) []*endpointInfo {
+	var endpoints []*endpointInfo
+	for _, epInfo := range b.endpoints.All() {
+		bucket1 := epInfo.callCounter.inactiveBucket
+		rv := bucket1.numSuccesses + bucket1.numFailures
+		if rv >= requestVolume {
+			endpoints = append(endpoints, epInfo)
+		}
+	}
+	return endpoints
+}
+
+// meanAndStdDev returns the mean and std dev of the fractions of successful
+// requests of the endpoints passed in.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) meanAndStdDev(endpoints []*endpointInfo) (float64, float64) {
+	var totalFractionOfSuccessfulRequests float64
+	var mean float64
+	for _, epInfo := range endpoints {
+		bucket := epInfo.callCounter.inactiveBucket
+		rv := bucket.numSuccesses + bucket.numFailures
+		totalFractionOfSuccessfulRequests += float64(bucket.numSuccesses) / float64(rv)
+	}
+	mean = totalFractionOfSuccessfulRequests / float64(len(endpoints))
+	var sumOfSquares float64
+	for _, epInfo := range endpoints {
+		bucket := epInfo.callCounter.inactiveBucket
+		rv := bucket.numSuccesses + bucket.numFailures
+		devFromMean := (float64(bucket.numSuccesses) / float64(rv)) - mean
+		sumOfSquares += devFromMean * devFromMean
+	}
+	variance := sumOfSquares / float64(len(endpoints))
+	return mean, math.Sqrt(variance)
+}
+
+// successRateAlgorithm ejects any endpoints where the success rate falls below
+// the other endpoints according to mean and standard deviation, and if overall
+// applicable from other set heuristics.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) successRateAlgorithm() {
+	endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.SuccessRateEjection.RequestVolume)
+	if len(endpointsToConsider) < int(b.cfg.SuccessRateEjection.MinimumHosts) {
+		return
+	}
+	mean, stddev := b.meanAndStdDev(endpointsToConsider)
+	ejectionCfg := b.cfg.SuccessRateEjection
+	for _, epInfo := range endpointsToConsider {
+		bucket := epInfo.callCounter.inactiveBucket
+		successRate := float64(bucket.numSuccesses) / float64(bucket.numSuccesses+bucket.numFailures)
+		requiredSuccessRate := mean - stddev*(float64(ejectionCfg.StdevFactor)/1000)
+		if successRate < requiredSuccessRate {
+			channelz.Infof(logger, b.channelzParent, "SuccessRate algorithm detected outlier: %s. Parameters: successRate=%f, mean=%f, stddev=%f, requiredSuccessRate=%f", epInfo, successRate, mean, stddev, requiredSuccessRate)
+			// Check if max ejection percentage would prevent ejection.
+			if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) {
+				// Record unenforced ejection due to max ejection percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "max_ejection_overflow")
+				continue
+			}
+			if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage {
+				b.ejectEndpoint(epInfo, "success_rate")
+			} else {
+				// Record unenforced ejection due to enforcement percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "enforcement_percentage")
+			}
+		}
+	}
+}
+
+// failurePercentageAlgorithm ejects any endpoints where the failure percentage
+// rate exceeds a set enforcement percentage, if overall applicable from other
+// set heuristics.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) failurePercentageAlgorithm() {
+	endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.FailurePercentageEjection.RequestVolume)
+	if len(endpointsToConsider) < int(b.cfg.FailurePercentageEjection.MinimumHosts) {
+		return
+	}
+
+	ejectionCfg := b.cfg.FailurePercentageEjection
+	for _, epInfo := range endpointsToConsider {
+		bucket := epInfo.callCounter.inactiveBucket
+		failurePercentage := (float64(bucket.numFailures) / float64(bucket.numSuccesses+bucket.numFailures)) * 100
+		if failurePercentage > float64(b.cfg.FailurePercentageEjection.Threshold) {
+			channelz.Infof(logger, b.channelzParent, "FailurePercentage algorithm detected outlier: %s, failurePercentage=%f", epInfo, failurePercentage)
+			// Check if max ejection percentage would prevent ejection.
+			if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) {
+				// Record unenforced ejection due to max ejection percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "max_ejection_overflow")
+				continue
+			}
+			if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage {
+				b.ejectEndpoint(epInfo, "failure_percentage")
+			} else {
+				// Record unenforced ejection due to enforcement percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "enforcement_percentage")
+			}
+		}
+	}
+}
+
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) ejectEndpoint(epInfo *endpointInfo, detectionMethod string) {
+	b.numEndpointsEjected++
+	epInfo.latestEjectionTimestamp = b.timerStartTime
+	epInfo.ejectionTimeMultiplier++
+	for _, sbw := range epInfo.sws {
+		sbw.eject()
+		channelz.Infof(logger, b.channelzParent, "Subchannel ejected: %s", sbw)
+	}
+
+	// Record the enforced ejection metric.
+	ejectionsEnforcedMetric.Record(b.metricsRecorder, 1, b.target, detectionMethod)
+}
+
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) unejectEndpoint(epInfo *endpointInfo) {
+	b.numEndpointsEjected--
+	epInfo.latestEjectionTimestamp = time.Time{}
+	for _, sbw := range epInfo.sws {
+		sbw.uneject()
+		channelz.Infof(logger, b.channelzParent, "Subchannel unejected: %s", sbw)
+	}
+}
+
+// synchronizingBalancerWrapper serializes calls into balancer (to uphold the
+// balancer.Balancer API guarantee of synchronous calls). It also ensures a
+// consistent order of locking mutexes when using SubConn listeners to avoid
+// deadlocks.
+type synchronizingBalancerWrapper struct {
+	// mu should not be used directly from outside this struct, instead use
+	// methods defined on the struct.
+	mu sync.Mutex
+	lb *gracefulswitch.Balancer
+}
+
+func (sbw *synchronizingBalancerWrapper) switchTo(builder balancer.Builder) error {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	return sbw.lb.SwitchTo(builder)
+}
+
+func (sbw *synchronizingBalancerWrapper) updateClientConnState(state balancer.ClientConnState) error {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	return sbw.lb.UpdateClientConnState(state)
+}
+
+func (sbw *synchronizingBalancerWrapper) resolverError(err error) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.ResolverError(err)
+}
+
+func (sbw *synchronizingBalancerWrapper) closeLB() {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.Close()
+}
+
+func (sbw *synchronizingBalancerWrapper) exitIdle() {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.ExitIdle()
+}
+
+func (sbw *synchronizingBalancerWrapper) updateSubConnHealthState(scw *subConnWrapper, scs balancer.SubConnState) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	scw.updateSubConnHealthState(scs)
+}
+
+func (sbw *synchronizingBalancerWrapper) updateSubConnState(scw *subConnWrapper, scs balancer.SubConnState) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	scw.updateSubConnConnectivityState(scs)
+}
+
+func (sbw *synchronizingBalancerWrapper) handleEjectionUpdate(u *ejectionUpdate) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	if u.isEjected {
+		u.scw.handleEjection()
+	} else {
+		u.scw.handleUnejection()
+	}
+}
+
+// endpointInfo contains the runtime information about an endpoint that pertains
+// to Outlier Detection. This struct and all of its fields is protected by
+// outlierDetectionBalancer.mu in the case where it is accessed through the
+// address or endpoint map. In the case of Picker callbacks, the writes to the
+// activeBucket of callCounter are protected by atomically loading and storing
+// unsafe.Pointers (see further explanation in incrementCounter()).
+type endpointInfo struct {
+	// The call result counter object.
+	callCounter *callCounter
+
+	// The latest ejection timestamp, or zero if the endpoint is currently not
+	// ejected.
+	latestEjectionTimestamp time.Time
+
+	// The current ejection time multiplier, starting at 0.
+	ejectionTimeMultiplier int64
+
+	// A list of subchannel wrapper objects that correspond to this endpoint.
+	sws []*subConnWrapper
+}
+
+func (a *endpointInfo) String() string {
+	var res strings.Builder
+	res.WriteString("[")
+	for _, sw := range a.sws {
+		res.WriteString(sw.String())
+	}
+	res.WriteString("]")
+	return res.String()
+}
+
+func newEndpointInfo() *endpointInfo {
+	return &endpointInfo{
+		callCounter: newCallCounter(),
+	}
+}

internal/xds/balancer/outlierdetection/balancer_ext_test.go

@@ -0,0 +1,518 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package outlierdetection_test
+
+import (
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// TestParseConfig verifies the ParseConfig() method in the Outlier Detection
+// Balancer.
+func (s) TestParseConfig(t *testing.T) {
+	const errParseConfigName = "errParseConfigBalancer"
+	stub.Register(errParseConfigName, stub.BalancerFuncs{
+		ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return nil, errors.New("some error")
+		},
+	})
+
+	parser := balancer.Get(outlierdetection.Name).(balancer.ConfigParser)
+	const (
+		defaultInterval                       = iserviceconfig.Duration(10 * time.Second)
+		defaultBaseEjectionTime               = iserviceconfig.Duration(30 * time.Second)
+		defaultMaxEjectionTime                = iserviceconfig.Duration(300 * time.Second)
+		defaultMaxEjectionPercent             = 10
+		defaultSuccessRateStdevFactor         = 1900
+		defaultEnforcingSuccessRate           = 100
+		defaultSuccessRateMinimumHosts        = 5
+		defaultSuccessRateRequestVolume       = 100
+		defaultFailurePercentageThreshold     = 85
+		defaultEnforcingFailurePercentage     = 0
+		defaultFailurePercentageMinimumHosts  = 5
+		defaultFailurePercentageRequestVolume = 50
+	)
+	tests := []struct {
+		name    string
+		input   string
+		wantCfg serviceconfig.LoadBalancingConfig
+		wantErr string
+	}{
+		{
+			name: "Default",
+			input: `{
+				"childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+
+		{
+			name: "TopLevelSet",
+			input: `{
+				"interval": "15s",
+				"maxEjectionTime": "350s",
+				"childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for unset fields.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           iserviceconfig.Duration(15 * time.Second),
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    iserviceconfig.Duration(350 * time.Second),
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Defaults",
+			input: `{
+				"successRateEjection": {},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get defaults of success-rate-ejection struct.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           defaultSuccessRateStdevFactor,
+					EnforcementPercentage: defaultEnforcingSuccessRate,
+					MinimumHosts:          defaultSuccessRateMinimumHosts,
+					RequestVolume:         defaultSuccessRateRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Partial",
+			input: `{
+				"successRateEjection": {
+					"stdevFactor": 1000,
+					"minimumHosts": 5
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for others in success rate
+			// ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1000,
+					EnforcementPercentage: defaultEnforcingSuccessRate,
+					MinimumHosts:          5,
+					RequestVolume:         defaultSuccessRateRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Full",
+			input: `{
+				"successRateEjection": {
+					"stdevFactor": 1000,
+					"enforcementPercentage": 50,
+					"minimumHosts": 5,
+					"requestVolume": 50
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1000,
+					EnforcementPercentage: 50,
+					MinimumHosts:          5,
+					RequestVolume:         50,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Defaults",
+			input: `{
+				"failurePercentageEjection": {},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get defaults of failure percentage ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             defaultFailurePercentageThreshold,
+					EnforcementPercentage: defaultEnforcingFailurePercentage,
+					MinimumHosts:          defaultFailurePercentageMinimumHosts,
+					RequestVolume:         defaultFailurePercentageRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Partial",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 80,
+					"minimumHosts": 10
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for others in success rate
+			// ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             80,
+					EnforcementPercentage: defaultEnforcingFailurePercentage,
+					MinimumHosts:          10,
+					RequestVolume:         defaultFailurePercentageRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Full",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 80,
+					"enforcementPercentage": 100,
+					"minimumHosts": 10,
+					"requestVolume": 40
+                },
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             80,
+					EnforcementPercentage: 100,
+					MinimumHosts:          10,
+					RequestVolume:         40,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{ // to make sure zero values aren't overwritten by defaults
+			name: "AllFields_Zero",
+			input: `{
+				"interval": "0s",
+				"baseEjectionTime": "0s",
+				"maxEjectionTime": "0s",
+				"maxEjectionPercent": 0,
+				"successRateEjection": {
+					"stdevFactor": 0,
+					"enforcementPercentage": 0,
+					"minimumHosts": 0,
+					"requestVolume": 0
+				},
+				"failurePercentageEjection": {
+					"threshold": 0,
+					"enforcementPercentage": 0,
+					"minimumHosts": 0,
+					"requestVolume": 0
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				SuccessRateEjection:       &outlierdetection.SuccessRateEjection{},
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "AllFields_Set",
+			input: `{
+				"interval": "10s",
+				"baseEjectionTime": "30s",
+				"maxEjectionTime": "300s",
+				"maxEjectionPercent": 10,
+				"successRateEjection": {
+					"stdevFactor": 1900,
+					"enforcementPercentage": 100,
+					"minimumHosts": 5,
+					"requestVolume": 100
+				},
+				"failurePercentageEjection": {
+					"threshold": 85,
+					"enforcementPercentage": 5,
+					"minimumHosts": 5,
+					"requestVolume": 50
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           iserviceconfig.Duration(10 * time.Second),
+				BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+				MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+				MaxEjectionPercent: 10,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1900,
+					EnforcementPercentage: 100,
+					MinimumHosts:          5,
+					RequestVolume:         100,
+				},
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             85,
+					EnforcementPercentage: 5,
+					MinimumHosts:          5,
+					RequestVolume:         50,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name:    "Interval_Negative",
+			input:   `{"interval": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.interval = -10s; must be >= 0",
+		},
+		{
+			name:    "BaseEjectionTime_Negative",
+			input:   `{"baseEjectionTime": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.base_ejection_time = -10s; must be >= 0",
+		},
+		{
+			name:    "MaxEjectionTime_Negative",
+			input:   `{"maxEjectionTime": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_time = -10s; must be >= 0",
+		},
+		{
+			name:    "MaxEjectionPercent_TooHigh",
+			input:   `{"maxEjectionPercent": 150}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_percent = 150; must be <= 100",
+		},
+		{
+			name: "SuccessRate_Enforcement_TooHigh",
+			input: `{
+				"successRateEjection": {
+					"enforcementPercentage": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = 150; must be <= 100",
+		},
+		{
+			name: "FailurePercentage_Threshold_TooHigh",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = 150; must be <= 100",
+		},
+		{
+			name: "FailurePercentage_Enforcement_TooHigh",
+			input: `{
+				"failurePercentageEjection": {
+					"enforcementPercentage": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = 150; must be <= 100",
+		},
+		{
+			name: "ChildPolicy_ParseError",
+			input: `{
+				"childPolicy": [
+				{
+					"errParseConfigBalancer": {
+						"cluster": "test_cluster"
+					}
+				}
+			]
+			}`,
+			wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"",
+		},
+		{
+			name: "ChildPolicy_NotSupported",
+			input: `{
+				"childPolicy": [
+				{
+					"doesNotExistBalancer": {
+						"cluster": "test_cluster"
+					}
+				}
+			]
+			}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input))
+			if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if (gotErr != nil) != (test.wantErr != "") {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if test.wantErr != "" {
+				return
+			}
+			if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" {
+				t.Fatalf("parseConfig(%v) got unexpected output, diff (-got +want): %v", string(test.input), diff)
+			}
+		})
+	}
+}

internal/xds/balancer/outlierdetection/callcounter.go

@@ -0,0 +1,66 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"sync/atomic"
+)
+
+type bucket struct {
+	numSuccesses uint32
+	numFailures  uint32
+}
+
+func newCallCounter() *callCounter {
+	cc := &callCounter{
+		inactiveBucket: &bucket{},
+	}
+	cc.activeBucket.Store(&bucket{})
+	return cc
+}
+
+// callCounter has two buckets, which each count successful and failing RPC's.
+// The activeBucket is used to actively count any finished RPC's, and the
+// inactiveBucket is populated with this activeBucket's data every interval for
+// use by the Outlier Detection algorithm.
+type callCounter struct {
+	// activeBucket updates every time a call finishes (from picker passed to
+	// Client Conn), so protect pointer read with atomic load of the pointer
+	// so picker does not have to grab a mutex per RPC, the critical path.
+	activeBucket   atomic.Pointer[bucket]
+	inactiveBucket *bucket
+}
+
+func (cc *callCounter) clear() {
+	cc.activeBucket.Store(&bucket{})
+	cc.inactiveBucket = &bucket{}
+}
+
+// "When the timer triggers, the inactive bucket is zeroed and swapped with the
+// active bucket. Then the inactive bucket contains the number of successes and
+// failures since the last time the timer triggered. Those numbers are used to
+// evaluate the ejection criteria." - A50.
+func (cc *callCounter) swap() {
+	ib := cc.inactiveBucket
+	*ib = bucket{}
+	ab := cc.activeBucket.Swap(ib)
+	cc.inactiveBucket = &bucket{
+		numSuccesses: atomic.LoadUint32(&ab.numSuccesses),
+		numFailures:  atomic.LoadUint32(&ab.numFailures),
+	}
+}

internal/xds/balancer/outlierdetection/callcounter_test.go

@@ -0,0 +1,92 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func (b1 *bucket) Equal(b2 *bucket) bool {
+	if b1 == nil && b2 == nil {
+		return true
+	}
+	if (b1 != nil) != (b2 != nil) {
+		return false
+	}
+	if b1.numSuccesses != b2.numSuccesses {
+		return false
+	}
+	return b1.numFailures == b2.numFailures
+}
+
+func (cc *callCounter) Equal(cc2 *callCounter) bool {
+	if cc == nil && cc2 == nil {
+		return true
+	}
+	if (cc != nil) != (cc2 != nil) {
+		return false
+	}
+	ab1 := cc.activeBucket.Load()
+	ab2 := cc2.activeBucket.Load()
+	if !ab1.Equal(ab2) {
+		return false
+	}
+	return cc.inactiveBucket.Equal(cc2.inactiveBucket)
+}
+
+// TestClear tests that clear on the call counter clears (everything set to 0)
+// the active and inactive buckets.
+func (s) TestClear(t *testing.T) {
+	cc := newCallCounter()
+	ab := cc.activeBucket.Load()
+	ab.numSuccesses = 1
+	ab.numFailures = 2
+	cc.inactiveBucket.numSuccesses = 4
+	cc.inactiveBucket.numFailures = 5
+	cc.clear()
+	// Both the active and inactive buckets should be cleared.
+	ccWant := newCallCounter()
+	if diff := cmp.Diff(cc, ccWant); diff != "" {
+		t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff)
+	}
+}
+
+// TestSwap tests that swap() on the callCounter successfully has the desired
+// end result of inactive bucket containing the previous active buckets data,
+// and the active bucket being cleared.
+func (s) TestSwap(t *testing.T) {
+	cc := newCallCounter()
+	ab := cc.activeBucket.Load()
+	ab.numSuccesses = 1
+	ab.numFailures = 2
+	cc.inactiveBucket.numSuccesses = 4
+	cc.inactiveBucket.numFailures = 5
+	ib := cc.inactiveBucket
+	cc.swap()
+	// Inactive should pick up active's data, active should be swapped to zeroed
+	// inactive.
+	ccWant := newCallCounter()
+	ccWant.inactiveBucket.numSuccesses = 1
+	ccWant.inactiveBucket.numFailures = 2
+	ccWant.activeBucket.Store(ib)
+	if diff := cmp.Diff(cc, ccWant); diff != "" {
+		t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff)
+	}
+}

internal/xds/balancer/outlierdetection/config.go

@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"encoding/json"
+	"time"
+
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// SuccessRateEjection is parameters for the success rate ejection algorithm.
+// This algorithm monitors the request success rate for all endpoints and ejects
+// individual endpoints whose success rates are statistical outliers.
+type SuccessRateEjection struct {
+	// StddevFactor is used to determine the ejection threshold for
+	// success rate outlier ejection. The ejection threshold is the difference
+	// between the mean success rate, and the product of this factor and the
+	// standard deviation of the mean success rate: mean - (stdev *
+	// success_rate_stdev_factor). This factor is divided by a thousand to get a
+	// double. That is, if the desired factor is 1.9, the runtime value should
+	// be 1900. Defaults to 1900.
+	StdevFactor uint32 `json:"stdevFactor,omitempty"`
+	// EnforcementPercentage is the % chance that a host will be actually ejected
+	// when an outlier status is detected through success rate statistics. This
+	// setting can be used to disable ejection or to ramp it up slowly. Defaults
+	// to 100.
+	EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"`
+	// MinimumHosts is the number of hosts in a cluster that must have enough
+	// request volume to detect success rate outliers. If the number of hosts is
+	// less than this setting, outlier detection via success rate statistics is
+	// not performed for any host in the cluster. Defaults to 5.
+	MinimumHosts uint32 `json:"minimumHosts,omitempty"`
+	// RequestVolume is the minimum number of total requests that must be
+	// collected in one interval (as defined by the interval duration above) to
+	// include this host in success rate based outlier detection. If the volume
+	// is lower than this setting, outlier detection via success rate statistics
+	// is not performed for that host. Defaults to 100.
+	RequestVolume uint32 `json:"requestVolume,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type successRateEjection SuccessRateEjection
+
+// UnmarshalJSON unmarshals JSON into SuccessRateEjection. If a
+// SuccessRateEjection field is not set, that field will get its default value.
+func (sre *SuccessRateEjection) UnmarshalJSON(j []byte) error {
+	sre.StdevFactor = 1900
+	sre.EnforcementPercentage = 100
+	sre.MinimumHosts = 5
+	sre.RequestVolume = 100
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*successRateEjection)(sre))
+}
+
+// Equal returns whether the SuccessRateEjection is the same with the parameter.
+func (sre *SuccessRateEjection) Equal(sre2 *SuccessRateEjection) bool {
+	if sre == nil && sre2 == nil {
+		return true
+	}
+	if (sre != nil) != (sre2 != nil) {
+		return false
+	}
+	if sre.StdevFactor != sre2.StdevFactor {
+		return false
+	}
+	if sre.EnforcementPercentage != sre2.EnforcementPercentage {
+		return false
+	}
+	if sre.MinimumHosts != sre2.MinimumHosts {
+		return false
+	}
+	return sre.RequestVolume == sre2.RequestVolume
+}
+
+// FailurePercentageEjection is parameters for the failure percentage algorithm.
+// This algorithm ejects individual endpoints whose failure rate is greater than
+// some threshold, independently of any other endpoint.
+type FailurePercentageEjection struct {
+	// Threshold is the failure percentage to use when determining failure
+	// percentage-based outlier detection. If the failure percentage of a given
+	// host is greater than or equal to this value, it will be ejected. Defaults
+	// to 85.
+	Threshold uint32 `json:"threshold,omitempty"`
+	// EnforcementPercentage is the % chance that a host will be actually
+	// ejected when an outlier status is detected through failure percentage
+	// statistics. This setting can be used to disable ejection or to ramp it up
+	// slowly. Defaults to 0.
+	EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"`
+	// MinimumHosts is the minimum number of hosts in a cluster in order to
+	// perform failure percentage-based ejection. If the total number of hosts
+	// in the cluster is less than this value, failure percentage-based ejection
+	// will not be performed. Defaults to 5.
+	MinimumHosts uint32 `json:"minimumHosts,omitempty"`
+	// RequestVolume is the minimum number of total requests that must be
+	// collected in one interval (as defined by the interval duration above) to
+	// perform failure percentage-based ejection for this host. If the volume is
+	// lower than this setting, failure percentage-based ejection will not be
+	// performed for this host. Defaults to 50.
+	RequestVolume uint32 `json:"requestVolume,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type failurePercentageEjection FailurePercentageEjection
+
+// UnmarshalJSON unmarshals JSON into FailurePercentageEjection. If a
+// FailurePercentageEjection field is not set, that field will get its default
+// value.
+func (fpe *FailurePercentageEjection) UnmarshalJSON(j []byte) error {
+	fpe.Threshold = 85
+	fpe.EnforcementPercentage = 0
+	fpe.MinimumHosts = 5
+	fpe.RequestVolume = 50
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*failurePercentageEjection)(fpe))
+}
+
+// Equal returns whether the FailurePercentageEjection is the same with the
+// parameter.
+func (fpe *FailurePercentageEjection) Equal(fpe2 *FailurePercentageEjection) bool {
+	if fpe == nil && fpe2 == nil {
+		return true
+	}
+	if (fpe != nil) != (fpe2 != nil) {
+		return false
+	}
+	if fpe.Threshold != fpe2.Threshold {
+		return false
+	}
+	if fpe.EnforcementPercentage != fpe2.EnforcementPercentage {
+		return false
+	}
+	if fpe.MinimumHosts != fpe2.MinimumHosts {
+		return false
+	}
+	return fpe.RequestVolume == fpe2.RequestVolume
+}
+
+// LBConfig is the config for the outlier detection balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+	// Interval is the time interval between ejection analysis sweeps. This can
+	// result in both new ejections as well as addresses being returned to
+	// service. Defaults to 10s.
+	Interval iserviceconfig.Duration `json:"interval,omitempty"`
+	// BaseEjectionTime is the base time that a host is ejected for. The real
+	// time is equal to the base time multiplied by the number of times the host
+	// has been ejected and is capped by MaxEjectionTime. Defaults to 30s.
+	BaseEjectionTime iserviceconfig.Duration `json:"baseEjectionTime,omitempty"`
+	// MaxEjectionTime is the maximum time that an endpoint is ejected for. If
+	// not specified, the default value (300s) or the BaseEjectionTime value is
+	// applied, whichever is larger.
+	MaxEjectionTime iserviceconfig.Duration `json:"maxEjectionTime,omitempty"`
+	// MaxEjectionPercent is the maximum % of an upstream cluster that can be
+	// ejected due to outlier detection. Defaults to 10% but will eject at least
+	// one host regardless of the value.
+	MaxEjectionPercent uint32 `json:"maxEjectionPercent,omitempty"`
+	// SuccessRateEjection is the parameters for the success rate ejection
+	// algorithm. If set, success rate ejections will be performed.
+	SuccessRateEjection *SuccessRateEjection `json:"successRateEjection,omitempty"`
+	// FailurePercentageEjection is the parameters for the failure percentage
+	// algorithm. If set, failure rate ejections will be performed.
+	FailurePercentageEjection *FailurePercentageEjection `json:"failurePercentageEjection,omitempty"`
+	// ChildPolicy is the config for the child policy.
+	ChildPolicy *iserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type lbConfig LBConfig
+
+// UnmarshalJSON unmarshals JSON into LBConfig. If a top level LBConfig field
+// (i.e. not next layer sre or fpe) is not set, that field will get its default
+// value. If sre or fpe is not set, it will stay unset, otherwise it will
+// unmarshal on those types populating with default values for their fields if
+// needed.
+func (lbc *LBConfig) UnmarshalJSON(j []byte) error {
+	// Default top layer values as documented in A50.
+	lbc.Interval = iserviceconfig.Duration(10 * time.Second)
+	lbc.BaseEjectionTime = iserviceconfig.Duration(30 * time.Second)
+	lbc.MaxEjectionTime = iserviceconfig.Duration(300 * time.Second)
+	lbc.MaxEjectionPercent = 10
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*lbConfig)(lbc))
+}
+
+// EqualIgnoringChildPolicy returns whether the LBConfig is same with the
+// parameter outside of the child policy, only comparing the Outlier Detection
+// specific configuration.
+func (lbc *LBConfig) EqualIgnoringChildPolicy(lbc2 *LBConfig) bool {
+	if lbc == nil && lbc2 == nil {
+		return true
+	}
+	if (lbc != nil) != (lbc2 != nil) {
+		return false
+	}
+	if lbc.Interval != lbc2.Interval {
+		return false
+	}
+	if lbc.BaseEjectionTime != lbc2.BaseEjectionTime {
+		return false
+	}
+	if lbc.MaxEjectionTime != lbc2.MaxEjectionTime {
+		return false
+	}
+	if lbc.MaxEjectionPercent != lbc2.MaxEjectionPercent {
+		return false
+	}
+	if !lbc.SuccessRateEjection.Equal(lbc2.SuccessRateEjection) {
+		return false
+	}
+	return lbc.FailurePercentageEjection.Equal(lbc2.FailurePercentageEjection)
+}

internal/xds/balancer/outlierdetection/config_test.go

@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSuccessRateEjection(t *testing.T) {
+	fields := map[string]bool{
+		"StdevFactor":           true,
+		"EnforcementPercentage": true,
+		"MinimumHosts":          true,
+		"RequestVolume":         true,
+	}
+	typ := reflect.TypeOf(SuccessRateEjection{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in SuccessRateEjection %q, update this test and Equal", n)
+		}
+	}
+}
+
+func TestEqualFieldsFailurePercentageEjection(t *testing.T) {
+	fields := map[string]bool{
+		"Threshold":             true,
+		"EnforcementPercentage": true,
+		"MinimumHosts":          true,
+		"RequestVolume":         true,
+	}
+	typ := reflect.TypeOf(FailurePercentageEjection{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in FailurePercentageEjection %q, update this test and Equal", n)
+		}
+	}
+}
+
+func TestEqualFieldsLBConfig(t *testing.T) {
+	fields := map[string]bool{
+		"LoadBalancingConfig":       true,
+		"Interval":                  true,
+		"BaseEjectionTime":          true,
+		"MaxEjectionTime":           true,
+		"MaxEjectionPercent":        true,
+		"SuccessRateEjection":       true,
+		"FailurePercentageEjection": true,
+		"ChildPolicy":               true,
+	}
+	typ := reflect.TypeOf(LBConfig{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in LBConfig %q, update this test and EqualIgnoringChildPolicy", n)
+		}
+	}
+}

internal/xds/balancer/outlierdetection/e2e_test/outlierdetection_test.go

@@ -0,0 +1,425 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package e2e_test contains e2e test cases for the Outlier Detection LB Policy.
+package e2e_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/balancer/outlierdetection" // To register helper functions which register/unregister Outlier Detection LB Policy.
+)
+
+var (
+	defaultTestTimeout = 5 * time.Second
+	leafPolicyName     = "round_robin"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// Setup spins up three test backends, each listening on a port on localhost.
+// Two of the backends are configured to always reply with an empty response and
+// no error and one is configured to always return an error.
+func setupBackends(t *testing.T) ([]string, func()) {
+	t.Helper()
+
+	backends := make([]*stubserver.StubServer, 3)
+	addresses := make([]string, 3)
+	// Construct and start 2 working backends.
+	for i := 0; i < 2; i++ {
+		backend := &stubserver.StubServer{
+			EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+				return &testpb.Empty{}, nil
+			},
+		}
+		if err := backend.StartServer(); err != nil {
+			t.Fatalf("Failed to start backend: %v", err)
+		}
+		t.Logf("Started good TestService backend at: %q", backend.Address)
+		backends[i] = backend
+		addresses[i] = backend.Address
+	}
+
+	// Construct and start a failing backend.
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return nil, errors.New("some error")
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started bad TestService backend at: %q", backend.Address)
+	backends[2] = backend
+	addresses[2] = backend.Address
+	cancel := func() {
+		for _, backend := range backends {
+			backend.Stop()
+		}
+	}
+	return addresses, cancel
+}
+
+// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn,
+// connected to a server exposing the test.grpc_testing.TestService, are
+// roundrobined across the given backend addresses.
+//
+// Returns a non-nil error if context deadline expires before RPCs start to get
+// roundrobined across the given backends.
+func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error {
+	wantAddrCount := make(map[string]int)
+	for _, addr := range addrs {
+		wantAddrCount[addr.Addr]++
+	}
+	gotAddrCount := make(map[string]int)
+	for ; ctx.Err() == nil; <-time.After(time.Millisecond) {
+		gotAddrCount = make(map[string]int)
+		// Perform 3 iterations.
+		var iterations [][]string
+		for i := 0; i < 3; i++ {
+			iteration := make([]string, len(addrs))
+			for c := 0; c < len(addrs); c++ {
+				var peer peer.Peer
+				client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer))
+				if peer.Addr != nil {
+					iteration[c] = peer.Addr.String()
+				}
+			}
+			iterations = append(iterations, iteration)
+		}
+		// Ensure the first iteration contains all addresses in addrs.
+		for _, addr := range iterations[0] {
+			gotAddrCount[addr]++
+		}
+		if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" {
+			continue
+		}
+		// Ensure all three iterations contain the same addresses.
+		if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) {
+			continue
+		}
+		return nil
+	}
+	return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v; got: %v", addrs, gotAddrCount)
+}
+
+// TestPickFirstHealthListenerDisabled verifies that outlier detection doesn't
+// eject subchannels created by pick_first when it is not under a petiole
+// policy. In that configuration, pick_first does not register a health
+// listener and ejection updates must not affect connectivity.
+func (s) TestPickFirstHealthListenerDisabled(t *testing.T) {
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return nil, errors.New("some error")
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	defer backend.Stop()
+
+	const serviceConfig = `{
+	  "loadBalancingConfig": [{
+		"outlier_detection_experimental": {
+		  "interval": "0.050s",
+		  "failurePercentageEjection": {
+			"threshold": 50,
+			"enforcementPercentage": 100,
+			"minimumHosts": 0,
+			"requestVolume": 2
+		  },
+		  "childPolicy": [{"pick_first": {}}]
+		}
+	  }]
+	}`
+	cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(serviceConfig))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	client.EmptyCall(ctx, &testpb.Empty{})
+	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+
+	for range 10 {
+		client.EmptyCall(ctx, &testpb.Empty{})
+	}
+
+	// Wait for three outlier detection intervals to verify that the subchannel is not ejected.
+	sCtx, sCancel := context.WithTimeout(ctx, 150*time.Millisecond)
+	defer sCancel()
+	testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready)
+}
+
+// TestOutlierDetectionAlgorithmsE2E tests the Outlier Detection Success Rate
+// and Failure Percentage algorithms in an e2e fashion. The Outlier Detection
+// Balancer is configured as the top level LB Policy of the channel with a Round
+// Robin child, and connects to three upstreams. Two of the upstreams are healthy and
+// one is unhealthy. The two algorithms should at some point eject the failing
+// upstream, causing RPC's to not be routed to that upstream, and only be
+// Round Robined across the two healthy upstreams. Other than the intervals the
+// unhealthy upstream is ejected, RPC's should regularly round robin
+// across all three upstreams.
+func (s) TestOutlierDetectionAlgorithmsE2E(t *testing.T) {
+	tests := []struct {
+		name     string
+		odscJSON string
+	}{
+		{
+			name: "Success Rate Algorithm",
+			odscJSON: fmt.Sprintf(`
+			{
+			  "loadBalancingConfig": [
+				{
+				  "outlier_detection_experimental": {
+					"interval": "0.050s",
+					"baseEjectionTime": "0.100s",
+					"maxEjectionTime": "300s",
+					"maxEjectionPercent": 33,
+					"successRateEjection": {
+						"stdevFactor": 50,
+						"enforcementPercentage": 100,
+						"minimumHosts": 3,
+						"requestVolume": 5
+					},
+					"childPolicy": [{"%s": {}}]
+				  }
+				}
+			  ]
+			}`, leafPolicyName),
+		},
+		{
+			name: "Failure Percentage Algorithm",
+			odscJSON: fmt.Sprintf(`
+			{
+			  "loadBalancingConfig": [
+				{
+				  "outlier_detection_experimental": {
+					"interval": "0.050s",
+					"baseEjectionTime": "0.100s",
+					"maxEjectionTime": "300s",
+					"maxEjectionPercent": 33,
+					"failurePercentageEjection": {
+						"threshold": 50,
+						"enforcementPercentage": 100,
+						"minimumHosts": 3,
+						"requestVolume": 5
+					},
+					"childPolicy": [{"%s": {}}
+					]
+				  }
+				}
+			  ]
+			}`, leafPolicyName),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			addresses, cancel := setupBackends(t)
+			defer cancel()
+
+			mr := manual.NewBuilderWithScheme("od-e2e")
+			defer mr.Close()
+
+			sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(test.odscJSON)
+			// The full list of addresses.
+			fullAddresses := []resolver.Address{
+				{Addr: addresses[0]},
+				{Addr: addresses[1]},
+				{Addr: addresses[2]},
+			}
+			mr.InitialState(resolver.State{
+				Addresses:     fullAddresses,
+				ServiceConfig: sc,
+			})
+
+			cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			testServiceClient := testgrpc.NewTestServiceClient(cc)
+
+			// At first, due to no statistics on each of the backends, the 3
+			// upstreams should all be round robined across.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+
+			// The addresses which don't return errors.
+			okAddresses := []resolver.Address{
+				{Addr: addresses[0]},
+				{Addr: addresses[1]},
+			}
+			// After calling the three upstreams, one of them constantly error
+			// and should eventually be ejected for a period of time. This
+			// period of time should cause the RPC's to be round robined only
+			// across the two that are healthy.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+
+			// The failing upstream isn't ejected indefinitely, and eventually
+			// should be unejected in subsequent iterations of the interval
+			// algorithm as per the spec for the two specific algorithms.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+		})
+	}
+}
+
+// TestNoopConfiguration tests the Outlier Detection Balancer configured with a
+// noop configuration. The noop configuration should cause the Outlier Detection
+// Balancer to not count RPC's, and thus never eject any upstreams and continue
+// to route to every upstream connected to, even if they continuously error.
+// Once the Outlier Detection Balancer gets reconfigured with configuration
+// requiring counting RPC's, the Outlier Detection Balancer should start
+// ejecting any upstreams as specified in the configuration.
+func (s) TestNoopConfiguration(t *testing.T) {
+	addresses, cancel := setupBackends(t)
+	defer cancel()
+
+	mr := manual.NewBuilderWithScheme("od-e2e")
+	defer mr.Close()
+
+	noopODServiceConfigJSON := fmt.Sprintf(`
+	{
+	  "loadBalancingConfig": [
+		{
+		  "outlier_detection_experimental": {
+			"interval": "0.050s",
+			"baseEjectionTime": "0.100s",
+			"maxEjectionTime": "300s",
+			"maxEjectionPercent": 33,
+			"childPolicy": [{"%s": {}}]
+		  }
+		}
+	  ]
+	}`, leafPolicyName)
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(noopODServiceConfigJSON)
+	// The full list of addresses.
+	fullAddresses := []resolver.Address{
+		{Addr: addresses[0]},
+		{Addr: addresses[1]},
+		{Addr: addresses[2]},
+	}
+	mr.InitialState(resolver.State{
+		Addresses:     fullAddresses,
+		ServiceConfig: sc,
+	})
+	cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testServiceClient := testgrpc.NewTestServiceClient(cc)
+
+	for i := 0; i < 2; i++ {
+		// Since the Outlier Detection Balancer starts with a noop
+		// configuration, it shouldn't count RPCs or eject any upstreams. Thus,
+		// even though an upstream it connects to constantly errors, it should
+		// continue to Round Robin across every upstream.
+		if err := checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+			t.Fatalf("error in expected round robin: %v", err)
+		}
+	}
+
+	// Reconfigure the Outlier Detection Balancer with a configuration that
+	// specifies to count RPC's and eject upstreams. Due to the balancer no
+	// longer being a noop, it should eject any unhealthy addresses as specified
+	// by the failure percentage portion of the configuration.
+	countingODServiceConfigJSON := fmt.Sprintf(`
+	{
+	  "loadBalancingConfig": [
+		{
+		  "outlier_detection_experimental": {
+			"interval": "0.050s",
+			"baseEjectionTime": "0.100s",
+			"maxEjectionTime": "300s",
+			"maxEjectionPercent": 33,
+			"failurePercentageEjection": {
+				"threshold": 50,
+				"enforcementPercentage": 100,
+				"minimumHosts": 3,
+				"requestVolume": 5
+			},
+			"childPolicy": [{"%s": {}}]
+		  }
+		}
+	  ]
+	}`, leafPolicyName)
+	sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(countingODServiceConfigJSON)
+
+	mr.UpdateState(resolver.State{
+		Addresses:     fullAddresses,
+		ServiceConfig: sc,
+	})
+
+	// At first on the reconfigured balancer, the balancer has no stats
+	// collected about upstreams. Thus, it should at first route across the full
+	// upstream list.
+	if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+
+	// The addresses which don't return errors.
+	okAddresses := []resolver.Address{
+		{Addr: addresses[0]},
+		{Addr: addresses[1]},
+	}
+	// Now that the reconfigured balancer has data about the failing upstream,
+	// it should eject the upstream and only route across the two healthy
+	// upstreams.
+	if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+}

internal/xds/balancer/outlierdetection/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package outlierdetection
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[outlier-detection-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *outlierDetectionBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/outlierdetection/subconn_wrapper.go

@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/resolver"
+)
+
+// subConnWrapper wraps every created SubConn in the Outlier Detection Balancer,
+// to help track the latest state update from the underlying SubConn, and also
+// whether or not this SubConn is ejected.
+type subConnWrapper struct {
+	balancer.SubConn
+	// endpointInfo is a pointer to the subConnWrapper's corresponding endpoint
+	// map entry, if the map entry exists.
+	endpointInfo *endpointInfo
+	// The following fields are set during object creation and read-only after
+	// that.
+
+	listener func(balancer.SubConnState)
+
+	scUpdateCh *buffer.Unbounded[any]
+
+	// The following fields are only referenced in the context of a work
+	// serializing buffer and don't need to be protected by a mutex.
+
+	// These two pieces of state will reach eventual consistency due to sync in
+	// run(), and child will always have the correctly updated SubConnState.
+
+	ejected bool
+
+	// addresses is the list of address(es) this SubConn was created with to
+	// help support any change in address(es)
+	addresses []resolver.Address
+	// latestHealthState is tracked to update the child policy during
+	// unejection.
+	latestHealthState balancer.SubConnState
+
+	// Access to the following fields are protected by a mutex. These fields
+	// should not be accessed from outside this file, instead use methods
+	// defined on the struct.
+	mu             sync.Mutex
+	healthListener func(balancer.SubConnState)
+	// latestReceivedConnectivityState is the SubConn's most recent connectivity
+	// state received. It may not be delivered to the child balancer yet. It is
+	// used to ensure a health listener is registered with the SubConn only when
+	// the SubConn is READY.
+	latestReceivedConnectivityState connectivity.State
+}
+
+// eject causes the wrapper to report a state update with the TRANSIENT_FAILURE
+// state, and to stop passing along updates from the underlying subchannel.
+func (scw *subConnWrapper) eject() {
+	scw.scUpdateCh.Put(&ejectionUpdate{
+		scw:       scw,
+		isEjected: true,
+	})
+}
+
+// uneject causes the wrapper to report a state update with the latest update
+// from the underlying subchannel, and resume passing along updates from the
+// underlying subchannel.
+func (scw *subConnWrapper) uneject() {
+	scw.scUpdateCh.Put(&ejectionUpdate{
+		scw:       scw,
+		isEjected: false,
+	})
+}
+
+func (scw *subConnWrapper) String() string {
+	return fmt.Sprintf("%+v", scw.addresses)
+}
+
+func (scw *subConnWrapper) RegisterHealthListener(listener func(balancer.SubConnState)) {
+	// gRPC currently supports two mechanisms that provide a health signal for
+	// a connection: client-side health checking and outlier detection. Earlier
+	// both these mechanisms signaled unhealthiness by setting the subchannel
+	// state to TRANSIENT_FAILURE. As part of the dualstack changes to make
+	// pick_first the universal leaf policy (see A61), both these mechanisms
+	// started using the new health listener to make health signal visible to
+	// the petiole policies without affecting the underlying connectivity
+	// management of the pick_first policy.
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+
+	if scw.latestReceivedConnectivityState != connectivity.Ready {
+		return
+	}
+	scw.healthListener = listener
+	if listener == nil {
+		scw.SubConn.RegisterHealthListener(nil)
+		return
+	}
+
+	scw.SubConn.RegisterHealthListener(func(scs balancer.SubConnState) {
+		scw.scUpdateCh.Put(&scHealthUpdate{
+			scw:   scw,
+			state: scs,
+		})
+	})
+}
+
+// updateSubConnHealthState stores the latest health state for unejection and
+// sends updates the health listener.
+func (scw *subConnWrapper) updateSubConnHealthState(scs balancer.SubConnState) {
+	scw.latestHealthState = scs
+	if scw.ejected {
+		return
+	}
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	if scw.healthListener != nil {
+		scw.healthListener(scs)
+	}
+}
+
+// updateSubConnConnectivityState stores the latest connectivity state for
+// unejection and updates the raw connectivity listener.
+func (scw *subConnWrapper) updateSubConnConnectivityState(scs balancer.SubConnState) {
+	if scw.listener != nil {
+		scw.listener(scs)
+	}
+}
+
+func (scw *subConnWrapper) clearHealthListener() {
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	scw.healthListener = nil
+}
+
+func (scw *subConnWrapper) handleUnejection() {
+	scw.ejected = false
+	// If scw.latestHealthState has never been written to will use the health
+	// state CONNECTING set during object creation.
+	scw.updateSubConnHealthState(scw.latestHealthState)
+}
+
+func (scw *subConnWrapper) handleEjection() {
+	scw.ejected = true
+	stateToUpdate := balancer.SubConnState{
+		ConnectivityState: connectivity.TransientFailure,
+	}
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	if scw.healthListener != nil {
+		scw.healthListener(stateToUpdate)
+	}
+}
+
+func (scw *subConnWrapper) setLatestConnectivityState(state connectivity.State) {
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	scw.latestReceivedConnectivityState = state
+}