snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

balancer/rls/balancer.go

@@ -0,0 +1,733 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package rls implements the RLS LB policy.
+package rls
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/backoff"
+	"google.golang.org/grpc/internal/balancergroup"
+	"google.golang.org/grpc/internal/buffer"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/resolver"
+)
+
+const (
+	// Name is the name of the RLS LB policy.
+	//
+	// It currently has an experimental suffix which would be removed once
+	// end-to-end testing of the policy is completed.
+	Name = internal.RLSLoadBalancingPolicyName
+	// Default frequency for data cache purging.
+	periodicCachePurgeFreq = time.Minute
+)
+
+var (
+	logger            = grpclog.Component("rls")
+	errBalancerClosed = errors.New("rls LB policy is closed")
+
+	// Below defined vars for overriding in unit tests.
+
+	// Default exponential backoff strategy for data cache entries.
+	defaultBackoffStrategy = backoff.Strategy(backoff.DefaultExponential)
+	// Ticker used for periodic data cache purging.
+	dataCachePurgeTicker = func() *time.Ticker { return time.NewTicker(periodicCachePurgeFreq) }
+	// We want every cache entry to live in the cache for at least this
+	// duration. If we encounter a cache entry whose minimum expiration time is
+	// in the future, we abort the LRU pass, which may temporarily leave the
+	// cache being too large. This is necessary to ensure that in cases where
+	// the cache is too small, when we receive an RLS Response, we keep the
+	// resulting cache entry around long enough for the pending incoming
+	// requests to be re-processed through the new Picker. If we didn't do this,
+	// then we'd risk throwing away each RLS response as we receive it, in which
+	// case we would fail to actually route any of our incoming requests.
+	minEvictDuration = 5 * time.Second
+
+	// Following functions are no-ops in actual code, but can be overridden in
+	// tests to give tests visibility into exactly when certain events happen.
+	clientConnUpdateHook = func() {}
+	dataCachePurgeHook   = func() {}
+	resetBackoffHook     = func() {}
+
+	cacheEntriesMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{
+		Name:        "grpc.lb.rls.cache_entries",
+		Description: "EXPERIMENTAL. Number of entries in the RLS cache.",
+		Unit:        "{entry}",
+		Labels:      []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.instance_uuid"},
+		Default:     false,
+	})
+	cacheSizeMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{
+		Name:        "grpc.lb.rls.cache_size",
+		Description: "EXPERIMENTAL. The current size of the RLS cache.",
+		Unit:        "By",
+		Labels:      []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.instance_uuid"},
+		Default:     false,
+	})
+	defaultTargetPicksMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "grpc.lb.rls.default_target_picks",
+		Description:    "EXPERIMENTAL. Number of LB picks sent to the default target.",
+		Unit:           "{pick}",
+		Labels:         []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.data_plane_target", "grpc.lb.pick_result"},
+		OptionalLabels: []string{"grpc.client.call.custom"},
+		Default:        false,
+	})
+	targetPicksMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "grpc.lb.rls.target_picks",
+		Description:    "EXPERIMENTAL. Number of LB picks sent to each RLS target. Note that if the default target is also returned by the RLS server, RPCs sent to that target from the cache will be counted in this metric, not in grpc.rls.default_target_picks.",
+		Unit:           "{pick}",
+		Labels:         []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.data_plane_target", "grpc.lb.pick_result"},
+		OptionalLabels: []string{"grpc.client.call.custom"},
+		Default:        false,
+	})
+	failedPicksMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "grpc.lb.rls.failed_picks",
+		Description:    "EXPERIMENTAL. Number of LB picks failed due to either a failed RLS request or the RLS channel being throttled.",
+		Unit:           "{pick}",
+		Labels:         []string{"grpc.target", "grpc.lb.rls.server_target"},
+		OptionalLabels: []string{"grpc.client.call.custom"},
+		Default:        false,
+	})
+)
+
+func init() {
+	balancer.Register(&rlsBB{})
+}
+
+type rlsBB struct{}
+
+func (rlsBB) Name() string {
+	return Name
+}
+
+func (rlsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	lb := &rlsBalancer{
+		closed:             grpcsync.NewEvent(),
+		done:               grpcsync.NewEvent(),
+		cc:                 cc,
+		bopts:              opts,
+		purgeTicker:        dataCachePurgeTicker(),
+		dataCachePurgeHook: dataCachePurgeHook,
+		lbCfg:              &lbConfig{},
+		pendingMap:         make(map[cacheKey]*backoffState),
+		childPolicies:      make(map[string]*childPolicyWrapper),
+		updateCh:           buffer.NewUnbounded[any](),
+	}
+	lb.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-experimental-lb %p] ", lb))
+	lb.dataCache = newDataCache(maxCacheSize, lb.logger, opts.Target.String())
+	metricsRecorder := cc.MetricsRecorder()
+	lb.unregisterMetricHandler = metricsRecorder.RegisterAsyncReporter(lb, cacheEntriesMetric, cacheSizeMetric)
+	lb.bg = balancergroup.New(balancergroup.Options{
+		CC:                      cc,
+		BuildOpts:               opts,
+		StateAggregator:         lb,
+		Logger:                  lb.logger,
+		SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies
+	})
+	go lb.run()
+	return lb
+}
+
+// rlsBalancer implements the RLS LB policy.
+type rlsBalancer struct {
+	closed             *grpcsync.Event // Fires when Close() is invoked. Guarded by stateMu.
+	done               *grpcsync.Event // Fires when Close() is done.
+	cc                 balancer.ClientConn
+	bopts              balancer.BuildOptions
+	purgeTicker        *time.Ticker
+	dataCachePurgeHook func()
+	logger             *internalgrpclog.PrefixLogger
+
+	// unregisterMetricHandler is the function to deregister the async metric reporter.
+	unregisterMetricHandler func()
+
+	// If both cacheMu and stateMu need to be acquired, the former must be
+	// acquired first to prevent a deadlock. This order restriction is due to the
+	// fact that in places where we need to acquire both the locks, we always
+	// start off reading the cache.
+
+	// cacheMu guards access to the data cache and pending requests map. We
+	// cannot use an RWMutex here since even an operation like
+	// dataCache.getEntry() modifies the underlying LRU, which is implemented as
+	// a doubly linked list.
+	cacheMu    sync.Mutex
+	dataCache  *dataCache                 // Cache of RLS data.
+	pendingMap map[cacheKey]*backoffState // Map of pending RLS requests.
+
+	// stateMu guards access to all LB policy state.
+	stateMu            sync.Mutex
+	lbCfg              *lbConfig        // Most recently received service config.
+	childPolicyBuilder balancer.Builder // Cached child policy builder.
+	resolverState      resolver.State   // Cached resolver state.
+	ctrlCh             *controlChannel  // Control channel to the RLS server.
+	bg                 *balancergroup.BalancerGroup
+	childPolicies      map[string]*childPolicyWrapper
+	defaultPolicy      *childPolicyWrapper
+	// A reference to the most recent picker sent to gRPC as part of a state
+	// update is cached in this field so that we can release the reference to the
+	// default child policy wrapper when a new picker is created. See
+	// sendNewPickerLocked() for details.
+	lastPicker *rlsPicker
+	// Set during UpdateClientConnState when pushing updates to child policies.
+	// Prevents state updates from child policies causing new pickers to be sent
+	// up the channel. Cleared after all child policies have processed the
+	// updates sent to them, after which a new picker is sent up the channel.
+	inhibitPickerUpdates bool
+
+	// Channel on which all updates are pushed. Processed in run().
+	updateCh *buffer.Unbounded[any]
+}
+
+type resumePickerUpdates struct {
+	done chan struct{}
+}
+
+// childPolicyIDAndState wraps a child policy id and its state update.
+type childPolicyIDAndState struct {
+	id    string
+	state balancer.State
+}
+
+type controlChannelReady struct{}
+
+// run is a long-running goroutine which handles all the updates that the
+// balancer wishes to handle. The appropriate updateHandler will push the update
+// on to a channel that this goroutine will select on, thereby the handling of
+// the update will happen asynchronously.
+func (b *rlsBalancer) run() {
+	// We exit out of the for loop below only after `Close()` has been invoked.
+	// Firing the done event here will ensure that Close() returns only after
+	// all goroutines are done.
+	defer func() { b.done.Fire() }()
+
+	// Wait for purgeDataCache() goroutine to exit before returning from here.
+	doneCh := make(chan struct{})
+	defer func() {
+		<-doneCh
+	}()
+	go b.purgeDataCache(doneCh)
+
+	for {
+		select {
+		case u, ok := <-b.updateCh.Get():
+			if !ok {
+				return
+			}
+			b.updateCh.Load()
+			switch update := u.(type) {
+			case childPolicyIDAndState:
+				b.handleChildPolicyStateUpdate(update.id, update.state)
+			case controlChannelReady:
+				b.logger.Infof("Resetting backoff state after control channel getting back to READY")
+				b.cacheMu.Lock()
+				updatePicker := b.dataCache.resetBackoffState(&backoffState{bs: defaultBackoffStrategy})
+				b.cacheMu.Unlock()
+				if updatePicker {
+					b.sendNewPicker()
+				}
+				resetBackoffHook()
+			case resumePickerUpdates:
+				b.stateMu.Lock()
+				b.logger.Infof("Resuming picker updates after config propagation to child policies")
+				b.inhibitPickerUpdates = false
+				b.sendNewPickerLocked()
+				close(update.done)
+				b.stateMu.Unlock()
+			default:
+				b.logger.Errorf("Unsupported update type %T", update)
+			}
+		case <-b.closed.Done():
+			return
+		}
+	}
+}
+
+// purgeDataCache is a long-running goroutine which periodically deletes expired
+// entries. An expired entry is one for which both the expiryTime and
+// backoffExpiryTime are in the past.
+func (b *rlsBalancer) purgeDataCache(doneCh chan struct{}) {
+	defer close(doneCh)
+
+	for {
+		select {
+		case <-b.closed.Done():
+			return
+		case <-b.purgeTicker.C:
+			b.cacheMu.Lock()
+			updatePicker := b.dataCache.evictExpiredEntries()
+			b.cacheMu.Unlock()
+			if updatePicker {
+				b.sendNewPicker()
+			}
+			b.dataCachePurgeHook()
+		}
+	}
+}
+
+func (b *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error {
+	defer clientConnUpdateHook()
+
+	b.stateMu.Lock()
+	if b.closed.HasFired() {
+		b.stateMu.Unlock()
+		b.logger.Warningf("Received service config after balancer close: %s", pretty.ToJSON(ccs.BalancerConfig))
+		return errBalancerClosed
+	}
+
+	newCfg := ccs.BalancerConfig.(*lbConfig)
+	if b.lbCfg.Equal(newCfg) {
+		b.stateMu.Unlock()
+		b.logger.Infof("New service config matches existing config")
+		return nil
+	}
+
+	b.logger.Infof("Delaying picker updates until config is propagated to and processed by child policies")
+	b.inhibitPickerUpdates = true
+
+	// When the RLS server name changes, the old control channel needs to be
+	// swapped out for a new one. All state associated with the throttling
+	// algorithm is stored on a per-control-channel basis; when we swap out
+	// channels, we also swap out the throttling state.
+	b.handleControlChannelUpdate(newCfg)
+
+	// Any changes to child policy name or configuration needs to be handled by
+	// either creating new child policies or pushing updates to existing ones.
+	b.resolverState = ccs.ResolverState
+	b.handleChildPolicyConfigUpdate(newCfg, &ccs)
+
+	// Resize the cache if the size in the config has changed.
+	resizeCache := newCfg.cacheSizeBytes != b.lbCfg.cacheSizeBytes
+
+	// Update the copy of the config in the LB policy before releasing the lock.
+	b.lbCfg = newCfg
+	b.stateMu.Unlock()
+
+	// We cannot do cache operations above because `cacheMu` needs to be grabbed
+	// before `stateMu` if we are to hold both locks at the same time.
+	b.cacheMu.Lock()
+	b.dataCache.updateRLSServerTarget(newCfg.lookupService)
+	if resizeCache {
+		// If the new config changes reduces the size of the data cache, we
+		// might have to evict entries to get the cache size down to the newly
+		// specified size. If we do evict an entry with valid backoff timer,
+		// the new picker needs to be sent to the channel to re-process any
+		// RPCs queued as a result of this backoff timer.
+		b.dataCache.resize(newCfg.cacheSizeBytes)
+	}
+	b.cacheMu.Unlock()
+	// Enqueue an event which will notify us when the above update has been
+	// propagated to all child policies, and the child policies have all
+	// processed their updates, and we have sent a picker update.
+	done := make(chan struct{})
+	b.updateCh.Put(resumePickerUpdates{done: done})
+	<-done
+	return nil
+}
+
+// handleControlChannelUpdate handles updates to service config fields which
+// influence the control channel to the RLS server.
+//
+// Caller must hold lb.stateMu.
+func (b *rlsBalancer) handleControlChannelUpdate(newCfg *lbConfig) {
+	if newCfg.lookupService == b.lbCfg.lookupService && newCfg.lookupServiceTimeout == b.lbCfg.lookupServiceTimeout {
+		return
+	}
+
+	// Create a new control channel and close the existing one.
+	b.logger.Infof("Creating control channel to RLS server at: %v", newCfg.lookupService)
+	backToReadyFn := func() {
+		b.updateCh.Put(controlChannelReady{})
+	}
+	ctrlCh, err := newControlChannel(newCfg.lookupService, newCfg.controlChannelServiceConfig, newCfg.lookupServiceTimeout, b.bopts, backToReadyFn)
+	if err != nil {
+		// This is very uncommon and usually represents a non-transient error.
+		// There is not much we can do here other than wait for another update
+		// which might fix things.
+		b.logger.Errorf("Failed to create control channel to %q: %v", newCfg.lookupService, err)
+		return
+	}
+	if b.ctrlCh != nil {
+		b.ctrlCh.close()
+	}
+	b.ctrlCh = ctrlCh
+}
+
+// handleChildPolicyConfigUpdate handles updates to service config fields which
+// influence child policy configuration.
+//
+// Caller must hold lb.stateMu.
+func (b *rlsBalancer) handleChildPolicyConfigUpdate(newCfg *lbConfig, ccs *balancer.ClientConnState) {
+	// Update child policy builder first since other steps are dependent on this.
+	if b.childPolicyBuilder == nil || b.childPolicyBuilder.Name() != newCfg.childPolicyName {
+		b.logger.Infof("Child policy changed to %q", newCfg.childPolicyName)
+		b.childPolicyBuilder = balancer.Get(newCfg.childPolicyName)
+		for _, cpw := range b.childPolicies {
+			// If the child policy has changed, we need to remove the old policy
+			// from the BalancerGroup and add a new one. The BalancerGroup takes
+			// care of closing the old one in this case.
+			b.bg.Remove(cpw.target)
+			b.bg.Add(cpw.target, b.childPolicyBuilder)
+		}
+	}
+
+	configSentToDefault := false
+	if b.lbCfg.defaultTarget != newCfg.defaultTarget {
+		// If the default target has changed, create a new childPolicyWrapper for
+		// the new target if required. If a new wrapper is created, add it to the
+		// childPolicies map and the BalancerGroup.
+		b.logger.Infof("Default target in LB config changing from %q to %q", b.lbCfg.defaultTarget, newCfg.defaultTarget)
+		cpw := b.childPolicies[newCfg.defaultTarget]
+		if cpw == nil {
+			cpw = newChildPolicyWrapper(newCfg.defaultTarget)
+			b.childPolicies[newCfg.defaultTarget] = cpw
+			b.bg.Add(newCfg.defaultTarget, b.childPolicyBuilder)
+			b.logger.Infof("Child policy %q added to BalancerGroup", newCfg.defaultTarget)
+		}
+		if err := b.buildAndPushChildPolicyConfigs(newCfg.defaultTarget, newCfg, ccs); err != nil {
+			cpw.lamify(err)
+		}
+
+		// If an old default exists, release its reference. If this was the last
+		// reference, remove the child policy from the BalancerGroup and remove the
+		// corresponding entry the childPolicies map.
+		if b.defaultPolicy != nil {
+			if b.defaultPolicy.releaseRef() {
+				delete(b.childPolicies, b.lbCfg.defaultTarget)
+				b.bg.Remove(b.defaultPolicy.target)
+			}
+		}
+		b.defaultPolicy = cpw
+		configSentToDefault = true
+	}
+
+	// No change in configuration affecting child policies. Return early.
+	if b.lbCfg.childPolicyName == newCfg.childPolicyName && b.lbCfg.childPolicyTargetField == newCfg.childPolicyTargetField && childPolicyConfigEqual(b.lbCfg.childPolicyConfig, newCfg.childPolicyConfig) {
+		return
+	}
+
+	// If fields affecting child policy configuration have changed, the changes
+	// are pushed to the childPolicyWrapper which handles them appropriately.
+	for _, cpw := range b.childPolicies {
+		if configSentToDefault && cpw.target == newCfg.defaultTarget {
+			// Default target has already been taken care of.
+			continue
+		}
+		if err := b.buildAndPushChildPolicyConfigs(cpw.target, newCfg, ccs); err != nil {
+			cpw.lamify(err)
+		}
+	}
+}
+
+// buildAndPushChildPolicyConfigs builds the final child policy configuration by
+// adding the `targetField` to the base child policy configuration received in
+// RLS LB policy configuration. The `targetField` is set to target and
+// configuration is pushed to the child policy through the BalancerGroup.
+//
+// Caller must hold lb.stateMu.
+func (b *rlsBalancer) buildAndPushChildPolicyConfigs(target string, newCfg *lbConfig, ccs *balancer.ClientConnState) error {
+	jsonTarget, err := json.Marshal(target)
+	if err != nil {
+		return fmt.Errorf("failed to marshal child policy target %q: %v", target, err)
+	}
+
+	config := newCfg.childPolicyConfig
+	targetField := newCfg.childPolicyTargetField
+	config[targetField] = jsonTarget
+	jsonCfg, err := json.Marshal(config)
+	if err != nil {
+		return fmt.Errorf("failed to marshal child policy config %+v: %v", config, err)
+	}
+
+	parser, _ := b.childPolicyBuilder.(balancer.ConfigParser)
+	parsedCfg, err := parser.ParseConfig(jsonCfg)
+	if err != nil {
+		return fmt.Errorf("childPolicy config parsing failed: %v", err)
+	}
+
+	state := balancer.ClientConnState{ResolverState: ccs.ResolverState, BalancerConfig: parsedCfg}
+	b.logger.Infof("Pushing new state to child policy %q: %+v", target, state)
+	if err := b.bg.UpdateClientConnState(target, state); err != nil {
+		b.logger.Warningf("UpdateClientConnState(%q, %+v) failed : %v", target, ccs, err)
+	}
+	return nil
+}
+
+func (b *rlsBalancer) ResolverError(err error) {
+	b.bg.ResolverError(err)
+}
+
+func (b *rlsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *rlsBalancer) Close() {
+	b.stateMu.Lock()
+	b.closed.Fire()
+	b.purgeTicker.Stop()
+	if b.ctrlCh != nil {
+		b.ctrlCh.close()
+	}
+	b.unregisterMetricHandler()
+	b.bg.Close()
+	b.stateMu.Unlock()
+
+	b.cacheMu.Lock()
+	b.dataCache.stop()
+	b.cacheMu.Unlock()
+
+	b.updateCh.Close()
+
+	<-b.done.Done()
+}
+
+func (b *rlsBalancer) ExitIdle() {
+	b.bg.ExitIdle()
+}
+
+// sendNewPickerLocked pushes a new picker on to the channel.
+//
+// Note that regardless of what connectivity state is reported, the policy will
+// return its own picker, and not a picker that unconditionally queues
+// (typically used for IDLE or CONNECTING) or a picker that unconditionally
+// fails (typically used for TRANSIENT_FAILURE). This is required because,
+// irrespective of the connectivity state, we need to able to perform RLS
+// lookups for incoming RPCs and affect the status of queued RPCs based on the
+// receipt of RLS responses.
+//
+// Caller must hold lb.stateMu.
+func (b *rlsBalancer) sendNewPickerLocked() {
+	aggregatedState := b.aggregatedConnectivityState()
+
+	// Acquire a separate reference for the picker. This is required to ensure
+	// that the wrapper held by the old picker is not closed when the default
+	// target changes in the config, and a new wrapper is created for the new
+	// default target. See handleChildPolicyConfigUpdate() for how config changes
+	// affecting the default target are handled.
+	if b.defaultPolicy != nil {
+		b.defaultPolicy.acquireRef()
+	}
+
+	picker := &rlsPicker{
+		kbm:             b.lbCfg.kbMap,
+		origEndpoint:    b.bopts.Target.Endpoint(),
+		lb:              b,
+		defaultPolicy:   b.defaultPolicy,
+		ctrlCh:          b.ctrlCh,
+		maxAge:          b.lbCfg.maxAge,
+		staleAge:        b.lbCfg.staleAge,
+		bg:              b.bg,
+		rlsServerTarget: b.lbCfg.lookupService,
+		grpcTarget:      b.bopts.Target.String(),
+		metricsRecorder: b.cc.MetricsRecorder(),
+	}
+	picker.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-picker %p] ", picker))
+	state := balancer.State{
+		ConnectivityState: aggregatedState,
+		Picker:            picker,
+	}
+
+	if !b.inhibitPickerUpdates {
+		b.logger.Infof("New balancer.State: %+v", state)
+		b.cc.UpdateState(state)
+	} else {
+		b.logger.Infof("Delaying picker update: %+v", state)
+	}
+
+	if b.lastPicker != nil {
+		if b.defaultPolicy != nil {
+			b.defaultPolicy.releaseRef()
+		}
+	}
+	b.lastPicker = picker
+}
+
+func (b *rlsBalancer) sendNewPicker() {
+	b.stateMu.Lock()
+	defer b.stateMu.Unlock()
+	if b.closed.HasFired() {
+		return
+	}
+	b.sendNewPickerLocked()
+}
+
+// The aggregated connectivity state reported is determined as follows:
+//   - If there is at least one child policy in state READY, the connectivity
+//     state is READY.
+//   - Otherwise, if there is at least one child policy in state CONNECTING, the
+//     connectivity state is CONNECTING.
+//   - Otherwise, if there is at least one child policy in state IDLE, the
+//     connectivity state is IDLE.
+//   - Otherwise, all child policies are in TRANSIENT_FAILURE, and the
+//     connectivity state is TRANSIENT_FAILURE.
+//
+// If the RLS policy has no child policies and no configured default target,
+// then we will report connectivity state IDLE.
+//
+// Caller must hold lb.stateMu.
+func (b *rlsBalancer) aggregatedConnectivityState() connectivity.State {
+	if len(b.childPolicies) == 0 && b.lbCfg.defaultTarget == "" {
+		return connectivity.Idle
+	}
+
+	var readyN, connectingN, idleN int
+	for _, cpw := range b.childPolicies {
+		state := (*balancer.State)(atomic.LoadPointer(&cpw.state))
+		switch state.ConnectivityState {
+		case connectivity.Ready:
+			readyN++
+		case connectivity.Connecting:
+			connectingN++
+		case connectivity.Idle:
+			idleN++
+		}
+	}
+
+	switch {
+	case readyN > 0:
+		return connectivity.Ready
+	case connectingN > 0:
+		return connectivity.Connecting
+	case idleN > 0:
+		return connectivity.Idle
+	default:
+		return connectivity.TransientFailure
+	}
+}
+
+// UpdateState is a implementation of the balancergroup.BalancerStateAggregator
+// interface. The actual state aggregation functionality is handled
+// asynchronously. This method only pushes the state update on to channel read
+// and dispatched by the run() goroutine.
+func (b *rlsBalancer) UpdateState(id string, state balancer.State) {
+	b.updateCh.Put(childPolicyIDAndState{id: id, state: state})
+}
+
+// handleChildPolicyStateUpdate provides the state aggregator functionality for
+// the BalancerGroup.
+//
+// This method is invoked by the BalancerGroup whenever a child policy sends a
+// state update. We cache the child policy's connectivity state and picker for
+// two reasons:
+//   - to suppress connectivity state transitions from TRANSIENT_FAILURE to states
+//     other than READY
+//   - to delegate picks to child policies
+func (b *rlsBalancer) handleChildPolicyStateUpdate(id string, newState balancer.State) {
+	b.stateMu.Lock()
+	defer b.stateMu.Unlock()
+
+	cpw := b.childPolicies[id]
+	if cpw == nil {
+		// All child policies start with an entry in the map. If ID is not in
+		// map, it's either been removed, or never existed.
+		b.logger.Warningf("Received state update %+v for missing child policy %q", newState, id)
+		return
+	}
+
+	oldState := (*balancer.State)(atomic.LoadPointer(&cpw.state))
+	if oldState.ConnectivityState == connectivity.TransientFailure && newState.ConnectivityState == connectivity.Connecting {
+		// Ignore state transitions from TRANSIENT_FAILURE to CONNECTING, and thus
+		// fail pending RPCs instead of queuing them indefinitely when all
+		// subChannels are failing, even if the subChannels are bouncing back and
+		// forth between CONNECTING and TRANSIENT_FAILURE.
+		return
+	}
+	atomic.StorePointer(&cpw.state, unsafe.Pointer(&newState))
+	b.logger.Infof("Child policy %q has new state %+v", id, newState)
+	b.sendNewPickerLocked()
+}
+
+// acquireChildPolicyReferences attempts to acquire references to
+// childPolicyWrappers corresponding to the passed in targets. If there is no
+// childPolicyWrapper corresponding to one of the targets, a new one is created
+// and added to the BalancerGroup.
+func (b *rlsBalancer) acquireChildPolicyReferences(targets []string) []*childPolicyWrapper {
+	b.stateMu.Lock()
+	var newChildPolicies []*childPolicyWrapper
+	for _, target := range targets {
+		// If the target exists in the LB policy's childPolicies map. a new
+		// reference is taken here and added to the new list.
+		if cpw := b.childPolicies[target]; cpw != nil {
+			cpw.acquireRef()
+			newChildPolicies = append(newChildPolicies, cpw)
+			continue
+		}
+
+		// If the target does not exist in the child policy map, then a new
+		// child policy wrapper is created and added to the new list.
+		cpw := newChildPolicyWrapper(target)
+		b.childPolicies[target] = cpw
+		b.bg.Add(target, b.childPolicyBuilder)
+		b.logger.Infof("Child policy %q added to BalancerGroup", target)
+		newChildPolicies = append(newChildPolicies, cpw)
+		if err := b.buildAndPushChildPolicyConfigs(target, b.lbCfg, &balancer.ClientConnState{
+			ResolverState: b.resolverState,
+		}); err != nil {
+			cpw.lamify(err)
+		}
+	}
+	b.stateMu.Unlock()
+	return newChildPolicies
+}
+
+// releaseChildPolicyReferences releases references to childPolicyWrappers
+// corresponding to the passed in targets. If the release reference was the last
+// one, the child policy is removed from the BalancerGroup.
+func (b *rlsBalancer) releaseChildPolicyReferences(targets []string) {
+	b.stateMu.Lock()
+	for _, target := range targets {
+		if cpw := b.childPolicies[target]; cpw.releaseRef() {
+			delete(b.childPolicies, cpw.target)
+			b.bg.Remove(cpw.target)
+		}
+	}
+	b.stateMu.Unlock()
+}
+
+// Report reports the metrics data to the provided recorder.
+func (b *rlsBalancer) Report(r estats.AsyncMetricsRecorder) error {
+	b.cacheMu.Lock()
+	currentSize := b.dataCache.currentSize
+	entriesLen := int64(len(b.dataCache.entries))
+	rlsServerTarget := b.dataCache.rlsServerTarget
+	grpcTarget := b.dataCache.grpcTarget
+	uuid := b.dataCache.uuid
+	shutdown := b.dataCache.shutdown.HasFired()
+	b.cacheMu.Unlock()
+
+	if shutdown {
+		return nil
+	}
+
+	cacheSizeMetric.Record(r, currentSize, grpcTarget, rlsServerTarget, uuid)
+	cacheEntriesMetric.Record(r, entriesLen, grpcTarget, rlsServerTarget, uuid)
+	return nil
+}

balancer/rls/cache.go

@@ -0,0 +1,378 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"container/list"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc/internal/backoff"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+)
+
+// cacheKey represents the key used to uniquely identify an entry in the data
+// cache and in the pending requests map.
+type cacheKey struct {
+	// path is the full path of the incoming RPC request.
+	path string
+	// keys is a stringified version of the RLS request key map built using the
+	// RLS keyBuilder. Since maps are not a type which is comparable in Go, it
+	// cannot be part of the key for another map (entries in the data cache and
+	// pending requests map are stored in maps).
+	keys string
+}
+
+// cacheEntry wraps all the data to be stored in a data cache entry.
+type cacheEntry struct {
+	// childPolicyWrappers contains the list of child policy wrappers
+	// corresponding to the targets returned by the RLS server for this entry.
+	childPolicyWrappers []*childPolicyWrapper
+	// headerData is received in the RLS response and is to be sent in the
+	// X-Google-RLS-Data header for matching RPCs.
+	headerData string
+	// expiryTime is the absolute time at which this cache entry stops
+	// being valid. When an RLS request succeeds, this is set to the current
+	// time plus the max_age field from the LB policy config.
+	expiryTime time.Time
+	// staleTime is the absolute time after which this cache entry will be
+	// proactively refreshed if an incoming RPC matches this entry. When an RLS
+	// request succeeds, this is set to the current time plus the stale_age from
+	// the LB policy config.
+	staleTime time.Time
+	// earliestEvictTime is the absolute time before which this entry should not
+	// be evicted from the cache. When a cache entry is created, this is set to
+	// the current time plus a default value of 5 seconds. This is required to
+	// make sure that a new entry added to the cache is not evicted before the
+	// RLS response arrives (usually when the cache is too small).
+	earliestEvictTime time.Time
+
+	// status stores the RPC status of the previous RLS request for this
+	// entry. Picks for entries with a non-nil value for this field are failed
+	// with the error stored here.
+	status error
+	// backoffState contains all backoff related state. When an RLS request
+	// succeeds, backoffState is reset. This state moves between the data cache
+	// and the pending requests map.
+	backoffState *backoffState
+	// backoffTime is the absolute time at which the backoff period for this
+	// entry ends. When an RLS request fails, this is set to the current time
+	// plus the backoff value returned by the backoffState. The backoff timer is
+	// also setup with this value. No new RLS requests are sent out for this
+	// entry until the backoff period ends.
+	//
+	// Set to zero time instant upon a successful RLS response.
+	backoffTime time.Time
+	// backoffExpiryTime is the absolute time at which an entry which has gone
+	// through backoff stops being valid.  When an RLS request fails, this is
+	// set to the current time plus twice the backoff time. The cache expiry
+	// timer will only delete entries for which both expiryTime and
+	// backoffExpiryTime are in the past.
+	//
+	// Set to zero time instant upon a successful RLS response.
+	backoffExpiryTime time.Time
+
+	// size stores the size of this cache entry. Used to enforce the cache size
+	// specified in the LB policy configuration.
+	size int64
+}
+
+// backoffState wraps all backoff related state associated with a cache entry.
+type backoffState struct {
+	// retries keeps track of the number of RLS failures, to be able to
+	// determine the amount of time to backoff before the next attempt.
+	retries int
+	// bs is the exponential backoff implementation which returns the amount of
+	// time to backoff, given the number of retries.
+	bs backoff.Strategy
+	// timer fires when the backoff period ends and incoming requests after this
+	// will trigger a new RLS request.
+	timer *time.Timer
+}
+
+// lru is a cache implementation with a least recently used eviction policy.
+// Internally it uses a doubly linked list, with the least recently used element
+// at the front of the list and the most recently used element at the back of
+// the list. The value stored in this cache will be of type `cacheKey`.
+//
+// It is not safe for concurrent access.
+type lru struct {
+	ll *list.List
+
+	// A map from the value stored in the lru to its underlying list element is
+	// maintained to have a clean API. Without this, a subset of the lru's API
+	// would accept/return cacheKey while another subset would accept/return
+	// list elements.
+	m map[cacheKey]*list.Element
+}
+
+// newLRU creates a new cache with a least recently used eviction policy.
+func newLRU() *lru {
+	return &lru{
+		ll: list.New(),
+		m:  make(map[cacheKey]*list.Element),
+	}
+}
+
+func (l *lru) addEntry(key cacheKey) {
+	e := l.ll.PushBack(key)
+	l.m[key] = e
+}
+
+func (l *lru) makeRecent(key cacheKey) {
+	e := l.m[key]
+	l.ll.MoveToBack(e)
+}
+
+func (l *lru) removeEntry(key cacheKey) {
+	e := l.m[key]
+	l.ll.Remove(e)
+	delete(l.m, key)
+}
+
+func (l *lru) getLeastRecentlyUsed() cacheKey {
+	e := l.ll.Front()
+	if e == nil {
+		return cacheKey{}
+	}
+	return e.Value.(cacheKey)
+}
+
+// dataCache contains a cache of RLS data used by the LB policy to make routing
+// decisions.
+//
+// The dataCache will be keyed by the request's path and keys, represented by
+// the `cacheKey` type. It will maintain the cache keys in an `lru` and the
+// cache data, represented by the `cacheEntry` type, in a native map.
+//
+// It is not safe for concurrent access.
+type dataCache struct {
+	maxSize         int64 // Maximum allowed size.
+	currentSize     int64 // Current size.
+	keys            *lru  // Cache keys maintained in lru order.
+	entries         map[cacheKey]*cacheEntry
+	logger          *internalgrpclog.PrefixLogger
+	shutdown        *grpcsync.Event
+	rlsServerTarget string
+
+	// Read only after initialization.
+	grpcTarget string
+	uuid       string
+}
+
+func newDataCache(size int64, logger *internalgrpclog.PrefixLogger, grpcTarget string) *dataCache {
+	return &dataCache{
+		maxSize:    size,
+		keys:       newLRU(),
+		entries:    make(map[cacheKey]*cacheEntry),
+		logger:     logger,
+		shutdown:   grpcsync.NewEvent(),
+		grpcTarget: grpcTarget,
+		uuid:       uuid.New().String(),
+	}
+}
+
+// updateRLSServerTarget updates the RLS Server Target the RLS Balancer is
+// configured with.
+func (dc *dataCache) updateRLSServerTarget(rlsServerTarget string) {
+	dc.rlsServerTarget = rlsServerTarget
+}
+
+// resize changes the maximum allowed size of the data cache.
+//
+// The return value indicates if an entry with a valid backoff timer was
+// evicted. This is important to the RLS LB policy which would send a new picker
+// on the channel to re-process any RPCs queued as a result of this backoff
+// timer.
+func (dc *dataCache) resize(size int64) (backoffCancelled bool) {
+	if dc.shutdown.HasFired() {
+		return false
+	}
+
+	backoffCancelled = false
+	for dc.currentSize > size {
+		key := dc.keys.getLeastRecentlyUsed()
+		entry, ok := dc.entries[key]
+		if !ok {
+			// This should never happen.
+			dc.logger.Errorf("cacheKey %+v not found in the cache while attempting to resize it", key)
+			break
+		}
+
+		// When we encounter a cache entry whose minimum expiration time is in
+		// the future, we abort the LRU pass, which may temporarily leave the
+		// cache being too large. This is necessary to ensure that in cases
+		// where the cache is too small, when we receive an RLS Response, we
+		// keep the resulting cache entry around long enough for the pending
+		// incoming requests to be re-processed through the new Picker. If we
+		// didn't do this, then we'd risk throwing away each RLS response as we
+		// receive it, in which case we would fail to actually route any of our
+		// incoming requests.
+		if entry.earliestEvictTime.After(time.Now()) {
+			dc.logger.Warningf("cachekey %+v is too recent to be evicted. Stopping cache resizing for now", key)
+			break
+		}
+
+		// Stop the backoff timer before evicting the entry.
+		if entry.backoffState != nil && entry.backoffState.timer != nil {
+			if entry.backoffState.timer.Stop() {
+				entry.backoffState.timer = nil
+				backoffCancelled = true
+			}
+		}
+		dc.deleteAndCleanup(key, entry)
+	}
+	dc.maxSize = size
+	return backoffCancelled
+}
+
+// evictExpiredEntries sweeps through the cache and deletes expired entries. An
+// expired entry is one for which both the `expiryTime` and `backoffExpiryTime`
+// fields are in the past.
+//
+// The return value indicates if any expired entries were evicted.
+//
+// The LB policy invokes this method periodically to purge expired entries.
+func (dc *dataCache) evictExpiredEntries() bool {
+	if dc.shutdown.HasFired() {
+		return false
+	}
+
+	evicted := false
+	for key, entry := range dc.entries {
+		// Only evict entries for which both the data expiration time and
+		// backoff expiration time fields are in the past.
+		now := time.Now()
+		if entry.expiryTime.After(now) || entry.backoffExpiryTime.After(now) {
+			continue
+		}
+		dc.deleteAndCleanup(key, entry)
+		evicted = true
+	}
+	return evicted
+}
+
+// resetBackoffState sweeps through the cache and for entries with a backoff
+// state, the backoff timer is cancelled and the backoff state is reset. The
+// return value indicates if any entries were mutated in this fashion.
+//
+// The LB policy invokes this method when the control channel moves from READY
+// to TRANSIENT_FAILURE back to READY. See `monitorConnectivityState` method on
+// the `controlChannel` type for more details.
+func (dc *dataCache) resetBackoffState(newBackoffState *backoffState) bool {
+	if dc.shutdown.HasFired() {
+		return false
+	}
+
+	backoffReset := false
+	for _, entry := range dc.entries {
+		if entry.backoffState == nil {
+			continue
+		}
+		if entry.backoffState.timer != nil {
+			entry.backoffState.timer.Stop()
+			entry.backoffState.timer = nil
+		}
+		entry.backoffState = &backoffState{bs: newBackoffState.bs}
+		entry.backoffTime = time.Time{}
+		entry.backoffExpiryTime = time.Time{}
+		backoffReset = true
+	}
+	return backoffReset
+}
+
+// addEntry adds a cache entry for the given key.
+//
+// Return value backoffCancelled indicates if a cache entry with a valid backoff
+// timer was evicted to make space for the current entry. This is important to
+// the RLS LB policy which would send a new picker on the channel to re-process
+// any RPCs queued as a result of this backoff timer.
+//
+// Return value ok indicates if entry was successfully added to the cache.
+func (dc *dataCache) addEntry(key cacheKey, entry *cacheEntry) (backoffCancelled bool, ok bool) {
+	if dc.shutdown.HasFired() {
+		return false, false
+	}
+
+	// Handle the extremely unlikely case that a single entry is bigger than the
+	// size of the cache.
+	if entry.size > dc.maxSize {
+		return false, false
+	}
+	dc.entries[key] = entry
+	dc.currentSize += entry.size
+	dc.keys.addEntry(key)
+	// If the new entry makes the cache go over its configured size, remove some
+	// old entries.
+	if dc.currentSize > dc.maxSize {
+		backoffCancelled = dc.resize(dc.maxSize)
+	}
+
+	return backoffCancelled, true
+}
+
+// updateEntrySize updates the size of a cache entry and the current size of the
+// data cache. An entry's size can change upon receipt of an RLS response.
+func (dc *dataCache) updateEntrySize(entry *cacheEntry, newSize int64) {
+	dc.currentSize -= entry.size
+	entry.size = newSize
+	dc.currentSize += entry.size
+
+}
+
+func (dc *dataCache) getEntry(key cacheKey) *cacheEntry {
+	if dc.shutdown.HasFired() {
+		return nil
+	}
+
+	entry, ok := dc.entries[key]
+	if !ok {
+		return nil
+	}
+	dc.keys.makeRecent(key)
+	return entry
+}
+
+func (dc *dataCache) removeEntryForTesting(key cacheKey) {
+	entry, ok := dc.entries[key]
+	if !ok {
+		return
+	}
+	dc.deleteAndCleanup(key, entry)
+}
+
+// deleteAndCleanup performs actions required at the time of deleting an entry
+// from the data cache.
+// - the entry is removed from the map of entries
+// - current size of the data cache is update
+// - the key is removed from the LRU
+func (dc *dataCache) deleteAndCleanup(key cacheKey, entry *cacheEntry) {
+	delete(dc.entries, key)
+	dc.currentSize -= entry.size
+	dc.keys.removeEntry(key)
+
+}
+
+func (dc *dataCache) stop() {
+	for key, entry := range dc.entries {
+		dc.deleteAndCleanup(key, entry)
+	}
+	dc.shutdown.Fire()
+}

balancer/rls/cache_test.go

@@ -0,0 +1,243 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/backoff"
+)
+
+var (
+	cacheKeys = []cacheKey{
+		{path: "0", keys: "a"},
+		{path: "1", keys: "b"},
+		{path: "2", keys: "c"},
+		{path: "3", keys: "d"},
+		{path: "4", keys: "e"},
+	}
+
+	longDuration  = 10 * time.Minute
+	shortDuration = 1 * time.Millisecond
+	cacheEntries  []*cacheEntry
+)
+
+func initCacheEntries() {
+	// All entries have a dummy size of 1 to simplify resize operations.
+	cacheEntries = []*cacheEntry{
+		{
+			// Entry is valid and minimum expiry time has not expired.
+			expiryTime:        time.Now().Add(longDuration),
+			earliestEvictTime: time.Now().Add(longDuration),
+			size:              1,
+		},
+		{
+			// Entry is valid and is in backoff.
+			expiryTime:   time.Now().Add(longDuration),
+			backoffTime:  time.Now().Add(longDuration),
+			backoffState: &backoffState{timer: time.NewTimer(longDuration)},
+			size:         1,
+		},
+		{
+			// Entry is valid, and not in backoff.
+			expiryTime: time.Now().Add(longDuration),
+			size:       1,
+		},
+		{
+			// Entry is invalid.
+			expiryTime: time.Time{}.Add(shortDuration),
+			size:       1,
+		},
+		{
+			// Entry is invalid valid and backoff has expired.
+			expiryTime:        time.Time{}.Add(shortDuration),
+			backoffExpiryTime: time.Time{}.Add(shortDuration),
+			size:              1,
+		},
+	}
+}
+
+func (s) TestLRU_BasicOperations(t *testing.T) {
+	initCacheEntries()
+	// Create an LRU and add some entries to it.
+	lru := newLRU()
+	for _, k := range cacheKeys {
+		lru.addEntry(k)
+	}
+
+	// Get the least recent entry. This should be the first entry we added.
+	if got, want := lru.getLeastRecentlyUsed(), cacheKeys[0]; got != want {
+		t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want)
+	}
+
+	// Iterate through the slice of keys we added earlier, making them the most
+	// recent entry, one at a time. The least recent entry at that point should
+	// be the next entry from our slice of keys.
+	for i, k := range cacheKeys {
+		lru.makeRecent(k)
+
+		lruIndex := (i + 1) % len(cacheKeys)
+		if got, want := lru.getLeastRecentlyUsed(), cacheKeys[lruIndex]; got != want {
+			t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want)
+		}
+	}
+
+	// Iterate through the slice of keys we added earlier, removing them one at
+	// a time The least recent entry at that point should be the next entry from
+	// our slice of keys, except for the last one because the lru will be empty.
+	for i, k := range cacheKeys {
+		lru.removeEntry(k)
+
+		var want cacheKey
+		if i < len(cacheKeys)-1 {
+			want = cacheKeys[i+1]
+		}
+		if got := lru.getLeastRecentlyUsed(); got != want {
+			t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want)
+		}
+	}
+}
+
+func (s) TestDataCache_BasicOperations(t *testing.T) {
+	initCacheEntries()
+	dc := newDataCache(5, nil, "")
+	for i, k := range cacheKeys {
+		dc.addEntry(k, cacheEntries[i])
+	}
+	for i, k := range cacheKeys {
+		entry := dc.getEntry(k)
+		if !cmp.Equal(entry, cacheEntries[i], cmp.AllowUnexported(cacheEntry{}, backoffState{}), cmpopts.IgnoreUnexported(time.Timer{})) {
+			t.Fatalf("Data cache lookup for key %v returned entry %v, want %v", k, entry, cacheEntries[i])
+		}
+	}
+}
+
+func (s) TestDataCache_AddForcesResize(t *testing.T) {
+	initCacheEntries()
+	dc := newDataCache(1, nil, "")
+
+	// The first entry in cacheEntries has a minimum expiry time in the future.
+	// This entry would stop the resize operation since we do not evict entries
+	// whose minimum expiration time is in the future. So, we do not use that
+	// entry in this test. The entry being added has a running backoff timer.
+	evicted, ok := dc.addEntry(cacheKeys[1], cacheEntries[1])
+	if evicted || !ok {
+		t.Fatalf("dataCache.addEntry() returned (%v, %v) want (false, true)", evicted, ok)
+	}
+
+	// Add another entry leading to the eviction of the above entry which has a
+	// running backoff timer. The first return value is expected to be true.
+	backoffCancelled, ok := dc.addEntry(cacheKeys[2], cacheEntries[2])
+	if !backoffCancelled || !ok {
+		t.Fatalf("dataCache.addEntry() returned (%v, %v) want (true, true)", backoffCancelled, ok)
+	}
+
+	// Add another entry leading to the eviction of the above entry which does not
+	// have a running backoff timer. This should evict the above entry, but the
+	// first return value is expected to be false.
+	backoffCancelled, ok = dc.addEntry(cacheKeys[3], cacheEntries[3])
+	if backoffCancelled || !ok {
+		t.Fatalf("dataCache.addEntry() returned (%v, %v) want (false, true)", backoffCancelled, ok)
+	}
+}
+
+func (s) TestDataCache_Resize(t *testing.T) {
+	initCacheEntries()
+	dc := newDataCache(5, nil, "")
+	for i, k := range cacheKeys {
+		dc.addEntry(k, cacheEntries[i])
+	}
+
+	// The first cache entry (with a key of cacheKeys[0]) that we added has an
+	// earliestEvictTime in the future. As part of the resize operation, we
+	// traverse the cache in least recently used order, and this will be first
+	// entry that we will encounter. And since the earliestEvictTime is in the
+	// future, the resize operation will stop, leaving the cache bigger than
+	// what was asked for.
+	if dc.resize(1) {
+		t.Fatalf("dataCache.resize() returned true, want false")
+	}
+	if dc.currentSize != 5 {
+		t.Fatalf("dataCache.size is %d, want 5", dc.currentSize)
+	}
+
+	// Remove the entry with earliestEvictTime in the future and retry the
+	// resize operation.
+	dc.removeEntryForTesting(cacheKeys[0])
+	if !dc.resize(1) {
+		t.Fatalf("dataCache.resize() returned false, want true")
+	}
+	if dc.currentSize != 1 {
+		t.Fatalf("dataCache.size is %d, want 1", dc.currentSize)
+	}
+}
+
+func (s) TestDataCache_EvictExpiredEntries(t *testing.T) {
+	initCacheEntries()
+	dc := newDataCache(5, nil, "")
+	for i, k := range cacheKeys {
+		dc.addEntry(k, cacheEntries[i])
+	}
+
+	// The last two entries in the cacheEntries list have expired, and will be
+	// evicted. The first three should still remain in the cache.
+	if !dc.evictExpiredEntries() {
+		t.Fatal("dataCache.evictExpiredEntries() returned false, want true")
+	}
+	if dc.currentSize != 3 {
+		t.Fatalf("dataCache.size is %d, want 3", dc.currentSize)
+	}
+	for i := 0; i < 3; i++ {
+		entry := dc.getEntry(cacheKeys[i])
+		if !cmp.Equal(entry, cacheEntries[i], cmp.AllowUnexported(cacheEntry{}, backoffState{}), cmpopts.IgnoreUnexported(time.Timer{})) {
+			t.Fatalf("Data cache lookup for key %v returned entry %v, want %v", cacheKeys[i], entry, cacheEntries[i])
+		}
+	}
+}
+
+func (s) TestDataCache_ResetBackoffState(t *testing.T) {
+	type fakeBackoff struct {
+		backoff.Strategy
+	}
+
+	initCacheEntries()
+	dc := newDataCache(5, nil, "")
+	for i, k := range cacheKeys {
+		dc.addEntry(k, cacheEntries[i])
+	}
+
+	newBackoffState := &backoffState{bs: &fakeBackoff{}}
+	if updatePicker := dc.resetBackoffState(newBackoffState); !updatePicker {
+		t.Fatal("dataCache.resetBackoffState() returned updatePicker is false, want true")
+	}
+
+	// Make sure that the entry with no backoff state was not touched.
+	if entry := dc.getEntry(cacheKeys[0]); cmp.Equal(entry.backoffState, newBackoffState, cmp.AllowUnexported(backoffState{})) {
+		t.Fatal("dataCache.resetBackoffState() touched entries without a valid backoffState")
+	}
+
+	// Make sure that the entry with a valid backoff state was reset.
+	entry := dc.getEntry(cacheKeys[1])
+	if diff := cmp.Diff(entry.backoffState, newBackoffState, cmp.AllowUnexported(backoffState{})); diff != "" {
+		t.Fatalf("unexpected diff in backoffState for cache entry after dataCache.resetBackoffState(): %s", diff)
+	}
+}

balancer/rls/child_policy.go

@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"fmt"
+	"sync/atomic"
+	"unsafe"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+// childPolicyWrapper is a reference counted wrapper around a child policy.
+//
+// The LB policy maintains a map of these wrappers keyed by the target returned
+// by RLS. When a target is seen for the first time, a child policy wrapper is
+// created for it and the wrapper is added to the child policy map. Each entry
+// in the data cache holds references to the corresponding child policy
+// wrappers. The LB policy also holds a reference to the child policy wrapper
+// for the default target specified in the LB Policy Configuration
+//
+// When a cache entry is evicted, it releases references to the child policy
+// wrappers that it contains. When all references have been released, the
+// wrapper is removed from the child policy map and is destroyed.
+//
+// The child policy wrapper also caches the connectivity state and most recent
+// picker from the child policy. Once the child policy wrapper reports
+// TRANSIENT_FAILURE, it will continue reporting that state until it goes READY;
+// transitions from TRANSIENT_FAILURE to CONNECTING are ignored.
+//
+// Whenever a child policy wrapper changes its connectivity state, the LB policy
+// returns a new picker to the channel, since the channel may need to re-process
+// the picks for queued RPCs.
+//
+// It is not safe for concurrent access.
+type childPolicyWrapper struct {
+	logger *internalgrpclog.PrefixLogger
+	target string // RLS target corresponding to this child policy.
+	refCnt int    // Reference count.
+
+	// Balancer state reported by the child policy. The RLS LB policy maintains
+	// these child policies in a BalancerGroup. The state reported by the child
+	// policy is pushed to the state aggregator (which is also implemented by the
+	// RLS LB policy) and cached here. See handleChildPolicyStateUpdate() for
+	// details on how the state aggregation is performed.
+	//
+	// While this field is written to by the LB policy, it is read by the picker
+	// at Pick time. Making this an atomic to enable the picker to read this value
+	// without a mutex.
+	state unsafe.Pointer // *balancer.State
+}
+
+// newChildPolicyWrapper creates a child policy wrapper for the given target,
+// and is initialized with one reference and starts off in CONNECTING state.
+func newChildPolicyWrapper(target string) *childPolicyWrapper {
+	c := &childPolicyWrapper{
+		target: target,
+		refCnt: 1,
+		state: unsafe.Pointer(&balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		}),
+	}
+	c.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-child-policy-wrapper %s %p] ", c.target, c))
+	c.logger.Infof("Created")
+	return c
+}
+
+// acquireRef increments the reference count on the child policy wrapper.
+func (c *childPolicyWrapper) acquireRef() {
+	c.refCnt++
+}
+
+// releaseRef decrements the reference count on the child policy wrapper. The
+// return value indicates whether the released reference was the last one.
+func (c *childPolicyWrapper) releaseRef() bool {
+	c.refCnt--
+	return c.refCnt == 0
+}
+
+// lamify causes the child policy wrapper to return a picker which will always
+// fail requests. This is used when the wrapper runs into errors when trying to
+// build and parse the child policy configuration.
+func (c *childPolicyWrapper) lamify(err error) {
+	c.logger.Warningf("Entering lame mode: %v", err)
+	atomic.StorePointer(&c.state, unsafe.Pointer(&balancer.State{
+		ConnectivityState: connectivity.TransientFailure,
+		Picker:            base.NewErrPicker(err),
+	}))
+}

balancer/rls/config.go

@@ -0,0 +1,330 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/rls/internal/keys"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/pretty"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+const (
+	// Default max_age if not specified (or greater than this value) in the
+	// service config.
+	maxMaxAge = 5 * time.Minute
+	// Upper limit for cache_size since we don't fully trust the service config.
+	maxCacheSize = 5 * 1024 * 1024 * 8 // 5MB in bytes
+	// Default lookup_service_timeout if not specified in the service config.
+	defaultLookupServiceTimeout = 10 * time.Second
+	// Default value for targetNameField in the child policy config during
+	// service config validation.
+	dummyChildPolicyTarget = "target_name_to_be_filled_in_later"
+)
+
+// lbConfig is the internal representation of the RLS LB policy's config.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+
+	cacheSizeBytes       int64 // Keep this field 64-bit aligned.
+	kbMap                keys.BuilderMap
+	lookupService        string
+	lookupServiceTimeout time.Duration
+	maxAge               time.Duration
+	staleAge             time.Duration
+	defaultTarget        string
+
+	childPolicyName             string
+	childPolicyConfig           map[string]json.RawMessage
+	childPolicyTargetField      string
+	controlChannelServiceConfig string
+}
+
+func (lbCfg *lbConfig) Equal(other *lbConfig) bool {
+	return lbCfg.kbMap.Equal(other.kbMap) &&
+		lbCfg.lookupService == other.lookupService &&
+		lbCfg.lookupServiceTimeout == other.lookupServiceTimeout &&
+		lbCfg.maxAge == other.maxAge &&
+		lbCfg.staleAge == other.staleAge &&
+		lbCfg.cacheSizeBytes == other.cacheSizeBytes &&
+		lbCfg.defaultTarget == other.defaultTarget &&
+		lbCfg.childPolicyName == other.childPolicyName &&
+		lbCfg.childPolicyTargetField == other.childPolicyTargetField &&
+		lbCfg.controlChannelServiceConfig == other.controlChannelServiceConfig &&
+		childPolicyConfigEqual(lbCfg.childPolicyConfig, other.childPolicyConfig)
+}
+
+func childPolicyConfigEqual(a, b map[string]json.RawMessage) bool {
+	if (b == nil) != (a == nil) {
+		return false
+	}
+	if len(b) != len(a) {
+		return false
+	}
+	for k, jsonA := range a {
+		jsonB, ok := b[k]
+		if !ok {
+			return false
+		}
+		if !bytes.Equal(jsonA, jsonB) {
+			return false
+		}
+	}
+	return true
+}
+
+// This struct resembles the JSON representation of the loadBalancing config
+// and makes it easier to unmarshal.
+type lbConfigJSON struct {
+	RouteLookupConfig                json.RawMessage
+	RouteLookupChannelServiceConfig  json.RawMessage
+	ChildPolicy                      []map[string]json.RawMessage
+	ChildPolicyConfigTargetFieldName string
+}
+
+// ParseConfig parses the JSON load balancer config provided into an
+// internal form or returns an error if the config is invalid.
+//
+//	 When parsing a config update, the following validations are performed:
+//	 - routeLookupConfig:
+//	   - grpc_keybuilders field:
+//	     - must have at least one entry
+//	     - must not have two entries with the same `Name`
+//	     - within each entry:
+//	       - must have at least one `Name`
+//	       - must not have a `Name` with the `service` field unset or empty
+//	       - within each `headers` entry:
+//	         - must not have `required_match` set
+//	         - must not have `key` unset or empty
+//	       - across all `headers`, `constant_keys` and `extra_keys` fields:
+//	         - must not have the same `key` specified twice
+//	         - no `key` must be the empty string
+//	   - `lookup_service` field must be set and must parse as a target URI
+//	   - if `max_age` > 5m, it should be set to 5 minutes
+//	   - if `stale_age` > `max_age`, ignore it
+//	   - if `stale_age` is set, then `max_age` must also be set
+//	   - ignore `valid_targets` field
+//	   - `cache_size_bytes` field must have a value greater than 0, and if its
+//	     value is greater than 5M, we cap it at 5M
+//
+//	- routeLookupChannelServiceConfig:
+//	  - if specified, must parse as valid service config
+//
+//	- childPolicy:
+//	  - must find a valid child policy with a valid config
+//
+//	- childPolicyConfigTargetFieldName:
+//	  - must be set and non-empty
+func (rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	if logger.V(2) {
+		logger.Infof("Received JSON service config: %v", pretty.ToJSON(c))
+	}
+
+	cfgJSON := &lbConfigJSON{}
+	if err := json.Unmarshal(c, cfgJSON); err != nil {
+		return nil, fmt.Errorf("rls: json unmarshal failed for service config %+v: %v", string(c), err)
+	}
+
+	m := protojson.UnmarshalOptions{DiscardUnknown: true}
+	rlsProto := &rlspb.RouteLookupConfig{}
+	if err := m.Unmarshal(cfgJSON.RouteLookupConfig, rlsProto); err != nil {
+		return nil, fmt.Errorf("rls: bad RouteLookupConfig proto %+v: %v", string(cfgJSON.RouteLookupConfig), err)
+	}
+	lbCfg, err := parseRLSProto(rlsProto)
+	if err != nil {
+		return nil, err
+	}
+
+	if sc := string(cfgJSON.RouteLookupChannelServiceConfig); sc != "" {
+		parsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(sc)
+		if parsed.Err != nil {
+			return nil, fmt.Errorf("rls: bad control channel service config %q: %v", sc, parsed.Err)
+		}
+		lbCfg.controlChannelServiceConfig = sc
+	}
+
+	if cfgJSON.ChildPolicyConfigTargetFieldName == "" {
+		return nil, fmt.Errorf("rls: childPolicyConfigTargetFieldName field is not set in service config %+v", string(c))
+	}
+	name, config, err := parseChildPolicyConfigs(cfgJSON.ChildPolicy, cfgJSON.ChildPolicyConfigTargetFieldName)
+	if err != nil {
+		return nil, err
+	}
+	lbCfg.childPolicyName = name
+	lbCfg.childPolicyConfig = config
+	lbCfg.childPolicyTargetField = cfgJSON.ChildPolicyConfigTargetFieldName
+	return lbCfg, nil
+}
+
+func parseRLSProto(rlsProto *rlspb.RouteLookupConfig) (*lbConfig, error) {
+	// Validations specified on the `grpc_keybuilders` field are performed here.
+	kbMap, err := keys.MakeBuilderMap(rlsProto)
+	if err != nil {
+		return nil, err
+	}
+
+	// `lookup_service` field must be set and must parse as a target URI.
+	lookupService := rlsProto.GetLookupService()
+	if lookupService == "" {
+		return nil, fmt.Errorf("rls: empty lookup_service in route lookup config %+v", rlsProto)
+	}
+	parsedTarget, err := url.Parse(lookupService)
+	if err != nil {
+		// url.Parse() fails if scheme is missing. Retry with default scheme.
+		parsedTarget, err = url.Parse(resolver.GetDefaultScheme() + ":///" + lookupService)
+		if err != nil {
+			return nil, fmt.Errorf("rls: invalid target URI in lookup_service %s", lookupService)
+		}
+	}
+	if parsedTarget.Scheme == "" {
+		parsedTarget.Scheme = resolver.GetDefaultScheme()
+	}
+	if resolver.Get(parsedTarget.Scheme) == nil {
+		return nil, fmt.Errorf("rls: unregistered scheme in lookup_service %s", lookupService)
+	}
+
+	lookupServiceTimeout, err := convertDuration(rlsProto.GetLookupServiceTimeout())
+	if err != nil {
+		return nil, fmt.Errorf("rls: failed to parse lookup_service_timeout in route lookup config %+v: %v", rlsProto, err)
+	}
+	if lookupServiceTimeout == 0 {
+		lookupServiceTimeout = defaultLookupServiceTimeout
+	}
+
+	// Validations performed here:
+	// - if `max_age` > 5m, it should be set to 5 minutes
+	//   only if stale age is not set
+	// - if `stale_age` > `max_age`, ignore it
+	// - if `stale_age` is set, then `max_age` must also be set
+	maxAgeSet := false
+	maxAge, err := convertDuration(rlsProto.GetMaxAge())
+	if err != nil {
+		return nil, fmt.Errorf("rls: failed to parse max_age in route lookup config %+v: %v", rlsProto, err)
+	}
+	if maxAge == 0 {
+		maxAge = maxMaxAge
+	} else {
+		maxAgeSet = true
+	}
+
+	staleAgeSet := false
+	staleAge, err := convertDuration(rlsProto.GetStaleAge())
+	if err != nil {
+		return nil, fmt.Errorf("rls: failed to parse staleAge in route lookup config %+v: %v", rlsProto, err)
+	}
+	if staleAge == 0 {
+		staleAge = maxMaxAge
+	} else {
+		staleAgeSet = true
+	}
+
+	if staleAgeSet && !maxAgeSet {
+		return nil, fmt.Errorf("rls: stale_age is set, but max_age is not in route lookup config %+v", rlsProto)
+	}
+	if staleAge > maxMaxAge {
+		staleAge = maxMaxAge
+	}
+	if !staleAgeSet && maxAge > maxMaxAge {
+		maxAge = maxMaxAge
+	}
+	if staleAge > maxAge {
+		staleAge = maxAge
+	}
+
+	// `cache_size_bytes` field must have a value greater than 0, and if its
+	// value is greater than 5M, we cap it at 5M
+	cacheSizeBytes := rlsProto.GetCacheSizeBytes()
+	if cacheSizeBytes <= 0 {
+		return nil, fmt.Errorf("rls: cache_size_bytes must be set to a non-zero value: %+v", rlsProto)
+	}
+	if cacheSizeBytes > maxCacheSize {
+		logger.Infof("rls: cache_size_bytes %v is too large, setting it to: %v", cacheSizeBytes, maxCacheSize)
+		cacheSizeBytes = maxCacheSize
+	}
+	return &lbConfig{
+		kbMap:                kbMap,
+		lookupService:        lookupService,
+		lookupServiceTimeout: lookupServiceTimeout,
+		maxAge:               maxAge,
+		staleAge:             staleAge,
+		cacheSizeBytes:       cacheSizeBytes,
+		defaultTarget:        rlsProto.GetDefaultTarget(),
+	}, nil
+}
+
+// parseChildPolicyConfigs iterates through the list of child policies and picks
+// the first registered policy and validates its config.
+func parseChildPolicyConfigs(childPolicies []map[string]json.RawMessage, targetFieldName string) (string, map[string]json.RawMessage, error) {
+	for i, config := range childPolicies {
+		if len(config) != 1 {
+			return "", nil, fmt.Errorf("rls: invalid childPolicy: entry %v does not contain exactly 1 policy/config pair: %q", i, config)
+		}
+
+		var name string
+		var rawCfg json.RawMessage
+		for name, rawCfg = range config {
+		}
+		builder := balancer.Get(name)
+		if builder == nil {
+			continue
+		}
+		parser, ok := builder.(balancer.ConfigParser)
+		if !ok {
+			return "", nil, fmt.Errorf("rls: childPolicy %q with config %q does not support config parsing", name, string(rawCfg))
+		}
+
+		// To validate child policy configs we do the following:
+		// - unmarshal the raw JSON bytes of the child policy config into a map
+		// - add an entry with key set to `target_field_name` and a dummy value
+		// - marshal the map back to JSON and parse the config using the parser
+		// retrieved previously
+		var childConfig map[string]json.RawMessage
+		if err := json.Unmarshal(rawCfg, &childConfig); err != nil {
+			return "", nil, fmt.Errorf("rls: json unmarshal failed for child policy config %q: %v", string(rawCfg), err)
+		}
+		childConfig[targetFieldName], _ = json.Marshal(dummyChildPolicyTarget)
+		jsonCfg, err := json.Marshal(childConfig)
+		if err != nil {
+			return "", nil, fmt.Errorf("rls: json marshal failed for child policy config {%+v}: %v", childConfig, err)
+		}
+		if _, err := parser.ParseConfig(jsonCfg); err != nil {
+			return "", nil, fmt.Errorf("rls: childPolicy config validation failed: %v", err)
+		}
+		return name, childConfig, nil
+	}
+	return "", nil, fmt.Errorf("rls: invalid childPolicy config: no supported policies found in %+v", childPolicies)
+}
+
+func convertDuration(d *durationpb.Duration) (time.Duration, error) {
+	if d == nil {
+		return 0, nil
+	}
+	return d.AsDuration(), d.CheckValid()
+}

balancer/rls/config_test.go

@@ -0,0 +1,515 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	_ "google.golang.org/grpc/balancer/grpclb"               // grpclb for config parsing.
+	_ "google.golang.org/grpc/internal/resolver/passthrough" // passthrough resolver.
+)
+
+// testEqual reports whether the lbCfgs a and b are equal. This is to be used
+// only from tests. This ignores the keyBuilderMap field because its internals
+// are not exported, and hence not possible to specify in the want section of
+// the test. This is fine because we already have tests to make sure that the
+// keyBuilder is parsed properly from the service config.
+func testEqual(a, b *lbConfig) bool {
+	return a.lookupService == b.lookupService &&
+		a.lookupServiceTimeout == b.lookupServiceTimeout &&
+		a.maxAge == b.maxAge &&
+		a.staleAge == b.staleAge &&
+		a.cacheSizeBytes == b.cacheSizeBytes &&
+		a.defaultTarget == b.defaultTarget &&
+		a.controlChannelServiceConfig == b.controlChannelServiceConfig &&
+		a.childPolicyName == b.childPolicyName &&
+		a.childPolicyTargetField == b.childPolicyTargetField &&
+		childPolicyConfigEqual(a.childPolicyConfig, b.childPolicyConfig)
+}
+
+// TestParseConfig verifies successful config parsing scenarios.
+func (s) TestParseConfig(t *testing.T) {
+	childPolicyTargetFieldVal, _ := json.Marshal(dummyChildPolicyTarget)
+	tests := []struct {
+		desc    string
+		input   []byte
+		wantCfg *lbConfig
+	}{
+		{
+			// This input validates a few cases:
+			// - A top-level unknown field should not fail.
+			// - An unknown field in routeLookupConfig proto should not fail.
+			// - lookupServiceTimeout is set to its default value, since it is not specified in the input.
+			// - maxAge is clamped to maxMaxAge if staleAge is not set.
+			// - staleAge is ignored because it is higher than maxAge in the input.
+			// - cacheSizeBytes is greater than the hard upper limit of 5MB
+			desc: "with transformations 1",
+			input: []byte(`{
+				"top-level-unknown-field": "unknown-value",
+				"routeLookupConfig": {
+					"unknown-field": "unknown-value",
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": ":///target",
+					"maxAge" : "500s",
+					"staleAge": "600s",
+					"cacheSizeBytes": 100000000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"childPolicy": [
+					{"cds_experimental": {"Cluster": "my-fav-cluster"}},
+					{"unknown-policy": {"unknown-field": "unknown-value"}},
+					{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}
+				],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantCfg: &lbConfig{
+				lookupService:          ":///target",
+				lookupServiceTimeout:   10 * time.Second,  // This is the default value.
+				maxAge:                 500 * time.Second, // Max age is not clamped when stale age is set.
+				staleAge:               300 * time.Second, // StaleAge is clamped because it was higher than maxMaxAge.
+				cacheSizeBytes:         maxCacheSize,
+				defaultTarget:          "passthrough:///default",
+				childPolicyName:        "grpclb",
+				childPolicyTargetField: "serviceName",
+				childPolicyConfig: map[string]json.RawMessage{
+					"childPolicy": json.RawMessage(`[{"pickfirst": {}}]`),
+					"serviceName": json.RawMessage(childPolicyTargetFieldVal),
+				},
+			},
+		},
+		{
+			desc: "maxAge not clamped when staleAge is set",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": ":///target",
+					"maxAge" : "500s",
+					"staleAge": "200s",
+					"cacheSizeBytes": 100000000
+				},
+				"childPolicy": [
+					{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}
+				],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantCfg: &lbConfig{
+				lookupService:          ":///target",
+				lookupServiceTimeout:   10 * time.Second,  // This is the default value.
+				maxAge:                 500 * time.Second, // Max age is not clamped when stale age is set.
+				staleAge:               200 * time.Second, // This is stale age within maxMaxAge.
+				cacheSizeBytes:         maxCacheSize,
+				childPolicyName:        "grpclb",
+				childPolicyTargetField: "serviceName",
+				childPolicyConfig: map[string]json.RawMessage{
+					"childPolicy": json.RawMessage(`[{"pickfirst": {}}]`),
+					"serviceName": json.RawMessage(childPolicyTargetFieldVal),
+				},
+			},
+		},
+		{
+			desc: "maxAge clamped when staleAge is not set",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": ":///target",
+					"maxAge" : "500s",
+					"cacheSizeBytes": 100000000
+				},
+				"childPolicy": [
+					{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}
+				],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantCfg: &lbConfig{
+				lookupService:          ":///target",
+				lookupServiceTimeout:   10 * time.Second,  // This is the default value.
+				maxAge:                 300 * time.Second, // Max age is clamped when stale age is not set.
+				staleAge:               300 * time.Second,
+				cacheSizeBytes:         maxCacheSize,
+				childPolicyName:        "grpclb",
+				childPolicyTargetField: "serviceName",
+				childPolicyConfig: map[string]json.RawMessage{
+					"childPolicy": json.RawMessage(`[{"pickfirst": {}}]`),
+					"serviceName": json.RawMessage(childPolicyTargetFieldVal),
+				},
+			},
+		},
+		{
+			desc: "without transformations",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "target",
+					"lookupServiceTimeout" : "100s",
+					"maxAge": "60s",
+					"staleAge" : "50s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"routeLookupChannelServiceConfig": {"loadBalancingConfig": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}]},
+				"childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantCfg: &lbConfig{
+				lookupService:               "target",
+				lookupServiceTimeout:        100 * time.Second,
+				maxAge:                      60 * time.Second,
+				staleAge:                    50 * time.Second,
+				cacheSizeBytes:              1000,
+				defaultTarget:               "passthrough:///default",
+				controlChannelServiceConfig: `{"loadBalancingConfig": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}]}`,
+				childPolicyName:             "grpclb",
+				childPolicyTargetField:      "serviceName",
+				childPolicyConfig: map[string]json.RawMessage{
+					"childPolicy": json.RawMessage(`[{"pickfirst": {}}]`),
+					"serviceName": json.RawMessage(childPolicyTargetFieldVal),
+				},
+			},
+		},
+	}
+
+	builder := rlsBB{}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			lbCfg, err := builder.ParseConfig(test.input)
+			if err != nil || !testEqual(lbCfg.(*lbConfig), test.wantCfg) {
+				t.Errorf("ParseConfig(%s) = {%+v, %v}, want {%+v, nil}", string(test.input), lbCfg, err, test.wantCfg)
+			}
+		})
+	}
+}
+
+// TestParseConfigErrors verifies config parsing failure scenarios.
+func (s) TestParseConfigErrors(t *testing.T) {
+	tests := []struct {
+		desc    string
+		input   []byte
+		wantErr string
+	}{
+		{
+			desc:    "empty input",
+			input:   nil,
+			wantErr: "rls: json unmarshal failed for service config",
+		},
+		{
+			desc:    "bad json",
+			input:   []byte(`bad bad json`),
+			wantErr: "rls: json unmarshal failed for service config",
+		},
+		{
+			desc: "bad grpcKeyBuilder",
+			input: []byte(`{
+					"routeLookupConfig": {
+						"grpcKeybuilders": [{
+							"names": [{"service": "service", "method": "method"}],
+							"headers": [{"key": "k1", "requiredMatch": true, "names": ["v1"]}]
+						}]
+					}
+				}`),
+			wantErr: "rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set",
+		},
+		{
+			desc: "empty lookup service",
+			input: []byte(`{
+					"routeLookupConfig": {
+						"grpcKeybuilders": [{
+							"names": [{"service": "service", "method": "method"}],
+							"headers": [{"key": "k1", "names": ["v1"]}]
+						}]
+					}
+				}`),
+			wantErr: "rls: empty lookup_service in route lookup config",
+		},
+		{
+			desc: "unregistered scheme in lookup service URI",
+			input: []byte(`{
+					"routeLookupConfig": {
+						"grpcKeybuilders": [{
+							"names": [{"service": "service", "method": "method"}],
+							"headers": [{"key": "k1", "names": ["v1"]}]
+						}],
+						"lookupService": "badScheme:///target"
+					}
+				}`),
+			wantErr: "rls: unregistered scheme in lookup_service",
+		},
+		{
+			desc: "invalid lookup service timeout",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "315576000001s"
+				}
+			}`),
+			wantErr: "google.protobuf.Duration value out of range",
+		},
+		{
+			desc: "invalid max age",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge" : "315576000001s"
+				}
+			}`),
+			wantErr: "google.protobuf.Duration value out of range",
+		},
+		{
+			desc: "invalid stale age",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge" : "10s",
+					"staleAge" : "315576000001s"
+				}
+			}`),
+			wantErr: "google.protobuf.Duration value out of range",
+		},
+		{
+			desc: "invalid max age stale age combo",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"staleAge" : "10s"
+				}
+			}`),
+			wantErr: "rls: stale_age is set, but max_age is not in route lookup config",
+		},
+		{
+			desc: "cache_size_bytes field is not set",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge": "30s",
+					"staleAge" : "25s",
+					"defaultTarget": "passthrough:///default"
+				},
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantErr: "rls: cache_size_bytes must be set to a non-zero value",
+		},
+		{
+			desc: "routeLookupChannelServiceConfig is not in service config format",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "target",
+					"lookupServiceTimeout" : "100s",
+					"maxAge": "60s",
+					"staleAge" : "50s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"routeLookupChannelServiceConfig": "unknown",
+				"childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantErr: "cannot unmarshal string into Go value of type grpc.jsonSC",
+		},
+		{
+			desc: "routeLookupChannelServiceConfig contains unknown LB policy",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "target",
+					"lookupServiceTimeout" : "100s",
+					"maxAge": "60s",
+					"staleAge" : "50s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"routeLookupChannelServiceConfig": {
+					"loadBalancingConfig": [{"not_a_balancer1": {} }, {"not_a_balancer2": {}}]
+				},
+				"childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantErr: "no supported policies found in config",
+		},
+		{
+			desc: "no child policy",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge": "30s",
+					"staleAge" : "25s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantErr: "rls: invalid childPolicy config: no supported policies found",
+		},
+		{
+			desc: "no known child policy",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge": "30s",
+					"staleAge" : "25s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"childPolicy": [
+					{"cds_experimental": {"Cluster": "my-fav-cluster"}},
+					{"unknown-policy": {"unknown-field": "unknown-value"}}
+				],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantErr: "rls: invalid childPolicy config: no supported policies found",
+		},
+		{
+			desc: "invalid child policy config - more than one entry in map",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge": "30s",
+					"staleAge" : "25s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"childPolicy": [
+					{
+						"cds_experimental": {"Cluster": "my-fav-cluster"},
+						"unknown-policy": {"unknown-field": "unknown-value"}
+					}
+				],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantErr: "does not contain exactly 1 policy/config pair",
+		},
+		{
+			desc: "no childPolicyConfigTargetFieldName",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge": "30s",
+					"staleAge" : "25s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"childPolicy": [
+					{"cds_experimental": {"Cluster": "my-fav-cluster"}},
+					{"unknown-policy": {"unknown-field": "unknown-value"}},
+					{"grpclb": {}}
+				]
+			}`),
+			wantErr: "rls: childPolicyConfigTargetFieldName field is not set in service config",
+		},
+		{
+			desc: "child policy config validation failure",
+			input: []byte(`{
+				"routeLookupConfig": {
+					"grpcKeybuilders": [{
+						"names": [{"service": "service", "method": "method"}],
+						"headers": [{"key": "k1", "names": ["v1"]}]
+					}],
+					"lookupService": "passthrough:///target",
+					"lookupServiceTimeout" : "10s",
+					"maxAge": "30s",
+					"staleAge" : "25s",
+					"cacheSizeBytes": 1000,
+					"defaultTarget": "passthrough:///default"
+				},
+				"childPolicy": [
+					{"cds_experimental": {"Cluster": "my-fav-cluster"}},
+					{"unknown-policy": {"unknown-field": "unknown-value"}},
+					{"grpclb": {"childPolicy": "not-an-array"}}
+				],
+				"childPolicyConfigTargetFieldName": "serviceName"
+			}`),
+			wantErr: "rls: childPolicy config validation failed",
+		},
+	}
+
+	builder := rlsBB{}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			lbCfg, err := builder.ParseConfig(test.input)
+			if lbCfg != nil || !strings.Contains(fmt.Sprint(err), test.wantErr) {
+				t.Errorf("ParseConfig(%s) = {%+v, %v}, want {nil, %s}", string(test.input), lbCfg, err, test.wantErr)
+			}
+		})
+	}
+}

balancer/rls/control_channel.go

@@ -0,0 +1,219 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/rls/internal/adaptive"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
+	rlsgrpc "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+)
+
+var newAdaptiveThrottler = func() adaptiveThrottler { return adaptive.New() }
+
+// newConnectivityStateSubscriber is a variable that can be overridden in tests
+// to wrap the connectivity state subscriber for testing purposes.
+var newConnectivityStateSubscriber = func(sub grpcsync.Subscriber) grpcsync.Subscriber {
+	return sub
+}
+
+type adaptiveThrottler interface {
+	ShouldThrottle() bool
+	RegisterBackendResponse(throttled bool)
+}
+
+// controlChannel is a wrapper around the gRPC channel to the RLS server
+// specified in the service config.
+type controlChannel struct {
+	// rpcTimeout specifies the timeout for the RouteLookup RPC call. The LB
+	// policy receives this value in its service config.
+	rpcTimeout time.Duration
+	// backToReadyFunc is a callback to be invoked when the connectivity state
+	// changes from READY --> TRANSIENT_FAILURE --> READY.
+	backToReadyFunc func()
+	// throttler in an adaptive throttling implementation used to avoid
+	// hammering the RLS service while it is overloaded or down.
+	throttler adaptiveThrottler
+
+	cc                      *grpc.ClientConn
+	client                  rlsgrpc.RouteLookupServiceClient
+	logger                  *internalgrpclog.PrefixLogger
+	dropConnStateSubscriber func()
+	seenTransientFailure    bool
+}
+
+// newControlChannel creates a controlChannel to rlsServerName and uses
+// serviceConfig, if non-empty, as the default service config for the underlying
+// gRPC channel.
+func newControlChannel(rlsServerName, serviceConfig string, rpcTimeout time.Duration, bOpts balancer.BuildOptions, backToReadyFunc func()) (*controlChannel, error) {
+	ctrlCh := &controlChannel{
+		rpcTimeout:      rpcTimeout,
+		backToReadyFunc: backToReadyFunc,
+		throttler:       newAdaptiveThrottler(),
+	}
+	ctrlCh.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-control-channel %p] ", ctrlCh))
+
+	dopts, err := ctrlCh.dialOpts(bOpts, serviceConfig)
+	if err != nil {
+		return nil, err
+	}
+	ctrlCh.cc, err = grpc.NewClient(rlsServerName, dopts...)
+	if err != nil {
+		return nil, err
+	}
+	// Subscribe to connectivity state before connecting to avoid missing initial
+	// updates, which are only delivered to active subscribers.
+	subscribe := internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())
+	ctrlCh.dropConnStateSubscriber = subscribe(ctrlCh.cc, newConnectivityStateSubscriber(ctrlCh))
+	ctrlCh.cc.Connect()
+	ctrlCh.client = rlsgrpc.NewRouteLookupServiceClient(ctrlCh.cc)
+	ctrlCh.logger.Infof("Control channel created to RLS server at: %v", rlsServerName)
+	return ctrlCh, nil
+}
+
+func (cc *controlChannel) OnMessage(msg any) {
+	st, ok := msg.(connectivity.State)
+	if !ok {
+		panic(fmt.Sprintf("Unexpected message type %T , wanted connectectivity.State type", msg))
+	}
+
+	switch st {
+	case connectivity.Ready:
+		// Only reset backoff when transitioning from TRANSIENT_FAILURE to READY.
+		// This indicates the RLS server has recovered from being unreachable, so
+		// we reset backoff state in all cache entries to allow pending RPCs to
+		// proceed immediately. We skip benign transitions like READY → IDLE → READY
+		// since those don't represent actual failures.
+		if cc.seenTransientFailure {
+			if cc.logger.V(2) {
+				cc.logger.Infof("Control channel back to READY after TRANSIENT_FAILURE")
+			}
+			cc.seenTransientFailure = false
+			if cc.backToReadyFunc != nil {
+				cc.backToReadyFunc()
+			}
+		} else {
+			if cc.logger.V(2) {
+				cc.logger.Infof("Control channel is READY")
+			}
+		}
+	case connectivity.TransientFailure:
+		// Track that we've entered TRANSIENT_FAILURE state so we know to reset
+		// backoffs when we recover to READY.
+		cc.logger.Warningf("Control channel is TRANSIENT_FAILURE")
+		cc.seenTransientFailure = true
+	default:
+		if cc.logger.V(2) {
+			cc.logger.Infof("Control channel connectivity state is %s", st)
+		}
+	}
+}
+
+// dialOpts constructs the dial options for the control plane channel.
+func (cc *controlChannel) dialOpts(bOpts balancer.BuildOptions, serviceConfig string) ([]grpc.DialOption, error) {
+	// The control plane channel will use the same authority as the parent
+	// channel for server authorization. This ensures that the identity of the
+	// RLS server and the identity of the backends is the same, so if the RLS
+	// config is injected by an attacker, it cannot cause leakage of private
+	// information contained in headers set by the application.
+	dopts := []grpc.DialOption{grpc.WithAuthority(bOpts.Authority)}
+	if bOpts.Dialer != nil {
+		dopts = append(dopts, grpc.WithContextDialer(bOpts.Dialer))
+	}
+	// The control channel will use the channel credentials from the parent
+	// channel, including any call creds associated with the channel creds.
+	var credsOpt grpc.DialOption
+	switch {
+	case bOpts.DialCreds != nil:
+		credsOpt = grpc.WithTransportCredentials(bOpts.DialCreds.Clone())
+	case bOpts.CredsBundle != nil:
+		// The "fallback" mode in google default credentials (which is the only
+		// type of credentials we expect to be used with RLS) uses TLS/ALTS
+		// creds for transport and uses the same call creds as that on the
+		// parent bundle.
+		bundle, err := bOpts.CredsBundle.NewWithMode(internal.CredsBundleModeFallback)
+		if err != nil {
+			return nil, err
+		}
+		credsOpt = grpc.WithCredentialsBundle(bundle)
+	default:
+		cc.logger.Warningf("no credentials available, using Insecure")
+		credsOpt = grpc.WithTransportCredentials(insecure.NewCredentials())
+	}
+	dopts = append(dopts, credsOpt)
+
+	// If the RLS LB policy's configuration specified a service config for the
+	// control channel, use that and disable service config fetching via the name
+	// resolver for the control channel.
+	if serviceConfig != "" {
+		cc.logger.Infof("Disabling service config from the name resolver and instead using: %s", serviceConfig)
+		dopts = append(dopts, grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(serviceConfig))
+	}
+
+	return dopts, nil
+}
+
+func (cc *controlChannel) close() {
+	cc.dropConnStateSubscriber()
+	cc.cc.Close()
+	cc.logger.Infof("Shutdown")
+}
+
+type lookupCallback func(targets []string, headerData string, err error)
+
+// lookup starts a RouteLookup RPC in a separate goroutine and returns the
+// results (and error, if any) in the provided callback.
+//
+// The returned boolean indicates whether the request was throttled by the
+// client-side adaptive throttling algorithm in which case the provided callback
+// will not be invoked.
+func (cc *controlChannel) lookup(reqKeys map[string]string, reason rlspb.RouteLookupRequest_Reason, staleHeaders string, cb lookupCallback) (throttled bool) {
+	if cc.throttler.ShouldThrottle() {
+		cc.logger.Infof("RLS request throttled by client-side adaptive throttling")
+		return true
+	}
+	go func() {
+		req := &rlspb.RouteLookupRequest{
+			TargetType:      "grpc",
+			KeyMap:          reqKeys,
+			Reason:          reason,
+			StaleHeaderData: staleHeaders,
+		}
+		if cc.logger.V(2) {
+			cc.logger.Infof("Sending RLS request %+v", pretty.ToJSON(req))
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), cc.rpcTimeout)
+		defer cancel()
+		resp, err := cc.client.RouteLookup(ctx, req)
+		cb(resp.GetTargets(), resp.GetHeaderData(), err)
+	}()
+	return false
+}

balancer/rls/control_channel_test.go

@@ -0,0 +1,465 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	rlstest "google.golang.org/grpc/internal/testutils/rls"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/testdata"
+	"google.golang.org/protobuf/proto"
+)
+
+// TestControlChannelThrottled tests the case where the adaptive throttler
+// indicates that the control channel needs to be throttled.
+func (s) TestControlChannelThrottled(t *testing.T) {
+	// Start an RLS server and set the throttler to always throttle requests.
+	rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil)
+	overrideAdaptiveThrottler(t, alwaysThrottlingThrottler())
+
+	// Create a control channel to the fake RLS server.
+	ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{}, nil)
+	if err != nil {
+		t.Fatalf("Failed to create control channel to RLS server: %v", err)
+	}
+	defer ctrlCh.close()
+
+	// Perform the lookup and expect the attempt to be throttled.
+	ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, nil)
+
+	select {
+	case <-rlsReqCh:
+		t.Fatal("RouteLookup RPC invoked when control channel is throttled")
+	case <-time.After(defaultTestShortTimeout):
+	}
+}
+
+// TestLookupFailure tests the case where the RLS server responds with an error.
+func (s) TestLookupFailure(t *testing.T) {
+	// Start an RLS server and set the throttler to never throttle requests.
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil)
+	overrideAdaptiveThrottler(t, neverThrottlingThrottler())
+
+	// Setup the RLS server to respond with errors.
+	rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse {
+		return &rlstest.RouteLookupResponse{Err: errors.New("rls failure")}
+	})
+
+	// Create a control channel to the fake RLS server.
+	ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{}, nil)
+	if err != nil {
+		t.Fatalf("Failed to create control channel to RLS server: %v", err)
+	}
+	defer ctrlCh.close()
+
+	// Perform the lookup and expect the callback to be invoked with an error.
+	errCh := make(chan error, 1)
+	ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) {
+		if err == nil {
+			errCh <- errors.New("rlsClient.lookup() succeeded, should have failed")
+			return
+		}
+		errCh <- nil
+	})
+
+	select {
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("timeout when waiting for lookup callback to be invoked")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// TestLookupDeadlineExceeded tests the case where the RLS server does not
+// respond within the configured rpc timeout.
+func (s) TestLookupDeadlineExceeded(t *testing.T) {
+	// A unary interceptor which returns a status error with DeadlineExceeded.
+	interceptor := func(context.Context, any, *grpc.UnaryServerInfo, grpc.UnaryHandler) (resp any, err error) {
+		return nil, status.Error(codes.DeadlineExceeded, "deadline exceeded")
+	}
+
+	// Start an RLS server and set the throttler to never throttle.
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor))
+	overrideAdaptiveThrottler(t, neverThrottlingThrottler())
+
+	// Create a control channel with a small deadline.
+	ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestShortTimeout, balancer.BuildOptions{}, nil)
+	if err != nil {
+		t.Fatalf("Failed to create control channel to RLS server: %v", err)
+	}
+	defer ctrlCh.close()
+
+	// Perform the lookup and expect the callback to be invoked with an error.
+	errCh := make(chan error, 1)
+	ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) {
+		if st, ok := status.FromError(err); !ok || st.Code() != codes.DeadlineExceeded {
+			errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, want %v", err, codes.DeadlineExceeded)
+			return
+		}
+		errCh <- nil
+	})
+
+	select {
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("timeout when waiting for lookup callback to be invoked")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// testCredsBundle wraps a test call creds and real transport creds.
+type testCredsBundle struct {
+	transportCreds credentials.TransportCredentials
+	callCreds      credentials.PerRPCCredentials
+}
+
+func (f *testCredsBundle) TransportCredentials() credentials.TransportCredentials {
+	return f.transportCreds
+}
+
+func (f *testCredsBundle) PerRPCCredentials() credentials.PerRPCCredentials {
+	return f.callCreds
+}
+
+func (f *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) {
+	if mode != internal.CredsBundleModeFallback {
+		return nil, fmt.Errorf("unsupported mode: %v", mode)
+	}
+	return &testCredsBundle{
+		transportCreds: f.transportCreds,
+		callCreds:      f.callCreds,
+	}, nil
+}
+
+var (
+	// Call creds sent by the testPerRPCCredentials on the client, and verified
+	// by an interceptor on the server.
+	perRPCCredsData = map[string]string{
+		"test-key":     "test-value",
+		"test-key-bin": string([]byte{1, 2, 3}),
+	}
+)
+
+type testPerRPCCredentials struct {
+	callCreds map[string]string
+}
+
+func (f *testPerRPCCredentials) GetRequestMetadata(context.Context, ...string) (map[string]string, error) {
+	return f.callCreds, nil
+}
+
+func (f *testPerRPCCredentials) RequireTransportSecurity() bool {
+	return true
+}
+
+// Unary server interceptor which validates if the RPC contains call credentials
+// which match `perRPCCredsData
+func callCredsValidatingServerInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return nil, status.Error(codes.PermissionDenied, "didn't find metadata in context")
+	}
+	for k, want := range perRPCCredsData {
+		got, ok := md[k]
+		if !ok {
+			return ctx, status.Errorf(codes.PermissionDenied, "didn't find call creds key %v in context", k)
+		}
+		if got[0] != want {
+			return ctx, status.Errorf(codes.PermissionDenied, "for key %v, got value %v, want %v", k, got, want)
+		}
+	}
+	return handler(ctx, req)
+}
+
+// makeTLSCreds is a test helper which creates a TLS based transport credentials
+// from files specified in the arguments.
+func makeTLSCreds(t *testing.T, certPath, keyPath, rootsPath string) credentials.TransportCredentials {
+	cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(%q, %q) failed: %v", certPath, keyPath, err)
+	}
+	b, err := os.ReadFile(testdata.Path(rootsPath))
+	if err != nil {
+		t.Fatalf("os.ReadFile(%q) failed: %v", rootsPath, err)
+	}
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(b) {
+		t.Fatal("failed to append certificates")
+	}
+	return credentials.NewTLS(&tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      roots,
+	})
+}
+
+const (
+	wantHeaderData  = "headerData"
+	staleHeaderData = "staleHeaderData"
+)
+
+var (
+	keyMap = map[string]string{
+		"k1": "v1",
+		"k2": "v2",
+	}
+	wantTargets   = []string{"us_east_1.firestore.googleapis.com"}
+	lookupRequest = &rlspb.RouteLookupRequest{
+		TargetType:      "grpc",
+		KeyMap:          keyMap,
+		Reason:          rlspb.RouteLookupRequest_REASON_MISS,
+		StaleHeaderData: staleHeaderData,
+	}
+	lookupResponse = &rlstest.RouteLookupResponse{
+		Resp: &rlspb.RouteLookupResponse{
+			Targets:    wantTargets,
+			HeaderData: wantHeaderData,
+		},
+	}
+)
+
+func testControlChannelCredsSuccess(t *testing.T, sopts []grpc.ServerOption, bopts balancer.BuildOptions) {
+	// Start an RLS server and set the throttler to never throttle requests.
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, sopts...)
+	overrideAdaptiveThrottler(t, neverThrottlingThrottler())
+
+	// Setup the RLS server to respond with a valid response.
+	rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse {
+		return lookupResponse
+	})
+
+	// Verify that the request received by the RLS matches the expected one.
+	rlsServer.SetRequestCallback(func(got *rlspb.RouteLookupRequest) {
+		if diff := cmp.Diff(lookupRequest, got, cmp.Comparer(proto.Equal)); diff != "" {
+			t.Errorf("RouteLookupRequest diff (-want, +got):\n%s", diff)
+		}
+	})
+
+	// Create a control channel to the fake server.
+	ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, bopts, nil)
+	if err != nil {
+		t.Fatalf("Failed to create control channel to RLS server: %v", err)
+	}
+	defer ctrlCh.close()
+
+	// Perform the lookup and expect a successful callback invocation.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	errCh := make(chan error, 1)
+	ctrlCh.lookup(keyMap, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(targets []string, headerData string, err error) {
+		if err != nil {
+			errCh <- fmt.Errorf("rlsClient.lookup() failed with err: %v", err)
+			return
+		}
+		if !cmp.Equal(targets, wantTargets) || headerData != wantHeaderData {
+			errCh <- fmt.Errorf("rlsClient.lookup() = (%v, %s), want (%v, %s)", targets, headerData, wantTargets, wantHeaderData)
+			return
+		}
+		errCh <- nil
+	})
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("timeout when waiting for lookup callback to be invoked")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// TestControlChannelCredsSuccess tests creation of the control channel with
+// different credentials, which are expected to succeed.
+func (s) TestControlChannelCredsSuccess(t *testing.T) {
+	serverCreds := makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	clientCreds := makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
+
+	tests := []struct {
+		name  string
+		sopts []grpc.ServerOption
+		bopts balancer.BuildOptions
+	}{
+		{
+			name:  "insecure",
+			sopts: nil,
+			bopts: balancer.BuildOptions{},
+		},
+		{
+			name:  "transport creds only",
+			sopts: []grpc.ServerOption{grpc.Creds(serverCreds)},
+			bopts: balancer.BuildOptions{
+				DialCreds: clientCreds,
+				Authority: "x.test.example.com",
+			},
+		},
+		{
+			name: "creds bundle",
+			sopts: []grpc.ServerOption{
+				grpc.Creds(serverCreds),
+				grpc.UnaryInterceptor(callCredsValidatingServerInterceptor),
+			},
+			bopts: balancer.BuildOptions{
+				CredsBundle: &testCredsBundle{
+					transportCreds: clientCreds,
+					callCreds:      &testPerRPCCredentials{callCreds: perRPCCredsData},
+				},
+				Authority: "x.test.example.com",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testControlChannelCredsSuccess(t, test.sopts, test.bopts)
+		})
+	}
+}
+
+func testControlChannelCredsFailure(t *testing.T, sopts []grpc.ServerOption, bopts balancer.BuildOptions, wantCode codes.Code, wantErrRegex *regexp.Regexp) {
+	// StartFakeRouteLookupServer a fake server.
+	//
+	// Start an RLS server and set the throttler to never throttle requests. The
+	// creds failures happen before the RPC handler on the server is invoked.
+	// So, there is need to setup the request and responses on the fake server.
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, sopts...)
+	overrideAdaptiveThrottler(t, neverThrottlingThrottler())
+
+	// Create the control channel to the fake server.
+	ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, bopts, nil)
+	if err != nil {
+		t.Fatalf("Failed to create control channel to RLS server: %v", err)
+	}
+	defer ctrlCh.close()
+
+	// Perform the lookup and expect the callback to be invoked with an error.
+	errCh := make(chan error, 1)
+	ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) {
+		if st, ok := status.FromError(err); !ok || st.Code() != wantCode || !wantErrRegex.MatchString(st.String()) {
+			errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, wantCode: %v, wantErr: %s", err, wantCode, wantErrRegex.String())
+			return
+		}
+		errCh <- nil
+	})
+
+	select {
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("timeout when waiting for lookup callback to be invoked")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// TestControlChannelCredsFailure tests creation of the control channel with
+// different credentials, which are expected to fail.
+func (s) TestControlChannelCredsFailure(t *testing.T) {
+	serverCreds := makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	clientCreds := makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
+
+	tests := []struct {
+		name         string
+		sopts        []grpc.ServerOption
+		bopts        balancer.BuildOptions
+		wantCode     codes.Code
+		wantErrRegex *regexp.Regexp
+	}{
+		{
+			name:  "transport creds authority mismatch",
+			sopts: []grpc.ServerOption{grpc.Creds(serverCreds)},
+			bopts: balancer.BuildOptions{
+				DialCreds: clientCreds,
+				Authority: "authority-mismatch",
+			},
+			wantCode:     codes.Unavailable,
+			wantErrRegex: regexp.MustCompile(`transport: authentication handshake failed: .* \*\.test\.example\.com.*authority-mismatch`),
+		},
+		{
+			name:  "transport creds handshake failure",
+			sopts: nil, // server expects insecure connection
+			bopts: balancer.BuildOptions{
+				DialCreds: clientCreds,
+				Authority: "x.test.example.com",
+			},
+			wantCode:     codes.Unavailable,
+			wantErrRegex: regexp.MustCompile("transport: authentication handshake failed: .*"),
+		},
+		{
+			name: "call creds mismatch",
+			sopts: []grpc.ServerOption{
+				grpc.Creds(serverCreds),
+				grpc.UnaryInterceptor(callCredsValidatingServerInterceptor), // server expects call creds
+			},
+			bopts: balancer.BuildOptions{
+				CredsBundle: &testCredsBundle{
+					transportCreds: clientCreds,
+					callCreds:      &testPerRPCCredentials{}, // sends no call creds
+				},
+				Authority: "x.test.example.com",
+			},
+			wantCode:     codes.PermissionDenied,
+			wantErrRegex: regexp.MustCompile("didn't find call creds"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testControlChannelCredsFailure(t, test.sopts, test.bopts, test.wantCode, test.wantErrRegex)
+		})
+	}
+}
+
+type unsupportedCredsBundle struct {
+	credentials.Bundle
+}
+
+func (*unsupportedCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) {
+	return nil, fmt.Errorf("unsupported mode: %v", mode)
+}
+
+// TestNewControlChannelUnsupportedCredsBundle tests the case where the control
+// channel is configured with a bundle which does not support the mode we use.
+func (s) TestNewControlChannelUnsupportedCredsBundle(t *testing.T) {
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil)
+
+	// Create the control channel to the fake server.
+	ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{CredsBundle: &unsupportedCredsBundle{}}, nil)
+	if err == nil {
+		ctrlCh.close()
+		t.Fatal("newControlChannel succeeded when expected to fail")
+	}
+}

balancer/rls/helpers_test.go

@@ -0,0 +1,299 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer/rls/internal/test/e2e"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/grpctest"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 100 * time.Millisecond
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// fakeBackoffStrategy is a fake implementation of the backoff.Strategy
+// interface, for tests to inject the backoff duration.
+type fakeBackoffStrategy struct {
+	backoff time.Duration
+}
+
+func (f *fakeBackoffStrategy) Backoff(int) time.Duration {
+	return f.backoff
+}
+
+// fakeThrottler is a fake implementation of the adaptiveThrottler interface.
+type fakeThrottler struct {
+	throttleFunc func() bool   // Fake throttler implementation.
+	throttleCh   chan struct{} // Invocation of ShouldThrottle signals here.
+}
+
+func (f *fakeThrottler) ShouldThrottle() bool {
+	select {
+	case <-f.throttleCh:
+	default:
+	}
+	f.throttleCh <- struct{}{}
+
+	return f.throttleFunc()
+}
+
+func (f *fakeThrottler) RegisterBackendResponse(bool) {}
+
+// alwaysThrottlingThrottler returns a fake throttler which always throttles.
+func alwaysThrottlingThrottler() *fakeThrottler {
+	return &fakeThrottler{
+		throttleFunc: func() bool { return true },
+		throttleCh:   make(chan struct{}, 1),
+	}
+}
+
+// neverThrottlingThrottler returns a fake throttler which never throttles.
+func neverThrottlingThrottler() *fakeThrottler {
+	return &fakeThrottler{
+		throttleFunc: func() bool { return false },
+		throttleCh:   make(chan struct{}, 1),
+	}
+}
+
+// oneTimeAllowingThrottler returns a fake throttler which does not throttle
+// requests until the client RPC succeeds, but throttles everything that comes
+// after. This is useful for tests which need to set up a valid cache entry
+// before testing other cases.
+func oneTimeAllowingThrottler(firstRPCDone *grpcsync.Event) *fakeThrottler {
+	return &fakeThrottler{
+		throttleFunc: firstRPCDone.HasFired,
+		throttleCh:   make(chan struct{}, 1),
+	}
+}
+
+func overrideAdaptiveThrottler(t *testing.T, f *fakeThrottler) {
+	origAdaptiveThrottler := newAdaptiveThrottler
+	newAdaptiveThrottler = func() adaptiveThrottler { return f }
+	t.Cleanup(func() { newAdaptiveThrottler = origAdaptiveThrottler })
+}
+
+// buildBasicRLSConfig constructs a basic service config for the RLS LB policy
+// with header matching rules. This expects the passed child policy name to
+// have been registered by the caller.
+func buildBasicRLSConfig(childPolicyName, rlsServerAddress string) *e2e.RLSConfig {
+	return &e2e.RLSConfig{
+		RouteLookupConfig: &rlspb.RouteLookupConfig{
+			GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+				{
+					Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}},
+					Headers: []*rlspb.NameMatcher{
+						{Key: "k1", Names: []string{"n1"}},
+						{Key: "k2", Names: []string{"n2"}},
+					},
+				},
+			},
+			LookupService:        rlsServerAddress,
+			LookupServiceTimeout: durationpb.New(defaultTestTimeout),
+			CacheSizeBytes:       1024,
+		},
+		RouteLookupChannelServiceConfig:  `{"loadBalancingConfig": [{"pick_first": {}}]}`,
+		ChildPolicy:                      &internalserviceconfig.BalancerConfig{Name: childPolicyName},
+		ChildPolicyConfigTargetFieldName: e2e.RLSChildPolicyTargetNameField,
+	}
+}
+
+// buildBasicRLSConfigWithChildPolicy constructs a very basic service config for
+// the RLS LB policy. It also registers a test LB policy which is capable of
+// being a child of the RLS LB policy.
+func buildBasicRLSConfigWithChildPolicy(t *testing.T, childPolicyName, rlsServerAddress string) *e2e.RLSConfig {
+	childPolicyName = "test-child-policy" + childPolicyName
+	e2e.RegisterRLSChildPolicy(childPolicyName, nil)
+	t.Logf("Registered child policy with name %q", childPolicyName)
+
+	return &e2e.RLSConfig{
+		RouteLookupConfig: &rlspb.RouteLookupConfig{
+			GrpcKeybuilders:      []*rlspb.GrpcKeyBuilder{{Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}}},
+			LookupService:        rlsServerAddress,
+			LookupServiceTimeout: durationpb.New(defaultTestTimeout),
+			CacheSizeBytes:       1024,
+		},
+		RouteLookupChannelServiceConfig:  `{"loadBalancingConfig": [{"pick_first": {}}]}`,
+		ChildPolicy:                      &internalserviceconfig.BalancerConfig{Name: childPolicyName},
+		ChildPolicyConfigTargetFieldName: e2e.RLSChildPolicyTargetNameField,
+	}
+}
+
+// startBackend starts a backend implementing the TestService on a local port.
+// It returns a channel for tests to get notified whenever an RPC is invoked on
+// the backend. This allows tests to ensure that RPCs reach expected backends.
+// Also returns the address of the backend.
+func startBackend(t *testing.T, sopts ...grpc.ServerOption) (rpcCh chan struct{}, address string) {
+	t.Helper()
+
+	rpcCh = make(chan struct{}, 1)
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			select {
+			case rpcCh <- struct{}{}:
+			default:
+			}
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := backend.StartServer(sopts...); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started TestService backend at: %q", backend.Address)
+	t.Cleanup(func() { backend.Stop() })
+	return rpcCh, backend.Address
+}
+
+// startManualResolverWithConfig registers and returns a manual resolver which
+// pushes the RLS LB policy's service config on the channel.
+func startManualResolverWithConfig(t *testing.T, rlsConfig *e2e.RLSConfig) *manual.Resolver {
+	t.Helper()
+
+	scJSON, err := rlsConfig.ServiceConfigJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON)
+	r := manual.NewBuilderWithScheme("rls-e2e")
+	r.InitialState(resolver.State{ServiceConfig: sc})
+	t.Cleanup(r.Close)
+	return r
+}
+
+// makeTestRPCAndExpectItToReachBackend is a test helper function which makes
+// the EmptyCall RPC on the given ClientConn and verifies that it reaches a
+// backend. The latter is accomplished by listening on the provided channel
+// which gets pushed to whenever the backend in question gets an RPC.
+//
+// There are many instances where it can take a while before the attempted RPC
+// reaches the expected backend. Examples include, but are not limited to:
+//   - control channel is changed in a config update. The RLS LB policy creates a
+//     new control channel, and sends a new picker to gRPC. But it takes a while
+//     before gRPC actually starts using the new picker.
+//   - test is waiting for a cache entry to expire after which we expect a
+//     different behavior because we have configured the fake RLS server to return
+//     different backends.
+//
+// Therefore, we do not return an error when the RPC fails. Instead, we wait for
+// the context to expire before failing.
+func makeTestRPCAndExpectItToReachBackend(ctx context.Context, t *testing.T, cc *grpc.ClientConn, ch chan struct{}) {
+	t.Helper()
+
+	// Drain the backend channel before performing the RPC to remove any
+	// notifications from previous RPCs.
+	select {
+	case <-ch:
+	default:
+	}
+
+	for {
+		if err := ctx.Err(); err != nil {
+			t.Fatalf("Timeout when waiting for RPCs to be routed to the given target: %v", err)
+		}
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		client := testgrpc.NewTestServiceClient(cc)
+		client.EmptyCall(sCtx, &testpb.Empty{})
+
+		select {
+		case <-sCtx.Done():
+		case <-ch:
+			sCancel()
+			return
+		}
+	}
+}
+
+// makeTestRPCAndVerifyError is a test helper function which makes the EmptyCall
+// RPC on the given ClientConn and verifies that the RPC fails with the given
+// status code and error.
+//
+// Similar to makeTestRPCAndExpectItToReachBackend, retries until expected
+// outcome is reached or the provided context has expired.
+func makeTestRPCAndVerifyError(ctx context.Context, t *testing.T, cc *grpc.ClientConn, wantCode codes.Code, wantErr error) {
+	t.Helper()
+
+	for {
+		if err := ctx.Err(); err != nil {
+			t.Fatalf("Timeout when waiting for RPCs to fail with given error: %v", err)
+		}
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		client := testgrpc.NewTestServiceClient(cc)
+		_, err := client.EmptyCall(sCtx, &testpb.Empty{})
+
+		// If the RPC fails with the expected code and expected error message (if
+		// one was provided), we return. Else we retry after blocking for a little
+		// while to ensure that we don't keep blasting away with RPCs.
+		if code := status.Code(err); code == wantCode {
+			if wantErr == nil || strings.Contains(err.Error(), wantErr.Error()) {
+				sCancel()
+				return
+			}
+		}
+		<-sCtx.Done()
+	}
+}
+
+// verifyRLSRequest is a test helper which listens on a channel to see if an RLS
+// request was received by the fake RLS server. Based on whether the test
+// expects a request to be sent out or not, it uses a different timeout.
+func verifyRLSRequest(t *testing.T, ch chan struct{}, wantRequest bool) {
+	t.Helper()
+
+	if wantRequest {
+		select {
+		case <-time.After(defaultTestTimeout):
+			t.Fatalf("Timeout when waiting for an RLS request to be sent out")
+		case <-ch:
+		}
+	} else {
+		select {
+		case <-time.After(defaultTestShortTimeout):
+		case <-ch:
+			t.Fatalf("RLS request sent out when not expecting one")
+		}
+	}
+}

balancer/rls/internal/adaptive/adaptive.go

@@ -0,0 +1,131 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package adaptive provides functionality for adaptive client-side throttling.
+package adaptive
+
+import (
+	rand "math/rand/v2"
+	"sync"
+	"time"
+)
+
+// For overriding in unittests.
+var (
+	timeNowFunc = time.Now
+	randFunc    = rand.Float64
+)
+
+const (
+	defaultDuration        = 30 * time.Second
+	defaultBins            = 100
+	defaultRatioForAccepts = 2.0
+	defaultRequestsPadding = 8.0
+)
+
+// Throttler implements a client-side throttling recommendation system. All
+// methods are safe for concurrent use by multiple goroutines.
+//
+// The throttler has the following knobs for which we will use defaults for
+// now. If there is a need to make them configurable at a later point in time,
+// support for the same will be added.
+//   - Duration: amount of recent history that will be taken into account for
+//     making client-side throttling decisions. A default of 30 seconds is used.
+//   - Bins: number of bins to be used for bucketing historical data. A default
+//     of 100 is used.
+//   - RatioForAccepts: ratio by which accepts are multiplied, typically a value
+//     slightly larger than 1.0. This is used to make the throttler behave as if
+//     the backend had accepted more requests than it actually has, which lets us
+//     err on the side of sending to the backend more requests than we think it
+//     will accept for the sake of speeding up the propagation of state. A
+//     default of 2.0 is used.
+//   - RequestsPadding: is used to decrease the (client-side) throttling
+//     probability in the low QPS regime (to speed up propagation of state), as
+//     well as to safeguard against hitting a client-side throttling probability
+//     of 100%. The weight of this value decreases as the number of requests in
+//     recent history grows. A default of 8 is used.
+//
+// The adaptive throttler attempts to estimate the probability that a request
+// will be throttled using recent history. Server requests (both throttled and
+// accepted) are registered with the throttler (via the RegisterBackendResponse
+// method), which then recommends client-side throttling (via the
+// ShouldThrottle method) with probability given by:
+// (requests - RatioForAccepts * accepts) / (requests + RequestsPadding)
+type Throttler struct {
+	ratioForAccepts float64
+	requestsPadding float64
+
+	// Number of total accepts and throttles in the lookback period.
+	mu        sync.Mutex
+	accepts   *lookback
+	throttles *lookback
+}
+
+// New initializes a new adaptive throttler with the default values.
+func New() *Throttler {
+	return newWithArgs(defaultDuration, defaultBins, defaultRatioForAccepts, defaultRequestsPadding)
+}
+
+// newWithArgs initializes a new adaptive throttler with the provided values.
+// Used only in unittests.
+func newWithArgs(duration time.Duration, bins int64, ratioForAccepts, requestsPadding float64) *Throttler {
+	return &Throttler{
+		ratioForAccepts: ratioForAccepts,
+		requestsPadding: requestsPadding,
+		accepts:         newLookback(bins, duration),
+		throttles:       newLookback(bins, duration),
+	}
+}
+
+// ShouldThrottle returns a probabilistic estimate of whether the server would
+// throttle the next request. This should be called for every request before
+// allowing it to hit the network. If the returned value is true, the request
+// should be aborted immediately (as if it had been throttled by the server).
+func (t *Throttler) ShouldThrottle() bool {
+	randomProbability := randFunc()
+	now := timeNowFunc()
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	accepts, throttles := float64(t.accepts.sum(now)), float64(t.throttles.sum(now))
+	requests := accepts + throttles
+	throttleProbability := (requests - t.ratioForAccepts*accepts) / (requests + t.requestsPadding)
+	if throttleProbability <= randomProbability {
+		return false
+	}
+
+	t.throttles.add(now, 1)
+	return true
+}
+
+// RegisterBackendResponse registers a response received from the backend for a
+// request allowed by ShouldThrottle. This should be called for every response
+// received from the backend (i.e., once for each request for which
+// ShouldThrottle returned false).
+func (t *Throttler) RegisterBackendResponse(throttled bool) {
+	now := timeNowFunc()
+
+	t.mu.Lock()
+	if throttled {
+		t.throttles.add(now, 1)
+	} else {
+		t.accepts.add(now, 1)
+	}
+	t.mu.Unlock()
+}

balancer/rls/internal/adaptive/adaptive_test.go

@@ -0,0 +1,231 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package adaptive
+
+import (
+	"sync"
+	"testing"
+	"time"
+)
+
+// stats returns a tuple with accepts, throttles for the current time.
+func (t *Throttler) stats() (int64, int64) {
+	now := timeNowFunc()
+
+	t.mu.Lock()
+	a, th := t.accepts.sum(now), t.throttles.sum(now)
+	t.mu.Unlock()
+	return a, th
+}
+
+// Enums for responses.
+const (
+	E = iota // No response
+	A        // Accepted
+	T        // Throttled
+)
+
+func TestRegisterBackendResponse(t *testing.T) {
+	testcases := []struct {
+		desc          string
+		bins          int64
+		ticks         []int64
+		responses     []int64
+		wantAccepts   []int64
+		wantThrottled []int64
+	}{
+		{
+			"Accumulate",
+			3,
+			[]int64{0, 1, 2}, // Ticks
+			[]int64{A, T, E}, // Responses
+			[]int64{1, 1, 1}, // Accepts
+			[]int64{0, 1, 1}, // Throttled
+		},
+		{
+			"LightTimeTravel",
+			3,
+			[]int64{1, 0, 2}, // Ticks
+			[]int64{A, T, E}, // Response
+			[]int64{1, 1, 1}, // Accepts
+			[]int64{0, 1, 1}, // Throttled
+		},
+		{
+			"HeavyTimeTravel",
+			3,
+			[]int64{8, 0, 9}, // Ticks
+			[]int64{A, A, A}, // Response
+			[]int64{1, 1, 2}, // Accepts
+			[]int64{0, 0, 0}, // Throttled
+		},
+		{
+			"Rollover",
+			1,
+			[]int64{0, 1, 2}, // Ticks
+			[]int64{A, T, E}, // Responses
+			[]int64{1, 0, 0}, // Accepts
+			[]int64{0, 1, 0}, // Throttled
+		},
+	}
+
+	m := mockClock{}
+	oldTimeNowFunc := timeNowFunc
+	timeNowFunc = m.Now
+	defer func() { timeNowFunc = oldTimeNowFunc }()
+
+	for _, test := range testcases {
+		t.Run(test.desc, func(t *testing.T) {
+			th := newWithArgs(time.Duration(test.bins), test.bins, 2.0, 8)
+			for i, tick := range test.ticks {
+				m.SetNanos(tick)
+
+				if test.responses[i] != E {
+					th.RegisterBackendResponse(test.responses[i] == T)
+				}
+
+				if gotAccepts, gotThrottled := th.stats(); gotAccepts != test.wantAccepts[i] || gotThrottled != test.wantThrottled[i] {
+					t.Errorf("th.stats() = {%d, %d} for index %d, want {%d, %d}", i, gotAccepts, gotThrottled, test.wantAccepts[i], test.wantThrottled[i])
+				}
+			}
+		})
+	}
+}
+
+func TestShouldThrottleOptions(t *testing.T) {
+	// ShouldThrottle should return true iff
+	//    (requests - RatioForAccepts * accepts) / (requests + RequestsPadding) <= p
+	// where p is a random number. For the purposes of this test it's fixed
+	// to 0.5.
+	responses := []int64{T, T, T, T, T, T, T, T, T, A, A, A, A, A, A, T, T, T, T}
+
+	n := false
+	y := true
+
+	testcases := []struct {
+		desc            string
+		ratioForAccepts float64
+		requestsPadding float64
+		want            []bool
+	}{
+		{
+			"Baseline",
+			1.1,
+			8,
+			[]bool{n, n, n, n, n, n, n, n, y, y, y, y, y, n, n, n, y, y, y},
+		},
+		{
+			"ChangePadding",
+			1.1,
+			7,
+			[]bool{n, n, n, n, n, n, n, y, y, y, y, y, y, y, y, y, y, y, y},
+		},
+		{
+			"ChangeRatioForAccepts",
+			1.4,
+			8,
+			[]bool{n, n, n, n, n, n, n, n, y, y, n, n, n, n, n, n, n, n, n},
+		},
+	}
+
+	m := mockClock{}
+	oldTimeNowFunc := timeNowFunc
+	timeNowFunc = m.Now
+	oldRandFunc := randFunc
+	randFunc = func() float64 { return 0.5 }
+	defer func() {
+		timeNowFunc = oldTimeNowFunc
+		randFunc = oldRandFunc
+	}()
+
+	for _, test := range testcases {
+		t.Run(test.desc, func(t *testing.T) {
+			m.SetNanos(0)
+			th := newWithArgs(time.Nanosecond, 1, test.ratioForAccepts, test.requestsPadding)
+			for i, response := range responses {
+				if response != E {
+					th.RegisterBackendResponse(response == T)
+				}
+				if got := th.ShouldThrottle(); got != test.want[i] {
+					t.Errorf("ShouldThrottle for index %d: got %v, want %v", i, got, test.want[i])
+				}
+			}
+		})
+	}
+}
+
+func TestParallel(t *testing.T) {
+	// Uses all the defaults which comes with a 30 second duration.
+	th := New()
+
+	testDuration := 2 * time.Second
+	numRoutines := 10
+	accepts := make([]int64, numRoutines)
+	throttles := make([]int64, numRoutines)
+	var wg sync.WaitGroup
+	for i := 0; i < numRoutines; i++ {
+		wg.Add(1)
+		go func(num int) {
+			defer wg.Done()
+
+			ticker := time.NewTicker(testDuration)
+			var accept int64
+			var throttle int64
+			for i := 0; ; i++ {
+				select {
+				case <-ticker.C:
+					ticker.Stop()
+					accepts[num] = accept
+					throttles[num] = throttle
+					return
+				default:
+					if i%2 == 0 {
+						th.RegisterBackendResponse(true)
+						throttle++
+					} else {
+						th.RegisterBackendResponse(false)
+						accept++
+					}
+				}
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	var wantAccepts, wantThrottles int64
+	for i := 0; i < numRoutines; i++ {
+		wantAccepts += accepts[i]
+		wantThrottles += throttles[i]
+	}
+
+	if gotAccepts, gotThrottles := th.stats(); gotAccepts != wantAccepts || gotThrottles != wantThrottles {
+		t.Errorf("th.stats() = {%d, %d}, want {%d, %d}", gotAccepts, gotThrottles, wantAccepts, wantThrottles)
+	}
+}
+
+type mockClock struct {
+	t time.Time
+}
+
+func (m *mockClock) Now() time.Time {
+	return m.t
+}
+
+func (m *mockClock) SetNanos(n int64) {
+	m.t = time.Unix(0, n)
+}

balancer/rls/internal/adaptive/lookback.go

@@ -0,0 +1,84 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package adaptive
+
+import "time"
+
+// lookback implements a moving sum over an int64 timeline.
+type lookback struct {
+	bins  int64         // Number of bins to use for lookback.
+	width time.Duration // Width of each bin.
+
+	head  int64   // Absolute bin index (time * bins / duration) of the current head bin.
+	total int64   // Sum over all the values in buf, within the lookback window behind head.
+	buf   []int64 // Ring buffer for keeping track of the sum elements.
+}
+
+// newLookback creates a new lookback for the given duration with a set number
+// of bins.
+func newLookback(bins int64, duration time.Duration) *lookback {
+	return &lookback{
+		bins:  bins,
+		width: duration / time.Duration(bins),
+		buf:   make([]int64, bins),
+	}
+}
+
+// add is used to increment the lookback sum.
+func (l *lookback) add(t time.Time, v int64) {
+	pos := l.advance(t)
+
+	if (l.head - pos) >= l.bins {
+		// Do not increment counters if pos is more than bins behind head.
+		return
+	}
+	l.buf[pos%l.bins] += v
+	l.total += v
+}
+
+// sum returns the sum of the lookback buffer at the given time or head,
+// whichever is greater.
+func (l *lookback) sum(t time.Time) int64 {
+	l.advance(t)
+	return l.total
+}
+
+// advance prepares the lookback buffer for calls to add() or sum() at time t.
+// If head is greater than t then the lookback buffer will be untouched. The
+// absolute bin index corresponding to t is returned. It will always be less
+// than or equal to head.
+func (l *lookback) advance(t time.Time) int64 {
+	ch := l.head                               // Current head bin index.
+	nh := t.UnixNano() / l.width.Nanoseconds() // New head bin index.
+
+	if nh <= ch {
+		// Either head unchanged or clock jitter (time has moved backwards). Do
+		// not advance.
+		return nh
+	}
+
+	jmax := min(l.bins, nh-ch)
+	for j := int64(0); j < jmax; j++ {
+		i := (ch + j + 1) % l.bins
+		l.total -= l.buf[i]
+		l.buf[i] = 0
+	}
+	l.head = nh
+	return nh
+}

balancer/rls/internal/adaptive/lookback_test.go

@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package adaptive
+
+import (
+	"testing"
+	"time"
+)
+
+func TestLookback(t *testing.T) {
+	makeTicks := func(offsets []int64) []time.Time {
+		var ticks []time.Time
+		now := time.Now()
+		for _, offset := range offsets {
+			ticks = append(ticks, now.Add(time.Duration(offset)))
+		}
+		return ticks
+	}
+
+	// lookback.add and lookback.sum behave correctly.
+	testcases := []struct {
+		desc   string
+		bins   int64
+		ticks  []time.Time
+		values []int64
+		want   []int64
+	}{
+		{
+			"Accumulate",
+			3,
+			makeTicks([]int64{0, 1, 2}), // Ticks
+			[]int64{1, 2, 3},            // Values
+			[]int64{1, 3, 6},            // Want
+		},
+		{
+			"LightTimeTravel",
+			3,
+			makeTicks([]int64{1, 0, 2}), // Ticks
+			[]int64{1, 2, 3},            // Values
+			[]int64{1, 3, 6},            // Want
+		},
+		{
+			"HeavyTimeTravel",
+			3,
+			makeTicks([]int64{8, 0, 9}), // Ticks
+			[]int64{1, 2, 3},            // Values
+			[]int64{1, 1, 4},            // Want
+		},
+		{
+			"Rollover",
+			1,
+			makeTicks([]int64{0, 1, 2}), // Ticks
+			[]int64{1, 2, 3},            // Values
+			[]int64{1, 2, 3},            // Want
+		},
+	}
+
+	for _, test := range testcases {
+		t.Run(test.desc, func(t *testing.T) {
+			lb := newLookback(test.bins, time.Duration(test.bins))
+			for i, tick := range test.ticks {
+				lb.add(tick, test.values[i])
+				if got := lb.sum(tick); got != test.want[i] {
+					t.Errorf("sum for index %d got %d, want %d", i, got, test.want[i])
+				}
+			}
+		})
+	}
+}

balancer/rls/internal/keys/builder.go

@@ -0,0 +1,267 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package keys provides functionality required to build RLS request keys.
+package keys
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	"google.golang.org/grpc/metadata"
+)
+
+// BuilderMap maps from request path to the key builder for that path.
+type BuilderMap map[string]builder
+
+// MakeBuilderMap parses the provided RouteLookupConfig proto and returns a map
+// from paths to key builders.
+func MakeBuilderMap(cfg *rlspb.RouteLookupConfig) (BuilderMap, error) {
+	kbs := cfg.GetGrpcKeybuilders()
+	if len(kbs) == 0 {
+		return nil, errors.New("rls: RouteLookupConfig does not contain any GrpcKeyBuilder")
+	}
+
+	bm := make(map[string]builder)
+	for _, kb := range kbs {
+		// Extract keys from `headers`, `constant_keys` and `extra_keys` fields
+		// and populate appropriate values in the builder struct. Also ensure
+		// that keys are not repeated.
+		var matchers []matcher
+		seenKeys := make(map[string]bool)
+		constantKeys := kb.GetConstantKeys()
+		for k := range kb.GetConstantKeys() {
+			seenKeys[k] = true
+		}
+		for _, h := range kb.GetHeaders() {
+			if h.GetRequiredMatch() {
+				return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set {%+v}", kbs)
+			}
+			key := h.GetKey()
+			if seenKeys[key] {
+				return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q across headers, constant_keys and extra_keys {%+v}", key, kbs)
+			}
+			seenKeys[key] = true
+			matchers = append(matchers, matcher{key: h.GetKey(), names: h.GetNames()})
+		}
+		if seenKeys[kb.GetExtraKeys().GetHost()] {
+			return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetHost(), kbs)
+		}
+		if seenKeys[kb.GetExtraKeys().GetService()] {
+			return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetService(), kbs)
+		}
+		if seenKeys[kb.GetExtraKeys().GetMethod()] {
+			return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetMethod(), kbs)
+		}
+		b := builder{
+			headerKeys:   matchers,
+			constantKeys: constantKeys,
+			hostKey:      kb.GetExtraKeys().GetHost(),
+			serviceKey:   kb.GetExtraKeys().GetService(),
+			methodKey:    kb.GetExtraKeys().GetMethod(),
+		}
+
+		// Store the builder created above in the BuilderMap based on the value
+		// of the `Names` field, which wraps incoming request's service and
+		// method. Also, ensure that there are no repeated `Names` field.
+		names := kb.GetNames()
+		if len(names) == 0 {
+			return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig does not contain any Name {%+v}", kbs)
+		}
+		for _, name := range names {
+			if name.GetService() == "" {
+				return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains a Name field with no Service {%+v}", kbs)
+			}
+			if strings.Contains(name.GetMethod(), `/`) {
+				return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains a method with a slash {%+v}", kbs)
+			}
+			path := "/" + name.GetService() + "/" + name.GetMethod()
+			if _, ok := bm[path]; ok {
+				return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated Name field {%+v}", kbs)
+			}
+			bm[path] = b
+		}
+	}
+	return bm, nil
+}
+
+// KeyMap represents the RLS keys to be used for a request.
+type KeyMap struct {
+	// Map is the representation of an RLS key as a Go map. This is used when
+	// an actual RLS request is to be sent out on the wire, since the
+	// RouteLookupRequest proto expects a Go map.
+	Map map[string]string
+	// Str is the representation of an RLS key as a string, sorted by keys.
+	// Since the RLS keys are part of the cache key in the request cache
+	// maintained by the RLS balancer, and Go maps cannot be used as keys for
+	// Go maps (the cache is implemented as a map), we need a stringified
+	// version of it.
+	Str string
+}
+
+// RLSKey builds the RLS keys to be used for the given request, identified by
+// the request path and the request headers stored in metadata.
+func (bm BuilderMap) RLSKey(md metadata.MD, host, path string) KeyMap {
+	// The path passed in is of the form "/service/method". The keyBuilderMap is
+	// indexed with keys of the form "/service/" or "/service/method". The service
+	// that we set in the keyMap (to be sent out in the RLS request) should not
+	// include any slashes though.
+	i := strings.LastIndex(path, "/")
+	service, method := path[:i+1], path[i+1:]
+	b, ok := bm[path]
+	if !ok {
+		b, ok = bm[service]
+		if !ok {
+			return KeyMap{}
+		}
+	}
+
+	kvMap := b.buildHeaderKeys(md)
+	if b.hostKey != "" {
+		kvMap[b.hostKey] = host
+	}
+	if b.serviceKey != "" {
+		kvMap[b.serviceKey] = strings.Trim(service, "/")
+	}
+	if b.methodKey != "" {
+		kvMap[b.methodKey] = method
+	}
+	for k, v := range b.constantKeys {
+		kvMap[k] = v
+	}
+	return KeyMap{Map: kvMap, Str: mapToString(kvMap)}
+}
+
+// Equal reports whether bm and am represent equivalent BuilderMaps.
+func (bm BuilderMap) Equal(am BuilderMap) bool {
+	if (bm == nil) != (am == nil) {
+		return false
+	}
+	if len(bm) != len(am) {
+		return false
+	}
+
+	for key, bBuilder := range bm {
+		aBuilder, ok := am[key]
+		if !ok {
+			return false
+		}
+		if !bBuilder.Equal(aBuilder) {
+			return false
+		}
+	}
+	return true
+}
+
+// builder provides the actual functionality of building RLS keys.
+type builder struct {
+	headerKeys   []matcher
+	constantKeys map[string]string
+	// The following keys mirror corresponding fields in `extra_keys`.
+	hostKey    string
+	serviceKey string
+	methodKey  string
+}
+
+// Equal reports whether b and a represent equivalent key builders.
+func (b builder) Equal(a builder) bool {
+	if len(b.headerKeys) != len(a.headerKeys) {
+		return false
+	}
+	// Protobuf serialization maintains the order of repeated fields. Matchers
+	// are specified as a repeated field inside the KeyBuilder proto. If the
+	// order changes, it means that the order in the protobuf changed. We report
+	// this case as not being equal even though the builders could possibly be
+	// functionally equal.
+	for i, bMatcher := range b.headerKeys {
+		aMatcher := a.headerKeys[i]
+		if !bMatcher.Equal(aMatcher) {
+			return false
+		}
+	}
+
+	if len(b.constantKeys) != len(a.constantKeys) {
+		return false
+	}
+	for k, v := range b.constantKeys {
+		if a.constantKeys[k] != v {
+			return false
+		}
+	}
+
+	return b.hostKey == a.hostKey && b.serviceKey == a.serviceKey && b.methodKey == a.methodKey
+}
+
+// matcher helps extract a key from request headers based on a given name.
+type matcher struct {
+	// The key used in the keyMap sent as part of the RLS request.
+	key string
+	// List of header names which can supply the value for this key.
+	names []string
+}
+
+// Equal reports if m and a are equivalent headerKeys.
+func (m matcher) Equal(a matcher) bool {
+	if m.key != a.key {
+		return false
+	}
+	if len(m.names) != len(a.names) {
+		return false
+	}
+	for i := 0; i < len(m.names); i++ {
+		if m.names[i] != a.names[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func (b builder) buildHeaderKeys(md metadata.MD) map[string]string {
+	kvMap := make(map[string]string)
+	if len(md) == 0 {
+		return kvMap
+	}
+	for _, m := range b.headerKeys {
+		for _, name := range m.names {
+			if vals := md.Get(name); vals != nil {
+				kvMap[m.key] = strings.Join(vals, ",")
+				break
+			}
+		}
+	}
+	return kvMap
+}
+
+func mapToString(kv map[string]string) string {
+	keys := make([]string, 0, len(kv))
+	for k := range kv {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var sb strings.Builder
+	for i, k := range keys {
+		if i != 0 {
+			fmt.Fprint(&sb, ",")
+		}
+		fmt.Fprintf(&sb, "%s=%s", k, kv[k])
+	}
+	return sb.String()
+}

balancer/rls/internal/keys/builder_test.go

@@ -0,0 +1,650 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package keys
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	"google.golang.org/grpc/metadata"
+)
+
+var (
+	goodKeyBuilder1 = &rlspb.GrpcKeyBuilder{
+		Names: []*rlspb.GrpcKeyBuilder_Name{
+			{Service: "gFoo"},
+		},
+		Headers: []*rlspb.NameMatcher{
+			{Key: "k1", Names: []string{"n1"}},
+			{Key: "k2", Names: []string{"n1"}},
+		},
+		ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{
+			Host:    "host",
+			Service: "service",
+			Method:  "method",
+		},
+		ConstantKeys: map[string]string{
+			"const-key-1": "const-val-1",
+			"const-key-2": "const-val-2",
+		},
+	}
+	goodKeyBuilder2 = &rlspb.GrpcKeyBuilder{
+		Names: []*rlspb.GrpcKeyBuilder_Name{
+			{Service: "gBar", Method: "method1"},
+			{Service: "gFoobar"},
+		},
+		Headers: []*rlspb.NameMatcher{
+			{Key: "k1", Names: []string{"n1", "n2"}},
+		},
+	}
+)
+
+func TestMakeBuilderMap(t *testing.T) {
+	gFooBuilder := builder{
+		headerKeys: []matcher{{key: "k1", names: []string{"n1"}}, {key: "k2", names: []string{"n1"}}},
+		constantKeys: map[string]string{
+			"const-key-1": "const-val-1",
+			"const-key-2": "const-val-2",
+		},
+		hostKey:    "host",
+		serviceKey: "service",
+		methodKey:  "method",
+	}
+	wantBuilderMap1 := map[string]builder{"/gFoo/": gFooBuilder}
+	wantBuilderMap2 := map[string]builder{
+		"/gFoo/":        gFooBuilder,
+		"/gBar/method1": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}},
+		"/gFoobar/":     {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}},
+	}
+
+	tests := []struct {
+		desc           string
+		cfg            *rlspb.RouteLookupConfig
+		wantBuilderMap BuilderMap
+	}{
+		{
+			desc: "One good GrpcKeyBuilder",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1},
+			},
+			wantBuilderMap: wantBuilderMap1,
+		},
+		{
+			desc: "Two good GrpcKeyBuilders",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1, goodKeyBuilder2},
+			},
+			wantBuilderMap: wantBuilderMap2,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			builderMap, err := MakeBuilderMap(test.cfg)
+			if err != nil || !builderMap.Equal(test.wantBuilderMap) {
+				t.Errorf("MakeBuilderMap(%+v) returned {%v, %v}, want: {%v, nil}", test.cfg, builderMap, err, test.wantBuilderMap)
+			}
+		})
+	}
+}
+
+func TestMakeBuilderMapErrors(t *testing.T) {
+	tests := []struct {
+		desc          string
+		cfg           *rlspb.RouteLookupConfig
+		wantErrPrefix string
+	}{
+		{
+			desc:          "No GrpcKeyBuilder",
+			cfg:           &rlspb.RouteLookupConfig{},
+			wantErrPrefix: "rls: RouteLookupConfig does not contain any GrpcKeyBuilder",
+		},
+		{
+			desc: "Two GrpcKeyBuilders with same Name",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1, goodKeyBuilder1},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated Name field",
+		},
+		{
+			desc: "GrpcKeyBuilder with empty Service field",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+					{
+						Names: []*rlspb.GrpcKeyBuilder_Name{
+							{Service: "bFoo", Method: "method1"},
+							{Service: "bBar"},
+							{Method: "method1"},
+						},
+						Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}},
+					},
+					goodKeyBuilder1,
+				},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains a Name field with no Service",
+		},
+		{
+			desc: "GrpcKeyBuilder with no Name",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{{}, goodKeyBuilder1},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig does not contain any Name",
+		},
+		{
+			desc: "GrpcKeyBuilder with requiredMatch field set",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+					{
+						Names:   []*rlspb.GrpcKeyBuilder_Name{{Service: "bFoo", Method: "method1"}},
+						Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}, RequiredMatch: true}},
+					},
+					goodKeyBuilder1,
+				},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set",
+		},
+		{
+			desc: "GrpcKeyBuilder two headers with same key",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+					{
+						Names: []*rlspb.GrpcKeyBuilder_Name{
+							{Service: "gBar", Method: "method1"},
+							{Service: "gFoobar"},
+						},
+						Headers: []*rlspb.NameMatcher{
+							{Key: "k1", Names: []string{"n1", "n2"}},
+							{Key: "k1", Names: []string{"n1", "n2"}},
+						},
+					},
+					goodKeyBuilder1,
+				},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" across headers, constant_keys and extra_keys",
+		},
+		{
+			desc: "GrpcKeyBuilder repeated keys across headers and constant_keys",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+					{
+						Names: []*rlspb.GrpcKeyBuilder_Name{
+							{Service: "gBar", Method: "method1"},
+							{Service: "gFoobar"},
+						},
+						Headers:      []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}},
+						ConstantKeys: map[string]string{"k1": "v1"},
+					},
+					goodKeyBuilder1,
+				},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" across headers, constant_keys and extra_keys",
+		},
+		{
+			desc: "GrpcKeyBuilder repeated keys across headers and extra_keys",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+					{
+						Names: []*rlspb.GrpcKeyBuilder_Name{
+							{Service: "gBar", Method: "method1"},
+							{Service: "gFoobar"},
+						},
+						Headers:   []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}},
+						ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{Method: "k1"},
+					},
+					goodKeyBuilder1,
+				},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" in extra_keys from constant_keys or headers",
+		},
+		{
+			desc: "GrpcKeyBuilder repeated keys across constant_keys and extra_keys",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+					{
+						Names: []*rlspb.GrpcKeyBuilder_Name{
+							{Service: "gBar", Method: "method1"},
+							{Service: "gFoobar"},
+						},
+						Headers:      []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}},
+						ConstantKeys: map[string]string{"host": "v1"},
+						ExtraKeys:    &rlspb.GrpcKeyBuilder_ExtraKeys{Host: "host"},
+					},
+					goodKeyBuilder1,
+				},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"host\" in extra_keys from constant_keys or headers",
+		},
+		{
+			desc: "GrpcKeyBuilder with slash in method name",
+			cfg: &rlspb.RouteLookupConfig{
+				GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{
+					{
+						Names:   []*rlspb.GrpcKeyBuilder_Name{{Service: "gBar", Method: "method1/foo"}},
+						Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}},
+					},
+				},
+			},
+			wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains a method with a slash",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			builderMap, err := MakeBuilderMap(test.cfg)
+			if builderMap != nil || !strings.HasPrefix(fmt.Sprint(err), test.wantErrPrefix) {
+				t.Errorf("MakeBuilderMap(%+v) returned {%v, %v}, want: {nil, %v}", test.cfg, builderMap, err, test.wantErrPrefix)
+			}
+		})
+	}
+}
+
+func TestRLSKey(t *testing.T) {
+	bm, err := MakeBuilderMap(&rlspb.RouteLookupConfig{
+		GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1, goodKeyBuilder2},
+	})
+	if err != nil {
+		t.Fatalf("MakeBuilderMap() failed: %v", err)
+	}
+
+	tests := []struct {
+		desc   string
+		path   string
+		md     metadata.MD
+		wantKM KeyMap
+	}{
+		{
+			// No keyBuilder is found for the provided service.
+			desc:   "service not found in key builder map",
+			path:   "/notFoundService/method",
+			md:     metadata.Pairs("n1", "v1", "n2", "v2"),
+			wantKM: KeyMap{},
+		},
+		{
+			// No keyBuilder is found for the provided method.
+			desc:   "method not found in key builder map",
+			path:   "/gBar/notFoundMethod",
+			md:     metadata.Pairs("n1", "v1", "n2", "v2"),
+			wantKM: KeyMap{},
+		},
+		{
+			// A keyBuilder is found, but none of the headers match.
+			desc:   "directPathMatch-NoMatchingKey",
+			path:   "/gBar/method1",
+			md:     metadata.Pairs("notMatchingKey", "v1"),
+			wantKM: KeyMap{Map: map[string]string{}, Str: ""},
+		},
+		{
+			// A keyBuilder is found, and a single headers matches.
+			desc:   "directPathMatch-SingleKey",
+			path:   "/gBar/method1",
+			md:     metadata.Pairs("n1", "v1"),
+			wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"},
+		},
+		{
+			// A keyBuilder is found, and multiple headers match, but the first
+			// match is chosen.
+			desc:   "directPathMatch-FirstMatchingKey",
+			path:   "/gBar/method1",
+			md:     metadata.Pairs("n2", "v2", "n1", "v1"),
+			wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"},
+		},
+		{
+			// A keyBuilder is found as a wildcard match, but none of the
+			// headers match.
+			desc:   "wildcardPathMatch-NoMatchingKey",
+			path:   "/gFoobar/method1",
+			md:     metadata.Pairs("notMatchingKey", "v1"),
+			wantKM: KeyMap{Map: map[string]string{}, Str: ""},
+		},
+		{
+			// A keyBuilder is found as a wildcard match, and a single headers
+			// matches.
+			desc:   "wildcardPathMatch-SingleKey",
+			path:   "/gFoobar/method1",
+			md:     metadata.Pairs("n1", "v1"),
+			wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"},
+		},
+		{
+			// A keyBuilder is found as a wildcard match, and multiple headers
+			// match, but the first match is chosen.
+			desc:   "wildcardPathMatch-FirstMatchingKey",
+			path:   "/gFoobar/method1",
+			md:     metadata.Pairs("n2", "v2", "n1", "v1"),
+			wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"},
+		},
+		{
+			// Multiple headerKeys find hits in the provided request headers.
+			desc: "multipleMatchers",
+			path: "/gFoo/method1",
+			md:   metadata.Pairs("n2", "v2", "n1", "v1"),
+			wantKM: KeyMap{
+				Map: map[string]string{
+					"const-key-1": "const-val-1",
+					"const-key-2": "const-val-2",
+					"host":        "dummy-host",
+					"service":     "gFoo",
+					"method":      "method1",
+					"k1":          "v1",
+					"k2":          "v1",
+				},
+				Str: "const-key-1=const-val-1,const-key-2=const-val-2,host=dummy-host,k1=v1,k2=v1,method=method1,service=gFoo",
+			},
+		},
+		{
+			// A match is found for a header which is specified multiple times.
+			// So, the values are joined with commas separating them.
+			desc:   "commaSeparated",
+			path:   "/gBar/method1",
+			md:     metadata.Pairs("n1", "v1", "n1", "v2", "n1", "v3"),
+			wantKM: KeyMap{Map: map[string]string{"k1": "v1,v2,v3"}, Str: "k1=v1,v2,v3"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if gotKM := bm.RLSKey(test.md, "dummy-host", test.path); !cmp.Equal(gotKM, test.wantKM) {
+				t.Errorf("RLSKey(%+v, %s) = %+v, want %+v", test.md, test.path, gotKM, test.wantKM)
+			}
+		})
+	}
+}
+
+func TestMapToString(t *testing.T) {
+	tests := []struct {
+		desc    string
+		input   map[string]string
+		wantStr string
+	}{
+		{
+			desc:    "empty map",
+			input:   nil,
+			wantStr: "",
+		},
+		{
+			desc: "one key",
+			input: map[string]string{
+				"k1": "v1",
+			},
+			wantStr: "k1=v1",
+		},
+		{
+			desc: "sorted keys",
+			input: map[string]string{
+				"k1": "v1",
+				"k2": "v2",
+				"k3": "v3",
+			},
+			wantStr: "k1=v1,k2=v2,k3=v3",
+		},
+		{
+			desc: "unsorted keys",
+			input: map[string]string{
+				"k3": "v3",
+				"k1": "v1",
+				"k2": "v2",
+			},
+			wantStr: "k1=v1,k2=v2,k3=v3",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if gotStr := mapToString(test.input); gotStr != test.wantStr {
+				t.Errorf("mapToString(%v) = %s, want %s", test.input, gotStr, test.wantStr)
+			}
+		})
+	}
+}
+
+func TestBuilderMapEqual(t *testing.T) {
+	tests := []struct {
+		desc      string
+		a         BuilderMap
+		b         BuilderMap
+		wantEqual bool
+	}{
+		{
+			desc:      "nil builder maps",
+			a:         nil,
+			b:         nil,
+			wantEqual: true,
+		},
+		{
+			desc:      "empty builder maps",
+			a:         make(map[string]builder),
+			b:         make(map[string]builder),
+			wantEqual: true,
+		},
+		{
+			desc:      "nil and non-nil builder maps",
+			a:         nil,
+			b:         map[string]builder{"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "empty and non-empty builder maps",
+			a:         make(map[string]builder),
+			b:         map[string]builder{"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}},
+			wantEqual: false,
+		},
+		{
+			desc: "different number of map keys",
+			a: map[string]builder{
+				"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			b: map[string]builder{
+				"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: false,
+		},
+		{
+			desc: "different map keys",
+			a: map[string]builder{
+				"/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			b: map[string]builder{
+				"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: false,
+		},
+		{
+			desc: "equal keys different values",
+			a: map[string]builder{
+				"/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}},
+			},
+			b: map[string]builder{
+				"/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: false,
+		},
+		{
+			desc: "good match",
+			a: map[string]builder{
+				"/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			b: map[string]builder{
+				"/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+				"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}},
+			},
+			wantEqual: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual {
+				t.Errorf("BuilderMap.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual)
+			}
+		})
+	}
+}
+
+func TestBuilderEqual(t *testing.T) {
+	tests := []struct {
+		desc      string
+		a         builder
+		b         builder
+		wantEqual bool
+	}{
+		{
+			desc:      "nil builders",
+			a:         builder{headerKeys: nil},
+			b:         builder{headerKeys: nil},
+			wantEqual: true,
+		},
+		{
+			desc:      "empty builders",
+			a:         builder{headerKeys: []matcher{}},
+			b:         builder{headerKeys: []matcher{}},
+			wantEqual: true,
+		},
+		{
+			desc:      "empty and non-empty builders",
+			a:         builder{headerKeys: []matcher{}},
+			b:         builder{headerKeys: []matcher{{key: "foo"}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "different number of headerKeys",
+			a:         builder{headerKeys: []matcher{{key: "foo"}, {key: "bar"}}},
+			b:         builder{headerKeys: []matcher{{key: "foo"}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "equal number but differing headerKeys",
+			a:         builder{headerKeys: []matcher{{key: "bar"}}},
+			b:         builder{headerKeys: []matcher{{key: "foo"}}},
+			wantEqual: false,
+		},
+		{
+			desc:      "different number of constantKeys",
+			a:         builder{constantKeys: map[string]string{"k1": "v1"}},
+			b:         builder{constantKeys: map[string]string{"k1": "v1", "k2": "v2"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "equal number but differing constantKeys",
+			a:         builder{constantKeys: map[string]string{"k1": "v1"}},
+			b:         builder{constantKeys: map[string]string{"k2": "v2"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "different hostKey",
+			a:         builder{hostKey: "host1"},
+			b:         builder{hostKey: "host2"},
+			wantEqual: false,
+		},
+		{
+			desc:      "different serviceKey",
+			a:         builder{hostKey: "service1"},
+			b:         builder{hostKey: "service2"},
+			wantEqual: false,
+		},
+		{
+			desc:      "different methodKey",
+			a:         builder{hostKey: "method1"},
+			b:         builder{hostKey: "method2"},
+			wantEqual: false,
+		},
+		{
+			desc: "equal",
+			a: builder{
+				headerKeys:   []matcher{{key: "foo"}},
+				constantKeys: map[string]string{"k1": "v1"},
+				hostKey:      "host",
+				serviceKey:   "/service/",
+				methodKey:    "method",
+			},
+			b: builder{
+				headerKeys:   []matcher{{key: "foo"}},
+				constantKeys: map[string]string{"k1": "v1"},
+				hostKey:      "host",
+				serviceKey:   "/service/",
+				methodKey:    "method",
+			},
+			wantEqual: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			t.Run(test.desc, func(t *testing.T) {
+				if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual {
+					t.Errorf("builder.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual)
+				}
+			})
+		})
+	}
+}
+
+// matcher helps extract a key from request headers based on a given name.
+func TestMatcherEqual(t *testing.T) {
+	tests := []struct {
+		desc      string
+		a         matcher
+		b         matcher
+		wantEqual bool
+	}{
+		{
+			desc:      "different keys",
+			a:         matcher{key: "foo"},
+			b:         matcher{key: "bar"},
+			wantEqual: false,
+		},
+		{
+			desc:      "different number of names",
+			a:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			b:         matcher{key: "foo", names: []string{"v1"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "equal number but differing names",
+			a:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			b:         matcher{key: "foo", names: []string{"v1", "v22"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "same names in different order",
+			a:         matcher{key: "foo", names: []string{"v2", "v1"}},
+			b:         matcher{key: "foo", names: []string{"v1", "v3"}},
+			wantEqual: false,
+		},
+		{
+			desc:      "good match",
+			a:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			b:         matcher{key: "foo", names: []string{"v1", "v2"}},
+			wantEqual: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual {
+				t.Errorf("matcher.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual)
+			}
+		})
+	}
+}

balancer/rls/internal/test/e2e/e2e.go

@@ -0,0 +1,20 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package e2e contains utilities for end-to-end RouteLookupService tests.
+package e2e

balancer/rls/internal/test/e2e/rls_child_policy.go

@@ -0,0 +1,131 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/pickfirst"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const (
+	// RLSChildPolicyTargetNameField is a top-level field name to add to the child
+	// policy's config, whose value is set to the target for the child policy.
+	RLSChildPolicyTargetNameField = "Backend"
+	// RLSChildPolicyBadTarget is a value which is considered a bad target by the
+	// child policy. This is useful to test bad child policy configuration.
+	RLSChildPolicyBadTarget = "bad-target"
+)
+
+// ErrParseConfigBadTarget is the error returned from ParseConfig when the
+// backend field is set to RLSChildPolicyBadTarget.
+var ErrParseConfigBadTarget = errors.New("backend field set to RLSChildPolicyBadTarget")
+
+// BalancerFuncs is a set of callbacks which get invoked when the corresponding
+// method on the child policy is invoked.
+type BalancerFuncs struct {
+	UpdateClientConnState func(cfg *RLSChildPolicyConfig) error
+	Close                 func()
+}
+
+// RegisterRLSChildPolicy registers a balancer builder with the given name, to
+// be used as a child policy for the RLS LB policy.
+//
+// The child policy uses a pickfirst balancer under the hood to send all traffic
+// to the single backend specified by the `RLSChildPolicyTargetNameField` field
+// in its configuration which looks like: {"Backend": "Backend-address"}.
+func RegisterRLSChildPolicy(name string, bf *BalancerFuncs) {
+	balancer.Register(bb{name: name, bf: bf})
+}
+
+type bb struct {
+	name string
+	bf   *BalancerFuncs
+}
+
+func (bb bb) Name() string { return bb.name }
+
+func (bb bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	pf := balancer.Get(pickfirst.Name)
+	b := &bal{
+		Balancer: pf.Build(cc, opts),
+		bf:       bb.bf,
+		done:     grpcsync.NewEvent(),
+	}
+	go b.run()
+	return b
+}
+
+func (bb bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	cfg := &RLSChildPolicyConfig{}
+	if err := json.Unmarshal(c, cfg); err != nil {
+		return nil, err
+	}
+	if cfg.Backend == RLSChildPolicyBadTarget {
+		return nil, ErrParseConfigBadTarget
+	}
+	return cfg, nil
+}
+
+type bal struct {
+	balancer.Balancer
+	bf   *BalancerFuncs
+	done *grpcsync.Event
+}
+
+// RLSChildPolicyConfig is the LB config for the test child policy.
+type RLSChildPolicyConfig struct {
+	serviceconfig.LoadBalancingConfig
+	Backend string // The target for which this child policy was created.
+	Random  string // A random field to test child policy config changes.
+}
+
+func (b *bal) UpdateClientConnState(c balancer.ClientConnState) error {
+	cfg, ok := c.BalancerConfig.(*RLSChildPolicyConfig)
+	if !ok {
+		return fmt.Errorf("received balancer config of type %T, want %T", c.BalancerConfig, &RLSChildPolicyConfig{})
+	}
+	if b.bf != nil && b.bf.UpdateClientConnState != nil {
+		b.bf.UpdateClientConnState(cfg)
+	}
+	return b.Balancer.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Addresses: []resolver.Address{{Addr: cfg.Backend}}},
+	})
+}
+
+func (b *bal) Close() {
+	b.Balancer.Close()
+	if b.bf != nil && b.bf.Close != nil {
+		b.bf.Close()
+	}
+	b.done.Fire()
+}
+
+// run is a dummy goroutine to make sure that child policies are closed at the
+// end of tests. If they are not closed, these goroutines will be picked up by
+// the leak checker and tests will fail.
+func (b *bal) run() {
+	<-b.done.Done()
+}

balancer/rls/internal/test/e2e/rls_lb_config.go

@@ -0,0 +1,103 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e
+
+import (
+	"errors"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+// RLSConfig is a utility type to build service config for the RLS LB policy.
+type RLSConfig struct {
+	RouteLookupConfig                *rlspb.RouteLookupConfig
+	RouteLookupChannelServiceConfig  string
+	ChildPolicy                      *internalserviceconfig.BalancerConfig
+	ChildPolicyConfigTargetFieldName string
+}
+
+// ServiceConfigJSON generates service config with a load balancing config
+// corresponding to the RLS LB policy.
+func (c *RLSConfig) ServiceConfigJSON() (string, error) {
+	m := protojson.MarshalOptions{
+		Multiline:     true,
+		Indent:        "  ",
+		UseProtoNames: true,
+	}
+	routeLookupCfg, err := m.Marshal(c.RouteLookupConfig)
+	if err != nil {
+		return "", err
+	}
+	childPolicy, err := c.ChildPolicy.MarshalJSON()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf(`
+{
+  "loadBalancingConfig": [
+    {
+      "rls_experimental": {
+        "routeLookupConfig": %s,
+				"routeLookupChannelServiceConfig": %s,
+        "childPolicy": %s,
+        "childPolicyConfigTargetFieldName": %q
+      }
+    }
+  ]
+}`, string(routeLookupCfg), c.RouteLookupChannelServiceConfig, string(childPolicy), c.ChildPolicyConfigTargetFieldName), nil
+}
+
+// LoadBalancingConfig generates load balancing config which can used as part of
+// a ClientConnState update to the RLS LB policy.
+func (c *RLSConfig) LoadBalancingConfig() (serviceconfig.LoadBalancingConfig, error) {
+	m := protojson.MarshalOptions{
+		Multiline:     true,
+		Indent:        "  ",
+		UseProtoNames: true,
+	}
+	routeLookupCfg, err := m.Marshal(c.RouteLookupConfig)
+	if err != nil {
+		return nil, err
+	}
+	childPolicy, err := c.ChildPolicy.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+	lbConfigJSON := fmt.Sprintf(`
+{
+  "routeLookupConfig": %s,
+  "routeLookupChannelServiceConfig": %s,
+  "childPolicy": %s,
+  "childPolicyConfigTargetFieldName": %q
+}`, string(routeLookupCfg), c.RouteLookupChannelServiceConfig, string(childPolicy), c.ChildPolicyConfigTargetFieldName)
+
+	builder := balancer.Get("rls_experimental")
+	if builder == nil {
+		return nil, errors.New("balancer builder not found for RLS LB policy")
+	}
+	parser := builder.(balancer.ConfigParser)
+	return parser.ParseConfig([]byte(lbConfigJSON))
+}

balancer/rls/metrics_test.go

@@ -0,0 +1,375 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rls
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/google/uuid"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	estats "google.golang.org/grpc/experimental/stats"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	"google.golang.org/grpc/internal/stubserver"
+	rlstest "google.golang.org/grpc/internal/testutils/rls"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/stats/opentelemetry"
+)
+
+func metricsDataFromReader(ctx context.Context, reader *metric.ManualReader) map[string]metricdata.Metrics {
+	rm := &metricdata.ResourceMetrics{}
+	reader.Collect(ctx, rm)
+	gotMetrics := map[string]metricdata.Metrics{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, m := range sm.Metrics {
+			gotMetrics[m.Name] = m
+		}
+	}
+	return gotMetrics
+}
+
+// TestRLSTargetPickMetric tests RLS Metrics in the case an RLS Balancer picks a
+// target from an RLS Response for a RPC. This should emit a
+// "grpc.lb.rls.target_picks" with certain labels and cache metrics with certain
+// labels.
+func (s) TestRLSTargetPickMetric(t *testing.T) {
+	// Overwrite the uuid random number generator to be deterministic.
+	uuid.SetRand(rand.New(rand.NewSource(1)))
+	defer uuid.SetRand(nil)
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil)
+	rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address)
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started TestService backend at: %q", backend.Address)
+	defer backend.Stop()
+
+	rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse {
+		return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backend.Address}}}
+	})
+	r := startManualResolverWithConfig(t, rlsConfig)
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	mo := opentelemetry.MetricsOptions{
+		MeterProvider:  provider,
+		Metrics:        opentelemetry.DefaultMetrics().Add("grpc.lb.rls.cache_entries", "grpc.lb.rls.cache_size", "grpc.lb.rls.default_target_picks", "grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"),
+		OptionalLabels: []string{"grpc.client.call.custom"},
+	}
+	grpcTarget := r.Scheme() + ":///"
+	cc, err := grpc.NewClient(grpcTarget, grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo}))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	wantMetrics := []metricdata.Metrics{
+		{
+			Name:        "grpc.lb.rls.target_picks",
+			Description: "EXPERIMENTAL. Number of LB picks sent to each RLS target. Note that if the default target is also returned by the RLS server, RPCs sent to that target from the cache will be counted in this metric, not in grpc.rls.default_target_picks.",
+			Unit:        "{pick}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.data_plane_target", backend.Address), attribute.String("grpc.lb.pick_result", "complete"), attribute.String("grpc.client.call.custom", "target-pick-custom")),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+
+		// Receives an empty RLS Response, so a single cache entry with no size.
+		{
+			Name:        "grpc.lb.rls.cache_entries",
+			Description: "EXPERIMENTAL. Number of entries in the RLS cache.",
+			Unit:        "{entry}",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")),
+						Value:      1,
+					},
+				},
+			},
+		},
+		{
+			Name:        "grpc.lb.rls.cache_size",
+			Description: "EXPERIMENTAL. The current size of the RLS cache.",
+			Unit:        "By",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")),
+						Value:      35,
+					},
+				},
+			},
+		},
+	}
+	client := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx = estats.NewContextWithCustomLabel(ctx, "target-pick-custom")
+	_, err = client.EmptyCall(ctx, &testpb.Empty{})
+	if err != nil {
+		t.Fatalf("client.EmptyCall failed with error: %v", err)
+	}
+
+	gotMetrics := metricsDataFromReader(ctx, reader)
+	for _, metric := range wantMetrics {
+		val, ok := gotMetrics[metric.Name]
+		if !ok {
+			t.Fatalf("Metric %v not present in recorded metrics", metric.Name)
+		}
+		if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) {
+			t.Fatalf("Metrics data type not equal for metric: %v", metric.Name)
+		}
+	}
+
+	// Only one pick was made, which was a target pick, so no default target
+	// pick or failed pick metric should emit.
+	for _, metric := range []string{"grpc.lb.rls.default_target_picks", "grpc.lb.rls.failed_picks"} {
+		if _, ok := gotMetrics[metric]; ok {
+			t.Fatalf("Metric %v present in recorded metrics", metric)
+		}
+	}
+}
+
+// TestRLSDefaultTargetPickMetric tests RLS Metrics in the case an RLS Balancer
+// falls back to the default target for an RPC. This should emit a
+// "grpc.lb.rls.default_target_picks" with certain labels and cache metrics with
+// certain labels.
+func (s) TestRLSDefaultTargetPickMetric(t *testing.T) {
+	// Overwrite the uuid random number generator to be deterministic.
+	uuid.SetRand(rand.New(rand.NewSource(1)))
+	defer uuid.SetRand(nil)
+
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil)
+	// Build RLS service config with a default target.
+	rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address)
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started TestService backend at: %q", backend.Address)
+	defer backend.Stop()
+	rlsConfig.RouteLookupConfig.DefaultTarget = backend.Address
+
+	r := startManualResolverWithConfig(t, rlsConfig)
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	mo := opentelemetry.MetricsOptions{
+		MeterProvider:  provider,
+		Metrics:        opentelemetry.DefaultMetrics().Add("grpc.lb.rls.cache_entries", "grpc.lb.rls.cache_size", "grpc.lb.rls.default_target_picks", "grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"),
+		OptionalLabels: []string{"grpc.client.call.custom"},
+	}
+	grpcTarget := r.Scheme() + ":///"
+	cc, err := grpc.NewClient(grpcTarget, grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo}))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	wantMetrics := []metricdata.Metrics{
+		{
+			Name:        "grpc.lb.rls.default_target_picks",
+			Description: "EXPERIMENTAL. Number of LB picks sent to the default target.",
+			Unit:        "{pick}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.data_plane_target", backend.Address), attribute.String("grpc.lb.pick_result", "complete"), attribute.String("grpc.client.call.custom", "default-target-pick-custom")),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		// Receives a RLS Response with target information, so a single cache
+		// entry with a certain size.
+		{
+			Name:        "grpc.lb.rls.cache_entries",
+			Description: "EXPERIMENTAL. Number of entries in the RLS cache.",
+			Unit:        "{entry}",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")),
+						Value:      1,
+					},
+				},
+			},
+		},
+		{
+			Name:        "grpc.lb.rls.cache_size",
+			Description: "EXPERIMENTAL. The current size of the RLS cache.",
+			Unit:        "By",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")),
+						Value:      0,
+					},
+				},
+			},
+		},
+	}
+	client := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx = estats.NewContextWithCustomLabel(ctx, "default-target-pick-custom")
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("client.EmptyCall failed with error: %v", err)
+	}
+
+	gotMetrics := metricsDataFromReader(ctx, reader)
+	for _, metric := range wantMetrics {
+		val, ok := gotMetrics[metric.Name]
+		if !ok {
+			t.Fatalf("Metric %v not present in recorded metrics", metric.Name)
+		}
+		if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) {
+			t.Fatalf("Metrics data type not equal for metric: %v", metric.Name)
+		}
+	}
+	// No target picks and failed pick metrics should be emitted, as the test
+	// made only one RPC which recorded as a default target pick.
+	for _, metric := range []string{"grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"} {
+		if _, ok := gotMetrics[metric]; ok {
+			t.Fatalf("Metric %v present in recorded metrics", metric)
+		}
+	}
+}
+
+// TestRLSFailedRPCMetric tests RLS Metrics in the case an RLS Balancer fails an
+// RPC due to an RLS failure. This should emit a
+// "grpc.lb.rls.default_target_picks" with certain labels and cache metrics with
+// certain labels.
+func (s) TestRLSFailedRPCMetric(t *testing.T) {
+	// Overwrite the uuid random number generator to be deterministic.
+	uuid.SetRand(rand.New(rand.NewSource(1)))
+	defer uuid.SetRand(nil)
+
+	rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil)
+	// Build an RLS config without a default target.
+	rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address)
+	// Register a manual resolver and push the RLS service config through it.
+	r := startManualResolverWithConfig(t, rlsConfig)
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	mo := opentelemetry.MetricsOptions{
+		MeterProvider:  provider,
+		Metrics:        opentelemetry.DefaultMetrics().Add("grpc.lb.rls.cache_entries", "grpc.lb.rls.cache_size", "grpc.lb.rls.default_target_picks", "grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"),
+		OptionalLabels: []string{"grpc.client.call.custom"},
+	}
+	grpcTarget := r.Scheme() + ":///"
+	cc, err := grpc.NewClient(grpcTarget, grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo}))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	wantMetrics := []metricdata.Metrics{
+		{
+			Name:        "grpc.lb.rls.failed_picks",
+			Description: "EXPERIMENTAL. Number of LB picks failed due to either a failed RLS request or the RLS channel being throttled.",
+			Unit:        "{pick}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.client.call.custom", "failed-pick-custom")),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		// Receives an empty RLS Response, so a single cache entry with no size.
+		{
+			Name:        "grpc.lb.rls.cache_entries",
+			Description: "EXPERIMENTAL. Number of entries in the RLS cache.",
+			Unit:        "{entry}",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")),
+						Value:      1,
+					},
+				},
+			},
+		},
+		{
+			Name:        "grpc.lb.rls.cache_size",
+			Description: "EXPERIMENTAL. The current size of the RLS cache.",
+			Unit:        "By",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")),
+						Value:      0,
+					},
+				},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx = estats.NewContextWithCustomLabel(ctx, "failed-pick-custom")
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatalf("client.EmptyCall error = %v, expected a non nil error", err)
+	}
+
+	gotMetrics := metricsDataFromReader(ctx, reader)
+	for _, metric := range wantMetrics {
+		val, ok := gotMetrics[metric.Name]
+		if !ok {
+			t.Fatalf("Metric %v not present in recorded metrics", metric.Name)
+		}
+		if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) {
+			t.Fatalf("Metrics data type not equal for metric: %v", metric.Name)
+		}
+	}
+	// Only one RPC was made, which was a failed pick due to an RLS failure, so
+	// no metrics for target picks or default target picks should have emitted.
+	for _, metric := range []string{"grpc.lb.rls.target_picks", "grpc.lb.rls.default_target_picks"} {
+		if _, ok := gotMetrics[metric]; ok {
+			t.Fatalf("Metric %v present in recorded metrics", metric)
+		}
+	}
+}

balancer/rls/picker.go

@@ -0,0 +1,401 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package rls
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/rls/internal/keys"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	estats "google.golang.org/grpc/experimental/stats"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+)
+
+var (
+	errRLSThrottled = errors.New("RLS call throttled at client side")
+
+	// Function to compute data cache entry size.
+	computeDataCacheEntrySize = dcEntrySize
+)
+
+// exitIdler wraps the only method on the BalancerGroup that the picker calls.
+type exitIdler interface {
+	ExitIdleOne(id string)
+}
+
+// rlsPicker selects the subConn to be used for a particular RPC. It does not
+// manage subConns directly and delegates to pickers provided by child policies.
+type rlsPicker struct {
+	// The keyBuilder map used to generate RLS keys for the RPC. This is built
+	// by the LB policy based on the received ServiceConfig.
+	kbm keys.BuilderMap
+	// Endpoint from the user's original dial target. Used to set the `host_key`
+	// field in `extra_keys`.
+	origEndpoint string
+
+	lb *rlsBalancer
+
+	// The picker is given its own copy of the below fields from the RLS LB policy
+	// to avoid having to grab the mutex on the latter.
+	rlsServerTarget string
+	grpcTarget      string
+	metricsRecorder estats.MetricsRecorder
+	defaultPolicy   *childPolicyWrapper // Child policy for the default target.
+	ctrlCh          *controlChannel     // Control channel to the RLS server.
+	maxAge          time.Duration       // Cache max age from LB config.
+	staleAge        time.Duration       // Cache stale age from LB config.
+	bg              exitIdler
+	logger          *internalgrpclog.PrefixLogger
+}
+
+// isFullMethodNameValid return true if name is of the form `/service/method`.
+func isFullMethodNameValid(name string) bool {
+	return strings.HasPrefix(name, "/") && strings.Count(name, "/") == 2
+}
+
+// Pick makes the routing decision for every outbound RPC.
+func (p *rlsPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	if name := info.FullMethodName; !isFullMethodNameValid(name) {
+		return balancer.PickResult{}, fmt.Errorf("rls: method name %q is not of the form '/service/method", name)
+	}
+
+	// Build the request's keys using the key builders from LB config.
+	md, _ := metadata.FromOutgoingContext(info.Ctx)
+	reqKeys := p.kbm.RLSKey(md, p.origEndpoint, info.FullMethodName)
+
+	p.lb.cacheMu.Lock()
+	var pr balancer.PickResult
+	var err error
+
+	// Record metrics without the cache mutex held, to prevent lock contention
+	// between concurrent RPC's and their Pick calls. Metrics Recording can
+	// potentially be expensive.
+	metricsCallback := func() {}
+	defer func() {
+		p.lb.cacheMu.Unlock()
+		metricsCallback()
+	}()
+
+	// Lookup data cache and pending request map using request path and keys.
+	cacheKey := cacheKey{path: info.FullMethodName, keys: reqKeys.Str}
+	dcEntry := p.lb.dataCache.getEntry(cacheKey)
+	pendingEntry := p.lb.pendingMap[cacheKey]
+	now := time.Now()
+
+	switch {
+	// No data cache entry. No pending request.
+	case dcEntry == nil && pendingEntry == nil:
+		throttled := p.sendRouteLookupRequestLocked(cacheKey, &backoffState{bs: defaultBackoffStrategy}, reqKeys.Map, rlspb.RouteLookupRequest_REASON_MISS, "")
+		if throttled {
+			pr, metricsCallback, err = p.useDefaultPickIfPossible(info, errRLSThrottled)
+			return pr, err
+		}
+		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+
+	// No data cache entry. Pending request exits.
+	case dcEntry == nil && pendingEntry != nil:
+		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+
+	// Data cache hit. No pending request.
+	case dcEntry != nil && pendingEntry == nil:
+		if dcEntry.expiryTime.After(now) {
+			if !dcEntry.staleTime.IsZero() && dcEntry.staleTime.Before(now) && dcEntry.backoffTime.Before(now) {
+				p.sendRouteLookupRequestLocked(cacheKey, dcEntry.backoffState, reqKeys.Map, rlspb.RouteLookupRequest_REASON_STALE, dcEntry.headerData)
+			}
+			// Delegate to child policies.
+			pr, metricsCallback, err = p.delegateToChildPoliciesLocked(dcEntry, info)
+			return pr, err
+		}
+
+		// We get here only if the data cache entry has expired. If entry is in
+		// backoff, delegate to default target or fail the pick.
+		if dcEntry.backoffState != nil && dcEntry.backoffTime.After(now) {
+			// Avoid propagating the status code received on control plane RPCs to the
+			// data plane which can lead to unexpected outcomes as we do not control
+			// the status code sent by the control plane. Propagating the status
+			// message received from the control plane is still fine, as it could be
+			// useful for debugging purposes.
+			st := dcEntry.status
+			pr, metricsCallback, err = p.useDefaultPickIfPossible(info, status.Error(codes.Unavailable, fmt.Sprintf("most recent error from RLS server: %v", st.Error())))
+			return pr, err
+		}
+
+		// We get here only if the entry has expired and is not in backoff.
+		throttled := p.sendRouteLookupRequestLocked(cacheKey, dcEntry.backoffState, reqKeys.Map, rlspb.RouteLookupRequest_REASON_MISS, "")
+		if throttled {
+			pr, metricsCallback, err = p.useDefaultPickIfPossible(info, errRLSThrottled)
+			return pr, err
+		}
+		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+
+	// Data cache hit. Pending request exists.
+	default:
+		if dcEntry.expiryTime.After(now) {
+			pr, metricsCallback, err = p.delegateToChildPoliciesLocked(dcEntry, info)
+			return pr, err
+		}
+		// Data cache entry has expired and pending request exists. Queue pick.
+		return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+	}
+}
+
+// errToPickResult is a helper function which converts the error value returned
+// by Pick() to a string that represents the pick result.
+func errToPickResult(err error) string {
+	if err == nil {
+		return "complete"
+	}
+	if errors.Is(err, balancer.ErrNoSubConnAvailable) {
+		return "queue"
+	}
+	if _, ok := status.FromError(err); ok {
+		return "drop"
+	}
+	return "fail"
+}
+
+// delegateToChildPoliciesLocked is a helper function which iterates through the
+// list of child policy wrappers in a cache entry and attempts to find a child
+// policy to which this RPC can be routed to. If all child policies are in
+// TRANSIENT_FAILURE, we delegate to the last child policy arbitrarily. Returns
+// a function to be invoked to record metrics.
+func (p *rlsPicker) delegateToChildPoliciesLocked(dcEntry *cacheEntry, info balancer.PickInfo) (balancer.PickResult, func(), error) {
+	const rlsDataHeaderName = "x-google-rls-data"
+	for i, cpw := range dcEntry.childPolicyWrappers {
+		state := (*balancer.State)(atomic.LoadPointer(&cpw.state))
+		// Delegate to the child policy if it is not in TRANSIENT_FAILURE, or if
+		// it is the last one (which handles the case of delegating to the last
+		// child picker if all child policies are in TRANSIENT_FAILURE).
+		if state.ConnectivityState != connectivity.TransientFailure || i == len(dcEntry.childPolicyWrappers)-1 {
+			// Any header data received from the RLS server is stored in the
+			// cache entry and needs to be sent to the actual backend in the
+			// X-Google-RLS-Data header.
+			res, err := state.Picker.Pick(info)
+			if err != nil {
+				pr := errToPickResult(err)
+				customLabel := estats.CustomLabelFromContext(info.Ctx)
+				return res, func() {
+					if pr == "queue" {
+						// Don't record metrics for queued Picks.
+						return
+					}
+					targetPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, cpw.target, pr, customLabel)
+				}, err
+			}
+
+			if res.Metadata == nil {
+				res.Metadata = metadata.Pairs(rlsDataHeaderName, dcEntry.headerData)
+			} else {
+				res.Metadata.Append(rlsDataHeaderName, dcEntry.headerData)
+			}
+			return res, func() {
+				customLabel := estats.CustomLabelFromContext(info.Ctx)
+				targetPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, cpw.target, "complete", customLabel)
+			}, nil
+		}
+	}
+
+	// In the unlikely event that we have a cache entry with no targets, we end up
+	// queueing the RPC.
+	return balancer.PickResult{}, func() {}, balancer.ErrNoSubConnAvailable
+}
+
+// useDefaultPickIfPossible is a helper method which delegates to the default
+// target if one is configured, or fails the pick with the given error. Returns
+// a function to be invoked to record metrics.
+func (p *rlsPicker) useDefaultPickIfPossible(info balancer.PickInfo, errOnNoDefault error) (balancer.PickResult, func(), error) {
+	customLabel := estats.CustomLabelFromContext(info.Ctx)
+
+	if p.defaultPolicy != nil {
+		state := (*balancer.State)(atomic.LoadPointer(&p.defaultPolicy.state))
+		res, err := state.Picker.Pick(info)
+		pr := errToPickResult(err)
+		return res, func() {
+			if pr == "queue" {
+				// Don't record metrics for queued Picks.
+				return
+			}
+			defaultTargetPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, p.defaultPolicy.target, pr, customLabel)
+		}, err
+	}
+
+	return balancer.PickResult{}, func() {
+		failedPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, customLabel)
+	}, errOnNoDefault
+}
+
+// sendRouteLookupRequestLocked adds an entry to the pending request map and
+// sends out an RLS request using the passed in arguments. Returns a value
+// indicating if the request was throttled by the client-side adaptive
+// throttler.
+func (p *rlsPicker) sendRouteLookupRequestLocked(cacheKey cacheKey, bs *backoffState, reqKeys map[string]string, reason rlspb.RouteLookupRequest_Reason, staleHeaders string) bool {
+	if p.lb.pendingMap[cacheKey] != nil {
+		return false
+	}
+
+	p.lb.pendingMap[cacheKey] = bs
+	throttled := p.ctrlCh.lookup(reqKeys, reason, staleHeaders, func(targets []string, headerData string, err error) {
+		p.handleRouteLookupResponse(cacheKey, targets, headerData, err)
+	})
+	if throttled {
+		delete(p.lb.pendingMap, cacheKey)
+	}
+	return throttled
+}
+
+// handleRouteLookupResponse is the callback invoked by the control channel upon
+// receipt of an RLS response. Modifies the data cache and pending requests map
+// and sends a new picker.
+//
+// Acquires the write-lock on the cache. Caller must not hold p.lb.cacheMu.
+func (p *rlsPicker) handleRouteLookupResponse(cacheKey cacheKey, targets []string, headerData string, err error) {
+	p.logger.Infof("Received RLS response for key %+v with targets %+v, headerData %q, err: %v", cacheKey, targets, headerData, err)
+
+	p.lb.cacheMu.Lock()
+	defer func() {
+		// Pending request map entry is unconditionally deleted since the request is
+		// no longer pending.
+		p.logger.Infof("Removing pending request entry for key %+v", cacheKey)
+		delete(p.lb.pendingMap, cacheKey)
+		p.lb.sendNewPicker()
+		p.lb.cacheMu.Unlock()
+	}()
+
+	// Lookup the data cache entry or create a new one.
+	dcEntry := p.lb.dataCache.getEntry(cacheKey)
+	if dcEntry == nil {
+		dcEntry = &cacheEntry{}
+		if _, ok := p.lb.dataCache.addEntry(cacheKey, dcEntry); !ok {
+			// This is a very unlikely case where we are unable to add a
+			// data cache entry. Log and leave.
+			p.logger.Warningf("Failed to add data cache entry for %+v", cacheKey)
+			return
+		}
+	}
+
+	// For failed requests, the data cache entry is modified as follows:
+	// - status is set to error returned from the control channel
+	// - current backoff state is available in the pending entry
+	//   - `retries` field is incremented and
+	//   - backoff state is moved to the data cache
+	// - backoffTime is set to the time indicated by the backoff state
+	// - backoffExpirationTime is set to twice the backoff time
+	// - backoffTimer is set to fire after backoffTime
+	//
+	// When a proactive cache refresh fails, this would leave the targets and the
+	// expiry time from the old entry unchanged. And this mean that the old valid
+	// entry would be used until expiration, and a new picker would be sent upon
+	// backoff expiry.
+	now := time.Now()
+
+	// "An RLS request is considered to have failed if it returns a non-OK
+	// status or the RLS response's targets list is non-empty." - RLS LB Policy
+	// design.
+	if len(targets) == 0 && err == nil {
+		err = fmt.Errorf("RLS response's target list does not contain any entries for key %+v", cacheKey)
+		// If err is set, rpc error from the control plane and no control plane
+		// configuration is why no targets were passed into this helper, no need
+		// to specify and tell the user this information.
+	}
+	if err != nil {
+		dcEntry.status = err
+		pendingEntry := p.lb.pendingMap[cacheKey]
+		pendingEntry.retries++
+		backoffTime := pendingEntry.bs.Backoff(pendingEntry.retries)
+		dcEntry.backoffState = pendingEntry
+		dcEntry.backoffTime = now.Add(backoffTime)
+		dcEntry.backoffExpiryTime = now.Add(2 * backoffTime)
+		if dcEntry.backoffState.timer != nil {
+			dcEntry.backoffState.timer.Stop()
+		}
+		dcEntry.backoffState.timer = time.AfterFunc(backoffTime, p.lb.sendNewPicker)
+		return
+	}
+
+	// For successful requests, the cache entry is modified as follows:
+	// - childPolicyWrappers is set to point to the child policy wrappers
+	//   associated with the targets specified in the received response
+	// - headerData is set to the value received in the response
+	// - expiryTime, stateTime and earliestEvictionTime are set
+	// - status is set to nil (OK status)
+	// - backoff state is cleared
+	p.setChildPolicyWrappersInCacheEntry(dcEntry, targets)
+	dcEntry.headerData = headerData
+	dcEntry.expiryTime = now.Add(p.maxAge)
+	if p.staleAge != 0 {
+		dcEntry.staleTime = now.Add(p.staleAge)
+	}
+	dcEntry.earliestEvictTime = now.Add(minEvictDuration)
+	dcEntry.status = nil
+	dcEntry.backoffState = &backoffState{bs: defaultBackoffStrategy}
+	dcEntry.backoffTime = time.Time{}
+	dcEntry.backoffExpiryTime = time.Time{}
+	p.lb.dataCache.updateEntrySize(dcEntry, computeDataCacheEntrySize(cacheKey, dcEntry))
+}
+
+// setChildPolicyWrappersInCacheEntry sets up the childPolicyWrappers field in
+// the cache entry to point to the child policy wrappers for the targets
+// specified in the RLS response.
+//
+// Caller must hold a write-lock on p.lb.cacheMu.
+func (p *rlsPicker) setChildPolicyWrappersInCacheEntry(dcEntry *cacheEntry, newTargets []string) {
+	// If the childPolicyWrappers field is already pointing to the right targets,
+	// then the field's value does not need to change.
+	targetsChanged := true
+	func() {
+		if cpws := dcEntry.childPolicyWrappers; cpws != nil {
+			if len(newTargets) != len(cpws) {
+				return
+			}
+			for i, target := range newTargets {
+				if cpws[i].target != target {
+					return
+				}
+			}
+			targetsChanged = false
+		}
+	}()
+	if !targetsChanged {
+		return
+	}
+
+	// If the childPolicyWrappers field is not already set to the right targets,
+	// then it must be reset. We construct a new list of child policies and
+	// then swap out the old list for the new one.
+	newChildPolicies := p.lb.acquireChildPolicyReferences(newTargets)
+	oldChildPolicyTargets := make([]string, len(dcEntry.childPolicyWrappers))
+	for i, cpw := range dcEntry.childPolicyWrappers {
+		oldChildPolicyTargets[i] = cpw.target
+	}
+	p.lb.releaseChildPolicyReferences(oldChildPolicyTargets)
+	dcEntry.childPolicyWrappers = newChildPolicies
+}
+
+func dcEntrySize(key cacheKey, entry *cacheEntry) int64 {
+	return int64(len(key.path) + len(key.keys) + len(entry.headerData))
+}